├── .gitignore ├── LICENSE ├── README.md ├── configs ├── NAS-PTB-BASE.config ├── NAS-WT2-BASE.config ├── cos1800.config ├── cos600.config ├── nas-cifar-cos-cut.config ├── nas-cifar-cos-cutB128.config ├── nas-cifar-cos-cutB64.config ├── nas-cifar-cos-cutB96.config ├── nas-cifar-cos-cutW1.config ├── nas-cifar-cos-cutW3.config ├── nas-cifar-cos-cutW5.config ├── nas-cifar-cos-nocut.config ├── nas-imagenet-B128.config ├── nas-imagenet-B256.config ├── nas-imagenet.config ├── pyramidC10.config ├── pyramidC100.config ├── resnet165.config └── resnet200.config ├── data ├── GDAS.pdf ├── GDAS.png ├── Get-PTB-WT2.sh ├── ImageNet-100.txt ├── README.md ├── classes.txt ├── compress.py ├── data │ ├── penn │ │ ├── test.txt │ │ ├── train.txt │ │ └── valid.txt │ └── wikitext-2 │ │ ├── test.txt │ │ ├── train.txt │ │ └── valid.txt ├── decompress.py ├── imagenet-results.png ├── load_data_CUHK-PEDES.py ├── logs │ ├── GDAS-F1-cifar10-cut-seed-6844.txt │ └── GDAS-V1-imagenet-seed-3993.txt ├── ps_mem.py ├── show-queue.sh ├── split-imagenet.py └── tiny-imagenet.py ├── exps-cnn ├── cvpr-vis.py ├── evaluate.py ├── train_base.py ├── train_utils.py ├── train_utils_imagenet.py └── vis-arch.py ├── exps-rnn ├── train_rnn_base.py └── train_rnn_utils.py ├── lib ├── datasets │ ├── LanguageDataset.py │ ├── MetaBatchSampler.py │ ├── TieredImageNet.py │ ├── __init__.py │ ├── get_dataset_with_transform.py │ ├── test_NLP.py │ └── test_dataset.py ├── nas │ ├── CifarNet.py │ ├── ImageNet.py │ ├── SE_Module.py │ ├── __init__.py │ ├── construct_utils.py │ ├── genotypes.py │ ├── head_utils.py │ └── operations.py ├── nas_rnn │ ├── __init__.py │ ├── basemodel.py │ ├── genotypes.py │ ├── model_search.py │ └── utils.py ├── scheduler │ ├── __init__.py │ ├── scheduler.py │ └── utils.py └── utils │ ├── __init__.py │ ├── draw_pts.py │ ├── evaluation_utils.py │ ├── flop_benchmark.py │ ├── gpu_manager.py │ ├── model_utils.py │ ├── save_meta.py │ └── utils.py ├── scripts-cluster ├── README.md ├── job-script.sh ├── submit.sh └── tmps │ └── .gitignore ├── scripts-cnn ├── train-cifar.sh └── train-imagenet.sh └── scripts-rnn ├── train-PTB.sh └── train-WT2.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # Pycharm project 92 | .idea 93 | snapshots 94 | *.pytorch 95 | *.tar.bz 96 | data 97 | .*.swp 98 | main_main.py 99 | *.pdf 100 | */*.pdf 101 | 102 | # Device 103 | scripts-nas/.nfs00* 104 | */.nfs00* 105 | *.DS_Store 106 | 107 | # logs and snapshots 108 | output 109 | logs 110 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Xuanyi Dong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## [Searching for A Robust Neural Architecture in Four GPU Hours](http://xuanyidong.com/publication/gradient-based-diff-sampler/) 2 | 3 | We propose A Gradient-based neural architecture search approach using Differentiable Architecture Sampler (GDAS). Please find details in [our paper](https://github.com/D-X-Y/GDAS/blob/master/data/GDAS.pdf). 4 | 5 | 6 | Figure-1. We utilize a DAG to represent the search space of a neural cell. Different operations (colored arrows) transform one node (square) to its intermediate features (little circles). Meanwhile, each node is the sum of the intermediate features transformed from the previous nodes. As indicated by the solid connections, the neural cell in the proposed GDAS is a sampled sub-graph of this DAG. Specifically, among the intermediate features between every two nodes, GDAS samples one feature in a differentiable way. 7 | 8 | ### Requirements 9 | - PyTorch 1.0.1 10 | - Python 3.6 11 | - opencv 12 | ``` 13 | conda install pytorch torchvision cuda100 -c pytorch 14 | ``` 15 | 16 | ### Usages 17 | 18 | Train the searched CNN on CIFAR 19 | ``` 20 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-cnn/train-cifar.sh GDAS_FG cifar10 cut 21 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-cnn/train-cifar.sh GDAS_F1 cifar10 cut 22 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-cnn/train-cifar.sh GDAS_V1 cifar100 cut 23 | ``` 24 | 25 | Train the searched CNN on ImageNet 26 | ``` 27 | CUDA_VISIBLE_DEVICES=0,1,2,3 bash ./scripts-cnn/train-imagenet.sh GDAS_F1 52 14 B128 -1 28 | CUDA_VISIBLE_DEVICES=0,1,2,3 bash ./scripts-cnn/train-imagenet.sh GDAS_V1 50 14 B256 -1 29 | ``` 30 | 31 | Evaluate a trained CNN model 32 | ``` 33 | CUDA_VISIBLE_DEVICES=0 python ./exps-cnn/evaluate.py --data_path $TORCH_HOME/cifar.python --checkpoint ${checkpoint-path} 34 | CUDA_VISIBLE_DEVICES=0 python ./exps-cnn/evaluate.py --data_path $TORCH_HOME/ILSVRC2012 --checkpoint ${checkpoint-path} 35 | CUDA_VISIBLE_DEVICES=0 python ./exps-cnn/evaluate.py --data_path $TORCH_HOME/ILSVRC2012 --checkpoint GDAS-V1-C50-N14-ImageNet.pth 36 | ``` 37 | 38 | Train the searched RNN 39 | ``` 40 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-PTB.sh DARTS_V1 41 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-PTB.sh DARTS_V2 42 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-PTB.sh GDAS 43 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-WT2.sh DARTS_V1 44 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-WT2.sh DARTS_V2 45 | CUDA_VISIBLE_DEVICES=0 bash ./scripts-rnn/train-WT2.sh GDAS 46 | ``` 47 | 48 | ### Training Logs 49 | You can find some training logs in [`./data/logs/`](https://github.com/D-X-Y/GDAS/tree/master/data/logs). 50 | You can also find some pre-trained models in [Google Driver](https://drive.google.com/open?id=1Ofhc49xC1PLIX4O708gJZ1ugzz4td_RJ). 51 | 52 | ### Experimental Results 53 | 54 | Figure-2. Top-1 and top-5 errors on ImageNet. 55 | 56 | ### Citation 57 | If you find that this project (GDAS) helps your research, please cite the paper: 58 | ``` 59 | @inproceedings{dong2019search, 60 | title={Searching for A Robust Neural Architecture in Four GPU Hours}, 61 | author={Dong, Xuanyi and Yang, Yi}, 62 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 63 | pages={1761--1770}, 64 | year={2019} 65 | } 66 | ``` 67 | -------------------------------------------------------------------------------- /configs/NAS-PTB-BASE.config: -------------------------------------------------------------------------------- 1 | { 2 | "data_name" : ["str", "PTB"], 3 | "data_path" : ["str", "./data/data/penn"], 4 | "emsize" : ["int", 850], 5 | "nhid" : ["int", 850], 6 | "nhidlast" : ["int", 850], 7 | "LR" : ["float", 20], 8 | "clip" : ["float", 0.25], 9 | "epochs" : ["int", 3000], 10 | "train_batch": ["int", 64], 11 | "eval_batch": ["int", 10], 12 | "test_batch": ["int", 1], 13 | "bptt" : ["int", 35], 14 | 15 | "dropout" : ["float", 0.75], 16 | "dropouth" : ["float", 0.25], 17 | "dropoutx" : ["float", 0.75], 18 | "dropouti" : ["float", 0.2], 19 | "dropoute" : ["float", 0.1], 20 | 21 | "nonmono" : ["int", 5], 22 | "alpha" : ["float", 0], 23 | "beta" : ["float", 1e-3], 24 | "wdecay" : ["float", 8e-7], 25 | 26 | "max_seq_len_delta" : ["int", 20] 27 | } 28 | -------------------------------------------------------------------------------- /configs/NAS-WT2-BASE.config: -------------------------------------------------------------------------------- 1 | { 2 | "data_name" : ["str", "WT2"], 3 | "data_path" : ["str", "./data/data/wikitext-2"], 4 | "emsize" : ["int", 700], 5 | "nhid" : ["int", 700], 6 | "nhidlast" : ["int", 700], 7 | "LR" : ["float", 20], 8 | "clip" : ["float", 0.25], 9 | "epochs" : ["int", 3000], 10 | "train_batch": ["int", 64], 11 | "eval_batch": ["int", 10], 12 | "test_batch": ["int", 1], 13 | "bptt" : ["int", 35], 14 | 15 | "dropout" : ["float", 0.75], 16 | "dropouth" : ["float", 0.15], 17 | "dropoutx" : ["float", 0.75], 18 | "dropouti" : ["float", 0.2], 19 | "dropoute" : ["float", 0.1], 20 | 21 | "nonmono" : ["int", 5], 22 | "alpha" : ["float", 0], 23 | "beta" : ["float", 1e-3], 24 | "wdecay" : ["float", 5e-7], 25 | 26 | "max_seq_len_delta" : ["int", 20] 27 | } 28 | -------------------------------------------------------------------------------- /configs/cos1800.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 1800], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.2] 8 | } 9 | -------------------------------------------------------------------------------- /configs/cos600.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.2] 8 | } 9 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cut.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutB128.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutB64.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 64], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutB96.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutW1.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutW3.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-cutW5.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 16], 13 | "drop_path_prob" : ["float", 0.2] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-cifar-cos-nocut.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "cosine"], 3 | "batch_size": ["int", 96], 4 | "epochs" : ["int", 600], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0003], 7 | "LR" : ["float", 0.025], 8 | "LR_MIN" : ["float", 0.0001], 9 | "auxiliary" : ["bool", 1], 10 | "auxiliary_weight" : ["float", 0.4], 11 | "grad_clip" : ["float", 5], 12 | "cutout" : ["int", 0], 13 | "drop_path_prob" : ["float", 0.3] 14 | } 15 | -------------------------------------------------------------------------------- /configs/nas-imagenet-B128.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /configs/nas-imagenet-B256.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 256], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /configs/nas-imagenet.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "steplr"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 250], 5 | "decay_period": ["int", 1], 6 | "gamma" : ["float", 0.97], 7 | "momentum" : ["float", 0.9], 8 | "decay" : ["float", 0.00003], 9 | "LR" : ["float", 0.1], 10 | "label_smooth": ["float", 0.1], 11 | "auxiliary" : ["bool", 1], 12 | "auxiliary_weight" : ["float", 0.4], 13 | "grad_clip" : ["float", 5], 14 | "drop_path_prob" : ["float", 0] 15 | } 16 | -------------------------------------------------------------------------------- /configs/pyramidC10.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 300], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.1], 8 | "milestones": ["int", [150, 225]], 9 | "gammas" : ["float", [0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /configs/pyramidC100.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 300], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.5], 8 | "milestones": ["int", [150, 225]], 9 | "gammas" : ["float", [0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /configs/resnet165.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 165], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0001], 7 | "LR" : ["float", 0.01], 8 | "milestones": ["int", [1, 83, 124]], 9 | "gammas" : ["float", [10, 0.1, 0.1]] 10 | } 11 | -------------------------------------------------------------------------------- /configs/resnet200.config: -------------------------------------------------------------------------------- 1 | { 2 | "type" : ["str", "multistep"], 3 | "batch_size": ["int", 128], 4 | "epochs" : ["int", 200], 5 | "momentum" : ["float", 0.9], 6 | "decay" : ["float", 0.0005], 7 | "LR" : ["float", 0.01], 8 | "milestones": ["int", [1 , 60, 120, 160]], 9 | "gammas" : ["float", [10, 0.2, 0.2, 0.2]] 10 | } 11 | -------------------------------------------------------------------------------- /data/GDAS.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csjunxu/GDAS/5eed8101a78d223a20a43494176051298b24ac3a/data/GDAS.pdf -------------------------------------------------------------------------------- /data/GDAS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csjunxu/GDAS/5eed8101a78d223a20a43494176051298b24ac3a/data/GDAS.png -------------------------------------------------------------------------------- /data/Get-PTB-WT2.sh: -------------------------------------------------------------------------------- 1 | # https://github.com/salesforce/awd-lstm-lm 2 | echo "=== Acquiring datasets ===" 3 | echo "---" 4 | mkdir -p save 5 | 6 | mkdir -p data 7 | cd data 8 | 9 | echo "- Downloading WikiText-2 (WT2)" 10 | wget --quiet --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip 11 | unzip -q wikitext-2-v1.zip 12 | cd wikitext-2 13 | mv wiki.train.tokens train.txt 14 | mv wiki.valid.tokens valid.txt 15 | mv wiki.test.tokens test.txt 16 | cd .. 17 | 18 | echo "- Downloading WikiText-103 (WT2)" 19 | wget --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip 20 | unzip -q wikitext-103-v1.zip 21 | cd wikitext-103 22 | mv wiki.train.tokens train.txt 23 | mv wiki.valid.tokens valid.txt 24 | mv wiki.test.tokens test.txt 25 | cd .. 26 | 27 | echo "- Downloading Penn Treebank (PTB)" 28 | wget --quiet --continue http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz 29 | tar -xzf simple-examples.tgz 30 | 31 | mkdir -p penn 32 | cd penn 33 | mv ../simple-examples/data/ptb.train.txt train.txt 34 | mv ../simple-examples/data/ptb.test.txt test.txt 35 | mv ../simple-examples/data/ptb.valid.txt valid.txt 36 | cd .. 37 | 38 | echo "- Downloading Penn Treebank (Character)" 39 | mkdir -p pennchar 40 | cd pennchar 41 | mv ../simple-examples/data/ptb.char.train.txt train.txt 42 | mv ../simple-examples/data/ptb.char.test.txt test.txt 43 | mv ../simple-examples/data/ptb.char.valid.txt valid.txt 44 | cd .. 45 | 46 | rm -rf simple-examples/ 47 | 48 | echo "---" 49 | echo "Happy language modeling :)" 50 | -------------------------------------------------------------------------------- /data/ImageNet-100.txt: -------------------------------------------------------------------------------- 1 | n01532829 2 | n01560419 3 | n01580077 4 | n01614925 5 | n01664065 6 | n01751748 7 | n01871265 8 | n01924916 9 | n02087394 10 | n02091134 11 | n02091244 12 | n02094433 13 | n02097209 14 | n02102040 15 | n02102480 16 | n02105251 17 | n02106662 18 | n02108422 19 | n02108551 20 | n02123597 21 | n02165105 22 | n02190166 23 | n02268853 24 | n02279972 25 | n02408429 26 | n02412080 27 | n02443114 28 | n02488702 29 | n02509815 30 | n02606052 31 | n02701002 32 | n02782093 33 | n02794156 34 | n02802426 35 | n02804414 36 | n02808440 37 | n02906734 38 | n02917067 39 | n02950826 40 | n02963159 41 | n03017168 42 | n03042490 43 | n03045698 44 | n03063689 45 | n03065424 46 | n03100240 47 | n03109150 48 | n03124170 49 | n03131574 50 | n03272562 51 | n03345487 52 | n03443371 53 | n03461385 54 | n03527444 55 | n03690938 56 | n03692522 57 | n03721384 58 | n03729826 59 | n03792782 60 | n03838899 61 | n03843555 62 | n03874293 63 | n03877472 64 | n03877845 65 | n03908618 66 | n03929660 67 | n03930630 68 | n03933933 69 | n03970156 70 | n03976657 71 | n03982430 72 | n04004767 73 | n04065272 74 | n04141975 75 | n04146614 76 | n04152593 77 | n04192698 78 | n04200800 79 | n04204347 80 | n04317175 81 | n04326547 82 | n04344873 83 | n04370456 84 | n04389033 85 | n04501370 86 | n04515003 87 | n04542943 88 | n04554684 89 | n04562935 90 | n04596742 91 | n04597913 92 | n04606251 93 | n07583066 94 | n07718472 95 | n07734744 96 | n07873807 97 | n07880968 98 | n09229709 99 | n12768682 100 | n12998815 101 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # ImageNet 2 | 3 | The class names of ImageNet-1K are in `classes.txt`. 4 | 5 | # A 100-class subset of ImageNet-1K : ImageNet-100 6 | 7 | The class names of ImageNet-100 are in `ImageNet-100.txt`. 8 | 9 | Run `python split-imagenet.py` will automatically create ImageNet-100 based on the data of ImageNet-1K. By default, we assume the data of ImageNet-1K locates at `~/.torch/ILSVRC2012`. If your data is in a different location, you need to modify line-19 and line-20 in `split-imagenet.py`. 10 | 11 | # Tiny-ImageNet 12 | The official website is [here](https://tiny-imagenet.herokuapp.com/). Please run `python tiny-imagenet.py` to generate the correct format of Tiny ImageNet for training. 13 | 14 | # PTB and WT2 15 | Run `bash Get-PTB-WT2.sh` to download the data. 16 | -------------------------------------------------------------------------------- /data/classes.txt: -------------------------------------------------------------------------------- 1 | n01440764 2 | n01443537 3 | n01484850 4 | n01491361 5 | n01494475 6 | n01496331 7 | n01498041 8 | n01514668 9 | n01514859 10 | n01518878 11 | n01530575 12 | n01531178 13 | n01532829 14 | n01534433 15 | n01537544 16 | n01558993 17 | n01560419 18 | n01580077 19 | n01582220 20 | n01592084 21 | n01601694 22 | n01608432 23 | n01614925 24 | n01616318 25 | n01622779 26 | n01629819 27 | n01630670 28 | n01631663 29 | n01632458 30 | n01632777 31 | n01641577 32 | n01644373 33 | n01644900 34 | n01664065 35 | n01665541 36 | n01667114 37 | n01667778 38 | n01669191 39 | n01675722 40 | n01677366 41 | n01682714 42 | n01685808 43 | n01687978 44 | n01688243 45 | n01689811 46 | n01692333 47 | n01693334 48 | n01694178 49 | n01695060 50 | n01697457 51 | n01698640 52 | n01704323 53 | n01728572 54 | n01728920 55 | n01729322 56 | n01729977 57 | n01734418 58 | n01735189 59 | n01737021 60 | n01739381 61 | n01740131 62 | n01742172 63 | n01744401 64 | n01748264 65 | n01749939 66 | n01751748 67 | n01753488 68 | n01755581 69 | n01756291 70 | n01768244 71 | n01770081 72 | n01770393 73 | n01773157 74 | n01773549 75 | n01773797 76 | n01774384 77 | n01774750 78 | n01775062 79 | n01776313 80 | n01784675 81 | n01795545 82 | n01796340 83 | n01797886 84 | n01798484 85 | n01806143 86 | n01806567 87 | n01807496 88 | n01817953 89 | n01818515 90 | n01819313 91 | n01820546 92 | n01824575 93 | n01828970 94 | n01829413 95 | n01833805 96 | n01843065 97 | n01843383 98 | n01847000 99 | n01855032 100 | n01855672 101 | n01860187 102 | n01871265 103 | n01872401 104 | n01873310 105 | n01877812 106 | n01882714 107 | n01883070 108 | n01910747 109 | n01914609 110 | n01917289 111 | n01924916 112 | n01930112 113 | n01943899 114 | n01944390 115 | n01945685 116 | n01950731 117 | n01955084 118 | n01968897 119 | n01978287 120 | n01978455 121 | n01980166 122 | n01981276 123 | n01983481 124 | n01984695 125 | n01985128 126 | n01986214 127 | n01990800 128 | n02002556 129 | n02002724 130 | n02006656 131 | n02007558 132 | n02009229 133 | n02009912 134 | n02011460 135 | n02012849 136 | n02013706 137 | n02017213 138 | n02018207 139 | n02018795 140 | n02025239 141 | n02027492 142 | n02028035 143 | n02033041 144 | n02037110 145 | n02051845 146 | n02056570 147 | n02058221 148 | n02066245 149 | n02071294 150 | n02074367 151 | n02077923 152 | n02085620 153 | n02085782 154 | n02085936 155 | n02086079 156 | n02086240 157 | n02086646 158 | n02086910 159 | n02087046 160 | n02087394 161 | n02088094 162 | n02088238 163 | n02088364 164 | n02088466 165 | n02088632 166 | n02089078 167 | n02089867 168 | n02089973 169 | n02090379 170 | n02090622 171 | n02090721 172 | n02091032 173 | n02091134 174 | n02091244 175 | n02091467 176 | n02091635 177 | n02091831 178 | n02092002 179 | n02092339 180 | n02093256 181 | n02093428 182 | n02093647 183 | n02093754 184 | n02093859 185 | n02093991 186 | n02094114 187 | n02094258 188 | n02094433 189 | n02095314 190 | n02095570 191 | n02095889 192 | n02096051 193 | n02096177 194 | n02096294 195 | n02096437 196 | n02096585 197 | n02097047 198 | n02097130 199 | n02097209 200 | n02097298 201 | n02097474 202 | n02097658 203 | n02098105 204 | n02098286 205 | n02098413 206 | n02099267 207 | n02099429 208 | n02099601 209 | n02099712 210 | n02099849 211 | n02100236 212 | n02100583 213 | n02100735 214 | n02100877 215 | n02101006 216 | n02101388 217 | n02101556 218 | n02102040 219 | n02102177 220 | n02102318 221 | n02102480 222 | n02102973 223 | n02104029 224 | n02104365 225 | n02105056 226 | n02105162 227 | n02105251 228 | n02105412 229 | n02105505 230 | n02105641 231 | n02105855 232 | n02106030 233 | n02106166 234 | n02106382 235 | n02106550 236 | n02106662 237 | n02107142 238 | n02107312 239 | n02107574 240 | n02107683 241 | n02107908 242 | n02108000 243 | n02108089 244 | n02108422 245 | n02108551 246 | n02108915 247 | n02109047 248 | n02109525 249 | n02109961 250 | n02110063 251 | n02110185 252 | n02110341 253 | n02110627 254 | n02110806 255 | n02110958 256 | n02111129 257 | n02111277 258 | n02111500 259 | n02111889 260 | n02112018 261 | n02112137 262 | n02112350 263 | n02112706 264 | n02113023 265 | n02113186 266 | n02113624 267 | n02113712 268 | n02113799 269 | n02113978 270 | n02114367 271 | n02114548 272 | n02114712 273 | n02114855 274 | n02115641 275 | n02115913 276 | n02116738 277 | n02117135 278 | n02119022 279 | n02119789 280 | n02120079 281 | n02120505 282 | n02123045 283 | n02123159 284 | n02123394 285 | n02123597 286 | n02124075 287 | n02125311 288 | n02127052 289 | n02128385 290 | n02128757 291 | n02128925 292 | n02129165 293 | n02129604 294 | n02130308 295 | n02132136 296 | n02133161 297 | n02134084 298 | n02134418 299 | n02137549 300 | n02138441 301 | n02165105 302 | n02165456 303 | n02167151 304 | n02168699 305 | n02169497 306 | n02172182 307 | n02174001 308 | n02177972 309 | n02190166 310 | n02206856 311 | n02219486 312 | n02226429 313 | n02229544 314 | n02231487 315 | n02233338 316 | n02236044 317 | n02256656 318 | n02259212 319 | n02264363 320 | n02268443 321 | n02268853 322 | n02276258 323 | n02277742 324 | n02279972 325 | n02280649 326 | n02281406 327 | n02281787 328 | n02317335 329 | n02319095 330 | n02321529 331 | n02325366 332 | n02326432 333 | n02328150 334 | n02342885 335 | n02346627 336 | n02356798 337 | n02361337 338 | n02363005 339 | n02364673 340 | n02389026 341 | n02391049 342 | n02395406 343 | n02396427 344 | n02397096 345 | n02398521 346 | n02403003 347 | n02408429 348 | n02410509 349 | n02412080 350 | n02415577 351 | n02417914 352 | n02422106 353 | n02422699 354 | n02423022 355 | n02437312 356 | n02437616 357 | n02441942 358 | n02442845 359 | n02443114 360 | n02443484 361 | n02444819 362 | n02445715 363 | n02447366 364 | n02454379 365 | n02457408 366 | n02480495 367 | n02480855 368 | n02481823 369 | n02483362 370 | n02483708 371 | n02484975 372 | n02486261 373 | n02486410 374 | n02487347 375 | n02488291 376 | n02488702 377 | n02489166 378 | n02490219 379 | n02492035 380 | n02492660 381 | n02493509 382 | n02493793 383 | n02494079 384 | n02497673 385 | n02500267 386 | n02504013 387 | n02504458 388 | n02509815 389 | n02510455 390 | n02514041 391 | n02526121 392 | n02536864 393 | n02606052 394 | n02607072 395 | n02640242 396 | n02641379 397 | n02643566 398 | n02655020 399 | n02666196 400 | n02667093 401 | n02669723 402 | n02672831 403 | n02676566 404 | n02687172 405 | n02690373 406 | n02692877 407 | n02699494 408 | n02701002 409 | n02704792 410 | n02708093 411 | n02727426 412 | n02730930 413 | n02747177 414 | n02749479 415 | n02769748 416 | n02776631 417 | n02777292 418 | n02782093 419 | n02783161 420 | n02786058 421 | n02787622 422 | n02788148 423 | n02790996 424 | n02791124 425 | n02791270 426 | n02793495 427 | n02794156 428 | n02795169 429 | n02797295 430 | n02799071 431 | n02802426 432 | n02804414 433 | n02804610 434 | n02807133 435 | n02808304 436 | n02808440 437 | n02814533 438 | n02814860 439 | n02815834 440 | n02817516 441 | n02823428 442 | n02823750 443 | n02825657 444 | n02834397 445 | n02835271 446 | n02837789 447 | n02840245 448 | n02841315 449 | n02843684 450 | n02859443 451 | n02860847 452 | n02865351 453 | n02869837 454 | n02870880 455 | n02871525 456 | n02877765 457 | n02879718 458 | n02883205 459 | n02892201 460 | n02892767 461 | n02894605 462 | n02895154 463 | n02906734 464 | n02909870 465 | n02910353 466 | n02916936 467 | n02917067 468 | n02927161 469 | n02930766 470 | n02939185 471 | n02948072 472 | n02950826 473 | n02951358 474 | n02951585 475 | n02963159 476 | n02965783 477 | n02966193 478 | n02966687 479 | n02971356 480 | n02974003 481 | n02977058 482 | n02978881 483 | n02979186 484 | n02980441 485 | n02981792 486 | n02988304 487 | n02992211 488 | n02992529 489 | n02999410 490 | n03000134 491 | n03000247 492 | n03000684 493 | n03014705 494 | n03016953 495 | n03017168 496 | n03018349 497 | n03026506 498 | n03028079 499 | n03032252 500 | n03041632 501 | n03042490 502 | n03045698 503 | n03047690 504 | n03062245 505 | n03063599 506 | n03063689 507 | n03065424 508 | n03075370 509 | n03085013 510 | n03089624 511 | n03095699 512 | n03100240 513 | n03109150 514 | n03110669 515 | n03124043 516 | n03124170 517 | n03125729 518 | n03126707 519 | n03127747 520 | n03127925 521 | n03131574 522 | n03133878 523 | n03134739 524 | n03141823 525 | n03146219 526 | n03160309 527 | n03179701 528 | n03180011 529 | n03187595 530 | n03188531 531 | n03196217 532 | n03197337 533 | n03201208 534 | n03207743 535 | n03207941 536 | n03208938 537 | n03216828 538 | n03218198 539 | n03220513 540 | n03223299 541 | n03240683 542 | n03249569 543 | n03250847 544 | n03255030 545 | n03259280 546 | n03271574 547 | n03272010 548 | n03272562 549 | n03290653 550 | n03291819 551 | n03297495 552 | n03314780 553 | n03325584 554 | n03337140 555 | n03344393 556 | n03345487 557 | n03347037 558 | n03355925 559 | n03372029 560 | n03376595 561 | n03379051 562 | n03384352 563 | n03388043 564 | n03388183 565 | n03388549 566 | n03393912 567 | n03394916 568 | n03400231 569 | n03404251 570 | n03417042 571 | n03424325 572 | n03425413 573 | n03443371 574 | n03444034 575 | n03445777 576 | n03445924 577 | n03447447 578 | n03447721 579 | n03450230 580 | n03452741 581 | n03457902 582 | n03459775 583 | n03461385 584 | n03467068 585 | n03476684 586 | n03476991 587 | n03478589 588 | n03481172 589 | n03482405 590 | n03483316 591 | n03485407 592 | n03485794 593 | n03492542 594 | n03494278 595 | n03495258 596 | n03496892 597 | n03498962 598 | n03527444 599 | n03529860 600 | n03530642 601 | n03532672 602 | n03534580 603 | n03535780 604 | n03538406 605 | n03544143 606 | n03584254 607 | n03584829 608 | n03590841 609 | n03594734 610 | n03594945 611 | n03595614 612 | n03598930 613 | n03599486 614 | n03602883 615 | n03617480 616 | n03623198 617 | n03627232 618 | n03630383 619 | n03633091 620 | n03637318 621 | n03642806 622 | n03649909 623 | n03657121 624 | n03658185 625 | n03661043 626 | n03662601 627 | n03666591 628 | n03670208 629 | n03673027 630 | n03676483 631 | n03680355 632 | n03690938 633 | n03691459 634 | n03692522 635 | n03697007 636 | n03706229 637 | n03709823 638 | n03710193 639 | n03710637 640 | n03710721 641 | n03717622 642 | n03720891 643 | n03721384 644 | n03724870 645 | n03729826 646 | n03733131 647 | n03733281 648 | n03733805 649 | n03742115 650 | n03743016 651 | n03759954 652 | n03761084 653 | n03763968 654 | n03764736 655 | n03769881 656 | n03770439 657 | n03770679 658 | n03773504 659 | n03775071 660 | n03775546 661 | n03776460 662 | n03777568 663 | n03777754 664 | n03781244 665 | n03782006 666 | n03785016 667 | n03786901 668 | n03787032 669 | n03788195 670 | n03788365 671 | n03791053 672 | n03792782 673 | n03792972 674 | n03793489 675 | n03794056 676 | n03796401 677 | n03803284 678 | n03804744 679 | n03814639 680 | n03814906 681 | n03825788 682 | n03832673 683 | n03837869 684 | n03838899 685 | n03840681 686 | n03841143 687 | n03843555 688 | n03854065 689 | n03857828 690 | n03866082 691 | n03868242 692 | n03868863 693 | n03871628 694 | n03873416 695 | n03874293 696 | n03874599 697 | n03876231 698 | n03877472 699 | n03877845 700 | n03884397 701 | n03887697 702 | n03888257 703 | n03888605 704 | n03891251 705 | n03891332 706 | n03895866 707 | n03899768 708 | n03902125 709 | n03903868 710 | n03908618 711 | n03908714 712 | n03916031 713 | n03920288 714 | n03924679 715 | n03929660 716 | n03929855 717 | n03930313 718 | n03930630 719 | n03933933 720 | n03935335 721 | n03937543 722 | n03938244 723 | n03942813 724 | n03944341 725 | n03947888 726 | n03950228 727 | n03954731 728 | n03956157 729 | n03958227 730 | n03961711 731 | n03967562 732 | n03970156 733 | n03976467 734 | n03976657 735 | n03977966 736 | n03980874 737 | n03982430 738 | n03983396 739 | n03991062 740 | n03992509 741 | n03995372 742 | n03998194 743 | n04004767 744 | n04005630 745 | n04008634 746 | n04009552 747 | n04019541 748 | n04023962 749 | n04026417 750 | n04033901 751 | n04033995 752 | n04037443 753 | n04039381 754 | n04040759 755 | n04041544 756 | n04044716 757 | n04049303 758 | n04065272 759 | n04067472 760 | n04069434 761 | n04070727 762 | n04074963 763 | n04081281 764 | n04086273 765 | n04090263 766 | n04099969 767 | n04111531 768 | n04116512 769 | n04118538 770 | n04118776 771 | n04120489 772 | n04125021 773 | n04127249 774 | n04131690 775 | n04133789 776 | n04136333 777 | n04141076 778 | n04141327 779 | n04141975 780 | n04146614 781 | n04147183 782 | n04149813 783 | n04152593 784 | n04153751 785 | n04154565 786 | n04162706 787 | n04179913 788 | n04192698 789 | n04200800 790 | n04201297 791 | n04204238 792 | n04204347 793 | n04208210 794 | n04209133 795 | n04209239 796 | n04228054 797 | n04229816 798 | n04235860 799 | n04238763 800 | n04239074 801 | n04243546 802 | n04251144 803 | n04252077 804 | n04252225 805 | n04254120 806 | n04254680 807 | n04254777 808 | n04258138 809 | n04259630 810 | n04263257 811 | n04264628 812 | n04265275 813 | n04266014 814 | n04270147 815 | n04273569 816 | n04275548 817 | n04277352 818 | n04285008 819 | n04286575 820 | n04296562 821 | n04310018 822 | n04311004 823 | n04311174 824 | n04317175 825 | n04325704 826 | n04326547 827 | n04328186 828 | n04330267 829 | n04332243 830 | n04335435 831 | n04336792 832 | n04344873 833 | n04346328 834 | n04347754 835 | n04350905 836 | n04355338 837 | n04355933 838 | n04356056 839 | n04357314 840 | n04366367 841 | n04367480 842 | n04370456 843 | n04371430 844 | n04371774 845 | n04372370 846 | n04376876 847 | n04380533 848 | n04389033 849 | n04392985 850 | n04398044 851 | n04399382 852 | n04404412 853 | n04409515 854 | n04417672 855 | n04418357 856 | n04423845 857 | n04428191 858 | n04429376 859 | n04435653 860 | n04442312 861 | n04443257 862 | n04447861 863 | n04456115 864 | n04458633 865 | n04461696 866 | n04462240 867 | n04465501 868 | n04467665 869 | n04476259 870 | n04479046 871 | n04482393 872 | n04483307 873 | n04485082 874 | n04486054 875 | n04487081 876 | n04487394 877 | n04493381 878 | n04501370 879 | n04505470 880 | n04507155 881 | n04509417 882 | n04515003 883 | n04517823 884 | n04522168 885 | n04523525 886 | n04525038 887 | n04525305 888 | n04532106 889 | n04532670 890 | n04536866 891 | n04540053 892 | n04542943 893 | n04548280 894 | n04548362 895 | n04550184 896 | n04552348 897 | n04553703 898 | n04554684 899 | n04557648 900 | n04560804 901 | n04562935 902 | n04579145 903 | n04579432 904 | n04584207 905 | n04589890 906 | n04590129 907 | n04591157 908 | n04591713 909 | n04592741 910 | n04596742 911 | n04597913 912 | n04599235 913 | n04604644 914 | n04606251 915 | n04612504 916 | n04613696 917 | n06359193 918 | n06596364 919 | n06785654 920 | n06794110 921 | n06874185 922 | n07248320 923 | n07565083 924 | n07579787 925 | n07583066 926 | n07584110 927 | n07590611 928 | n07613480 929 | n07614500 930 | n07615774 931 | n07684084 932 | n07693725 933 | n07695742 934 | n07697313 935 | n07697537 936 | n07711569 937 | n07714571 938 | n07714990 939 | n07715103 940 | n07716358 941 | n07716906 942 | n07717410 943 | n07717556 944 | n07718472 945 | n07718747 946 | n07720875 947 | n07730033 948 | n07734744 949 | n07742313 950 | n07745940 951 | n07747607 952 | n07749582 953 | n07753113 954 | n07753275 955 | n07753592 956 | n07754684 957 | n07760859 958 | n07768694 959 | n07802026 960 | n07831146 961 | n07836838 962 | n07860988 963 | n07871810 964 | n07873807 965 | n07875152 966 | n07880968 967 | n07892512 968 | n07920052 969 | n07930864 970 | n07932039 971 | n09193705 972 | n09229709 973 | n09246464 974 | n09256479 975 | n09288635 976 | n09332890 977 | n09399592 978 | n09421951 979 | n09428293 980 | n09468604 981 | n09472597 982 | n09835506 983 | n10148035 984 | n10565667 985 | n11879895 986 | n11939491 987 | n12057211 988 | n12144580 989 | n12267677 990 | n12620546 991 | n12768682 992 | n12985857 993 | n12998815 994 | n13037406 995 | n13040303 996 | n13044778 997 | n13052670 998 | n13054560 999 | n13133613 1000 | n15075141 1001 | -------------------------------------------------------------------------------- /data/compress.py: -------------------------------------------------------------------------------- 1 | # python ./data/compress.py $TORCH_HOME/ILSVRC2012/ $TORCH_HOME/ILSVRC2012-TAR tar 2 | # python ./data/compress.py $TORCH_HOME/ILSVRC2012/ $TORCH_HOME/ILSVRC2012-ZIP zip 3 | import os, sys 4 | from pathlib import Path 5 | 6 | 7 | def command(prefix, cmd): 8 | print ('{:}{:}'.format(prefix, cmd)) 9 | os.system(cmd) 10 | 11 | 12 | def main(source, destination, xtype): 13 | assert source.exists(), '{:} does not exist'.format(source) 14 | assert (source/'train').exists(), '{:}/train does not exist'.format(source) 15 | assert (source/'val' ).exists(), '{:}/val does not exist'.format(source) 16 | source = source.resolve() 17 | destination = destination.resolve() 18 | destination.mkdir(parents=True, exist_ok=True) 19 | os.system('rm -rf {:}'.format(destination)) 20 | destination.mkdir(parents=True, exist_ok=True) 21 | (destination/'train').mkdir(parents=True, exist_ok=True) 22 | 23 | subdirs = list( (source / 'train').glob('n*') ) 24 | assert len(subdirs) == 1000, 'ILSVRC2012 should contain 1000 classes instead of {:}.'.format( len(subdirs) ) 25 | if xtype == 'tar' : command('', 'tar -cf {:} -C {:} val'.format(destination/'val.tar', source)) 26 | elif xtype == 'zip': command('', '(cd {:} ; zip -r {:} val)'.format(source, destination/'val.zip')) 27 | else: raise ValueError('invalid compress type : {:}'.format(xtype)) 28 | for idx, subdir in enumerate(subdirs): 29 | name = subdir.name 30 | if xtype == 'tar' : command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), 'tar -cf {:} -C {:} {:}'.format(destination/'train'/'{:}.tar'.format(name), source / 'train', name)) 31 | elif xtype == 'zip': command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), '(cd {:}; zip -r {:} {:})'.format(source / 'train', destination/'train'/'{:}.zip'.format(name), name)) 32 | else: raise ValueError('invalid compress type : {:}'.format(xtype)) 33 | 34 | 35 | if __name__ == '__main__': 36 | assert len(sys.argv) == 4, 'invalid argv : {:}'.format(sys.argv) 37 | source, destination = Path(sys.argv[1]), Path(sys.argv[2]) 38 | main(source, destination, sys.argv[3]) 39 | -------------------------------------------------------------------------------- /data/decompress.py: -------------------------------------------------------------------------------- 1 | # python ./data/decompress.py $TORCH_HOME/ILSVRC2012-TAR/ ./data/data/ILSVRC2012 tar 2 | # python ./data/decompress.py $TORCH_HOME/ILSVRC2012-ZIP/ ./data/data/ILSVRC2012 zip 3 | import os, gc, sys 4 | from pathlib import Path 5 | import multiprocessing 6 | 7 | 8 | def execute(cmds, idx, num): 9 | #print ('{:03d} :: {:03d} :: {:03d}'.format(idx, num, len(cmds))) 10 | for i, cmd in enumerate(cmds): 11 | if i % num == idx: 12 | print ('{:03d} :: {:03d} :: {:03d}/{:03d} : {:}'.format(idx, num, i, len(cmds), cmd)) 13 | os.system(cmd) 14 | 15 | 16 | def command(prefix, cmd): 17 | #print ('{:}{:}'.format(prefix, cmd)) 18 | #if execute: os.system(cmd) 19 | #xcmd = '(echo {:} $(date +\"%Y-%h-%d--%T\") \"PID:\"$$; {:}; sleep 0.1s)'.format(prefix, cmd) 20 | #xcmd = '(echo {:} $(date +\"%Y-%h-%d--%T\") \"PID:\"$$; {:}; sleep 0.1s; pmap $$; echo \"\")'.format(prefix, cmd) 21 | #xcmd = '(echo {:} $(date +\"%Y-%h-%d--%T\") \"PID:\"$$; {:}; sleep 0.1s; pmap $$; echo \"\")'.format(prefix, cmd) 22 | xcmd = '(echo {:} $(date +\"%Y-%h-%d--%T\") \"PID:\"$$; {:}; sleep 0.1s)'.format(prefix, cmd) 23 | return xcmd 24 | 25 | 26 | def mkILSVRC2012(destination): 27 | destination = destination.resolve() 28 | destination.mkdir(parents=True, exist_ok=True) 29 | os.system('rm -rf {:}'.format(destination)) 30 | destination.mkdir(parents=True, exist_ok=True) 31 | (destination/'train').mkdir(parents=True, exist_ok=True) 32 | 33 | 34 | def main(source, destination, xtype): 35 | assert source.exists(), '{:} does not exist'.format(source) 36 | assert (source/'train' ).exists(), '{:}/train does not exist'.format(source) 37 | if xtype == 'tar' : assert (source/'val.tar').exists(), '{:}/val does not exist'.format(source) 38 | elif xtype == 'zip': assert (source/'val.zip').exists(), '{:}/val does not exist'.format(source) 39 | else : raise ValueError('invalid unzip type : {:}'.format(xtype)) 40 | #assert num_process > 0, 'invalid num_process : {:}'.format(num_process) 41 | source = source.resolve() 42 | mkILSVRC2012(destination) 43 | 44 | subdirs = list( (source / 'train').glob('n*') ) 45 | all_commands = [] 46 | assert len(subdirs) == 1000, 'ILSVRC2012 should contain 1000 classes instead of {:}.'.format( len(subdirs) ) 47 | for idx, subdir in enumerate(subdirs): 48 | name = subdir.name 49 | if xtype == 'tar' : cmd = command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), 'tar -xf {:} -C {:}'.format(source/'train'/'{:}'.format(name), destination / 'train')) 50 | elif xtype == 'zip': cmd = command('{:03d}/{:03d}-th: '.format(idx, len(subdirs)), 'unzip -qd {:} {:}'.format(destination / 'train', source/'train'/'{:}'.format(name))) 51 | else : raise ValueError('invalid unzip type : {:}'.format(xtype)) 52 | all_commands.append( cmd ) 53 | if xtype == 'tar' : cmd = command('', 'tar -xf {:} -C {:}'.format(source/'val.tar', destination)) 54 | elif xtype == 'zip': cmd = command('', 'unzip -qd {:} {:}'.format(destination, source/'val.zip')) 55 | else : raise ValueError('invalid unzip type : {:}'.format(xtype)) 56 | all_commands.append( cmd ) 57 | #print ('Collect all commands done : {:} lines'.format( len(all_commands) )) 58 | 59 | for i, cmd in enumerate(all_commands): 60 | print(cmd) 61 | # os.system(cmd) 62 | # print ('{:03d}/{:03d} : {:}'.format(i, len(all_commands), cmd)) 63 | # gc.collect() 64 | 65 | """ 66 | records = [] 67 | for i in range(num_process): 68 | process = multiprocessing.Process(target=execute, args=(all_commands, i, num_process)) 69 | process.start() 70 | records.append(process) 71 | for process in records: 72 | process.join() 73 | """ 74 | 75 | 76 | if __name__ == '__main__': 77 | assert len(sys.argv) == 4, 'invalid argv : {:}'.format(sys.argv) 78 | source, destination = Path(sys.argv[1]), Path(sys.argv[2]) 79 | #num_process = int(sys.argv[3]) 80 | if sys.argv[3] == 'wget': 81 | with open(source) as f: 82 | content = f.readlines() 83 | content = [x.strip() for x in content] 84 | assert len(content) == 1000, 'invalid lines={:} from {:}'.format( len(content), source ) 85 | mkILSVRC2012(destination) 86 | all_commands = [] 87 | cmd = command('make-val', 'wget -q http://10.127.2.44:8000/ILSVRC2012-TAR/val.tar --directory-prefix={:} ; tar -xf {:} -C {:} ; rm {:}'.format(destination, destination / 'val.tar', destination, destination / 'val.tar')) 88 | all_commands.append(cmd) 89 | for idx, name in enumerate(content): 90 | cmd = command('{:03d}/{:03d}-th: '.format(idx, len(content)), 'wget -q http://10.127.2.44:8000/ILSVRC2012-TAR/train/{:}.tar --directory-prefix={:} ; tar -xf {:}.tar -C {:} ; rm {:}.tar'.format(name, destination / 'train', destination / 'train' / name, destination / 'train', destination / 'train' / name)) 91 | all_commands.append(cmd) 92 | for i, cmd in enumerate(all_commands): print(cmd) 93 | else: 94 | main(source, destination, sys.argv[3]) 95 | -------------------------------------------------------------------------------- /data/imagenet-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csjunxu/GDAS/5eed8101a78d223a20a43494176051298b24ac3a/data/imagenet-results.png -------------------------------------------------------------------------------- /data/load_data_CUHK-PEDES.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | def main(): 4 | xpath = 'caption_all.json' 5 | with open(xpath, 'r') as cfile: 6 | cap_data = json.load(cfile) 7 | print ('There are {:} images'.format( len(cap_data) )) 8 | IDs = set() 9 | for idx, data in enumerate( cap_data ): 10 | IDs.add( data['id'] ) 11 | assert len( data['captions'] ) > 0, 'invalid {:}-th caption length : {:} {:}'.format(idx, data['captions'], len(data['captions'])) 12 | print ('IDs :: min={:}, max={:}, num={:}'.format(min(IDs), max(IDs), len(IDs))) 13 | 14 | if __name__ == '__main__': 15 | main() 16 | -------------------------------------------------------------------------------- /data/ps_mem.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Try to determine how much RAM is currently being used per program. 4 | # Note per _program_, not per process. So for example this script 5 | # will report RAM used by all httpd process together. In detail it reports: 6 | # sum(private RAM for program processes) + sum(Shared RAM for program processes) 7 | # The shared RAM is problematic to calculate, and this script automatically 8 | # selects the most accurate method available for your kernel. 9 | 10 | # Licence: LGPLv2 11 | # Author: P@draigBrady.com 12 | # Source: http://www.pixelbeat.org/scripts/ps_mem.py 13 | 14 | # V1.0 06 Jul 2005 Initial release 15 | # V1.1 11 Aug 2006 root permission required for accuracy 16 | # V1.2 08 Nov 2006 Add total to output 17 | # Use KiB,MiB,... for units rather than K,M,... 18 | # V1.3 22 Nov 2006 Ignore shared col from /proc/$pid/statm for 19 | # 2.6 kernels up to and including 2.6.9. 20 | # There it represented the total file backed extent 21 | # V1.4 23 Nov 2006 Remove total from output as it's meaningless 22 | # (the shared values overlap with other programs). 23 | # Display the shared column. This extra info is 24 | # useful, especially as it overlaps between programs. 25 | # V1.5 26 Mar 2007 Remove redundant recursion from human() 26 | # V1.6 05 Jun 2007 Also report number of processes with a given name. 27 | # Patch from riccardo.murri@gmail.com 28 | # V1.7 20 Sep 2007 Use PSS from /proc/$pid/smaps if available, which 29 | # fixes some over-estimation and allows totalling. 30 | # Enumerate the PIDs directly rather than using ps, 31 | # which fixes the possible race between reading 32 | # RSS with ps, and shared memory with this program. 33 | # Also we can show non truncated command names. 34 | # V1.8 28 Sep 2007 More accurate matching for stats in /proc/$pid/smaps 35 | # as otherwise could match libraries causing a crash. 36 | # Patch from patrice.bouchand.fedora@gmail.com 37 | # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. 38 | # Reported by Andrey Borzenkov 39 | # V3.13 17 Sep 2018 40 | # http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py 41 | 42 | # Notes: 43 | # 44 | # All interpreted programs where the interpreter is started 45 | # by the shell or with env, will be merged to the interpreter 46 | # (as that's what's given to exec). For e.g. all python programs 47 | # starting with "#!/usr/bin/env python" will be grouped under python. 48 | # You can change this by using the full command line but that will 49 | # have the undesirable affect of splitting up programs started with 50 | # differing parameters (for e.g. mingetty tty[1-6]). 51 | # 52 | # For 2.6 kernels up to and including 2.6.13 and later 2.4 redhat kernels 53 | # (rmap vm without smaps) it can not be accurately determined how many pages 54 | # are shared between processes in general or within a program in our case: 55 | # http://lkml.org/lkml/2005/7/6/250 56 | # A warning is printed if overestimation is possible. 57 | # In addition for 2.6 kernels up to 2.6.9 inclusive, the shared 58 | # value in /proc/$pid/statm is the total file-backed extent of a process. 59 | # We ignore that, introducing more overestimation, again printing a warning. 60 | # Since kernel 2.6.23-rc8-mm1 PSS is available in smaps, which allows 61 | # us to calculate a more accurate value for the total RAM used by programs. 62 | # 63 | # Programs that use CLONE_VM without CLONE_THREAD are discounted by assuming 64 | # they're the only programs that have the same /proc/$PID/smaps file for 65 | # each instance. This will fail if there are multiple real instances of a 66 | # program that then use CLONE_VM without CLONE_THREAD, or if a clone changes 67 | # its memory map while we're checksumming each /proc/$PID/smaps. 68 | # 69 | # I don't take account of memory allocated for a program 70 | # by other programs. For e.g. memory used in the X server for 71 | # a program could be determined, but is not. 72 | # 73 | # FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/ 74 | # FreeBSD 8.0 supports up to a level of Linux 2.6.16 75 | 76 | import getopt 77 | import time 78 | import errno 79 | import os 80 | import sys 81 | 82 | # The following exits cleanly on Ctrl-C or EPIPE 83 | # while treating other exceptions as before. 84 | def std_exceptions(etype, value, tb): 85 | sys.excepthook = sys.__excepthook__ 86 | if issubclass(etype, KeyboardInterrupt): 87 | pass 88 | elif issubclass(etype, IOError) and value.errno == errno.EPIPE: 89 | pass 90 | else: 91 | sys.__excepthook__(etype, value, tb) 92 | sys.excepthook = std_exceptions 93 | 94 | # 95 | # Define some global variables 96 | # 97 | 98 | PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB 99 | our_pid = os.getpid() 100 | 101 | have_pss = 0 102 | have_swap_pss = 0 103 | 104 | class Unbuffered(object): 105 | def __init__(self, stream): 106 | self.stream = stream 107 | def write(self, data): 108 | self.stream.write(data) 109 | self.stream.flush() 110 | def close(self): 111 | self.stream.close() 112 | def flush(self): 113 | self.stream.flush() 114 | 115 | class Proc: 116 | def __init__(self): 117 | uname = os.uname() 118 | if uname[0] == "FreeBSD": 119 | self.proc = '/compat/linux/proc' 120 | else: 121 | self.proc = '/proc' 122 | 123 | def path(self, *args): 124 | return os.path.join(self.proc, *(str(a) for a in args)) 125 | 126 | def open(self, *args): 127 | try: 128 | if sys.version_info < (3,): 129 | return open(self.path(*args)) 130 | else: 131 | return open(self.path(*args), errors='ignore') 132 | except (IOError, OSError): 133 | val = sys.exc_info()[1] 134 | if (val.errno == errno.ENOENT or # kernel thread or process gone 135 | val.errno == errno.EPERM or 136 | val.errno == errno.EACCES): 137 | raise LookupError 138 | raise 139 | 140 | proc = Proc() 141 | 142 | 143 | # 144 | # Functions 145 | # 146 | 147 | def parse_options(): 148 | try: 149 | long_options = [ 150 | 'split-args', 151 | 'help', 152 | 'version', 153 | 'total', 154 | 'discriminate-by-pid', 155 | 'swap' 156 | ] 157 | opts, args = getopt.getopt(sys.argv[1:], "shtdSp:w:", long_options) 158 | except getopt.GetoptError: 159 | sys.stderr.write(help()) 160 | sys.exit(3) 161 | 162 | if len(args): 163 | sys.stderr.write("Extraneous arguments: %s\n" % args) 164 | sys.exit(3) 165 | 166 | # ps_mem.py options 167 | split_args = False 168 | pids_to_show = None 169 | discriminate_by_pid = False 170 | show_swap = False 171 | watch = None 172 | only_total = False 173 | 174 | for o, a in opts: 175 | if o in ('-s', '--split-args'): 176 | split_args = True 177 | if o in ('-t', '--total'): 178 | only_total = True 179 | if o in ('-d', '--discriminate-by-pid'): 180 | discriminate_by_pid = True 181 | if o in ('-S', '--swap'): 182 | show_swap = True 183 | if o in ('-h', '--help'): 184 | sys.stdout.write(help()) 185 | sys.exit(0) 186 | if o in ('--version'): 187 | sys.stdout.write('3.13'+'\n') 188 | sys.exit(0) 189 | if o in ('-p',): 190 | try: 191 | pids_to_show = [int(x) for x in a.split(',')] 192 | except: 193 | sys.stderr.write(help()) 194 | sys.exit(3) 195 | if o in ('-w',): 196 | try: 197 | watch = int(a) 198 | except: 199 | sys.stderr.write(help()) 200 | sys.exit(3) 201 | 202 | return ( 203 | split_args, 204 | pids_to_show, 205 | watch, 206 | only_total, 207 | discriminate_by_pid, 208 | show_swap 209 | ) 210 | 211 | 212 | def help(): 213 | help_msg = 'Usage: ps_mem [OPTION]...\n' \ 214 | 'Show program core memory usage\n' \ 215 | '\n' \ 216 | ' -h, -help Show this help\n' \ 217 | ' -p [,pid2,...pidN] Only show memory usage PIDs in the '\ 218 | 'specified list\n' \ 219 | ' -s, --split-args Show and separate by, all command line'\ 220 | ' arguments\n' \ 221 | ' -t, --total Show only the total value\n' \ 222 | ' -d, --discriminate-by-pid Show by process rather than by program\n' \ 223 | ' -S, --swap Show swap information\n' \ 224 | ' -w Measure and show process memory every'\ 225 | ' N seconds\n' 226 | 227 | return help_msg 228 | 229 | 230 | # (major,minor,release) 231 | def kernel_ver(): 232 | kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] 233 | last = len(kv) 234 | if last == 2: 235 | kv.append('0') 236 | last -= 1 237 | while last > 0: 238 | for char in "-_": 239 | kv[last] = kv[last].split(char)[0] 240 | try: 241 | int(kv[last]) 242 | except: 243 | kv[last] = 0 244 | last -= 1 245 | return (int(kv[0]), int(kv[1]), int(kv[2])) 246 | 247 | 248 | #return Private,Shared,Swap(Pss),unique_id 249 | #Note shared is always a subset of rss (trs is not always) 250 | def getMemStats(pid): 251 | global have_pss 252 | global have_swap_pss 253 | mem_id = pid #unique 254 | Private_lines = [] 255 | Shared_lines = [] 256 | Pss_lines = [] 257 | Rss = (int(proc.open(pid, 'statm').readline().split()[1]) 258 | * PAGESIZE) 259 | Swap_lines = [] 260 | Swap_pss_lines = [] 261 | 262 | Swap = 0 263 | 264 | if os.path.exists(proc.path(pid, 'smaps')): # stat 265 | smaps = 'smaps' 266 | if os.path.exists(proc.path(pid, 'smaps_rollup')): 267 | smaps = 'smaps_rollup' # faster to process 268 | lines = proc.open(pid, smaps).readlines() # open 269 | # Note we checksum smaps as maps is usually but 270 | # not always different for separate processes. 271 | mem_id = hash(''.join(lines)) 272 | for line in lines: 273 | if line.startswith("Shared"): 274 | Shared_lines.append(line) 275 | elif line.startswith("Private"): 276 | Private_lines.append(line) 277 | elif line.startswith("Pss"): 278 | have_pss = 1 279 | Pss_lines.append(line) 280 | elif line.startswith("Swap:"): 281 | Swap_lines.append(line) 282 | elif line.startswith("SwapPss:"): 283 | have_swap_pss = 1 284 | Swap_pss_lines.append(line) 285 | Shared = sum([int(line.split()[1]) for line in Shared_lines]) 286 | Private = sum([int(line.split()[1]) for line in Private_lines]) 287 | #Note Shared + Private = Rss above 288 | #The Rss in smaps includes video card mem etc. 289 | if have_pss: 290 | pss_adjust = 0.5 # add 0.5KiB as this avg error due to truncation 291 | Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) 292 | Shared = Pss - Private 293 | if have_swap_pss: 294 | # The kernel supports SwapPss, that shows proportional swap share. 295 | # Note that Swap - SwapPss is not Private Swap. 296 | Swap = sum([int(line.split()[1]) for line in Swap_pss_lines]) 297 | else: 298 | # Note that Swap = Private swap + Shared swap. 299 | Swap = sum([int(line.split()[1]) for line in Swap_lines]) 300 | elif (2,6,1) <= kernel_ver() <= (2,6,9): 301 | Shared = 0 #lots of overestimation, but what can we do? 302 | Private = Rss 303 | else: 304 | Shared = int(proc.open(pid, 'statm').readline().split()[2]) 305 | Shared *= PAGESIZE 306 | Private = Rss - Shared 307 | return (Private, Shared, Swap, mem_id) 308 | 309 | 310 | def getCmdName(pid, split_args, discriminate_by_pid, exe_only=False): 311 | cmdline = proc.open(pid, 'cmdline').read().split("\0") 312 | if cmdline[-1] == '' and len(cmdline) > 1: 313 | cmdline = cmdline[:-1] 314 | 315 | path = proc.path(pid, 'exe') 316 | try: 317 | path = os.readlink(path) 318 | # Some symlink targets were seen to contain NULs on RHEL 5 at least 319 | # https://github.com/pixelb/scripts/pull/10, so take string up to NUL 320 | path = path.split('\0')[0] 321 | except OSError: 322 | val = sys.exc_info()[1] 323 | if (val.errno == errno.ENOENT or # either kernel thread or process gone 324 | val.errno == errno.EPERM or 325 | val.errno == errno.EACCES): 326 | raise LookupError 327 | raise 328 | 329 | if split_args: 330 | return ' '.join(cmdline).replace('\n', ' ') 331 | if path.endswith(" (deleted)"): 332 | path = path[:-10] 333 | if os.path.exists(path): 334 | path += " [updated]" 335 | else: 336 | #The path could be have prelink stuff so try cmdline 337 | #which might have the full path present. This helped for: 338 | #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted) 339 | if os.path.exists(cmdline[0]): 340 | path = cmdline[0] + " [updated]" 341 | else: 342 | path += " [deleted]" 343 | exe = os.path.basename(path) 344 | if exe_only: return exe 345 | 346 | proc_status = proc.open(pid, 'status').readlines() 347 | cmd = proc_status[0][6:-1] 348 | if exe.startswith(cmd): 349 | cmd = exe #show non truncated version 350 | #Note because we show the non truncated name 351 | #one can have separated programs as follows: 352 | #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) 353 | # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin 354 | else: 355 | #Lookup the parent's exe and use that if matching 356 | #which will merge "Web Content" with "firefox" for example 357 | ppid = 0 358 | for l in range(10): 359 | ps_line = proc_status[l] 360 | if ps_line.startswith('PPid:'): 361 | ppid = int(ps_line[6:-1]) 362 | break 363 | if ppid: 364 | p_exe = getCmdName(ppid, False, False, exe_only=True) 365 | if exe == p_exe: 366 | cmd = exe 367 | if sys.version_info >= (3,): 368 | cmd = cmd.encode(errors='replace').decode() 369 | if discriminate_by_pid: 370 | cmd = '%s [%d]' % (cmd, pid) 371 | return cmd 372 | 373 | 374 | #The following matches "du -h" output 375 | #see also human.py 376 | def human(num, power="Ki", units=None): 377 | if units is None: 378 | powers = ["Ki", "Mi", "Gi", "Ti"] 379 | while num >= 1000: #4 digits 380 | num /= 1024.0 381 | power = powers[powers.index(power)+1] 382 | return "%.1f %sB" % (num, power) 383 | else: 384 | return "%.f" % ((num * 1024) / units) 385 | 386 | 387 | def cmd_with_count(cmd, count): 388 | if count > 1: 389 | return "%s (%u)" % (cmd, count) 390 | else: 391 | return cmd 392 | 393 | #Warn of possible inaccuracies 394 | #RAM: 395 | #2 = accurate & can total 396 | #1 = accurate only considering each process in isolation 397 | #0 = some shared mem not reported 398 | #-1= all shared mem not reported 399 | #SWAP: 400 | #2 = accurate & can total 401 | #1 = accurate only considering each process in isolation 402 | #-1= not available 403 | def val_accuracy(show_swap): 404 | """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" 405 | kv = kernel_ver() 406 | pid = os.getpid() 407 | swap_accuracy = -1 408 | if kv[:2] == (2,4): 409 | if proc.open('meminfo').read().find("Inact_") == -1: 410 | return 1, swap_accuracy 411 | return 0, swap_accuracy 412 | elif kv[:2] == (2,6): 413 | if os.path.exists(proc.path(pid, 'smaps')): 414 | swap_accuracy = 1 415 | if proc.open(pid, 'smaps').read().find("Pss:")!=-1: 416 | return 2, swap_accuracy 417 | else: 418 | return 1, swap_accuracy 419 | if (2,6,1) <= kv <= (2,6,9): 420 | return -1, swap_accuracy 421 | return 0, swap_accuracy 422 | elif kv[0] > 2 and os.path.exists(proc.path(pid, 'smaps')): 423 | swap_accuracy = 1 424 | if show_swap and proc.open(pid, 'smaps').read().find("SwapPss:")!=-1: 425 | swap_accuracy = 2 426 | return 2, swap_accuracy 427 | else: 428 | return 1, swap_accuracy 429 | 430 | def show_val_accuracy( ram_inacc, swap_inacc, only_total, show_swap ): 431 | level = ("Warning","Error")[only_total] 432 | 433 | # Only show significant warnings 434 | if not show_swap: 435 | swap_inacc = 2 436 | elif only_total: 437 | ram_inacc = 2 438 | 439 | if ram_inacc == -1: 440 | sys.stderr.write( 441 | "%s: Shared memory is not reported by this system.\n" % level 442 | ) 443 | sys.stderr.write( 444 | "Values reported will be too large, and totals are not reported\n" 445 | ) 446 | elif ram_inacc == 0: 447 | sys.stderr.write( 448 | "%s: Shared memory is not reported accurately by this system.\n" % level 449 | ) 450 | sys.stderr.write( 451 | "Values reported could be too large, and totals are not reported\n" 452 | ) 453 | elif ram_inacc == 1: 454 | sys.stderr.write( 455 | "%s: Shared memory is slightly over-estimated by this system\n" 456 | "for each program, so totals are not reported.\n" % level 457 | ) 458 | 459 | if swap_inacc == -1: 460 | sys.stderr.write( 461 | "%s: Swap is not reported by this system.\n" % level 462 | ) 463 | elif swap_inacc == 1: 464 | sys.stderr.write( 465 | "%s: Swap is over-estimated by this system for each program,\n" 466 | "so totals are not reported.\n" % level 467 | ) 468 | 469 | sys.stderr.close() 470 | if only_total: 471 | if show_swap: 472 | accuracy = swap_inacc 473 | else: 474 | accuracy = ram_inacc 475 | if accuracy != 2: 476 | sys.exit(1) 477 | 478 | 479 | def get_memory_usage(pids_to_show, split_args, discriminate_by_pid, 480 | include_self=False, only_self=False): 481 | cmds = {} 482 | shareds = {} 483 | mem_ids = {} 484 | count = {} 485 | swaps = {} 486 | for pid in os.listdir(proc.path('')): 487 | if not pid.isdigit(): 488 | continue 489 | pid = int(pid) 490 | 491 | # Some filters 492 | if only_self and pid != our_pid: 493 | continue 494 | if pid == our_pid and not include_self: 495 | continue 496 | if pids_to_show is not None and pid not in pids_to_show: 497 | continue 498 | 499 | try: 500 | cmd = getCmdName(pid, split_args, discriminate_by_pid) 501 | except LookupError: 502 | #operation not permitted 503 | #kernel threads don't have exe links or 504 | #process gone 505 | continue 506 | 507 | try: 508 | private, shared, swap, mem_id = getMemStats(pid) 509 | except RuntimeError: 510 | continue #process gone 511 | if shareds.get(cmd): 512 | if have_pss: #add shared portion of PSS together 513 | shareds[cmd] += shared 514 | elif shareds[cmd] < shared: #just take largest shared val 515 | shareds[cmd] = shared 516 | else: 517 | shareds[cmd] = shared 518 | cmds[cmd] = cmds.setdefault(cmd, 0) + private 519 | if cmd in count: 520 | count[cmd] += 1 521 | else: 522 | count[cmd] = 1 523 | mem_ids.setdefault(cmd, {}).update({mem_id: None}) 524 | 525 | # Swap (overcounting for now...) 526 | swaps[cmd] = swaps.setdefault(cmd, 0) + swap 527 | 528 | # Total swaped mem for each program 529 | total_swap = 0 530 | 531 | # Add shared mem for each program 532 | total = 0 533 | 534 | for cmd in cmds: 535 | cmd_count = count[cmd] 536 | if len(mem_ids[cmd]) == 1 and cmd_count > 1: 537 | # Assume this program is using CLONE_VM without CLONE_THREAD 538 | # so only account for one of the processes 539 | cmds[cmd] /= cmd_count 540 | if have_pss: 541 | shareds[cmd] /= cmd_count 542 | cmds[cmd] = cmds[cmd] + shareds[cmd] 543 | total += cmds[cmd] # valid if PSS available 544 | total_swap += swaps[cmd] 545 | 546 | sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) 547 | sorted_cmds = [x for x in sorted_cmds if x[1]] 548 | 549 | return sorted_cmds, shareds, count, total, swaps, total_swap 550 | 551 | def print_header(show_swap, discriminate_by_pid): 552 | output_string = " Private + Shared = RAM used" 553 | if show_swap: 554 | output_string += " Swap used" 555 | output_string += "\tProgram" 556 | if discriminate_by_pid: 557 | output_string += "[pid]" 558 | output_string += "\n\n" 559 | sys.stdout.write(output_string) 560 | 561 | 562 | def print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap, 563 | show_swap): 564 | for cmd in sorted_cmds: 565 | 566 | output_string = "%9s + %9s = %9s" 567 | output_data = (human(cmd[1]-shareds[cmd[0]]), 568 | human(shareds[cmd[0]]), human(cmd[1])) 569 | if show_swap: 570 | output_string += " %9s" 571 | output_data += (human(swaps[cmd[0]]),) 572 | output_string += "\t%s\n" 573 | output_data += (cmd_with_count(cmd[0], count[cmd[0]]),) 574 | 575 | sys.stdout.write(output_string % output_data) 576 | 577 | # Only show totals if appropriate 578 | if have_swap_pss and show_swap: # kernel will have_pss 579 | sys.stdout.write("%s\n%s%9s%s%9s\n%s\n" % 580 | ("-" * 45, " " * 24, human(total), " " * 3, 581 | human(total_swap), "=" * 45)) 582 | elif have_pss: 583 | sys.stdout.write("%s\n%s%9s\n%s\n" % 584 | ("-" * 33, " " * 24, human(total), "=" * 33)) 585 | 586 | 587 | def verify_environment(pids_to_show): 588 | if os.geteuid() != 0 and not pids_to_show: 589 | sys.stderr.write("Sorry, root permission required, or specify pids with -p\n") 590 | sys.stderr.close() 591 | sys.exit(1) 592 | 593 | try: 594 | kernel_ver() 595 | except (IOError, OSError): 596 | val = sys.exc_info()[1] 597 | if val.errno == errno.ENOENT: 598 | sys.stderr.write( 599 | "Couldn't access " + proc.path('') + "\n" 600 | "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") 601 | sys.exit(2) 602 | else: 603 | raise 604 | 605 | def main(): 606 | # Force the stdout and stderr streams to be unbuffered 607 | sys.stdout = Unbuffered(sys.stdout) 608 | sys.stderr = Unbuffered(sys.stderr) 609 | 610 | split_args, pids_to_show, watch, only_total, discriminate_by_pid, \ 611 | show_swap = parse_options() 612 | 613 | verify_environment(pids_to_show) 614 | 615 | if not only_total: 616 | print_header(show_swap, discriminate_by_pid) 617 | 618 | if watch is not None: 619 | try: 620 | sorted_cmds = True 621 | while sorted_cmds: 622 | sorted_cmds, shareds, count, total, swaps, total_swap = \ 623 | get_memory_usage(pids_to_show, split_args, 624 | discriminate_by_pid) 625 | if only_total and show_swap and have_swap_pss: 626 | sys.stdout.write(human(total_swap, units=1)+'\n') 627 | elif only_total and not show_swap and have_pss: 628 | sys.stdout.write(human(total, units=1)+'\n') 629 | elif not only_total: 630 | print_memory_usage(sorted_cmds, shareds, count, total, 631 | swaps, total_swap, show_swap) 632 | 633 | sys.stdout.flush() 634 | time.sleep(watch) 635 | else: 636 | sys.stdout.write('Process does not exist anymore.\n') 637 | except KeyboardInterrupt: 638 | pass 639 | else: 640 | # This is the default behavior 641 | sorted_cmds, shareds, count, total, swaps, total_swap = \ 642 | get_memory_usage(pids_to_show, split_args, 643 | discriminate_by_pid) 644 | if only_total and show_swap and have_swap_pss: 645 | sys.stdout.write(human(total_swap, units=1)+'\n') 646 | elif only_total and not show_swap and have_pss: 647 | sys.stdout.write(human(total, units=1)+'\n') 648 | elif not only_total: 649 | print_memory_usage(sorted_cmds, shareds, count, total, swaps, 650 | total_swap, show_swap) 651 | 652 | # We must close explicitly, so that any EPIPE exception 653 | # is handled by our excepthook, rather than the default 654 | # one which is reenabled after this script finishes. 655 | sys.stdout.close() 656 | 657 | ram_accuracy, swap_accuracy = val_accuracy( show_swap ) 658 | show_val_accuracy( ram_accuracy, swap_accuracy, only_total, show_swap ) 659 | 660 | if __name__ == '__main__': 661 | main() 662 | -------------------------------------------------------------------------------- /data/show-queue.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Show High-priority 3 | echo '-------------------------------' 4 | echo 'Queue in high-priority clusters' 5 | echo '-------------------------------' 6 | queues="yq01-v100-box-1-8 yq01-v100-box-idl-2-8" 7 | for queue in ${queues} 8 | do 9 | showjob -p ${queue} 10 | sleep 0.3s 11 | done 12 | 13 | echo '-------------------------------' 14 | echo 'Queue in low-priority clusters' 15 | echo '-------------------------------' 16 | 17 | #queues="yq01-p40-3-8 yq01-p40-2-8 yq01-p40-box-1-8 yq01-v100-box-2-8" 18 | queues="yq01-p40-3-8 yq01-p40-box-1-8 yq01-v100-box-2-8" 19 | for queue in ${queues} 20 | do 21 | showjob -p ${queue} 22 | sleep 0.3s 23 | done 24 | 25 | 26 | echo '-------------------------------' 27 | echo 'Queue for other IDL teams' 28 | echo '-------------------------------' 29 | 30 | queues="yq01-v100-box-idl-8 yq01-v100-box-idl-3-8" 31 | for queue in ${queues} 32 | do 33 | showjob -p ${queue} 34 | sleep 0.3s 35 | done 36 | -------------------------------------------------------------------------------- /data/split-imagenet.py: -------------------------------------------------------------------------------- 1 | import os, sys, random 2 | from pathlib import Path 3 | 4 | 5 | def sample_100_cls(): 6 | with open('classes.txt') as f: 7 | content = f.readlines() 8 | content = [x.strip() for x in content] 9 | random.seed(111) 10 | classes = random.sample(content, 100) 11 | classes.sort() 12 | with open('ImageNet-100.txt', 'w') as f: 13 | for cls in classes: f.write('{:}\n'.format(cls)) 14 | print('-'*100) 15 | 16 | 17 | if __name__ == "__main__": 18 | #sample_100_cls() 19 | IN1K_root = Path.home() / '.torch' / 'ILSVRC2012' 20 | IN100_root = Path.home() / '.torch' / 'ILSVRC2012-100' 21 | assert IN1K_root.exists(), 'ImageNet directory does not exist : {:}'.format(IN1K_root) 22 | print ('Create soft link from ImageNet directory into : {:}'.format(IN100_root)) 23 | with open('ImageNet-100.txt', 'r') as f: 24 | classes = f.readlines() 25 | classes = [x.strip() for x in classes] 26 | for sub in ['train', 'val']: 27 | xdir = IN100_root / sub 28 | if not xdir.exists(): xdir.mkdir(parents=True, exist_ok=True) 29 | 30 | for idx, cls in enumerate(classes): 31 | xdir = IN1K_root / 'train' / cls 32 | assert xdir.exists(), '{:} does not exist'.format(xdir) 33 | os.system('ln -s {:} {:}'.format(xdir, IN100_root / 'train' / cls)) 34 | 35 | xdir = IN1K_root / 'val' / cls 36 | assert xdir.exists(), '{:} does not exist'.format(xdir) 37 | os.system('ln -s {:} {:}'.format(xdir, IN100_root / 'val' / cls)) 38 | -------------------------------------------------------------------------------- /data/tiny-imagenet.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | from pathlib import Path 3 | 4 | url = "http://cs231n.stanford.edu/tiny-imagenet-200.zip" 5 | 6 | def load_val(): 7 | path = 'tiny-imagenet-200/val/val_annotations.txt' 8 | cfile = open(path, 'r') 9 | content = cfile.readlines() 10 | content = [x.strip().split('\t') for x in content] 11 | cfile.close() 12 | images = [x[0] for x in content] 13 | labels = [x[1] for x in content] 14 | return images, labels 15 | 16 | def main(): 17 | os.system("wget {:}".format(url)) 18 | os.system("rm -rf tiny-imagenet-200") 19 | os.system("unzip -o tiny-imagenet-200.zip") 20 | images, labels = load_val() 21 | savedir = 'tiny-imagenet-200/new_val' 22 | if not os.path.exists(savedir): os.makedirs(savedir) 23 | for image, label in zip(images, labels): 24 | cdir = savedir + '/' + label 25 | if not os.path.exists(cdir): os.makedirs(cdir) 26 | ori_path = 'tiny-imagenet-200/val/images/' + image 27 | os.system("cp {:} {:}".format(ori_path, cdir)) 28 | os.system("rm -rf tiny-imagenet-200/val") 29 | os.system("mv {:} tiny-imagenet-200/val".format(savedir)) 30 | 31 | def generate_salt_pepper(): 32 | targetdir = Path('tiny-imagenet-200/val') 33 | noisedir = Path('tiny-imagenet-200/val-noise') 34 | assert targetdir.exists(), '{:} does not exist'.format(targetdir) 35 | from imgaug import augmenters as iaa 36 | import cv2 37 | aug = iaa.SaltAndPepper(p=0.2) 38 | 39 | for sub in targetdir.iterdir(): 40 | if not sub.is_dir(): continue 41 | subdir = noisedir / sub.name 42 | if not subdir.exists(): os.makedirs('{:}'.format(subdir)) 43 | images = sub.glob('*.JPEG') 44 | for image in images: 45 | I = cv2.imread(str(image)) 46 | Inoise = aug.augment_image(I) 47 | savepath = subdir / image.name 48 | cv2.imwrite(str(savepath), Inoise) 49 | print ('{:} done'.format(sub)) 50 | 51 | if __name__ == "__main__": 52 | #main() 53 | generate_salt_pepper() 54 | -------------------------------------------------------------------------------- /exps-cnn/cvpr-vis.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | # python ./exps-nas/cvpr-vis.py --save_dir ./snapshots/NAS-VIS/ 5 | import os, sys, time, glob, random, argparse 6 | import numpy as np 7 | from copy import deepcopy 8 | import torch 9 | from pathlib import Path 10 | lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() 11 | if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) 12 | from nas import DMS_V1, DMS_F1 13 | from nas_rnn import DARTS_V2, GDAS 14 | from graphviz import Digraph 15 | 16 | parser = argparse.ArgumentParser("Visualize the Networks") 17 | parser.add_argument('--save_dir', type=str, help='The directory to save the network plot.') 18 | args = parser.parse_args() 19 | 20 | 21 | def plot_cnn(genotype, filename): 22 | g = Digraph( 23 | format='pdf', 24 | edge_attr=dict(fontsize='20', fontname="times"), 25 | node_attr=dict(style='filled', shape='rect', align='center', fontsize='20', height='0.5', width='0.5', penwidth='2', fontname="times"), 26 | engine='dot') 27 | g.body.extend(['rankdir=LR']) 28 | 29 | g.node("c_{k-2}", fillcolor='darkseagreen2') 30 | g.node("c_{k-1}", fillcolor='darkseagreen2') 31 | assert len(genotype) % 2 == 0, '{:}'.format(genotype) 32 | steps = len(genotype) // 2 33 | 34 | for i in range(steps): 35 | g.node(str(i), fillcolor='lightblue') 36 | 37 | for i in range(steps): 38 | for k in [2*i, 2*i + 1]: 39 | op, j, weight = genotype[k] 40 | if j == 0: 41 | u = "c_{k-2}" 42 | elif j == 1: 43 | u = "c_{k-1}" 44 | else: 45 | u = str(j-2) 46 | v = str(i) 47 | g.edge(u, v, label=op, fillcolor="gray") 48 | 49 | g.node("c_{k}", fillcolor='palegoldenrod') 50 | for i in range(steps): 51 | g.edge(str(i), "c_{k}", fillcolor="gray") 52 | 53 | g.render(filename, view=False) 54 | 55 | def plot_rnn(genotype, filename): 56 | g = Digraph( 57 | format='pdf', 58 | edge_attr=dict(fontsize='20', fontname="times"), 59 | node_attr=dict(style='filled', shape='rect', align='center', fontsize='20', height='0.5', width='0.5', penwidth='2', fontname="times"), 60 | engine='dot') 61 | g.body.extend(['rankdir=LR']) 62 | 63 | g.node("x_{t}", fillcolor='darkseagreen2') 64 | g.node("h_{t-1}", fillcolor='darkseagreen2') 65 | g.node("0", fillcolor='lightblue') 66 | g.edge("x_{t}", "0", fillcolor="gray") 67 | g.edge("h_{t-1}", "0", fillcolor="gray") 68 | steps = len(genotype) 69 | 70 | for i in range(1, steps + 1): 71 | g.node(str(i), fillcolor='lightblue') 72 | 73 | for i, (op, j) in enumerate(genotype): 74 | g.edge(str(j), str(i + 1), label=op, fillcolor="gray") 75 | 76 | g.node("h_{t}", fillcolor='palegoldenrod') 77 | for i in range(1, steps + 1): 78 | g.edge(str(i), "h_{t}", fillcolor="gray") 79 | 80 | g.render(filename, view=False) 81 | 82 | 83 | if __name__ == '__main__': 84 | save_dir = Path(args.save_dir) 85 | 86 | save_path = str(save_dir / 'DMS_V1-normal') 87 | plot_cnn(DMS_V1.normal, save_path) 88 | save_path = str(save_dir / 'DMS_V1-reduce') 89 | plot_cnn(DMS_V1.reduce, save_path) 90 | save_path = str(save_dir / 'DMS_F1-normal') 91 | plot_cnn(DMS_F1.normal, save_path) 92 | 93 | save_path = str(save_dir / 'DARTS-V2-RNN') 94 | plot_rnn(DARTS_V2.recurrent, save_path) 95 | 96 | save_path = str(save_dir / 'GDAS-V1-RNN') 97 | plot_rnn(GDAS.recurrent, save_path) 98 | -------------------------------------------------------------------------------- /exps-cnn/evaluate.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | # For evaluating the learned model 5 | import os, sys, time, glob, random, argparse 6 | import numpy as np 7 | from copy import deepcopy 8 | import torch 9 | import torch.nn as nn 10 | import torch.nn.functional as F 11 | import torchvision.datasets as dset 12 | import torch.backends.cudnn as cudnn 13 | import torchvision.transforms as transforms 14 | from pathlib import Path 15 | lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() 16 | if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) 17 | from utils import AverageMeter, time_string, convert_secs2time 18 | from utils import print_log, obtain_accuracy 19 | from utils import Cutout, count_parameters_in_MB 20 | from nas import model_types as models 21 | from train_utils import main_procedure 22 | from train_utils_imagenet import main_procedure_imagenet 23 | from scheduler import load_config 24 | 25 | 26 | parser = argparse.ArgumentParser("Evaluate-CNN") 27 | parser.add_argument('--data_path', type=str, help='Path to dataset.') 28 | parser.add_argument('--checkpoint', type=str, help='Choose between Cifar10/100 and ImageNet.') 29 | args = parser.parse_args() 30 | 31 | assert torch.cuda.is_available(), 'torch.cuda is not available' 32 | 33 | 34 | def main(): 35 | 36 | assert os.path.isdir( args.data_path ), 'invalid data-path : {:}'.format(args.data_path) 37 | assert os.path.isfile( args.checkpoint ), 'invalid checkpoint : {:}'.format(args.checkpoint) 38 | 39 | checkpoint = torch.load( args.checkpoint ) 40 | xargs = checkpoint['args'] 41 | config = load_config(xargs.model_config) 42 | genotype = models[xargs.arch] 43 | 44 | # clear GPU cache 45 | torch.cuda.empty_cache() 46 | if xargs.dataset == 'imagenet': 47 | main_procedure_imagenet(config, args.data_path, xargs, genotype, xargs.init_channels, xargs.layers, checkpoint['state_dict'], None) 48 | else: 49 | main_procedure(config, xargs.dataset, args.data_path, xargs, genotype, xargs.init_channels, xargs.layers, checkpoint['state_dict'], None) 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /exps-cnn/train_base.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, time, glob, random, argparse 5 | import numpy as np 6 | from copy import deepcopy 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torchvision.datasets as dset 11 | import torch.backends.cudnn as cudnn 12 | import torchvision.transforms as transforms 13 | from pathlib import Path 14 | lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() 15 | if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) 16 | from utils import AverageMeter, time_string, convert_secs2time 17 | from utils import print_log, obtain_accuracy 18 | from utils import Cutout, count_parameters_in_MB 19 | from nas import model_types as models 20 | from train_utils import main_procedure 21 | from train_utils_imagenet import main_procedure_imagenet 22 | from scheduler import load_config 23 | 24 | 25 | parser = argparse.ArgumentParser("Train-CNN") 26 | parser.add_argument('--data_path', type=str, help='Path to dataset') 27 | parser.add_argument('--dataset', type=str, choices=['imagenet', 'cifar10', 'cifar100'], help='Choose between Cifar10/100 and ImageNet.') 28 | parser.add_argument('--arch', type=str, choices=models.keys(), help='the searched model.') 29 | # 30 | parser.add_argument('--grad_clip', type=float, help='gradient clipping') 31 | parser.add_argument('--model_config', type=str , help='the model configuration') 32 | parser.add_argument('--init_channels', type=int , help='the initial number of channels') 33 | parser.add_argument('--layers', type=int , help='the number of layers.') 34 | 35 | # log 36 | parser.add_argument('--workers', type=int, default=2, help='number of data loading workers (default: 2)') 37 | parser.add_argument('--save_path', type=str, help='Folder to save checkpoints and log.') 38 | parser.add_argument('--print_freq', type=int, help='print frequency (default: 200)') 39 | parser.add_argument('--manualSeed', type=int, help='manual seed') 40 | args = parser.parse_args() 41 | 42 | if 'CUDA_VISIBLE_DEVICES' not in os.environ: print('Can not find CUDA_VISIBLE_DEVICES in os.environ') 43 | else : print('Find CUDA_VISIBLE_DEVICES={:}'.format(os.environ['CUDA_VISIBLE_DEVICES'])) 44 | 45 | assert torch.cuda.is_available(), 'torch.cuda is not available' 46 | 47 | 48 | if args.manualSeed is None or args.manualSeed < 0: 49 | args.manualSeed = random.randint(1, 10000) 50 | random.seed(args.manualSeed) 51 | cudnn.benchmark = True 52 | cudnn.enabled = True 53 | torch.manual_seed(args.manualSeed) 54 | torch.cuda.manual_seed_all(args.manualSeed) 55 | 56 | 57 | def main(): 58 | 59 | # Init logger 60 | #args.save_path = os.path.join(args.save_path, 'seed-{:}'.format(args.manualSeed)) 61 | if not os.path.isdir(args.save_path): 62 | os.makedirs(args.save_path) 63 | log = open(os.path.join(args.save_path, 'seed-{:}-log.txt'.format(args.manualSeed)), 'w') 64 | print_log('Save Path : {:}'.format(args.save_path), log) 65 | state = {k: v for k, v in args._get_kwargs()} 66 | print_log(state, log) 67 | print_log("Random Seed : {:}".format(args.manualSeed), log) 68 | print_log("Python version : {:}".format(sys.version.replace('\n', ' ')), log) 69 | print_log("Torch version : {:}".format(torch.__version__), log) 70 | print_log("CUDA version : {:}".format(torch.version.cuda), log) 71 | print_log("cuDNN version : {:}".format(cudnn.version()), log) 72 | print_log("Num of GPUs : {:}".format(torch.cuda.device_count()), log) 73 | args.dataset = args.dataset.lower() 74 | 75 | config = load_config(args.model_config) 76 | genotype = models[args.arch] 77 | print_log('configuration : {:}'.format(config), log) 78 | print_log('genotype : {:}'.format(genotype), log) 79 | # clear GPU cache 80 | torch.cuda.empty_cache() 81 | if args.dataset == 'imagenet': 82 | main_procedure_imagenet(config, args.data_path, args, genotype, args.init_channels, args.layers, None, log) 83 | else: 84 | main_procedure(config, args.dataset, args.data_path, args, genotype, args.init_channels, args.layers, None, log) 85 | log.close() 86 | 87 | 88 | if __name__ == '__main__': 89 | main() 90 | -------------------------------------------------------------------------------- /exps-cnn/train_utils.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, time 5 | from copy import deepcopy 6 | import torch 7 | import torchvision.transforms as transforms 8 | from shutil import copyfile 9 | 10 | from utils import print_log, obtain_accuracy, AverageMeter 11 | from utils import time_string, convert_secs2time 12 | from utils import count_parameters_in_MB 13 | from utils import Cutout 14 | from nas import NetworkCIFAR as Network 15 | from datasets import get_datasets 16 | 17 | 18 | def obtain_best(accuracies): 19 | if len(accuracies) == 0: return (0, 0) 20 | tops = [value for key, value in accuracies.items()] 21 | s2b = sorted( tops ) 22 | return s2b[-1] 23 | 24 | 25 | def main_procedure(config, dataset, data_path, args, genotype, init_channels, layers, pure_evaluate, log): 26 | 27 | train_data, test_data, class_num = get_datasets(dataset, data_path, config.cutout) 28 | 29 | print_log('-------------------------------------- main-procedure', log) 30 | print_log('config : {:}'.format(config), log) 31 | print_log('genotype : {:}'.format(genotype), log) 32 | print_log('init_channels : {:}'.format(init_channels), log) 33 | print_log('layers : {:}'.format(layers), log) 34 | print_log('class_num : {:}'.format(class_num), log) 35 | basemodel = Network(init_channels, class_num, layers, config.auxiliary, genotype) 36 | model = torch.nn.DataParallel(basemodel).cuda() 37 | 38 | total_param, aux_param = count_parameters_in_MB(basemodel), count_parameters_in_MB(basemodel.auxiliary_param()) 39 | print_log('Network =>\n{:}'.format(basemodel), log) 40 | print_log('Parameters : {:} - {:} = {:.3f} MB'.format(total_param, aux_param, total_param - aux_param), log) 41 | print_log('config : {:}'.format(config), log) 42 | print_log('genotype : {:}'.format(genotype), log) 43 | print_log('args : {:}'.format(args), log) 44 | print_log('Train-Dataset : {:}'.format(train_data), log) 45 | print_log('Train-Trans : {:}'.format(train_data.transform), log) 46 | print_log('Test--Dataset : {:}'.format(test_data ), log) 47 | print_log('Test--Trans : {:}'.format(test_data.transform ), log) 48 | 49 | 50 | train_loader = torch.utils.data.DataLoader(train_data, batch_size=config.batch_size, shuffle=True, 51 | num_workers=args.workers, pin_memory=True) 52 | test_loader = torch.utils.data.DataLoader(test_data , batch_size=config.batch_size, shuffle=False, 53 | num_workers=args.workers, pin_memory=True) 54 | 55 | criterion = torch.nn.CrossEntropyLoss().cuda() 56 | 57 | optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay) 58 | #optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nestero=True) 59 | if config.type == 'cosine': 60 | scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(config.epochs), eta_min=float(config.LR_MIN)) 61 | else: 62 | raise ValueError('Can not find the schedular type : {:}'.format(config.type)) 63 | 64 | 65 | checkpoint_path = os.path.join(args.save_path, 'seed-{:}-checkpoint-{:}-model.pth'.format(args.manualSeed, dataset)) 66 | checkpoint_best = os.path.join(args.save_path, 'seed-{:}-checkpoint-{:}-best.pth'.format(args.manualSeed, dataset)) 67 | if pure_evaluate: 68 | print_log('-'*20 + 'Pure Evaluation' + '-'*20, log) 69 | basemodel.load_state_dict( pure_evaluate ) 70 | with torch.no_grad(): 71 | valid_acc1, valid_acc5, valid_los = _train(test_loader, model, criterion, optimizer, 'test', -1, config, args.print_freq, log) 72 | return (valid_acc1, valid_acc5) 73 | elif os.path.isfile(checkpoint_path): 74 | checkpoint = torch.load( checkpoint_path ) 75 | start_epoch = checkpoint['epoch'] 76 | basemodel.load_state_dict(checkpoint['state_dict']) 77 | optimizer.load_state_dict(checkpoint['optimizer']) 78 | scheduler.load_state_dict(checkpoint['scheduler']) 79 | accuracies = checkpoint['accuracies'] 80 | print_log('Load checkpoint from {:} with start-epoch = {:}'.format(checkpoint_path, start_epoch), log) 81 | else: 82 | start_epoch, accuracies = 0, {} 83 | print_log('Train model from scratch without pre-trained model or snapshot', log) 84 | 85 | 86 | # Main loop 87 | start_time, epoch_time = time.time(), AverageMeter() 88 | for epoch in range(start_epoch, config.epochs): 89 | scheduler.step() 90 | 91 | need_time = convert_secs2time(epoch_time.val * (config.epochs-epoch), True) 92 | print_log("\n==>>{:s} [Epoch={:03d}/{:03d}] {:s} LR={:6.4f} ~ {:6.4f}, Batch={:d}".format(time_string(), epoch, config.epochs, need_time, min(scheduler.get_lr()), max(scheduler.get_lr()), config.batch_size), log) 93 | 94 | basemodel.update_drop_path(config.drop_path_prob * epoch / config.epochs) 95 | 96 | train_acc1, train_acc5, train_los = _train(train_loader, model, criterion, optimizer, 'train', epoch, config, args.print_freq, log) 97 | 98 | with torch.no_grad(): 99 | valid_acc1, valid_acc5, valid_los = _train(test_loader, model, criterion, optimizer, 'test', epoch, config, args.print_freq, log) 100 | accuracies[epoch] = (valid_acc1, valid_acc5) 101 | 102 | torch.save({'epoch' : epoch + 1, 103 | 'args' : deepcopy(args), 104 | 'state_dict': basemodel.state_dict(), 105 | 'optimizer' : optimizer.state_dict(), 106 | 'scheduler' : scheduler.state_dict(), 107 | 'accuracies': accuracies}, 108 | checkpoint_path) 109 | best_acc = obtain_best( accuracies ) 110 | if accuracies[epoch] == best_acc: copyfile(checkpoint_path, checkpoint_best) 111 | print_log('----> Best Accuracy : Acc@1={:.2f}, Acc@5={:.2f}, Error@1={:.2f}, Error@5={:.2f}'.format(best_acc[0], best_acc[1], 100-best_acc[0], 100-best_acc[1]), log) 112 | print_log('----> Save into {:}'.format(checkpoint_path), log) 113 | 114 | # measure elapsed time 115 | epoch_time.update(time.time() - start_time) 116 | start_time = time.time() 117 | return obtain_best( accuracies ) 118 | 119 | 120 | def _train(xloader, model, criterion, optimizer, mode, epoch, config, print_freq, log): 121 | data_time, batch_time, losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter() 122 | if mode == 'train': 123 | model.train() 124 | elif mode == 'test': 125 | model.eval() 126 | else: raise ValueError("The mode is not right : {:}".format(mode)) 127 | 128 | end = time.time() 129 | for i, (inputs, targets) in enumerate(xloader): 130 | # measure data loading time 131 | data_time.update(time.time() - end) 132 | # calculate prediction and loss 133 | targets = targets.cuda(non_blocking=True) 134 | 135 | if mode == 'train': optimizer.zero_grad() 136 | 137 | if config.auxiliary and model.training: 138 | logits, logits_aux = model(inputs) 139 | else: 140 | logits = model(inputs) 141 | 142 | loss = criterion(logits, targets) 143 | if config.auxiliary and model.training: 144 | loss_aux = criterion(logits_aux, targets) 145 | loss += config.auxiliary_weight * loss_aux 146 | 147 | if mode == 'train': 148 | loss.backward() 149 | if config.grad_clip > 0: 150 | torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) 151 | optimizer.step() 152 | # record 153 | prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) 154 | losses.update(loss.item(), inputs.size(0)) 155 | top1.update (prec1.item(), inputs.size(0)) 156 | top5.update (prec5.item(), inputs.size(0)) 157 | 158 | # measure elapsed time 159 | batch_time.update(time.time() - end) 160 | end = time.time() 161 | 162 | if i % print_freq == 0 or (i+1) == len(xloader): 163 | Sstr = ' {:5s}'.format(mode) + time_string() + ' Epoch: [{:03d}][{:03d}/{:03d}]'.format(epoch, i, len(xloader)) 164 | Tstr = 'Time {batch_time.val:.2f} ({batch_time.avg:.2f}) Data {data_time.val:.2f} ({data_time.avg:.2f})'.format(batch_time=batch_time, data_time=data_time) 165 | Lstr = 'Loss {loss.val:.3f} ({loss.avg:.3f}) Prec@1 {top1.val:.2f} ({top1.avg:.2f}) Prec@5 {top5.val:.2f} ({top5.avg:.2f})'.format(loss=losses, top1=top1, top5=top5) 166 | print_log(Sstr + ' ' + Tstr + ' ' + Lstr, log) 167 | 168 | print_log ('{TIME:} **{mode:}** Prec@1 {top1.avg:.2f} Prec@5 {top5.avg:.2f} Error@1 {error1:.2f} Error@5 {error5:.2f} Loss:{loss:.3f}'.format(TIME=time_string(), mode=mode, top1=top1, top5=top5, error1=100-top1.avg, error5=100-top5.avg, loss=losses.avg), log) 169 | return top1.avg, top5.avg, losses.avg 170 | -------------------------------------------------------------------------------- /exps-cnn/train_utils_imagenet.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, time 5 | from copy import deepcopy 6 | import torch 7 | import torch.nn as nn 8 | import torchvision.transforms as transforms 9 | from shutil import copyfile 10 | 11 | from utils import print_log, obtain_accuracy, AverageMeter 12 | from utils import time_string, convert_secs2time 13 | from utils import count_parameters_in_MB 14 | from utils import print_FLOPs 15 | from utils import Cutout 16 | from nas import NetworkImageNet as Network 17 | from datasets import get_datasets 18 | 19 | 20 | def obtain_best(accuracies): 21 | if len(accuracies) == 0: return (0, 0) 22 | tops = [value for key, value in accuracies.items()] 23 | s2b = sorted( tops ) 24 | return s2b[-1] 25 | 26 | 27 | class CrossEntropyLabelSmooth(nn.Module): 28 | 29 | def __init__(self, num_classes, epsilon): 30 | super(CrossEntropyLabelSmooth, self).__init__() 31 | self.num_classes = num_classes 32 | self.epsilon = epsilon 33 | self.logsoftmax = nn.LogSoftmax(dim=1) 34 | 35 | def forward(self, inputs, targets): 36 | log_probs = self.logsoftmax(inputs) 37 | targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1) 38 | targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes 39 | loss = (-targets * log_probs).mean(0).sum() 40 | return loss 41 | 42 | 43 | def main_procedure_imagenet(config, data_path, args, genotype, init_channels, layers, pure_evaluate, log): 44 | 45 | # training data and testing data 46 | train_data, valid_data, class_num = get_datasets('imagenet-1k', data_path, -1) 47 | 48 | train_queue = torch.utils.data.DataLoader( 49 | train_data, batch_size=config.batch_size, shuffle= True, pin_memory=True, num_workers=args.workers) 50 | 51 | valid_queue = torch.utils.data.DataLoader( 52 | valid_data, batch_size=config.batch_size, shuffle=False, pin_memory=True, num_workers=args.workers) 53 | 54 | print_log('-------------------------------------- main-procedure', log) 55 | print_log('config : {:}'.format(config), log) 56 | print_log('genotype : {:}'.format(genotype), log) 57 | print_log('init_channels : {:}'.format(init_channels), log) 58 | print_log('layers : {:}'.format(layers), log) 59 | print_log('class_num : {:}'.format(class_num), log) 60 | basemodel = Network(init_channels, class_num, layers, config.auxiliary, genotype) 61 | model = torch.nn.DataParallel(basemodel).cuda() 62 | 63 | total_param, aux_param = count_parameters_in_MB(basemodel), count_parameters_in_MB(basemodel.auxiliary_param()) 64 | print_log('Network =>\n{:}'.format(basemodel), log) 65 | print_FLOPs(basemodel, (1,3,224,224), [print_log, log]) 66 | print_log('Parameters : {:} - {:} = {:.3f} MB'.format(total_param, aux_param, total_param - aux_param), log) 67 | print_log('config : {:}'.format(config), log) 68 | print_log('genotype : {:}'.format(genotype), log) 69 | print_log('Train-Dataset : {:}'.format(train_data), log) 70 | print_log('Valid--Dataset : {:}'.format(valid_data), log) 71 | print_log('Args : {:}'.format(args), log) 72 | 73 | 74 | criterion = torch.nn.CrossEntropyLoss().cuda() 75 | criterion_smooth = CrossEntropyLabelSmooth(class_num, config.label_smooth).cuda() 76 | 77 | 78 | optimizer = torch.optim.SGD(model.parameters(), config.LR, momentum=config.momentum, weight_decay=config.decay, nesterov=True) 79 | if config.type == 'cosine': 80 | scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(config.epochs)) 81 | elif config.type == 'steplr': 82 | scheduler = torch.optim.lr_scheduler.StepLR(optimizer, config.decay_period, gamma=config.gamma) 83 | else: 84 | raise ValueError('Can not find the schedular type : {:}'.format(config.type)) 85 | 86 | 87 | checkpoint_path = os.path.join(args.save_path, 'seed-{:}-checkpoint-imagenet-model.pth'.format(args.manualSeed)) 88 | checkpoint_best = os.path.join(args.save_path, 'seed-{:}-checkpoint-imagenet-best.pth'.format(args.manualSeed)) 89 | 90 | if pure_evaluate: 91 | print_log('-'*20 + 'Pure Evaluation' + '-'*20, log) 92 | basemodel.load_state_dict( pure_evaluate ) 93 | with torch.no_grad(): 94 | valid_acc1, valid_acc5, valid_los = _train(valid_queue, model, criterion, None, 'test' , -1, config, args.print_freq, log) 95 | return (valid_acc1, valid_acc5) 96 | elif os.path.isfile(checkpoint_path): 97 | checkpoint = torch.load( checkpoint_path ) 98 | start_epoch = checkpoint['epoch'] 99 | basemodel.load_state_dict(checkpoint['state_dict']) 100 | optimizer.load_state_dict(checkpoint['optimizer']) 101 | scheduler.load_state_dict(checkpoint['scheduler']) 102 | accuracies = checkpoint['accuracies'] 103 | print_log('Load checkpoint from {:} with start-epoch = {:}'.format(checkpoint_path, start_epoch), log) 104 | else: 105 | start_epoch, accuracies = 0, {} 106 | print_log('Train model from scratch without pre-trained model or snapshot', log) 107 | 108 | 109 | # Main loop 110 | start_time, epoch_time = time.time(), AverageMeter() 111 | for epoch in range(start_epoch, config.epochs): 112 | scheduler.step() 113 | 114 | basemodel.update_drop_path(config.drop_path_prob * epoch / config.epochs) 115 | 116 | need_time = convert_secs2time(epoch_time.val * (config.epochs-epoch), True) 117 | print_log("\n==>>{:s} [Epoch={:03d}/{:03d}] {:s} LR={:6.4f} ~ {:6.4f}, Batch={:d}, Drop-Path-Prob={:}".format(time_string(), epoch, config.epochs, need_time, min(scheduler.get_lr()), max(scheduler.get_lr()), config.batch_size, basemodel.get_drop_path()), log) 118 | 119 | train_acc1, train_acc5, train_los = _train(train_queue, model, criterion_smooth, optimizer, 'train', epoch, config, args.print_freq, log) 120 | 121 | with torch.no_grad(): 122 | valid_acc1, valid_acc5, valid_los = _train(valid_queue, model, criterion, None, 'test' , epoch, config, args.print_freq, log) 123 | accuracies[epoch] = (valid_acc1, valid_acc5) 124 | 125 | torch.save({'epoch' : epoch + 1, 126 | 'args' : deepcopy(args), 127 | 'state_dict': basemodel.state_dict(), 128 | 'optimizer' : optimizer.state_dict(), 129 | 'scheduler' : scheduler.state_dict(), 130 | 'accuracies': accuracies}, 131 | checkpoint_path) 132 | best_acc = obtain_best( accuracies ) 133 | if accuracies[epoch] == best_acc: copyfile(checkpoint_path, checkpoint_best) 134 | print_log('----> Best Accuracy : Acc@1={:.2f}, Acc@5={:.2f}, Error@1={:.2f}, Error@5={:.2f}'.format(best_acc[0], best_acc[1], 100-best_acc[0], 100-best_acc[1]), log) 135 | print_log('----> Save into {:}'.format(checkpoint_path), log) 136 | 137 | # measure elapsed time 138 | epoch_time.update(time.time() - start_time) 139 | start_time = time.time() 140 | return obtain_best( accuracies ) 141 | 142 | 143 | def _train(xloader, model, criterion, optimizer, mode, epoch, config, print_freq, log): 144 | data_time, batch_time, losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter() 145 | if mode == 'train': 146 | model.train() 147 | elif mode == 'test': 148 | model.eval() 149 | else: raise ValueError("The mode is not right : {:}".format(mode)) 150 | 151 | end = time.time() 152 | for i, (inputs, targets) in enumerate(xloader): 153 | # measure data loading time 154 | data_time.update(time.time() - end) 155 | # calculate prediction and loss 156 | targets = targets.cuda(non_blocking=True) 157 | 158 | if mode == 'train': optimizer.zero_grad() 159 | 160 | if config.auxiliary and model.training: 161 | logits, logits_aux = model(inputs) 162 | else: 163 | logits = model(inputs) 164 | 165 | loss = criterion(logits, targets) 166 | if config.auxiliary and model.training: 167 | loss_aux = criterion(logits_aux, targets) 168 | loss += config.auxiliary_weight * loss_aux 169 | 170 | if mode == 'train': 171 | loss.backward() 172 | if config.grad_clip > 0: 173 | torch.nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip) 174 | optimizer.step() 175 | # record 176 | prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) 177 | losses.update(loss.item(), inputs.size(0)) 178 | top1.update (prec1.item(), inputs.size(0)) 179 | top5.update (prec5.item(), inputs.size(0)) 180 | 181 | # measure elapsed time 182 | batch_time.update(time.time() - end) 183 | end = time.time() 184 | 185 | if i % print_freq == 0 or (i+1) == len(xloader): 186 | Sstr = ' {:5s}'.format(mode) + time_string() + ' Epoch: [{:03d}][{:03d}/{:03d}]'.format(epoch, i, len(xloader)) 187 | Tstr = 'Time {batch_time.val:.2f} ({batch_time.avg:.2f}) Data {data_time.val:.2f} ({data_time.avg:.2f})'.format(batch_time=batch_time, data_time=data_time) 188 | Lstr = 'Loss {loss.val:.3f} ({loss.avg:.3f}) Prec@1 {top1.val:.2f} ({top1.avg:.2f}) Prec@5 {top5.val:.2f} ({top5.avg:.2f})'.format(loss=losses, top1=top1, top5=top5) 189 | print_log(Sstr + ' ' + Tstr + ' ' + Lstr, log) 190 | 191 | print_log ('{TIME:} **{mode:}** Prec@1 {top1.avg:.2f} Prec@5 {top5.avg:.2f} Error@1 {error1:.2f} Error@5 {error5:.2f} Loss:{loss:.3f}'.format(TIME=time_string(), mode=mode, top1=top1, top5=top5, error1=100-top1.avg, error5=100-top5.avg, loss=losses.avg), log) 192 | return top1.avg, top5.avg, losses.avg 193 | -------------------------------------------------------------------------------- /exps-cnn/vis-arch.py: -------------------------------------------------------------------------------- 1 | import os, sys, time, glob, random, argparse 2 | import numpy as np 3 | from copy import deepcopy 4 | import torch 5 | from pathlib import Path 6 | lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() 7 | if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) 8 | from graphviz import Digraph 9 | 10 | parser = argparse.ArgumentParser("Visualize the Networks") 11 | parser.add_argument('--checkpoint', type=str, help='The path to the checkpoint.') 12 | parser.add_argument('--save_dir', type=str, help='The directory to save the network plot.') 13 | args = parser.parse_args() 14 | 15 | 16 | def plot(genotype, filename): 17 | g = Digraph( 18 | format='pdf', 19 | edge_attr=dict(fontsize='20', fontname="times"), 20 | node_attr=dict(style='filled', shape='rect', align='center', fontsize='20', height='0.5', width='0.5', penwidth='2', fontname="times"), 21 | engine='dot') 22 | g.body.extend(['rankdir=LR']) 23 | 24 | g.node("c_{k-2}", fillcolor='darkseagreen2') 25 | g.node("c_{k-1}", fillcolor='darkseagreen2') 26 | assert len(genotype) % 2 == 0 27 | steps = len(genotype) // 2 28 | 29 | for i in range(steps): 30 | g.node(str(i), fillcolor='lightblue') 31 | 32 | for i in range(steps): 33 | for k in [2*i, 2*i + 1]: 34 | op, j, weight = genotype[k] 35 | if j == 0: 36 | u = "c_{k-2}" 37 | elif j == 1: 38 | u = "c_{k-1}" 39 | else: 40 | u = str(j-2) 41 | v = str(i) 42 | g.edge(u, v, label=op, fillcolor="gray") 43 | 44 | g.node("c_{k}", fillcolor='palegoldenrod') 45 | for i in range(steps): 46 | g.edge(str(i), "c_{k}", fillcolor="gray") 47 | 48 | g.render(filename, view=False) 49 | 50 | 51 | if __name__ == '__main__': 52 | checkpoint = args.checkpoint 53 | assert os.path.isfile(checkpoint), 'Invalid path for checkpoint : {:}'.format(checkpoint) 54 | checkpoint = torch.load( checkpoint, map_location='cpu' ) 55 | genotypes = checkpoint['genotypes'] 56 | save_dir = Path(args.save_dir) 57 | subs = ['normal', 'reduce'] 58 | for sub in subs: 59 | if not (save_dir / sub).exists(): 60 | (save_dir / sub).mkdir(parents=True, exist_ok=True) 61 | 62 | for key, network in genotypes.items(): 63 | save_path = str(save_dir / 'normal' / 'epoch-{:03d}'.format( int(key) )) 64 | print('save into {:}'.format(save_path)) 65 | plot(network.normal, save_path) 66 | 67 | save_path = str(save_dir / 'reduce' / 'epoch-{:03d}'.format( int(key) )) 68 | print('save into {:}'.format(save_path)) 69 | plot(network.reduce, save_path) 70 | -------------------------------------------------------------------------------- /exps-rnn/train_rnn_base.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, gc, sys, math, time, glob, random, argparse 5 | import numpy as np 6 | from copy import deepcopy 7 | import torch 8 | import torch.nn as nn 9 | import torch.nn.functional as F 10 | import torchvision.datasets as dset 11 | import torch.backends.cudnn as cudnn 12 | import torchvision.transforms as transforms 13 | import multiprocessing 14 | from pathlib import Path 15 | lib_dir = (Path(__file__).parent / '..' / 'lib').resolve() 16 | print ('lib-dir : {:}'.format(lib_dir)) 17 | if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir)) 18 | from utils import AverageMeter, time_string, time_file_str, convert_secs2time 19 | from utils import print_log, obtain_accuracy 20 | from utils import count_parameters_in_MB 21 | from nas_rnn import DARTS_V1, DARTS_V2, GDAS 22 | from train_rnn_utils import main_procedure 23 | from scheduler import load_config 24 | 25 | Networks = {'DARTS_V1': DARTS_V1, 26 | 'DARTS_V2': DARTS_V2, 27 | 'GDAS' : GDAS} 28 | 29 | parser = argparse.ArgumentParser("RNN") 30 | parser.add_argument('--arch', type=str, choices=Networks.keys(), help='the network architecture') 31 | parser.add_argument('--config_path', type=str, help='the training configure for the discovered model') 32 | # log 33 | parser.add_argument('--save_path', type=str, help='Folder to save checkpoints and log.') 34 | parser.add_argument('--print_freq', type=int, help='print frequency (default: 200)') 35 | parser.add_argument('--manualSeed', type=int, help='manual seed') 36 | parser.add_argument('--threads', type=int, default=4, help='the number of threads') 37 | args = parser.parse_args() 38 | 39 | assert torch.cuda.is_available(), 'torch.cuda is not available' 40 | 41 | if args.manualSeed is None: 42 | args.manualSeed = random.randint(1, 10000) 43 | random.seed(args.manualSeed) 44 | cudnn.benchmark = True 45 | cudnn.enabled = True 46 | torch.manual_seed(args.manualSeed) 47 | torch.cuda.manual_seed_all(args.manualSeed) 48 | torch.set_num_threads(args.threads) 49 | 50 | def main(): 51 | 52 | # Init logger 53 | args.save_path = os.path.join(args.save_path, 'seed-{:}'.format(args.manualSeed)) 54 | if not os.path.isdir(args.save_path): 55 | os.makedirs(args.save_path) 56 | log = open(os.path.join(args.save_path, 'log-seed-{:}-{:}.txt'.format(args.manualSeed, time_file_str())), 'w') 57 | print_log('save path : {:}'.format(args.save_path), log) 58 | state = {k: v for k, v in args._get_kwargs()} 59 | print_log(state, log) 60 | print_log("Random Seed: {}".format(args.manualSeed), log) 61 | print_log("Python version : {}".format(sys.version.replace('\n', ' ')), log) 62 | print_log("Torch version : {}".format(torch.__version__), log) 63 | print_log("CUDA version : {}".format(torch.version.cuda), log) 64 | print_log("cuDNN version : {}".format(cudnn.version()), log) 65 | print_log("Num of GPUs : {}".format(torch.cuda.device_count()), log) 66 | print_log("Num of CPUs : {}".format(multiprocessing.cpu_count()), log) 67 | 68 | config = load_config( args.config_path ) 69 | genotype = Networks[ args.arch ] 70 | 71 | main_procedure(config, genotype, args.save_path, args.print_freq, log) 72 | log.close() 73 | 74 | 75 | if __name__ == '__main__': 76 | main() 77 | -------------------------------------------------------------------------------- /exps-rnn/train_rnn_utils.py: -------------------------------------------------------------------------------- 1 | # Modified from https://github.com/quark0/darts 2 | import os, gc, sys, time, math 3 | import numpy as np 4 | from copy import deepcopy 5 | import torch 6 | import torch.nn as nn 7 | from utils import print_log, obtain_accuracy, AverageMeter 8 | from utils import time_string, convert_secs2time 9 | from utils import count_parameters_in_MB 10 | from datasets import Corpus 11 | from nas_rnn import batchify, get_batch, repackage_hidden 12 | from nas_rnn import DARTSCell, RNNModel 13 | 14 | 15 | def obtain_best(accuracies): 16 | if len(accuracies) == 0: return (0, 0) 17 | tops = [value for key, value in accuracies.items()] 18 | s2b = sorted( tops ) 19 | return s2b[-1] 20 | 21 | 22 | def main_procedure(config, genotype, save_dir, print_freq, log): 23 | 24 | print_log('-'*90, log) 25 | print_log('save-dir : {:}'.format(save_dir), log) 26 | print_log('genotype : {:}'.format(genotype), log) 27 | print_log('config : {:}'.format(config), log) 28 | 29 | corpus = Corpus(config.data_path) 30 | train_data = batchify(corpus.train, config.train_batch, True) 31 | valid_data = batchify(corpus.valid, config.eval_batch , True) 32 | test_data = batchify(corpus.test, config.test_batch , True) 33 | ntokens = len(corpus.dictionary) 34 | print_log("Train--Data Size : {:}".format(train_data.size()), log) 35 | print_log("Valid--Data Size : {:}".format(valid_data.size()), log) 36 | print_log("Test---Data Size : {:}".format( test_data.size()), log) 37 | print_log("ntokens = {:}".format(ntokens), log) 38 | 39 | model = RNNModel(ntokens, config.emsize, config.nhid, config.nhidlast, 40 | config.dropout, config.dropouth, config.dropoutx, config.dropouti, config.dropoute, 41 | cell_cls=DARTSCell, genotype=genotype) 42 | model = model.cuda() 43 | print_log('Network =>\n{:}'.format(model), log) 44 | print_log('Genotype : {:}'.format(genotype), log) 45 | print_log('Parameters : {:.3f} MB'.format(count_parameters_in_MB(model)), log) 46 | 47 | checkpoint_path = os.path.join(save_dir, 'checkpoint-{:}.pth'.format(config.data_name)) 48 | 49 | Soptimizer = torch.optim.SGD (model.parameters(), lr=config.LR, weight_decay=config.wdecay) 50 | Aoptimizer = torch.optim.ASGD(model.parameters(), lr=config.LR, t0=0, lambd=0., weight_decay=config.wdecay) 51 | if os.path.isfile(checkpoint_path): 52 | checkpoint = torch.load(checkpoint_path) 53 | model.load_state_dict( checkpoint['state_dict'] ) 54 | Soptimizer.load_state_dict( checkpoint['SGD_optimizer'] ) 55 | Aoptimizer.load_state_dict( checkpoint['ASGD_optimizer'] ) 56 | epoch = checkpoint['epoch'] 57 | use_asgd = checkpoint['use_asgd'] 58 | print_log('load checkpoint from {:} and start train from {:}'.format(checkpoint_path, epoch), log) 59 | else: 60 | epoch, use_asgd = 0, False 61 | 62 | start_time, epoch_time = time.time(), AverageMeter() 63 | valid_loss_from_sgd, losses = [], {-1 : 1e9} 64 | while epoch < config.epochs: 65 | need_time = convert_secs2time(epoch_time.val * (config.epochs-epoch), True) 66 | print_log("\n==>>{:s} [Epoch={:04d}/{:04d}] {:}".format(time_string(), epoch, config.epochs, need_time), log) 67 | if use_asgd : optimizer = Aoptimizer 68 | else : optimizer = Soptimizer 69 | 70 | try: 71 | Dtime, Btime = train(model, optimizer, corpus, train_data, config, epoch, print_freq, log) 72 | except: 73 | torch.cuda.empty_cache() 74 | checkpoint = torch.load(checkpoint_path) 75 | model.load_state_dict( checkpoint['state_dict'] ) 76 | Soptimizer.load_state_dict( checkpoint['SGD_optimizer'] ) 77 | Aoptimizer.load_state_dict( checkpoint['ASGD_optimizer'] ) 78 | epoch = checkpoint['epoch'] 79 | use_asgd = checkpoint['use_asgd'] 80 | valid_loss_from_sgd = checkpoint['valid_loss_from_sgd'] 81 | continue 82 | if use_asgd: 83 | tmp = {} 84 | for prm in model.parameters(): 85 | tmp[prm] = prm.data.clone() 86 | prm.data = Aoptimizer.state[prm]['ax'].clone() 87 | 88 | val_loss = evaluate(model, corpus, valid_data, config.eval_batch, config.bptt) 89 | 90 | for prm in model.parameters(): 91 | prm.data = tmp[prm].clone() 92 | else: 93 | val_loss = evaluate(model, corpus, valid_data, config.eval_batch, config.bptt) 94 | if len(valid_loss_from_sgd) > config.nonmono and val_loss > min(valid_loss_from_sgd): 95 | use_asgd = True 96 | valid_loss_from_sgd.append( val_loss ) 97 | 98 | print_log('{:} end of epoch {:3d} with {:} | valid loss {:5.2f} | valid ppl {:8.2f}'.format(time_string(), epoch, 'ASGD' if use_asgd else 'SGD', val_loss, math.exp(val_loss)), log) 99 | 100 | if val_loss < min(losses.values()): 101 | if use_asgd: 102 | tmp = {} 103 | for prm in model.parameters(): 104 | tmp[prm] = prm.data.clone() 105 | prm.data = Aoptimizer.state[prm]['ax'].clone() 106 | torch.save({'epoch' : epoch, 107 | 'use_asgd' : use_asgd, 108 | 'valid_loss_from_sgd': valid_loss_from_sgd, 109 | 'state_dict': model.state_dict(), 110 | 'SGD_optimizer' : Soptimizer.state_dict(), 111 | 'ASGD_optimizer': Aoptimizer.state_dict()}, 112 | checkpoint_path) 113 | if use_asgd: 114 | for prm in model.parameters(): 115 | prm.data = tmp[prm].clone() 116 | print_log('save into {:}'.format(checkpoint_path), log) 117 | if use_asgd: 118 | tmp = {} 119 | for prm in model.parameters(): 120 | tmp[prm] = prm.data.clone() 121 | prm.data = Aoptimizer.state[prm]['ax'].clone() 122 | test_loss = evaluate(model, corpus, test_data, config.test_batch, config.bptt) 123 | if use_asgd: 124 | for prm in model.parameters(): 125 | prm.data = tmp[prm].clone() 126 | print_log('| epoch={:03d} | test loss {:5.2f} | test ppl {:8.2f}'.format(epoch, test_loss, math.exp(test_loss)), log) 127 | losses[epoch] = val_loss 128 | epoch = epoch + 1 129 | # measure elapsed time 130 | epoch_time.update(time.time() - start_time) 131 | start_time = time.time() 132 | 133 | 134 | print_log('--------------------- Finish Training ----------------', log) 135 | checkpoint = torch.load(checkpoint_path) 136 | model.load_state_dict( checkpoint['state_dict'] ) 137 | test_loss = evaluate(model, corpus, test_data , config.test_batch, config.bptt) 138 | print_log('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(test_loss, math.exp(test_loss)), log) 139 | vali_loss = evaluate(model, corpus, valid_data, config.eval_batch, config.bptt) 140 | print_log('| End of training | valid loss {:5.2f} | valid ppl {:8.2f}'.format(vali_loss, math.exp(vali_loss)), log) 141 | 142 | 143 | 144 | def evaluate(model, corpus, data_source, batch_size, bptt): 145 | # Turn on evaluation mode which disables dropout. 146 | model.eval() 147 | total_loss, total_length = 0.0, 0.0 148 | with torch.no_grad(): 149 | ntokens = len(corpus.dictionary) 150 | hidden = model.init_hidden(batch_size) 151 | for i in range(0, data_source.size(0) - 1, bptt): 152 | data, targets = get_batch(data_source, i, bptt) 153 | targets = targets.view(-1) 154 | 155 | log_prob, hidden = model(data, hidden) 156 | loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets) 157 | 158 | total_loss += loss.item() * len(data) 159 | total_length += len(data) 160 | hidden = repackage_hidden(hidden) 161 | return total_loss / total_length 162 | 163 | 164 | 165 | def train(model, optimizer, corpus, train_data, config, epoch, print_freq, log): 166 | # Turn on training mode which enables dropout. 167 | total_loss, data_time, batch_time = 0, AverageMeter(), AverageMeter() 168 | start_time = time.time() 169 | ntokens = len(corpus.dictionary) 170 | 171 | hidden_train = model.init_hidden(config.train_batch) 172 | 173 | batch, i = 0, 0 174 | while i < train_data.size(0) - 1 - 1: 175 | bptt = config.bptt if np.random.random() < 0.95 else config.bptt / 2. 176 | # Prevent excessively small or negative sequence lengths 177 | seq_len = max(5, int(np.random.normal(bptt, 5))) 178 | # There's a very small chance that it could select a very long sequence length resulting in OOM 179 | seq_len = min(seq_len, config.bptt + config.max_seq_len_delta) 180 | 181 | 182 | lr2 = optimizer.param_groups[0]['lr'] 183 | optimizer.param_groups[0]['lr'] = lr2 * seq_len / config.bptt 184 | 185 | model.train() 186 | data, targets = get_batch(train_data, i, seq_len) 187 | targets = targets.contiguous().view(-1) 188 | # count data preparation time 189 | data_time.update(time.time() - start_time) 190 | 191 | optimizer.zero_grad() 192 | hidden_train = repackage_hidden(hidden_train) 193 | log_prob, hidden_train, rnn_hs, dropped_rnn_hs = model(data, hidden_train, return_h=True) 194 | raw_loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets) 195 | 196 | loss = raw_loss 197 | # Activiation Regularization 198 | if config.alpha > 0: 199 | loss = loss + sum(config.alpha * dropped_rnn_h.pow(2).mean() for dropped_rnn_h in dropped_rnn_hs[-1:]) 200 | # Temporal Activation Regularization (slowness) 201 | loss = loss + sum(config.beta * (rnn_h[1:] - rnn_h[:-1]).pow(2).mean() for rnn_h in rnn_hs[-1:]) 202 | loss.backward() 203 | torch.nn.utils.clip_grad_norm_(model.parameters(), config.clip) 204 | optimizer.step() 205 | 206 | gc.collect() 207 | 208 | optimizer.param_groups[0]['lr'] = lr2 209 | 210 | total_loss += raw_loss.item() 211 | assert torch.isnan(loss) == False, '--- Epoch={:04d} :: {:03d}/{:03d} Get Loss = Nan'.format(epoch, batch, len(train_data)//config.bptt) 212 | 213 | batch_time.update(time.time() - start_time) 214 | start_time = time.time() 215 | batch, i = batch + 1, i + seq_len 216 | 217 | if batch % print_freq == 0: 218 | cur_loss = total_loss / print_freq 219 | print_log(' >> Epoch: {:04d} :: {:03d}/{:03d} || loss = {:5.2f}, ppl = {:8.2f}'.format(epoch, batch, len(train_data) // config.bptt, cur_loss, math.exp(cur_loss)), log) 220 | total_loss = 0 221 | return data_time.sum, batch_time.sum 222 | -------------------------------------------------------------------------------- /lib/datasets/LanguageDataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | from collections import Counter 5 | 6 | 7 | class Dictionary(object): 8 | def __init__(self): 9 | self.word2idx = {} 10 | self.idx2word = [] 11 | self.counter = Counter() 12 | self.total = 0 13 | 14 | def add_word(self, word): 15 | if word not in self.word2idx: 16 | self.idx2word.append(word) 17 | self.word2idx[word] = len(self.idx2word) - 1 18 | token_id = self.word2idx[word] 19 | self.counter[token_id] += 1 20 | self.total += 1 21 | return self.word2idx[word] 22 | 23 | def __len__(self): 24 | return len(self.idx2word) 25 | 26 | 27 | class Corpus(object): 28 | def __init__(self, path): 29 | self.dictionary = Dictionary() 30 | self.train = self.tokenize(os.path.join(path, 'train.txt')) 31 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) 32 | self.test = self.tokenize(os.path.join(path, 'test.txt')) 33 | 34 | def tokenize(self, path): 35 | """Tokenizes a text file.""" 36 | assert os.path.exists(path) 37 | # Add words to the dictionary 38 | with open(path, 'r', encoding='utf-8') as f: 39 | tokens = 0 40 | for line in f: 41 | words = line.split() + [''] 42 | tokens += len(words) 43 | for word in words: 44 | self.dictionary.add_word(word) 45 | 46 | # Tokenize file content 47 | with open(path, 'r', encoding='utf-8') as f: 48 | ids = torch.LongTensor(tokens) 49 | token = 0 50 | for line in f: 51 | words = line.split() + [''] 52 | for word in words: 53 | ids[token] = self.dictionary.word2idx[word] 54 | token += 1 55 | 56 | return ids 57 | 58 | class SentCorpus(object): 59 | def __init__(self, path): 60 | self.dictionary = Dictionary() 61 | self.train = self.tokenize(os.path.join(path, 'train.txt')) 62 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) 63 | self.test = self.tokenize(os.path.join(path, 'test.txt')) 64 | 65 | def tokenize(self, path): 66 | """Tokenizes a text file.""" 67 | assert os.path.exists(path) 68 | # Add words to the dictionary 69 | with open(path, 'r', encoding='utf-8') as f: 70 | tokens = 0 71 | for line in f: 72 | words = line.split() + [''] 73 | tokens += len(words) 74 | for word in words: 75 | self.dictionary.add_word(word) 76 | 77 | # Tokenize file content 78 | sents = [] 79 | with open(path, 'r', encoding='utf-8') as f: 80 | for line in f: 81 | if not line: 82 | continue 83 | words = line.split() + [''] 84 | sent = torch.LongTensor(len(words)) 85 | for i, word in enumerate(words): 86 | sent[i] = self.dictionary.word2idx[word] 87 | sents.append(sent) 88 | 89 | return sents 90 | 91 | class BatchSentLoader(object): 92 | def __init__(self, sents, batch_size, pad_id=0, cuda=False, volatile=False): 93 | self.sents = sents 94 | self.batch_size = batch_size 95 | self.sort_sents = sorted(sents, key=lambda x: x.size(0)) 96 | self.cuda = cuda 97 | self.volatile = volatile 98 | self.pad_id = pad_id 99 | 100 | def __next__(self): 101 | if self.idx >= len(self.sort_sents): 102 | raise StopIteration 103 | 104 | batch_size = min(self.batch_size, len(self.sort_sents)-self.idx) 105 | batch = self.sort_sents[self.idx:self.idx+batch_size] 106 | max_len = max([s.size(0) for s in batch]) 107 | tensor = torch.LongTensor(max_len, batch_size).fill_(self.pad_id) 108 | for i in range(len(batch)): 109 | s = batch[i] 110 | tensor[:s.size(0),i].copy_(s) 111 | if self.cuda: 112 | tensor = tensor.cuda() 113 | 114 | self.idx += batch_size 115 | 116 | return tensor 117 | 118 | next = __next__ 119 | 120 | def __iter__(self): 121 | self.idx = 0 122 | return self 123 | -------------------------------------------------------------------------------- /lib/datasets/MetaBatchSampler.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import numpy as np 3 | import torch 4 | 5 | 6 | class MetaBatchSampler(object): 7 | 8 | def __init__(self, labels, classes_per_it, num_samples, iterations): 9 | ''' 10 | Initialize MetaBatchSampler 11 | Args: 12 | - labels: an iterable containing all the labels for the current dataset 13 | samples indexes will be infered from this iterable. 14 | - classes_per_it: number of random classes for each iteration 15 | - num_samples: number of samples for each iteration for each class (support + query) 16 | - iterations: number of iterations (episodes) per epoch 17 | ''' 18 | super(MetaBatchSampler, self).__init__() 19 | self.labels = labels.copy() 20 | self.classes_per_it = classes_per_it 21 | self.sample_per_class = num_samples 22 | self.iterations = iterations 23 | 24 | self.classes, self.counts = np.unique(self.labels, return_counts=True) 25 | assert len(self.classes) == np.max(self.classes) + 1 and np.min(self.classes) == 0 26 | assert classes_per_it < len(self.classes), '{:} vs. {:}'.format(classes_per_it, len(self.classes)) 27 | self.classes = torch.LongTensor(self.classes) 28 | 29 | # create a matrix, indexes, of dim: classes X max(elements per class) 30 | # fill it with nans 31 | # for every class c, fill the relative row with the indices samples belonging to c 32 | # in numel_per_class we store the number of samples for each class/row 33 | self.indexes = { x.item() : [] for x in self.classes } 34 | indexes = { x.item() : [] for x in self.classes } 35 | 36 | for idx, label in enumerate(self.labels): 37 | indexes[ label.item() ].append( idx ) 38 | for key, value in indexes.items(): 39 | self.indexes[ key ] = torch.LongTensor( value ) 40 | 41 | 42 | def __iter__(self): 43 | # yield a batch of indexes 44 | spc = self.sample_per_class 45 | cpi = self.classes_per_it 46 | 47 | for it in range(self.iterations): 48 | batch_size = spc * cpi 49 | batch = torch.LongTensor(batch_size) 50 | assert cpi < len(self.classes), '{:} vs. {:}'.format(cpi, len(self.classes)) 51 | c_idxs = torch.randperm(len(self.classes))[:cpi] 52 | 53 | for i, cls in enumerate(self.classes[c_idxs]): 54 | s = slice(i * spc, (i + 1) * spc) 55 | num = self.indexes[ cls.item() ].nelement() 56 | assert spc < num, '{:} vs. {:}'.format(spc, num) 57 | sample_idxs = torch.randperm( num )[:spc] 58 | batch[s] = self.indexes[ cls.item() ][sample_idxs] 59 | 60 | batch = batch[torch.randperm(len(batch))] 61 | yield batch 62 | 63 | def __len__(self): 64 | # returns the number of iterations (episodes) per epoch 65 | return self.iterations 66 | -------------------------------------------------------------------------------- /lib/datasets/TieredImageNet.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from PIL import Image 4 | import pickle as pkl 5 | import os, cv2, csv, glob 6 | import torch 7 | import torch.utils.data as data 8 | 9 | 10 | class TieredImageNet(data.Dataset): 11 | 12 | def __init__(self, root_dir, split, transform=None): 13 | self.split = split 14 | self.root_dir = root_dir 15 | self.transform = transform 16 | splits = split.split('-') 17 | 18 | images, labels, last = [], [], 0 19 | for split in splits: 20 | labels_name = '{:}/{:}_labels.pkl'.format(self.root_dir, split) 21 | images_name = '{:}/{:}_images.npz'.format(self.root_dir, split) 22 | # decompress images if npz not exits 23 | if not os.path.exists(images_name): 24 | png_pkl = images_name[:-4] + '_png.pkl' 25 | if os.path.exists(png_pkl): 26 | decompress(images_name, png_pkl) 27 | else: 28 | raise ValueError('png_pkl {:} not exits'.format( png_pkl )) 29 | assert os.path.exists(images_name) and os.path.exists(labels_name), '{:} & {:}'.format(images_name, labels_name) 30 | print ("Prepare {:} done".format(images_name)) 31 | try: 32 | with open(labels_name) as f: 33 | data = pkl.load(f) 34 | label_specific = data["label_specific"] 35 | except: 36 | with open(labels_name, 'rb') as f: 37 | data = pkl.load(f, encoding='bytes') 38 | label_specific = data[b'label_specific'] 39 | with np.load(images_name, mmap_mode="r", encoding='latin1') as data: 40 | image_data = data["images"] 41 | images.append( image_data ) 42 | label_specific = label_specific + last 43 | labels.append( label_specific ) 44 | last = np.max(label_specific) + 1 45 | print ("Load {:} done, with image shape = {:}, label shape = {:}, [{:} ~ {:}]".format(images_name, image_data.shape, label_specific.shape, np.min(label_specific), np.max(label_specific))) 46 | images, labels = np.concatenate(images), np.concatenate(labels) 47 | 48 | self.images = images 49 | self.labels = labels 50 | self.n_classes = int( np.max(labels) + 1 ) 51 | self.dict_index_label = {} 52 | for cls in range(self.n_classes): 53 | idxs = np.where(labels==cls)[0] 54 | self.dict_index_label[cls] = idxs 55 | self.length = len(labels) 56 | print ("There are {:} images, {:} labels [{:} ~ {:}]".format(images.shape, labels.shape, np.min(labels), np.max(labels))) 57 | 58 | 59 | def __repr__(self): 60 | return ('{name}(length={length}, classes={n_classes})'.format(name=self.__class__.__name__, **self.__dict__)) 61 | 62 | def __len__(self): 63 | return self.length 64 | 65 | def __getitem__(self, index): 66 | assert index >= 0 and index < self.length, 'invalid index = {:}'.format(index) 67 | image = self.images[index].copy() 68 | label = int(self.labels[index]) 69 | image = Image.fromarray(image[:,:,::-1].astype('uint8'), 'RGB') 70 | if self.transform is not None: 71 | image = self.transform( image ) 72 | return image, label 73 | 74 | 75 | 76 | 77 | def decompress(path, output): 78 | with open(output, 'rb') as f: 79 | array = pkl.load(f, encoding='bytes') 80 | images = np.zeros([len(array), 84, 84, 3], dtype=np.uint8) 81 | for ii, item in enumerate(array): 82 | im = cv2.imdecode(item, 1) 83 | images[ii] = im 84 | np.savez(path, images=images) 85 | -------------------------------------------------------------------------------- /lib/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .MetaBatchSampler import MetaBatchSampler 5 | from .TieredImageNet import TieredImageNet 6 | from .LanguageDataset import Corpus 7 | from .get_dataset_with_transform import get_datasets 8 | -------------------------------------------------------------------------------- /lib/datasets/get_dataset_with_transform.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, torch 5 | import os.path as osp 6 | import torchvision.datasets as dset 7 | import torch.backends.cudnn as cudnn 8 | import torchvision.transforms as transforms 9 | 10 | from utils import Cutout 11 | from .TieredImageNet import TieredImageNet 12 | 13 | 14 | Dataset2Class = {'cifar10' : 10, 15 | 'cifar100': 100, 16 | 'tiered' : -1, 17 | 'imagenet-1k' : 1000, 18 | 'imagenet-100': 100} 19 | 20 | 21 | def get_datasets(name, root, cutout): 22 | 23 | # Mean + Std 24 | if name == 'cifar10': 25 | mean = [x / 255 for x in [125.3, 123.0, 113.9]] 26 | std = [x / 255 for x in [63.0, 62.1, 66.7]] 27 | elif name == 'cifar100': 28 | mean = [x / 255 for x in [129.3, 124.1, 112.4]] 29 | std = [x / 255 for x in [68.2, 65.4, 70.4]] 30 | elif name == 'tiered': 31 | mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 32 | elif name == 'imagenet-1k' or name == 'imagenet-100': 33 | mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 34 | else: raise TypeError("Unknow dataset : {:}".format(name)) 35 | 36 | 37 | # Data Argumentation 38 | if name == 'cifar10' or name == 'cifar100': 39 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), 40 | transforms.Normalize(mean, std)] 41 | if cutout > 0 : lists += [Cutout(cutout)] 42 | train_transform = transforms.Compose(lists) 43 | test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)]) 44 | elif name == 'tiered': 45 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(80, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)] 46 | if cutout > 0 : lists += [Cutout(cutout)] 47 | train_transform = transforms.Compose(lists) 48 | test_transform = transforms.Compose([transforms.CenterCrop(80), transforms.ToTensor(), transforms.Normalize(mean, std)]) 49 | elif name == 'imagenet-1k' or name == 'imagenet-100': 50 | normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) 51 | train_transform = transforms.Compose([ 52 | transforms.RandomResizedCrop(224), 53 | transforms.RandomHorizontalFlip(), 54 | transforms.ColorJitter( 55 | brightness=0.4, 56 | contrast=0.4, 57 | saturation=0.4, 58 | hue=0.2), 59 | transforms.ToTensor(), 60 | normalize, 61 | ]) 62 | test_transform = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]) 63 | else: raise TypeError("Unknow dataset : {:}".format(name)) 64 | 65 | if name == 'cifar10': 66 | train_data = dset.CIFAR10 (root, train=True , transform=train_transform, download=True) 67 | test_data = dset.CIFAR10 (root, train=False, transform=test_transform , download=True) 68 | elif name == 'cifar100': 69 | train_data = dset.CIFAR100(root, train=True , transform=train_transform, download=True) 70 | test_data = dset.CIFAR100(root, train=False, transform=test_transform , download=True) 71 | elif name == 'imagenet-1k' or name == 'imagenet-100': 72 | train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform) 73 | test_data = dset.ImageFolder(osp.join(root, 'val'), test_transform) 74 | else: raise TypeError("Unknow dataset : {:}".format(name)) 75 | 76 | class_num = Dataset2Class[name] 77 | return train_data, test_data, class_num 78 | -------------------------------------------------------------------------------- /lib/datasets/test_NLP.py: -------------------------------------------------------------------------------- 1 | import os, sys, torch 2 | 3 | from LanguageDataset import SentCorpus, BatchSentLoader 4 | 5 | if __name__ == '__main__': 6 | path = '../../data/data/penn' 7 | corpus = SentCorpus( path ) 8 | loader = BatchSentLoader(corpus.test, 10) 9 | for i, d in enumerate(loader): 10 | print('{:} :: {:}'.format(i, d.size())) 11 | -------------------------------------------------------------------------------- /lib/datasets/test_dataset.py: -------------------------------------------------------------------------------- 1 | import os, sys, torch 2 | import torchvision.transforms as transforms 3 | 4 | from TieredImageNet import TieredImageNet 5 | from MetaBatchSampler import MetaBatchSampler 6 | 7 | root_dir = os.environ['TORCH_HOME'] + '/tiered-imagenet' 8 | print ('root : {:}'.format(root_dir)) 9 | means, stds = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] 10 | 11 | lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(84, padding=8), transforms.ToTensor(), transforms.Normalize(means, stds)] 12 | transform = transforms.Compose(lists) 13 | 14 | dataset = TieredImageNet(root_dir, 'val-test', transform) 15 | image, label = dataset[111] 16 | print ('image shape = {:}, label = {:}'.format(image.size(), label)) 17 | print ('image : min = {:}, max = {:} ||| label : {:}'.format(image.min(), image.max(), label)) 18 | 19 | 20 | sampler = MetaBatchSampler(dataset.labels, 250, 100, 10) 21 | 22 | dataloader = torch.utils.data.DataLoader(dataset, batch_sampler=sampler) 23 | 24 | print ('the length of dataset : {:}'.format( len(dataset) )) 25 | print ('the length of loader : {:}'.format( len(dataloader) )) 26 | 27 | for images, labels in dataloader: 28 | print ('images : {:}'.format( images.size() )) 29 | print ('labels : {:}'.format( labels.size() )) 30 | for i in range(3): 31 | print ('image-value-[{:}] : {:} ~ {:}, mean={:}, std={:}'.format(i, images[:,i].min(), images[:,i].max(), images[:,i].mean(), images[:,i].std())) 32 | 33 | print('-----') 34 | -------------------------------------------------------------------------------- /lib/nas/CifarNet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .construct_utils import Cell, Transition 4 | 5 | class AuxiliaryHeadCIFAR(nn.Module): 6 | 7 | def __init__(self, C, num_classes): 8 | """assuming input size 8x8""" 9 | super(AuxiliaryHeadCIFAR, self).__init__() 10 | self.features = nn.Sequential( 11 | nn.ReLU(inplace=True), 12 | nn.AvgPool2d(5, stride=3, padding=0, count_include_pad=False), # image size = 2 x 2 13 | nn.Conv2d(C, 128, 1, bias=False), 14 | nn.BatchNorm2d(128), 15 | nn.ReLU(inplace=True), 16 | nn.Conv2d(128, 768, 2, bias=False), 17 | nn.BatchNorm2d(768), 18 | nn.ReLU(inplace=True) 19 | ) 20 | self.classifier = nn.Linear(768, num_classes) 21 | 22 | def forward(self, x): 23 | x = self.features(x) 24 | x = self.classifier(x.view(x.size(0),-1)) 25 | return x 26 | 27 | 28 | class NetworkCIFAR(nn.Module): 29 | 30 | def __init__(self, C, num_classes, layers, auxiliary, genotype): 31 | super(NetworkCIFAR, self).__init__() 32 | self._layers = layers 33 | 34 | stem_multiplier = 3 35 | C_curr = stem_multiplier*C 36 | self.stem = nn.Sequential( 37 | nn.Conv2d(3, C_curr, 3, padding=1, bias=False), 38 | nn.BatchNorm2d(C_curr) 39 | ) 40 | 41 | C_prev_prev, C_prev, C_curr = C_curr, C_curr, C 42 | self.cells = nn.ModuleList() 43 | reduction_prev = False 44 | for i in range(layers): 45 | if i in [layers//3, 2*layers//3]: 46 | C_curr *= 2 47 | reduction = True 48 | else: 49 | reduction = False 50 | if reduction and genotype.reduce is None: 51 | cell = Transition(C_prev_prev, C_prev, C_curr, reduction_prev) 52 | else: 53 | cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) 54 | reduction_prev = reduction 55 | self.cells.append( cell ) 56 | C_prev_prev, C_prev = C_prev, cell.multiplier*C_curr 57 | if i == 2*layers//3: 58 | C_to_auxiliary = C_prev 59 | 60 | if auxiliary: 61 | self.auxiliary_head = AuxiliaryHeadCIFAR(C_to_auxiliary, num_classes) 62 | else: 63 | self.auxiliary_head = None 64 | self.global_pooling = nn.AdaptiveAvgPool2d(1) 65 | self.classifier = nn.Linear(C_prev, num_classes) 66 | self.drop_path_prob = -1 67 | 68 | def update_drop_path(self, drop_path_prob): 69 | self.drop_path_prob = drop_path_prob 70 | 71 | def auxiliary_param(self): 72 | if self.auxiliary_head is None: return [] 73 | else: return list( self.auxiliary_head.parameters() ) 74 | 75 | def forward(self, inputs): 76 | s0 = s1 = self.stem(inputs) 77 | for i, cell in enumerate(self.cells): 78 | s0, s1 = s1, cell(s0, s1, self.drop_path_prob) 79 | if i == 2*self._layers//3: 80 | if self.auxiliary_head and self.training: 81 | logits_aux = self.auxiliary_head(s1) 82 | out = self.global_pooling(s1) 83 | out = out.view(out.size(0), -1) 84 | logits = self.classifier(out) 85 | 86 | if self.auxiliary_head and self.training: 87 | return logits, logits_aux 88 | else: 89 | return logits 90 | -------------------------------------------------------------------------------- /lib/nas/ImageNet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .construct_utils import Cell, Transition 4 | 5 | class AuxiliaryHeadImageNet(nn.Module): 6 | 7 | def __init__(self, C, num_classes): 8 | """assuming input size 14x14""" 9 | super(AuxiliaryHeadImageNet, self).__init__() 10 | self.features = nn.Sequential( 11 | nn.ReLU(inplace=True), 12 | nn.AvgPool2d(5, stride=2, padding=0, count_include_pad=False), 13 | nn.Conv2d(C, 128, 1, bias=False), 14 | nn.BatchNorm2d(128), 15 | nn.ReLU(inplace=True), 16 | nn.Conv2d(128, 768, 2, bias=False), 17 | # NOTE: This batchnorm was omitted in my earlier implementation due to a typo. 18 | # Commenting it out for consistency with the experiments in the paper. 19 | # nn.BatchNorm2d(768), 20 | nn.ReLU(inplace=True) 21 | ) 22 | self.classifier = nn.Linear(768, num_classes) 23 | 24 | def forward(self, x): 25 | x = self.features(x) 26 | x = self.classifier(x.view(x.size(0),-1)) 27 | return x 28 | 29 | 30 | 31 | 32 | class NetworkImageNet(nn.Module): 33 | 34 | def __init__(self, C, num_classes, layers, auxiliary, genotype): 35 | super(NetworkImageNet, self).__init__() 36 | self._layers = layers 37 | 38 | self.stem0 = nn.Sequential( 39 | nn.Conv2d(3, C // 2, kernel_size=3, stride=2, padding=1, bias=False), 40 | nn.BatchNorm2d(C // 2), 41 | nn.ReLU(inplace=True), 42 | nn.Conv2d(C // 2, C, 3, stride=2, padding=1, bias=False), 43 | nn.BatchNorm2d(C), 44 | ) 45 | 46 | self.stem1 = nn.Sequential( 47 | nn.ReLU(inplace=True), 48 | nn.Conv2d(C, C, 3, stride=2, padding=1, bias=False), 49 | nn.BatchNorm2d(C), 50 | ) 51 | 52 | C_prev_prev, C_prev, C_curr = C, C, C 53 | 54 | self.cells = nn.ModuleList() 55 | reduction_prev = True 56 | for i in range(layers): 57 | if i in [layers // 3, 2 * layers // 3]: 58 | C_curr *= 2 59 | reduction = True 60 | else: 61 | reduction = False 62 | if reduction and genotype.reduce is None: 63 | cell = Transition(C_prev_prev, C_prev, C_curr, reduction_prev) 64 | else: 65 | cell = Cell(genotype, C_prev_prev, C_prev, C_curr, reduction, reduction_prev) 66 | reduction_prev = reduction 67 | self.cells += [cell] 68 | C_prev_prev, C_prev = C_prev, cell.multiplier * C_curr 69 | if i == 2 * layers // 3: 70 | C_to_auxiliary = C_prev 71 | 72 | if auxiliary: 73 | self.auxiliary_head = AuxiliaryHeadImageNet(C_to_auxiliary, num_classes) 74 | else: 75 | self.auxiliary_head = None 76 | self.global_pooling = nn.AvgPool2d(7) 77 | self.classifier = nn.Linear(C_prev, num_classes) 78 | self.drop_path_prob = -1 79 | 80 | def update_drop_path(self, drop_path_prob): 81 | self.drop_path_prob = drop_path_prob 82 | 83 | def get_drop_path(self): 84 | return self.drop_path_prob 85 | 86 | def auxiliary_param(self): 87 | if self.auxiliary_head is None: return [] 88 | else: return list( self.auxiliary_head.parameters() ) 89 | 90 | def forward(self, input): 91 | s0 = self.stem0(input) 92 | s1 = self.stem1(s0) 93 | for i, cell in enumerate(self.cells): 94 | s0, s1 = s1, cell(s0, s1, self.drop_path_prob) 95 | #print ('{:} : {:} - {:}'.format(i, s0.size(), s1.size())) 96 | if i == 2 * self._layers // 3: 97 | if self.auxiliary_head and self.training: 98 | logits_aux = self.auxiliary_head(s1) 99 | out = self.global_pooling(s1) 100 | logits = self.classifier(out.view(out.size(0), -1)) 101 | if self.auxiliary_head and self.training: 102 | return logits, logits_aux 103 | else: 104 | return logits 105 | -------------------------------------------------------------------------------- /lib/nas/SE_Module.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | # Squeeze and Excitation module 4 | 5 | class SqEx(nn.Module): 6 | 7 | def __init__(self, n_features, reduction=16): 8 | super(SqEx, self).__init__() 9 | 10 | if n_features % reduction != 0: 11 | raise ValueError('n_features must be divisible by reduction (default = 16)') 12 | 13 | self.linear1 = nn.Linear(n_features, n_features // reduction, bias=True) 14 | self.nonlin1 = nn.ReLU(inplace=True) 15 | self.linear2 = nn.Linear(n_features // reduction, n_features, bias=True) 16 | self.nonlin2 = nn.Sigmoid() 17 | 18 | def forward(self, x): 19 | 20 | y = F.avg_pool2d(x, kernel_size=x.size()[2:4]) 21 | y = y.permute(0, 2, 3, 1) 22 | y = self.nonlin1(self.linear1(y)) 23 | y = self.nonlin2(self.linear2(y)) 24 | y = y.permute(0, 3, 1, 2) 25 | y = x * y 26 | return y 27 | 28 | -------------------------------------------------------------------------------- /lib/nas/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .CifarNet import NetworkCIFAR 5 | from .ImageNet import NetworkImageNet 6 | 7 | # genotypes 8 | from .genotypes import model_types 9 | 10 | from .construct_utils import return_alphas_str 11 | -------------------------------------------------------------------------------- /lib/nas/construct_utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from .operations import OPS, FactorizedReduce, ReLUConvBN, Identity 6 | 7 | 8 | def random_select(length, ratio): 9 | clist = [] 10 | index = random.randint(0, length-1) 11 | for i in range(length): 12 | if i == index or random.random() < ratio: 13 | clist.append( 1 ) 14 | else: 15 | clist.append( 0 ) 16 | return clist 17 | 18 | 19 | def all_select(length): 20 | return [1 for i in range(length)] 21 | 22 | 23 | def drop_path(x, drop_prob): 24 | if drop_prob > 0.: 25 | keep_prob = 1. - drop_prob 26 | mask = x.new_zeros(x.size(0), 1, 1, 1) 27 | mask = mask.bernoulli_(keep_prob) 28 | x.div_(keep_prob) 29 | x.mul_(mask) 30 | return x 31 | 32 | 33 | def return_alphas_str(basemodel): 34 | string = 'normal : {:}'.format( F.softmax(basemodel.alphas_normal, dim=-1) ) 35 | if hasattr(basemodel, 'alphas_reduce'): 36 | string = string + '\nreduce : {:}'.format( F.softmax(basemodel.alphas_reduce, dim=-1) ) 37 | return string 38 | 39 | 40 | class Cell(nn.Module): 41 | 42 | def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev): 43 | super(Cell, self).__init__() 44 | print(C_prev_prev, C_prev, C) 45 | 46 | if reduction_prev: 47 | self.preprocess0 = FactorizedReduce(C_prev_prev, C) 48 | else: 49 | self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0) 50 | self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0) 51 | 52 | if reduction: 53 | op_names, indices, values = zip(*genotype.reduce) 54 | concat = genotype.reduce_concat 55 | else: 56 | op_names, indices, values = zip(*genotype.normal) 57 | concat = genotype.normal_concat 58 | self._compile(C, op_names, indices, values, concat, reduction) 59 | 60 | def _compile(self, C, op_names, indices, values, concat, reduction): 61 | assert len(op_names) == len(indices) 62 | self._steps = len(op_names) // 2 63 | self._concat = concat 64 | self.multiplier = len(concat) 65 | 66 | self._ops = nn.ModuleList() 67 | for name, index in zip(op_names, indices): 68 | stride = 2 if reduction and index < 2 else 1 69 | op = OPS[name](C, stride, True) 70 | self._ops.append( op ) 71 | self._indices = indices 72 | self._values = values 73 | 74 | def forward(self, s0, s1, drop_prob): 75 | s0 = self.preprocess0(s0) 76 | s1 = self.preprocess1(s1) 77 | 78 | states = [s0, s1] 79 | for i in range(self._steps): 80 | h1 = states[self._indices[2*i]] 81 | h2 = states[self._indices[2*i+1]] 82 | op1 = self._ops[2*i] 83 | op2 = self._ops[2*i+1] 84 | h1 = op1(h1) 85 | h2 = op2(h2) 86 | if self.training and drop_prob > 0.: 87 | if not isinstance(op1, Identity): 88 | h1 = drop_path(h1, drop_prob) 89 | if not isinstance(op2, Identity): 90 | h2 = drop_path(h2, drop_prob) 91 | 92 | s = h1 + h2 93 | 94 | states += [s] 95 | return torch.cat([states[i] for i in self._concat], dim=1) 96 | 97 | 98 | 99 | class Transition(nn.Module): 100 | 101 | def __init__(self, C_prev_prev, C_prev, C, reduction_prev, multiplier=4): 102 | super(Transition, self).__init__() 103 | if reduction_prev: 104 | self.preprocess0 = FactorizedReduce(C_prev_prev, C) 105 | else: 106 | self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0) 107 | self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0) 108 | self.multiplier = multiplier 109 | 110 | self.reduction = True 111 | self.ops1 = nn.ModuleList( 112 | [nn.Sequential( 113 | nn.ReLU(inplace=False), 114 | nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), 115 | nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), 116 | nn.BatchNorm2d(C, affine=True), 117 | nn.ReLU(inplace=False), 118 | nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), 119 | nn.BatchNorm2d(C, affine=True)), 120 | nn.Sequential( 121 | nn.ReLU(inplace=False), 122 | nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), 123 | nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), 124 | nn.BatchNorm2d(C, affine=True), 125 | nn.ReLU(inplace=False), 126 | nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), 127 | nn.BatchNorm2d(C, affine=True))]) 128 | 129 | self.ops2 = nn.ModuleList( 130 | [nn.Sequential( 131 | nn.MaxPool2d(3, stride=2, padding=1), 132 | nn.BatchNorm2d(C, affine=True)), 133 | nn.Sequential( 134 | nn.MaxPool2d(3, stride=2, padding=1), 135 | nn.BatchNorm2d(C, affine=True))]) 136 | 137 | 138 | def forward(self, s0, s1, drop_prob = -1): 139 | s0 = self.preprocess0(s0) 140 | s1 = self.preprocess1(s1) 141 | 142 | X0 = self.ops1[0] (s0) 143 | X1 = self.ops1[1] (s1) 144 | if self.training and drop_prob > 0.: 145 | X0, X1 = drop_path(X0, drop_prob), drop_path(X1, drop_prob) 146 | 147 | #X2 = self.ops2[0] (X0+X1) 148 | X2 = self.ops2[0] (s0) 149 | X3 = self.ops2[1] (s1) 150 | if self.training and drop_prob > 0.: 151 | X2, X3 = drop_path(X2, drop_prob), drop_path(X3, drop_prob) 152 | return torch.cat([X0, X1, X2, X3], dim=1) 153 | -------------------------------------------------------------------------------- /lib/nas/genotypes.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat') 4 | 5 | PRIMITIVES = [ 6 | 'none', 7 | 'max_pool_3x3', 8 | 'avg_pool_3x3', 9 | 'skip_connect', 10 | 'sep_conv_3x3', 11 | 'sep_conv_5x5', 12 | 'dil_conv_3x3', 13 | 'dil_conv_5x5' 14 | ] 15 | 16 | NASNet = Genotype( 17 | normal = [ 18 | ('sep_conv_5x5', 1, 1.0), 19 | ('sep_conv_3x3', 0, 1.0), 20 | ('sep_conv_5x5', 0, 1.0), 21 | ('sep_conv_3x3', 0, 1.0), 22 | ('avg_pool_3x3', 1, 1.0), 23 | ('skip_connect', 0, 1.0), 24 | ('avg_pool_3x3', 0, 1.0), 25 | ('avg_pool_3x3', 0, 1.0), 26 | ('sep_conv_3x3', 1, 1.0), 27 | ('skip_connect', 1, 1.0), 28 | ], 29 | normal_concat = [2, 3, 4, 5, 6], 30 | reduce = [ 31 | ('sep_conv_5x5', 1, 1.0), 32 | ('sep_conv_7x7', 0, 1.0), 33 | ('max_pool_3x3', 1, 1.0), 34 | ('sep_conv_7x7', 0, 1.0), 35 | ('avg_pool_3x3', 1, 1.0), 36 | ('sep_conv_5x5', 0, 1.0), 37 | ('skip_connect', 3, 1.0), 38 | ('avg_pool_3x3', 2, 1.0), 39 | ('sep_conv_3x3', 2, 1.0), 40 | ('max_pool_3x3', 1, 1.0), 41 | ], 42 | reduce_concat = [4, 5, 6], 43 | ) 44 | 45 | AmoebaNet = Genotype( 46 | normal = [ 47 | ('avg_pool_3x3', 0, 1.0), 48 | ('max_pool_3x3', 1, 1.0), 49 | ('sep_conv_3x3', 0, 1.0), 50 | ('sep_conv_5x5', 2, 1.0), 51 | ('sep_conv_3x3', 0, 1.0), 52 | ('avg_pool_3x3', 3, 1.0), 53 | ('sep_conv_3x3', 1, 1.0), 54 | ('skip_connect', 1, 1.0), 55 | ('skip_connect', 0, 1.0), 56 | ('avg_pool_3x3', 1, 1.0), 57 | ], 58 | normal_concat = [4, 5, 6], 59 | reduce = [ 60 | ('avg_pool_3x3', 0, 1.0), 61 | ('sep_conv_3x3', 1, 1.0), 62 | ('max_pool_3x3', 0, 1.0), 63 | ('sep_conv_7x7', 2, 1.0), 64 | ('sep_conv_7x7', 0, 1.0), 65 | ('avg_pool_3x3', 1, 1.0), 66 | ('max_pool_3x3', 0, 1.0), 67 | ('max_pool_3x3', 1, 1.0), 68 | ('conv_7x1_1x7', 0, 1.0), 69 | ('sep_conv_3x3', 5, 1.0), 70 | ], 71 | reduce_concat = [3, 4, 6] 72 | ) 73 | 74 | DARTS_V1 = Genotype( 75 | normal=[ 76 | ('sep_conv_3x3', 1, 1.0), 77 | ('sep_conv_3x3', 0, 1.0), 78 | ('skip_connect', 0, 1.0), 79 | ('sep_conv_3x3', 1, 1.0), 80 | ('skip_connect', 0, 1.0), 81 | ('sep_conv_3x3', 1, 1.0), 82 | ('sep_conv_3x3', 0, 1.0), 83 | ('skip_connect', 2, 1.0)], 84 | normal_concat=[2, 3, 4, 5], 85 | reduce=[ 86 | ('max_pool_3x3', 0, 1.0), 87 | ('max_pool_3x3', 1, 1.0), 88 | ('skip_connect', 2, 1.0), 89 | ('max_pool_3x3', 0, 1.0), 90 | ('max_pool_3x3', 0, 1.0), 91 | ('skip_connect', 2, 1.0), 92 | ('skip_connect', 2, 1.0), 93 | ('avg_pool_3x3', 0, 1.0)], 94 | reduce_concat=[2, 3, 4, 5] 95 | ) 96 | 97 | DARTS_V2 = Genotype( 98 | normal=[ 99 | ('sep_conv_3x3', 0, 1.0), 100 | ('sep_conv_3x3', 1, 1.0), 101 | ('sep_conv_3x3', 0, 1.0), 102 | ('sep_conv_3x3', 1, 1.0), 103 | ('sep_conv_3x3', 1, 1.0), 104 | ('skip_connect', 0, 1.0), 105 | ('skip_connect', 0, 1.0), 106 | ('dil_conv_3x3', 2, 1.0)], 107 | normal_concat=[2, 3, 4, 5], 108 | reduce=[ 109 | ('max_pool_3x3', 0, 1.0), 110 | ('max_pool_3x3', 1, 1.0), 111 | ('skip_connect', 2, 1.0), 112 | ('max_pool_3x3', 1, 1.0), 113 | ('max_pool_3x3', 0, 1.0), 114 | ('skip_connect', 2, 1.0), 115 | ('skip_connect', 2, 1.0), 116 | ('max_pool_3x3', 1, 1.0)], 117 | reduce_concat=[2, 3, 4, 5] 118 | ) 119 | 120 | PNASNet = Genotype( 121 | normal = [ 122 | ('sep_conv_5x5', 0, 1.0), 123 | ('max_pool_3x3', 0, 1.0), 124 | ('sep_conv_7x7', 1, 1.0), 125 | ('max_pool_3x3', 1, 1.0), 126 | ('sep_conv_5x5', 1, 1.0), 127 | ('sep_conv_3x3', 1, 1.0), 128 | ('sep_conv_3x3', 4, 1.0), 129 | ('max_pool_3x3', 1, 1.0), 130 | ('sep_conv_3x3', 0, 1.0), 131 | ('skip_connect', 1, 1.0), 132 | ], 133 | normal_concat = [2, 3, 4, 5, 6], 134 | reduce = [ 135 | ('sep_conv_5x5', 0, 1.0), 136 | ('max_pool_3x3', 0, 1.0), 137 | ('sep_conv_7x7', 1, 1.0), 138 | ('max_pool_3x3', 1, 1.0), 139 | ('sep_conv_5x5', 1, 1.0), 140 | ('sep_conv_3x3', 1, 1.0), 141 | ('sep_conv_3x3', 4, 1.0), 142 | ('max_pool_3x3', 1, 1.0), 143 | ('sep_conv_3x3', 0, 1.0), 144 | ('skip_connect', 1, 1.0), 145 | ], 146 | reduce_concat = [2, 3, 4, 5, 6], 147 | ) 148 | 149 | # https://arxiv.org/pdf/1802.03268.pdf 150 | ENASNet = Genotype( 151 | normal = [ 152 | ('sep_conv_3x3', 1, 1.0), 153 | ('skip_connect', 1, 1.0), 154 | ('sep_conv_5x5', 1, 1.0), 155 | ('skip_connect', 0, 1.0), 156 | ('avg_pool_3x3', 0, 1.0), 157 | ('sep_conv_3x3', 1, 1.0), 158 | ('sep_conv_3x3', 0, 1.0), 159 | ('avg_pool_3x3', 1, 1.0), 160 | ('sep_conv_5x5', 1, 1.0), 161 | ('avg_pool_3x3', 0, 1.0), 162 | ], 163 | normal_concat = [2, 3, 4, 5, 6], 164 | reduce = [ 165 | ('sep_conv_5x5', 0, 1.0), 166 | ('sep_conv_3x3', 1, 1.0), # 2 167 | ('sep_conv_3x3', 1, 1.0), 168 | ('avg_pool_3x3', 1, 1.0), # 3 169 | ('sep_conv_3x3', 1, 1.0), 170 | ('avg_pool_3x3', 1, 1.0), # 4 171 | ('avg_pool_3x3', 1, 1.0), 172 | ('sep_conv_5x5', 4, 1.0), # 5 173 | ('sep_conv_3x3', 5, 1.0), 174 | ('sep_conv_5x5', 0, 1.0), 175 | ], 176 | reduce_concat = [2, 3, 4, 5, 6], 177 | ) 178 | 179 | DARTS = DARTS_V2 180 | 181 | # Search by normal and reduce 182 | GDAS_V1 = Genotype( 183 | normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)], 184 | normal_concat=range(2, 6), 185 | reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)], 186 | reduce_concat=range(2, 6) 187 | ) 188 | 189 | # Search by normal and fixing reduction 190 | GDAS_F1 = Genotype( 191 | normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)], 192 | normal_concat=[2, 3, 4, 5], 193 | reduce=None, 194 | reduce_concat=[2, 3, 4, 5], 195 | ) 196 | 197 | # Combine DMS_V1 and DMS_F1 198 | GDAS_GF = Genotype( 199 | normal=[('skip_connect', 0, 0.13017432391643524), ('skip_connect', 1, 0.12947972118854523), ('skip_connect', 0, 0.13062666356563568), ('sep_conv_5x5', 2, 0.12980839610099792), ('sep_conv_3x3', 3, 0.12923765182495117), ('skip_connect', 0, 0.12901571393013), ('sep_conv_5x5', 4, 0.12938997149467468), ('sep_conv_3x3', 3, 0.1289220005273819)], 200 | normal_concat=range(2, 6), 201 | reduce=None, 202 | reduce_concat=range(2, 6) 203 | ) 204 | GDAS_FG = Genotype( 205 | normal=[('skip_connect', 0, 0.16), ('skip_connect', 1, 0.13), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.17), ('sep_conv_3x3', 2, 0.15), ('skip_connect', 0, 0.16), ('sep_conv_3x3', 2, 0.15)], 206 | normal_concat=range(2, 6), 207 | reduce=[('sep_conv_5x5', 0, 0.12862831354141235), ('sep_conv_3x3', 1, 0.12783904373645782), ('sep_conv_5x5', 2, 0.12725995481014252), ('sep_conv_5x5', 1, 0.12705285847187042), ('dil_conv_5x5', 2, 0.12797553837299347), ('sep_conv_3x3', 1, 0.12737272679805756), ('sep_conv_5x5', 0, 0.12833961844444275), ('sep_conv_5x5', 1, 0.12758426368236542)], 208 | reduce_concat=range(2, 6) 209 | ) 210 | 211 | PDARTS = Genotype( 212 | normal=[ 213 | ('skip_connect', 0, 1.0), 214 | ('dil_conv_3x3', 1, 1.0), 215 | ('skip_connect', 0, 1.0), 216 | ('sep_conv_3x3', 1, 1.0), 217 | ('sep_conv_3x3', 1, 1.0), 218 | ('sep_conv_3x3', 3, 1.0), 219 | ('sep_conv_3x3', 0, 1.0), 220 | ('dil_conv_5x5', 4, 1.0)], 221 | normal_concat=range(2, 6), 222 | reduce=[ 223 | ('avg_pool_3x3', 0, 1.0), 224 | ('sep_conv_5x5', 1, 1.0), 225 | ('sep_conv_3x3', 0, 1.0), 226 | ('dil_conv_5x5', 2, 1.0), 227 | ('max_pool_3x3', 0, 1.0), 228 | ('dil_conv_3x3', 1, 1.0), 229 | ('dil_conv_3x3', 1, 1.0), 230 | ('dil_conv_5x5', 3, 1.0)], 231 | reduce_concat=range(2, 6) 232 | ) 233 | 234 | 235 | model_types = {'DARTS_V1': DARTS_V1, 236 | 'DARTS_V2': DARTS_V2, 237 | 'NASNet' : NASNet, 238 | 'PNASNet' : PNASNet, 239 | 'AmoebaNet': AmoebaNet, 240 | 'ENASNet' : ENASNet, 241 | 'PDARTS' : PDARTS, 242 | 'GDAS_V1' : GDAS_V1, 243 | 'GDAS_F1' : GDAS_F1, 244 | 'GDAS_GF' : GDAS_GF, 245 | 'GDAS_FG' : GDAS_FG} 246 | -------------------------------------------------------------------------------- /lib/nas/head_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | 5 | class ImageNetHEAD(nn.Sequential): 6 | def __init__(self, C, stride=2): 7 | super(ImageNetHEAD, self).__init__() 8 | self.add_module('conv1', nn.Conv2d(3, C // 2, kernel_size=3, stride=2, padding=1, bias=False)) 9 | self.add_module('bn1' , nn.BatchNorm2d(C // 2)) 10 | self.add_module('relu1', nn.ReLU(inplace=True)) 11 | self.add_module('conv2', nn.Conv2d(C // 2, C, kernel_size=3, stride=stride, padding=1, bias=False)) 12 | self.add_module('bn2' , nn.BatchNorm2d(C)) 13 | 14 | 15 | class CifarHEAD(nn.Sequential): 16 | def __init__(self, C): 17 | super(CifarHEAD, self).__init__() 18 | self.add_module('conv', nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False)) 19 | self.add_module('bn', nn.BatchNorm2d(C)) 20 | -------------------------------------------------------------------------------- /lib/nas/operations.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | OPS = { 5 | 'none' : lambda C, stride, affine: Zero(stride), 6 | 'avg_pool_3x3' : lambda C, stride, affine: nn.Sequential( 7 | nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False), 8 | nn.BatchNorm2d(C, affine=False) ), 9 | 'max_pool_3x3' : lambda C, stride, affine: nn.Sequential( 10 | nn.MaxPool2d(3, stride=stride, padding=1), 11 | nn.BatchNorm2d(C, affine=False) ), 12 | 'skip_connect' : lambda C, stride, affine: Identity() if stride == 1 else FactorizedReduce(C, C, affine=affine), 13 | 'sep_conv_3x3' : lambda C, stride, affine: SepConv(C, C, 3, stride, 1, affine=affine), 14 | 'sep_conv_5x5' : lambda C, stride, affine: SepConv(C, C, 5, stride, 2, affine=affine), 15 | 'sep_conv_7x7' : lambda C, stride, affine: SepConv(C, C, 7, stride, 3, affine=affine), 16 | 'dil_conv_3x3' : lambda C, stride, affine: DilConv(C, C, 3, stride, 2, 2, affine=affine), 17 | 'dil_conv_5x5' : lambda C, stride, affine: DilConv(C, C, 5, stride, 4, 2, affine=affine), 18 | 'conv_7x1_1x7' : lambda C, stride, affine: Conv717(C, C, stride, affine), 19 | } 20 | 21 | class Conv717(nn.Module): 22 | 23 | def __init__(self, C_in, C_out, stride, affine): 24 | super(Conv717, self).__init__() 25 | self.op = nn.Sequential( 26 | nn.ReLU(inplace=False), 27 | nn.Conv2d(C_in , C_out, (1,7), stride=(1, stride), padding=(0, 3), bias=False), 28 | nn.Conv2d(C_out, C_out, (7,1), stride=(stride, 1), padding=(3, 0), bias=False), 29 | nn.BatchNorm2d(C_out, affine=affine) 30 | ) 31 | 32 | def forward(self, x): 33 | return self.op(x) 34 | 35 | 36 | class ReLUConvBN(nn.Module): 37 | 38 | def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): 39 | super(ReLUConvBN, self).__init__() 40 | self.op = nn.Sequential( 41 | nn.ReLU(inplace=False), 42 | nn.Conv2d(C_in, C_out, kernel_size, stride=stride, padding=padding, bias=False), 43 | nn.BatchNorm2d(C_out, affine=affine) 44 | ) 45 | 46 | def forward(self, x): 47 | return self.op(x) 48 | 49 | 50 | class DilConv(nn.Module): 51 | 52 | def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine=True): 53 | super(DilConv, self).__init__() 54 | self.op = nn.Sequential( 55 | nn.ReLU(inplace=False), 56 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=C_in, bias=False), 57 | nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), 58 | nn.BatchNorm2d(C_out, affine=affine), 59 | ) 60 | 61 | def forward(self, x): 62 | return self.op(x) 63 | 64 | 65 | class SepConv(nn.Module): 66 | 67 | def __init__(self, C_in, C_out, kernel_size, stride, padding, affine=True): 68 | super(SepConv, self).__init__() 69 | self.op = nn.Sequential( 70 | nn.ReLU(inplace=False), 71 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, groups=C_in, bias=False), 72 | nn.Conv2d(C_in, C_in, kernel_size=1, padding=0, bias=False), 73 | nn.BatchNorm2d(C_in, affine=affine), 74 | nn.ReLU(inplace=False), 75 | nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=1, padding=padding, groups=C_in, bias=False), 76 | nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=False), 77 | nn.BatchNorm2d(C_out, affine=affine), 78 | ) 79 | 80 | def forward(self, x): 81 | return self.op(x) 82 | 83 | 84 | class Identity(nn.Module): 85 | 86 | def __init__(self): 87 | super(Identity, self).__init__() 88 | 89 | def forward(self, x): 90 | return x 91 | 92 | 93 | class Zero(nn.Module): 94 | 95 | def __init__(self, stride): 96 | super(Zero, self).__init__() 97 | self.stride = stride 98 | 99 | def forward(self, x): 100 | if self.stride == 1: 101 | return x.mul(0.) 102 | return x[:,:,::self.stride,::self.stride].mul(0.) 103 | 104 | 105 | class FactorizedReduce(nn.Module): 106 | 107 | def __init__(self, C_in, C_out, affine=True): 108 | super(FactorizedReduce, self).__init__() 109 | assert C_out % 2 == 0 110 | self.relu = nn.ReLU(inplace=False) 111 | self.conv_1 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) 112 | self.conv_2 = nn.Conv2d(C_in, C_out // 2, 1, stride=2, padding=0, bias=False) 113 | self.bn = nn.BatchNorm2d(C_out, affine=affine) 114 | self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) 115 | 116 | 117 | def forward(self, x): 118 | x = self.relu(x) 119 | y = self.pad(x) 120 | out = torch.cat([self.conv_1(x), self.conv_2(y[:,:,1:,1:])], dim=1) 121 | out = self.bn(out) 122 | return out 123 | -------------------------------------------------------------------------------- /lib/nas_rnn/__init__.py: -------------------------------------------------------------------------------- 1 | # utils 2 | from .utils import batchify, get_batch, repackage_hidden 3 | # models 4 | from .model_search import RNNModelSearch 5 | from .model_search import DARTSCellSearch 6 | from .basemodel import DARTSCell, RNNModel 7 | # architecture 8 | from .genotypes import DARTS_V1, DARTS_V2 9 | from .genotypes import GDAS 10 | -------------------------------------------------------------------------------- /lib/nas_rnn/basemodel.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from .genotypes import STEPS 6 | from .utils import mask2d, LockedDropout, embedded_dropout 7 | 8 | 9 | INITRANGE = 0.04 10 | 11 | def none_func(x): 12 | return x * 0 13 | 14 | 15 | class DARTSCell(nn.Module): 16 | 17 | def __init__(self, ninp, nhid, dropouth, dropoutx, genotype): 18 | super(DARTSCell, self).__init__() 19 | self.nhid = nhid 20 | self.dropouth = dropouth 21 | self.dropoutx = dropoutx 22 | self.genotype = genotype 23 | 24 | # genotype is None when doing arch search 25 | steps = len(self.genotype.recurrent) if self.genotype is not None else STEPS 26 | self._W0 = nn.Parameter(torch.Tensor(ninp+nhid, 2*nhid).uniform_(-INITRANGE, INITRANGE)) 27 | self._Ws = nn.ParameterList([ 28 | nn.Parameter(torch.Tensor(nhid, 2*nhid).uniform_(-INITRANGE, INITRANGE)) for i in range(steps) 29 | ]) 30 | 31 | def forward(self, inputs, hidden, arch_probs): 32 | T, B = inputs.size(0), inputs.size(1) 33 | 34 | if self.training: 35 | x_mask = mask2d(B, inputs.size(2), keep_prob=1.-self.dropoutx) 36 | h_mask = mask2d(B, hidden.size(2), keep_prob=1.-self.dropouth) 37 | else: 38 | x_mask = h_mask = None 39 | 40 | hidden = hidden[0] 41 | hiddens = [] 42 | for t in range(T): 43 | hidden = self.cell(inputs[t], hidden, x_mask, h_mask, arch_probs) 44 | hiddens.append(hidden) 45 | hiddens = torch.stack(hiddens) 46 | return hiddens, hiddens[-1].unsqueeze(0) 47 | 48 | def _compute_init_state(self, x, h_prev, x_mask, h_mask): 49 | if self.training: 50 | xh_prev = torch.cat([x * x_mask, h_prev * h_mask], dim=-1) 51 | else: 52 | xh_prev = torch.cat([x, h_prev], dim=-1) 53 | c0, h0 = torch.split(xh_prev.mm(self._W0), self.nhid, dim=-1) 54 | c0 = c0.sigmoid() 55 | h0 = h0.tanh() 56 | s0 = h_prev + c0 * (h0-h_prev) 57 | return s0 58 | 59 | def _get_activation(self, name): 60 | if name == 'tanh': 61 | f = torch.tanh 62 | elif name == 'relu': 63 | f = torch.relu 64 | elif name == 'sigmoid': 65 | f = torch.sigmoid 66 | elif name == 'identity': 67 | f = lambda x: x 68 | elif name == 'none': 69 | f = none_func 70 | else: 71 | raise NotImplementedError 72 | return f 73 | 74 | def cell(self, x, h_prev, x_mask, h_mask, _): 75 | s0 = self._compute_init_state(x, h_prev, x_mask, h_mask) 76 | 77 | states = [s0] 78 | for i, (name, pred) in enumerate(self.genotype.recurrent): 79 | s_prev = states[pred] 80 | if self.training: 81 | ch = (s_prev * h_mask).mm(self._Ws[i]) 82 | else: 83 | ch = s_prev.mm(self._Ws[i]) 84 | c, h = torch.split(ch, self.nhid, dim=-1) 85 | c = c.sigmoid() 86 | fn = self._get_activation(name) 87 | h = fn(h) 88 | s = s_prev + c * (h-s_prev) 89 | states += [s] 90 | output = torch.mean(torch.stack([states[i] for i in self.genotype.concat], -1), -1) 91 | return output 92 | 93 | 94 | class RNNModel(nn.Module): 95 | """Container module with an encoder, a recurrent module, and a decoder.""" 96 | def __init__(self, ntoken, ninp, nhid, nhidlast, 97 | dropout=0.5, dropouth=0.5, dropoutx=0.5, dropouti=0.5, dropoute=0.1, 98 | cell_cls=None, genotype=None): 99 | super(RNNModel, self).__init__() 100 | self.lockdrop = LockedDropout() 101 | self.encoder = nn.Embedding(ntoken, ninp) 102 | 103 | assert ninp == nhid == nhidlast 104 | if cell_cls == DARTSCell: 105 | assert genotype is not None 106 | rnns = [cell_cls(ninp, nhid, dropouth, dropoutx, genotype)] 107 | else: 108 | assert genotype is None 109 | rnns = [cell_cls(ninp, nhid, dropouth, dropoutx)] 110 | 111 | self.rnns = torch.nn.ModuleList(rnns) 112 | self.decoder = nn.Linear(ninp, ntoken) 113 | self.decoder.weight = self.encoder.weight 114 | self.init_weights() 115 | self.arch_weights = None 116 | 117 | self.ninp = ninp 118 | self.nhid = nhid 119 | self.nhidlast = nhidlast 120 | self.dropout = dropout 121 | self.dropouti = dropouti 122 | self.dropoute = dropoute 123 | self.ntoken = ntoken 124 | self.cell_cls = cell_cls 125 | # acceleration 126 | self.tau = None 127 | self.use_gumbel = False 128 | 129 | def set_gumbel(self, use_gumbel, set_check): 130 | self.use_gumbel = use_gumbel 131 | for i, rnn in enumerate(self.rnns): 132 | rnn.set_check(set_check) 133 | 134 | def set_tau(self, tau): 135 | self.tau = tau 136 | 137 | def get_tau(self): 138 | return self.tau 139 | 140 | def init_weights(self): 141 | self.encoder.weight.data.uniform_(-INITRANGE, INITRANGE) 142 | self.decoder.bias.data.fill_(0) 143 | self.decoder.weight.data.uniform_(-INITRANGE, INITRANGE) 144 | 145 | def forward(self, input, hidden, return_h=False): 146 | batch_size = input.size(1) 147 | 148 | emb = embedded_dropout(self.encoder, input, dropout=self.dropoute if self.training else 0) 149 | emb = self.lockdrop(emb, self.dropouti) 150 | 151 | raw_output = emb 152 | new_hidden = [] 153 | raw_outputs = [] 154 | outputs = [] 155 | if self.arch_weights is None: 156 | arch_probs = None 157 | else: 158 | if self.use_gumbel: arch_probs = F.gumbel_softmax(self.arch_weights, self.tau, False) 159 | else : arch_probs = F.softmax(self.arch_weights, dim=-1) 160 | 161 | for l, rnn in enumerate(self.rnns): 162 | current_input = raw_output 163 | raw_output, new_h = rnn(raw_output, hidden[l], arch_probs) 164 | new_hidden.append(new_h) 165 | raw_outputs.append(raw_output) 166 | hidden = new_hidden 167 | 168 | output = self.lockdrop(raw_output, self.dropout) 169 | outputs.append(output) 170 | 171 | logit = self.decoder(output.view(-1, self.ninp)) 172 | log_prob = nn.functional.log_softmax(logit, dim=-1) 173 | model_output = log_prob 174 | model_output = model_output.view(-1, batch_size, self.ntoken) 175 | 176 | if return_h: return model_output, hidden, raw_outputs, outputs 177 | else : return model_output, hidden 178 | 179 | def init_hidden(self, bsz): 180 | weight = next(self.parameters()).clone() 181 | return [weight.new(1, bsz, self.nhid).zero_()] 182 | -------------------------------------------------------------------------------- /lib/nas_rnn/genotypes.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | Genotype = namedtuple('Genotype', 'recurrent concat') 4 | 5 | PRIMITIVES = [ 6 | 'none', 7 | 'tanh', 8 | 'relu', 9 | 'sigmoid', 10 | 'identity' 11 | ] 12 | STEPS = 8 13 | CONCAT = 8 14 | 15 | ENAS = Genotype( 16 | recurrent = [ 17 | ('tanh', 0), 18 | ('tanh', 1), 19 | ('relu', 1), 20 | ('tanh', 3), 21 | ('tanh', 3), 22 | ('relu', 3), 23 | ('relu', 4), 24 | ('relu', 7), 25 | ('relu', 8), 26 | ('relu', 8), 27 | ('relu', 8), 28 | ], 29 | concat = [2, 5, 6, 9, 10, 11] 30 | ) 31 | 32 | DARTS_V1 = Genotype( 33 | recurrent = [ 34 | ('relu', 0), 35 | ('relu', 1), 36 | ('tanh', 2), 37 | ('relu', 3), ('relu', 4), ('identity', 1), ('relu', 5), ('relu', 1) 38 | ], 39 | concat=range(1, 9) 40 | ) 41 | 42 | DARTS_V2 = Genotype( 43 | recurrent = [ 44 | ('sigmoid', 0), ('relu', 1), ('relu', 1), 45 | ('identity', 1), ('tanh', 2), ('sigmoid', 5), 46 | ('tanh', 3), ('relu', 5) 47 | ], 48 | concat=range(1, 9) 49 | ) 50 | 51 | GDAS = Genotype( 52 | recurrent=[('relu', 0), ('relu', 0), ('identity', 1), ('relu', 1), ('tanh', 0), ('relu', 2), ('identity', 4), ('identity', 2)], 53 | concat=range(1, 9) 54 | ) 55 | 56 | -------------------------------------------------------------------------------- /lib/nas_rnn/model_search.py: -------------------------------------------------------------------------------- 1 | import copy, torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from collections import namedtuple 5 | from .genotypes import PRIMITIVES, STEPS, CONCAT, Genotype 6 | from .basemodel import DARTSCell, RNNModel 7 | 8 | 9 | class DARTSCellSearch(DARTSCell): 10 | 11 | def __init__(self, ninp, nhid, dropouth, dropoutx): 12 | super(DARTSCellSearch, self).__init__(ninp, nhid, dropouth, dropoutx, genotype=None) 13 | self.bn = nn.BatchNorm1d(nhid, affine=False) 14 | self.check_zero = False 15 | 16 | def set_check(self, check_zero): 17 | self.check_zero = check_zero 18 | 19 | def cell(self, x, h_prev, x_mask, h_mask, arch_probs): 20 | s0 = self._compute_init_state(x, h_prev, x_mask, h_mask) 21 | s0 = self.bn(s0) 22 | if self.check_zero: 23 | arch_probs_cpu = arch_probs.cpu().tolist() 24 | #arch_probs = F.softmax(self.weights, dim=-1) 25 | 26 | offset = 0 27 | states = s0.unsqueeze(0) 28 | for i in range(STEPS): 29 | if self.training: 30 | masked_states = states * h_mask.unsqueeze(0) 31 | else: 32 | masked_states = states 33 | ch = masked_states.view(-1, self.nhid).mm(self._Ws[i]).view(i+1, -1, 2*self.nhid) 34 | c, h = torch.split(ch, self.nhid, dim=-1) 35 | c = c.sigmoid() 36 | 37 | s = torch.zeros_like(s0) 38 | for k, name in enumerate(PRIMITIVES): 39 | if name == 'none': 40 | continue 41 | fn = self._get_activation(name) 42 | unweighted = states + c * (fn(h) - states) 43 | if self.check_zero: 44 | INDEX, INDDX = [], [] 45 | for jj in range(offset, offset+i+1): 46 | if arch_probs_cpu[jj][k] > 0: 47 | INDEX.append(jj) 48 | INDDX.append(jj-offset) 49 | if len(INDEX) == 0: continue 50 | s += torch.sum(arch_probs[INDEX, k].unsqueeze(-1).unsqueeze(-1) * unweighted[INDDX, :, :], dim=0) 51 | else: 52 | s += torch.sum(arch_probs[offset:offset+i+1, k].unsqueeze(-1).unsqueeze(-1) * unweighted, dim=0) 53 | s = self.bn(s) 54 | states = torch.cat([states, s.unsqueeze(0)], 0) 55 | offset += i+1 56 | output = torch.mean(states[-CONCAT:], dim=0) 57 | return output 58 | 59 | 60 | class RNNModelSearch(RNNModel): 61 | 62 | def __init__(self, *args): 63 | super(RNNModelSearch, self).__init__(*args) 64 | self._args = copy.deepcopy( args ) 65 | 66 | k = sum(i for i in range(1, STEPS+1)) 67 | self.arch_weights = nn.Parameter(torch.Tensor(k, len(PRIMITIVES))) 68 | nn.init.normal_(self.arch_weights, 0, 0.001) 69 | 70 | def base_parameters(self): 71 | lists = list(self.lockdrop.parameters()) 72 | lists += list(self.encoder.parameters()) 73 | lists += list(self.rnns.parameters()) 74 | lists += list(self.decoder.parameters()) 75 | return lists 76 | 77 | def arch_parameters(self): 78 | return [self.arch_weights] 79 | 80 | def genotype(self): 81 | 82 | def _parse(probs): 83 | gene = [] 84 | start = 0 85 | for i in range(STEPS): 86 | end = start + i + 1 87 | W = probs[start:end].copy() 88 | #j = sorted(range(i + 1), key=lambda x: -max(W[x][k] for k in range(len(W[x])) if k != PRIMITIVES.index('none')))[0] 89 | j = sorted(range(i + 1), key=lambda x: -max(W[x][k] for k in range(len(W[x])) ))[0] 90 | k_best = None 91 | for k in range(len(W[j])): 92 | #if k != PRIMITIVES.index('none'): 93 | # if k_best is None or W[j][k] > W[j][k_best]: 94 | # k_best = k 95 | if k_best is None or W[j][k] > W[j][k_best]: 96 | k_best = k 97 | gene.append((PRIMITIVES[k_best], j)) 98 | start = end 99 | return gene 100 | 101 | with torch.no_grad(): 102 | gene = _parse(F.softmax(self.arch_weights, dim=-1).cpu().numpy()) 103 | genotype = Genotype(recurrent=gene, concat=list(range(STEPS+1)[-CONCAT:])) 104 | return genotype 105 | -------------------------------------------------------------------------------- /lib/nas_rnn/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import os, shutil 4 | import numpy as np 5 | 6 | 7 | def repackage_hidden(h): 8 | if isinstance(h, torch.Tensor): 9 | return h.detach() 10 | else: 11 | return tuple(repackage_hidden(v) for v in h) 12 | 13 | 14 | def batchify(data, bsz, use_cuda): 15 | nbatch = data.size(0) // bsz 16 | data = data.narrow(0, 0, nbatch * bsz) 17 | data = data.view(bsz, -1).t().contiguous() 18 | if use_cuda: return data.cuda() 19 | else : return data 20 | 21 | 22 | def get_batch(source, i, seq_len): 23 | seq_len = min(seq_len, len(source) - 1 - i) 24 | data = source[i:i+seq_len].clone() 25 | target = source[i+1:i+1+seq_len].clone() 26 | return data, target 27 | 28 | 29 | 30 | def embedded_dropout(embed, words, dropout=0.1, scale=None): 31 | if dropout: 32 | mask = embed.weight.data.new().resize_((embed.weight.size(0), 1)).bernoulli_(1 - dropout).expand_as(embed.weight) / (1 - dropout) 33 | mask.requires_grad_(True) 34 | masked_embed_weight = mask * embed.weight 35 | else: 36 | masked_embed_weight = embed.weight 37 | if scale: 38 | masked_embed_weight = scale.expand_as(masked_embed_weight) * masked_embed_weight 39 | 40 | padding_idx = embed.padding_idx 41 | if padding_idx is None: 42 | padding_idx = -1 43 | X = torch.nn.functional.embedding( 44 | words, masked_embed_weight, 45 | padding_idx, embed.max_norm, embed.norm_type, 46 | embed.scale_grad_by_freq, embed.sparse) 47 | return X 48 | 49 | 50 | class LockedDropout(nn.Module): 51 | def __init__(self): 52 | super(LockedDropout, self).__init__() 53 | 54 | def forward(self, x, dropout=0.5): 55 | if not self.training or not dropout: 56 | return x 57 | m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - dropout) 58 | mask = m.div_(1 - dropout).detach() 59 | mask = mask.expand_as(x) 60 | return mask * x 61 | 62 | 63 | def mask2d(B, D, keep_prob, cuda=True): 64 | m = torch.floor(torch.rand(B, D) + keep_prob) / keep_prob 65 | if cuda: return m.cuda() 66 | else : return m 67 | -------------------------------------------------------------------------------- /lib/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .utils import load_config 5 | from .scheduler import MultiStepLR, obtain_scheduler 6 | -------------------------------------------------------------------------------- /lib/scheduler/scheduler.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import torch 5 | from bisect import bisect_right 6 | 7 | 8 | class MultiStepLR(torch.optim.lr_scheduler._LRScheduler): 9 | 10 | def __init__(self, optimizer, milestones, gammas, last_epoch=-1): 11 | if not list(milestones) == sorted(milestones): 12 | raise ValueError('Milestones should be a list of' 13 | ' increasing integers. Got {:}', milestones) 14 | assert len(milestones) == len(gammas), '{:} vs {:}'.format(milestones, gammas) 15 | self.milestones = milestones 16 | self.gammas = gammas 17 | super(MultiStepLR, self).__init__(optimizer, last_epoch) 18 | 19 | def get_lr(self): 20 | LR = 1 21 | for x in self.gammas[:bisect_right(self.milestones, self.last_epoch)]: LR = LR * x 22 | return [base_lr * LR for base_lr in self.base_lrs] 23 | 24 | 25 | def obtain_scheduler(config, optimizer): 26 | if config.type == 'multistep': 27 | scheduler = MultiStepLR(optimizer, milestones=config.milestones, gammas=config.gammas) 28 | elif config.type == 'cosine': 29 | scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, config.epochs) 30 | else: 31 | raise ValueError('Unknown learning rate scheduler type : {:}'.format(config.type)) 32 | return scheduler 33 | -------------------------------------------------------------------------------- /lib/scheduler/utils.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, json 5 | from pathlib import Path 6 | from collections import namedtuple 7 | 8 | support_types = ('str', 'int', 'bool', 'float') 9 | 10 | def convert_param(original_lists): 11 | assert isinstance(original_lists, list), 'The type is not right : {:}'.format(original_lists) 12 | ctype, value = original_lists[0], original_lists[1] 13 | assert ctype in support_types, 'Ctype={:}, support={:}'.format(ctype, support_types) 14 | is_list = isinstance(value, list) 15 | if not is_list: value = [value] 16 | outs = [] 17 | for x in value: 18 | if ctype == 'int': 19 | x = int(x) 20 | elif ctype == 'str': 21 | x = str(x) 22 | elif ctype == 'bool': 23 | x = bool(int(x)) 24 | elif ctype == 'float': 25 | x = float(x) 26 | else: 27 | raise TypeError('Does not know this type : {:}'.format(ctype)) 28 | outs.append(x) 29 | if not is_list: outs = outs[0] 30 | return outs 31 | 32 | def load_config(path): 33 | path = str(path) 34 | assert os.path.exists(path), 'Can not find {:}'.format(path) 35 | # Reading data back 36 | with open(path, 'r') as f: 37 | data = json.load(f) 38 | f.close() 39 | content = { k: convert_param(v) for k,v in data.items()} 40 | Arguments = namedtuple('Configure', ' '.join(content.keys())) 41 | content = Arguments(**content) 42 | return content 43 | -------------------------------------------------------------------------------- /lib/utils/__init__.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | from .utils import AverageMeter, RecorderMeter, convert_secs2time 5 | from .utils import time_file_str, time_string 6 | from .utils import test_imagenet_data 7 | from .utils import print_log 8 | from .evaluation_utils import obtain_accuracy 9 | #from .draw_pts import draw_points 10 | from .gpu_manager import GPUManager 11 | 12 | from .save_meta import Save_Meta 13 | 14 | from .model_utils import count_parameters_in_MB 15 | from .model_utils import Cutout 16 | from .flop_benchmark import print_FLOPs 17 | -------------------------------------------------------------------------------- /lib/utils/draw_pts.py: -------------------------------------------------------------------------------- 1 | import os, sys, time 2 | import numpy as np 3 | import matplotlib 4 | import random 5 | matplotlib.use('agg') 6 | import matplotlib.pyplot as plt 7 | import matplotlib.cm as cm 8 | 9 | def draw_points(points, labels, save_path): 10 | title = 'the visualized features' 11 | dpi = 100 12 | width, height = 1000, 1000 13 | legend_fontsize = 10 14 | figsize = width / float(dpi), height / float(dpi) 15 | fig = plt.figure(figsize=figsize) 16 | 17 | classes = np.unique(labels).tolist() 18 | colors = cm.rainbow(np.linspace(0, 1, len(classes))) 19 | 20 | legends = [] 21 | legendnames = [] 22 | 23 | for cls, c in zip(classes, colors): 24 | 25 | indexes = labels == cls 26 | ptss = points[indexes, :] 27 | x = ptss[:,0] 28 | y = ptss[:,1] 29 | if cls % 2 == 0: marker = 'x' 30 | else: marker = 'o' 31 | legend = plt.scatter(x, y, color=c, s=1, marker=marker) 32 | legendname = '{:02d}'.format(cls+1) 33 | legends.append( legend ) 34 | legendnames.append( legendname ) 35 | 36 | plt.legend(legends, legendnames, scatterpoints=1, ncol=5, fontsize=8) 37 | 38 | if save_path is not None: 39 | fig.savefig(save_path, dpi=dpi, bbox_inches='tight') 40 | print ('---- save figure {} into {}'.format(title, save_path)) 41 | plt.close(fig) 42 | -------------------------------------------------------------------------------- /lib/utils/evaluation_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def obtain_accuracy(output, target, topk=(1,)): 4 | """Computes the precision@k for the specified values of k""" 5 | maxk = max(topk) 6 | batch_size = target.size(0) 7 | 8 | _, pred = output.topk(maxk, 1, True, True) 9 | pred = pred.t() 10 | correct = pred.eq(target.view(1, -1).expand_as(pred)) 11 | 12 | res = [] 13 | for k in topk: 14 | correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) 15 | res.append(correct_k.mul_(100.0 / batch_size)) 16 | return res 17 | -------------------------------------------------------------------------------- /lib/utils/flop_benchmark.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | # modified from https://github.com/warmspringwinds/pytorch-segmentation-detection/blob/master/pytorch_segmentation_detection/utils/flops_benchmark.py 5 | import copy, torch 6 | 7 | def print_FLOPs(model, shape, logs): 8 | print_log, log = logs 9 | model = copy.deepcopy( model ) 10 | 11 | model = add_flops_counting_methods(model) 12 | model = model.cuda() 13 | model.eval() 14 | 15 | cache_inputs = torch.zeros(*shape).cuda() 16 | #print_log('In the calculating function : cache input size : {:}'.format(cache_inputs.size()), log) 17 | _ = model(cache_inputs) 18 | FLOPs = compute_average_flops_cost( model ) / 1e6 19 | print_log('FLOPs : {:} MB'.format(FLOPs), log) 20 | torch.cuda.empty_cache() 21 | 22 | 23 | # ---- Public functions 24 | def add_flops_counting_methods( model ): 25 | model.__batch_counter__ = 0 26 | add_batch_counter_hook_function( model ) 27 | model.apply( add_flops_counter_variable_or_reset ) 28 | model.apply( add_flops_counter_hook_function ) 29 | return model 30 | 31 | 32 | 33 | def compute_average_flops_cost(model): 34 | """ 35 | A method that will be available after add_flops_counting_methods() is called on a desired net object. 36 | Returns current mean flops consumption per image. 37 | """ 38 | batches_count = model.__batch_counter__ 39 | flops_sum = 0 40 | for module in model.modules(): 41 | if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear): 42 | flops_sum += module.__flops__ 43 | return flops_sum / batches_count 44 | 45 | 46 | # ---- Internal functions 47 | def pool_flops_counter_hook(pool_module, inputs, output): 48 | batch_size = inputs[0].size(0) 49 | kernel_size = pool_module.kernel_size 50 | out_C, output_height, output_width = output.shape[1:] 51 | assert out_C == inputs[0].size(1), '{:} vs. {:}'.format(out_C, inputs[0].size()) 52 | 53 | overall_flops = batch_size * out_C * output_height * output_width * kernel_size * kernel_size 54 | pool_module.__flops__ += overall_flops 55 | 56 | 57 | def fc_flops_counter_hook(fc_module, inputs, output): 58 | batch_size = inputs[0].size(0) 59 | xin, xout = fc_module.in_features, fc_module.out_features 60 | assert xin == inputs[0].size(1) and xout == output.size(1), 'IO=({:}, {:})'.format(xin, xout) 61 | overall_flops = batch_size * xin * xout 62 | if fc_module.bias is not None: 63 | overall_flops += batch_size * xout 64 | fc_module.__flops__ += overall_flops 65 | 66 | 67 | def conv_flops_counter_hook(conv_module, inputs, output): 68 | batch_size = inputs[0].size(0) 69 | output_height, output_width = output.shape[2:] 70 | 71 | kernel_height, kernel_width = conv_module.kernel_size 72 | in_channels = conv_module.in_channels 73 | out_channels = conv_module.out_channels 74 | groups = conv_module.groups 75 | conv_per_position_flops = kernel_height * kernel_width * in_channels * out_channels / groups 76 | 77 | active_elements_count = batch_size * output_height * output_width 78 | overall_flops = conv_per_position_flops * active_elements_count 79 | 80 | if conv_module.bias is not None: 81 | overall_flops += out_channels * active_elements_count 82 | conv_module.__flops__ += overall_flops 83 | 84 | 85 | def batch_counter_hook(module, inputs, output): 86 | # Can have multiple inputs, getting the first one 87 | inputs = inputs[0] 88 | batch_size = inputs.shape[0] 89 | module.__batch_counter__ += batch_size 90 | 91 | 92 | def add_batch_counter_hook_function(module): 93 | if not hasattr(module, '__batch_counter_handle__'): 94 | handle = module.register_forward_hook(batch_counter_hook) 95 | module.__batch_counter_handle__ = handle 96 | 97 | 98 | def add_flops_counter_variable_or_reset(module): 99 | if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ 100 | or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): 101 | module.__flops__ = 0 102 | 103 | 104 | def add_flops_counter_hook_function(module): 105 | if isinstance(module, torch.nn.Conv2d): 106 | if not hasattr(module, '__flops_handle__'): 107 | handle = module.register_forward_hook(conv_flops_counter_hook) 108 | module.__flops_handle__ = handle 109 | elif isinstance(module, torch.nn.Linear): 110 | if not hasattr(module, '__flops_handle__'): 111 | handle = module.register_forward_hook(fc_flops_counter_hook) 112 | module.__flops_handle__ = handle 113 | elif isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): 114 | if not hasattr(module, '__flops_handle__'): 115 | handle = module.register_forward_hook(pool_flops_counter_hook) 116 | module.__flops_handle__ = handle 117 | -------------------------------------------------------------------------------- /lib/utils/gpu_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | class GPUManager(): 4 | queries = ('index', 'gpu_name', 'memory.free', 'memory.used', 'memory.total', 'power.draw', 'power.limit') 5 | 6 | def __init__(self): 7 | all_gpus = self.query_gpu(False) 8 | 9 | def get_info(self, ctype): 10 | cmd = 'nvidia-smi --query-gpu={} --format=csv,noheader'.format(ctype) 11 | lines = os.popen(cmd).readlines() 12 | lines = [line.strip('\n') for line in lines] 13 | return lines 14 | 15 | def query_gpu(self, show=True): 16 | num_gpus = len( self.get_info('index') ) 17 | all_gpus = [ {} for i in range(num_gpus) ] 18 | for query in self.queries: 19 | infos = self.get_info(query) 20 | for idx, info in enumerate(infos): 21 | all_gpus[idx][query] = info 22 | 23 | if 'CUDA_VISIBLE_DEVICES' in os.environ: 24 | CUDA_VISIBLE_DEVICES = os.environ['CUDA_VISIBLE_DEVICES'].split(',') 25 | selected_gpus = [] 26 | for idx, CUDA_VISIBLE_DEVICE in enumerate(CUDA_VISIBLE_DEVICES): 27 | find = False 28 | for gpu in all_gpus: 29 | if gpu['index'] == CUDA_VISIBLE_DEVICE: 30 | assert find==False, 'Duplicate cuda device index : {}'.format(CUDA_VISIBLE_DEVICE) 31 | find = True 32 | selected_gpus.append( gpu.copy() ) 33 | selected_gpus[-1]['index'] = '{}'.format(idx) 34 | assert find, 'Does not find the device : {}'.format(CUDA_VISIBLE_DEVICE) 35 | all_gpus = selected_gpus 36 | 37 | if show: 38 | allstrings = '' 39 | for gpu in all_gpus: 40 | string = '| ' 41 | for query in self.queries: 42 | if query.find('memory') == 0: xinfo = '{:>9}'.format(gpu[query]) 43 | else: xinfo = gpu[query] 44 | string = string + query + ' : ' + xinfo + ' | ' 45 | allstrings = allstrings + string + '\n' 46 | return allstrings 47 | else: 48 | return all_gpus 49 | 50 | def select_by_memory(self, numbers=1): 51 | all_gpus = self.query_gpu(False) 52 | assert numbers <= len(all_gpus), 'Require {} gpus more than you have'.format(numbers) 53 | alls = [] 54 | for idx, gpu in enumerate(all_gpus): 55 | free_memory = gpu['memory.free'] 56 | free_memory = free_memory.split(' ')[0] 57 | free_memory = int(free_memory) 58 | index = gpu['index'] 59 | alls.append((free_memory, index)) 60 | alls.sort(reverse = True) 61 | alls = [ int(alls[i][1]) for i in range(numbers) ] 62 | return sorted(alls) 63 | 64 | """ 65 | if __name__ == '__main__': 66 | manager = GPUManager() 67 | manager.query_gpu(True) 68 | indexes = manager.select_by_memory(3) 69 | print (indexes) 70 | """ 71 | -------------------------------------------------------------------------------- /lib/utils/model_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | 5 | 6 | def count_parameters_in_MB(model): 7 | if isinstance(model, nn.Module): 8 | return np.sum(np.prod(v.size()) for v in model.parameters())/1e6 9 | else: 10 | return np.sum(np.prod(v.size()) for v in model)/1e6 11 | 12 | 13 | class Cutout(object): 14 | def __init__(self, length): 15 | self.length = length 16 | 17 | def __repr__(self): 18 | return ('{name}(length={length})'.format(name=self.__class__.__name__, **self.__dict__)) 19 | 20 | def __call__(self, img): 21 | h, w = img.size(1), img.size(2) 22 | mask = np.ones((h, w), np.float32) 23 | y = np.random.randint(h) 24 | x = np.random.randint(w) 25 | 26 | y1 = np.clip(y - self.length // 2, 0, h) 27 | y2 = np.clip(y + self.length // 2, 0, h) 28 | x1 = np.clip(x - self.length // 2, 0, w) 29 | x2 = np.clip(x + self.length // 2, 0, w) 30 | 31 | mask[y1: y2, x1: x2] = 0. 32 | mask = torch.from_numpy(mask) 33 | mask = mask.expand_as(img) 34 | img *= mask 35 | return img 36 | -------------------------------------------------------------------------------- /lib/utils/save_meta.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import torch 5 | import os, sys 6 | import os.path as osp 7 | import numpy as np 8 | 9 | def tensor2np(x): 10 | if isinstance(x, np.ndarray): return x 11 | if x.is_cuda: x = x.cpu() 12 | return x.numpy() 13 | 14 | class Save_Meta(): 15 | 16 | def __init__(self): 17 | self.reset() 18 | 19 | def __repr__(self): 20 | return ('{name}'.format(name=self.__class__.__name__)+'(number of data = {})'.format(len(self))) 21 | 22 | def reset(self): 23 | self.predictions = [] 24 | self.groundtruth = [] 25 | 26 | def __len__(self): 27 | return len(self.predictions) 28 | 29 | def append(self, _pred, _ground): 30 | _pred, _ground = tensor2np(_pred), tensor2np(_ground) 31 | assert _ground.shape[0] == _pred.shape[0] and len(_pred.shape) == 2 and len(_ground.shape) == 1, 'The shapes are wrong : {} & {}'.format(_pred.shape, _ground.shape) 32 | self.predictions.append(_pred) 33 | self.groundtruth.append(_ground) 34 | 35 | def save(self, save_dir, filename, test=True): 36 | meta = {'predictions': self.predictions, 37 | 'groundtruth': self.groundtruth} 38 | filename = osp.join(save_dir, filename) 39 | torch.save(meta, filename) 40 | if test: 41 | predictions = np.concatenate(self.predictions) 42 | groundtruth = np.concatenate(self.groundtruth) 43 | predictions = np.argmax(predictions, axis=1) 44 | accuracy = np.sum(groundtruth==predictions) * 100.0 / predictions.size 45 | else: 46 | accuracy = None 47 | print ('save save_meta into {} with accuracy = {}'.format(filename, accuracy)) 48 | 49 | def load(self, filename): 50 | assert os.path.isfile(filename), '{} is not a file'.format(filename) 51 | checkpoint = torch.load(filename) 52 | self.predictions = checkpoint['predictions'] 53 | self.groundtruth = checkpoint['groundtruth'] 54 | -------------------------------------------------------------------------------- /lib/utils/utils.py: -------------------------------------------------------------------------------- 1 | ################################################## 2 | # Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # 3 | ################################################## 4 | import os, sys, time 5 | import numpy as np 6 | import random 7 | 8 | class AverageMeter(object): 9 | """Computes and stores the average and current value""" 10 | def __init__(self): 11 | self.reset() 12 | 13 | def reset(self): 14 | self.val = 0 15 | self.avg = 0 16 | self.sum = 0 17 | self.count = 0 18 | 19 | def update(self, val, n=1): 20 | self.val = val 21 | self.sum += val * n 22 | self.count += n 23 | self.avg = self.sum / self.count 24 | 25 | 26 | class RecorderMeter(object): 27 | """Computes and stores the minimum loss value and its epoch index""" 28 | def __init__(self, total_epoch): 29 | self.reset(total_epoch) 30 | 31 | def reset(self, total_epoch): 32 | assert total_epoch > 0 33 | self.total_epoch = total_epoch 34 | self.current_epoch = 0 35 | self.epoch_losses = np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] 36 | self.epoch_losses = self.epoch_losses - 1 37 | 38 | self.epoch_accuracy= np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] 39 | self.epoch_accuracy= self.epoch_accuracy 40 | 41 | def update(self, idx, train_loss, train_acc, val_loss, val_acc): 42 | assert idx >= 0 and idx < self.total_epoch, 'total_epoch : {} , but update with the {} index'.format(self.total_epoch, idx) 43 | self.epoch_losses [idx, 0] = train_loss 44 | self.epoch_losses [idx, 1] = val_loss 45 | self.epoch_accuracy[idx, 0] = train_acc 46 | self.epoch_accuracy[idx, 1] = val_acc 47 | self.current_epoch = idx + 1 48 | return self.max_accuracy(False) == self.epoch_accuracy[idx, 1] 49 | 50 | def max_accuracy(self, istrain): 51 | if self.current_epoch <= 0: return 0 52 | if istrain: return self.epoch_accuracy[:self.current_epoch, 0].max() 53 | else: return self.epoch_accuracy[:self.current_epoch, 1].max() 54 | 55 | def plot_curve(self, save_path): 56 | import matplotlib 57 | matplotlib.use('agg') 58 | import matplotlib.pyplot as plt 59 | title = 'the accuracy/loss curve of train/val' 60 | dpi = 100 61 | width, height = 1600, 1000 62 | legend_fontsize = 10 63 | figsize = width / float(dpi), height / float(dpi) 64 | 65 | fig = plt.figure(figsize=figsize) 66 | x_axis = np.array([i for i in range(self.total_epoch)]) # epochs 67 | y_axis = np.zeros(self.total_epoch) 68 | 69 | plt.xlim(0, self.total_epoch) 70 | plt.ylim(0, 100) 71 | interval_y = 5 72 | interval_x = 5 73 | plt.xticks(np.arange(0, self.total_epoch + interval_x, interval_x)) 74 | plt.yticks(np.arange(0, 100 + interval_y, interval_y)) 75 | plt.grid() 76 | plt.title(title, fontsize=20) 77 | plt.xlabel('the training epoch', fontsize=16) 78 | plt.ylabel('accuracy', fontsize=16) 79 | 80 | y_axis[:] = self.epoch_accuracy[:, 0] 81 | plt.plot(x_axis, y_axis, color='g', linestyle='-', label='train-accuracy', lw=2) 82 | plt.legend(loc=4, fontsize=legend_fontsize) 83 | 84 | y_axis[:] = self.epoch_accuracy[:, 1] 85 | plt.plot(x_axis, y_axis, color='y', linestyle='-', label='valid-accuracy', lw=2) 86 | plt.legend(loc=4, fontsize=legend_fontsize) 87 | 88 | 89 | y_axis[:] = self.epoch_losses[:, 0] 90 | plt.plot(x_axis, y_axis*50, color='g', linestyle=':', label='train-loss-x50', lw=2) 91 | plt.legend(loc=4, fontsize=legend_fontsize) 92 | 93 | y_axis[:] = self.epoch_losses[:, 1] 94 | plt.plot(x_axis, y_axis*50, color='y', linestyle=':', label='valid-loss-x50', lw=2) 95 | plt.legend(loc=4, fontsize=legend_fontsize) 96 | 97 | if save_path is not None: 98 | fig.savefig(save_path, dpi=dpi, bbox_inches='tight') 99 | print ('---- save figure {} into {}'.format(title, save_path)) 100 | plt.close(fig) 101 | 102 | def print_log(print_string, log): 103 | print ("{:}".format(print_string)) 104 | if log is not None: 105 | log.write('{}\n'.format(print_string)) 106 | log.flush() 107 | 108 | def time_file_str(): 109 | ISOTIMEFORMAT='%Y-%m-%d' 110 | string = '{}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) 111 | return string + '-{}'.format(random.randint(1, 10000)) 112 | 113 | def time_string(): 114 | ISOTIMEFORMAT='%Y-%m-%d-%X' 115 | string = '[{}]'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) 116 | return string 117 | 118 | def convert_secs2time(epoch_time, return_str=False): 119 | need_hour = int(epoch_time / 3600) 120 | need_mins = int((epoch_time - 3600*need_hour) / 60) 121 | need_secs = int(epoch_time - 3600*need_hour - 60*need_mins) 122 | if return_str == False: 123 | return need_hour, need_mins, need_secs 124 | else: 125 | return '[Need: {:02d}:{:02d}:{:02d}]'.format(need_hour, need_mins, need_secs) 126 | 127 | def test_imagenet_data(imagenet): 128 | total_length = len(imagenet) 129 | assert total_length == 1281166 or total_length == 50000, 'The length of ImageNet is wrong : {}'.format(total_length) 130 | map_id = {} 131 | for index in range(total_length): 132 | path, target = imagenet.imgs[index] 133 | folder, image_name = os.path.split(path) 134 | _, folder = os.path.split(folder) 135 | if folder not in map_id: 136 | map_id[folder] = target 137 | else: 138 | assert map_id[folder] == target, 'Class : {} is not {}'.format(folder, target) 139 | assert image_name.find(folder) == 0, '{} is wrong.'.format(path) 140 | print ('Check ImageNet Dataset OK') 141 | -------------------------------------------------------------------------------- /scripts-cluster/README.md: -------------------------------------------------------------------------------- 1 | # Commands on Cluster 2 | 3 | ## RNN 4 | ``` 5 | bash scripts-cluster/submit.sh yq01-v100-box-idl-2-8 WT2-GDAS 1 "bash ./scripts-rnn/train-WT2.sh GDAS" 6 | bash scripts-cluster/submit.sh yq01-v100-box-idl-2-8 PTB-GDAS 1 "bash ./scripts-rnn/train-PTB.sh GDAS" 7 | ``` 8 | 9 | ## CNN 10 | ``` 11 | bash scripts-cluster/submit.sh yq01-v100-box-idl-2-8 CIFAR10-CUT-GDAS-F1 1 "bash ./scripts-cnn/train-cifar.sh GDAS_F1 cifar10 cut" 12 | bash scripts-cluster/submit.sh yq01-v100-box-idl-2-8 IMAGENET-GDAS-F1 1 "bash ./scripts-cnn/train-imagenet.sh GDAS_F1 52 14" 13 | bash scripts-cluster/submit.sh yq01-v100-box-idl-2-8 IMAGENET-GDAS-V1 1 "bash ./scripts-cnn/train-imagenet.sh GDAS_V1 50 14" 14 | ``` 15 | -------------------------------------------------------------------------------- /scripts-cluster/job-script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | echo "CHECK-DATA-DIR START" 4 | sh /home/HGCP_Program/software-install/afs_mount/bin/afs_mount.sh \ 5 | COMM_KM_Data COMM_km_2018 \ 6 | `pwd`/hadoop-data \ 7 | afs://xingtian.afs.baidu.com:9902/user/COMM_KM_Data/dongxuanyi/datasets 8 | 9 | export TORCH_HOME="./data/data/" 10 | tar -xf ./hadoop-data/cifar.python.tar -C ${TORCH_HOME} 11 | 12 | cifar_dir="${TORCH_HOME}/cifar.python" 13 | if [ -d ${cifar_dir} ]; then 14 | echo "Find cifar-dir: "${cifar_dir} 15 | else 16 | echo "Can not find cifar-dir: "${cifar_dir} 17 | exit 1 18 | fi 19 | echo "CHECK-DATA-DIR DONE" 20 | 21 | PID=$$ 22 | 23 | # config python 24 | PYTHON_ENV=py36_pytorch1.0_env0.1.3.tar.gz 25 | wget -e "http_proxy=cp01-sys-hic-gpu-02.cp01:8888" http://cp01-sys-hic-gpu-02.cp01/HGCP_DEMO/$PYTHON_ENV > screen.log 2>&1 26 | tar xzf $PYTHON_ENV 27 | 28 | echo "JOB-PID : "${PID} 29 | echo "JOB-PWD : "$(pwd) 30 | echo "JOB-files : "$(ls) 31 | echo "JOB-CUDA_VISIBLE_DEVICES: " ${CUDA_VISIBLE_DEVICES} 32 | 33 | ./env/bin/python --version 34 | echo "JOB-TORCH_HOME: "${TORCH_HOME} 35 | 36 | # real commands 37 | -------------------------------------------------------------------------------- /scripts-cluster/submit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # bash ./scripts-cluster/submit.sh ${QUEUE} ${JOB-NAME} ${GPUs} 3 | #find -name "._*" | xargs rm -rf 4 | ODIR=$(pwd) 5 | FDIR=$(cd $(dirname $0); pwd) 6 | echo "Bash-Dir : "${ODIR} 7 | echo "File-Dir : "${FDIR} 8 | echo "File-Name : "${0} 9 | 10 | if [ "$#" -ne 4 ] ;then 11 | echo "Input illegal number of parameters " $# 12 | echo "Need 4 parameters for the queue-name, the job-name, and the number-of-GPUs" 13 | exit 1 14 | fi 15 | find -name "__pycache__" | xargs rm -rf 16 | 17 | QUEUE=$1 18 | NAME=$2 19 | GPUs=$3 20 | CMD=$4 21 | TIME=$(date +"%Y-%h-%d--%T") 22 | TIME="${TIME//:/-}" 23 | 24 | JOB_SCRIPT="${FDIR}/tmps/job-${TIME}.sh" 25 | HDFS_DIR="/user/COMM_KM_Data/${USER}/logs/alljobs/${NAME}-${TIME}" 26 | echo "JOB-SCRIPT: "${JOB_SCRIPT} 27 | 28 | cat ${FDIR}/job-script.sh > ${JOB_SCRIPT} 29 | echo ${CMD} >> ${JOB_SCRIPT} 30 | 31 | ${HDP} -mkdir ${HDFS_DIR} 32 | echo "Create "${HDFS_DIR}" done!" 33 | sleep 1s 34 | 35 | HGCP_CLIENT_BIN="${HOME}/.hgcp/software-install/HGCP_client/bin" 36 | 37 | ${HGCP_CLIENT_BIN}/submit \ 38 | --hdfs afs://xingtian.afs.baidu.com:9902 \ 39 | --hdfs-user COMM_KM_Data \ 40 | --hdfs-passwd COMM_km_2018 \ 41 | --hdfs-path ${HDFS_DIR} \ 42 | --file-dir ./ \ 43 | --job-name ${NAME} \ 44 | --queue-name ${QUEUE} \ 45 | --num-nodes 1 \ 46 | --num-task-pernode 1 \ 47 | --gpu-pnode ${GPUs} \ 48 | --time-limit 0 \ 49 | --job-script ${JOB_SCRIPT} 50 | 51 | #--job-script ${FDIR}/job-script.sh 52 | #echo "JOB-SCRIPT: " ${JOB_SCRIPT} 53 | -------------------------------------------------------------------------------- /scripts-cluster/tmps/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /scripts-cnn/train-cifar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | # bash scripts-cnn/train-cifar.sh GDAS cifar10 cut 3 | if [ "$#" -ne 3 ] ;then 4 | echo "Input illegal number of parameters " $# 5 | echo "Need 3 parameters for the architecture, and the dataset-name, and the cutout" 6 | exit 1 7 | fi 8 | if [ "$TORCH_HOME" = "" ]; then 9 | echo "Must set TORCH_HOME envoriment variable for data dir saving" 10 | exit 1 11 | else 12 | echo "TORCH_HOME : $TORCH_HOME" 13 | fi 14 | 15 | arch=$1 16 | dataset=$2 17 | cutout=$3 18 | SAVED=./output/NAS-CNN/${arch}-${dataset}-${cutout}-E600 19 | 20 | PY_C="./env/bin/python" 21 | 22 | if [ ! -f ${PY_C} ]; then 23 | echo "Local Run with Python: "`which python` 24 | PY_C="python" 25 | else 26 | echo "Cluster Run with Python: "${PY_C} 27 | fi 28 | 29 | ${PY_C} --version 30 | 31 | ${PY_C} ./exps-cnn/train_base.py \ 32 | --data_path $TORCH_HOME/cifar.python \ 33 | --dataset ${dataset} --arch ${arch} \ 34 | --save_path ${SAVED} \ 35 | --grad_clip 5 \ 36 | --init_channels 36 --layers 20 \ 37 | --model_config ./configs/nas-cifar-cos-${cutout}.config \ 38 | --print_freq 100 --workers 6 39 | -------------------------------------------------------------------------------- /scripts-cnn/train-imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | if [ "$#" -ne 5 ] ;then 3 | echo "Input illegal number of parameters " $# 4 | echo "Need 5 parameters for the architecture, and the channel, and the layers, and the batch-size, and the seed" 5 | exit 1 6 | fi 7 | if [ "$TORCH_HOME" = "" ]; then 8 | echo "Must set TORCH_HOME envoriment variable for data dir saving" 9 | exit 1 10 | else 11 | echo "TORCH_HOME : $TORCH_HOME" 12 | fi 13 | 14 | arch=$1 15 | dataset=imagenet 16 | channels=$2 17 | layers=$3 18 | BATCH=$4 19 | seed=$5 20 | SAVED=./output/NAS-CNN/${arch}-${dataset}-C${channels}-L${layers}-${BATCH}-E250 21 | 22 | PY_C="./env/bin/python" 23 | #PY_C="$CONDA_PYTHON_EXE" 24 | 25 | if [ ! -f ${PY_C} ]; then 26 | echo "Local Run with Python: "`which python` 27 | PY_C="python" 28 | else 29 | echo "Cluster Run with Python: "${PY_C} 30 | echo "Unzip ILSVRC2012" 31 | tar --version 32 | tar -xf ./hadoop-data/ILSVRC2012.tar -C ${TORCH_HOME} 33 | #commands="./data/data/get_imagenet.sh" 34 | #${PY_C} ./data/decompress.py ./hadoop-data/ILSVRC2012-TAR ./data/data/ILSVRC2012 tar > ${commands} 35 | #${PY_C} ./data/decompress.py ./hadoop-data/ILSVRC2012-ZIP ./data/data/ILSVRC2012 zip > ./data/data/get_imagenet.sh 36 | #bash ./data/data/get_imagenet.sh 37 | #count=0 38 | #while read -r line; do 39 | # temp_file="./data/data/TEMP-${count}.sh" 40 | # echo "${line}" > ${temp_file} 41 | # bash ${temp_file} 42 | # count=$((count+1)) 43 | #${PY_C} ./data/ps_mem.py -p $$ 44 | # free -g 45 | #done < "${commands}" 46 | #wget http://10.127.2.44:8000/ILSVRC2012.tar --directory-prefix=${TORCH_HOME} 47 | #${PY_C} ./data/decompress.py ./data/classes.txt ${TORCH_HOME}/ILSVRC2012 wget > ${commands} 48 | #count=0 49 | #while read -r line; do 50 | # temp_file="./data/data/TEMP-${count}.sh" 51 | # echo "${line}" > ${temp_file} 52 | # bash ${temp_file} 53 | # count=$((count+1)) 54 | #${PY_C} ./data/ps_mem.py -p $$ 55 | # free -g 56 | #done < "${commands}" 57 | #echo "Copy ILSVRC2012 done" 58 | #tar -xvf ${TORCH_HOME}/ILSVRC2012.tar -C ${TORCH_HOME} 59 | #rm ${TORCH_HOME}/ILSVRC2012.tar 60 | echo "Unzip ILSVRC2012 done" 61 | fi 62 | 63 | ${PY_C} --version 64 | 65 | ${PY_C} ./exps-cnn/train_base.py \ 66 | --data_path $TORCH_HOME/ILSVRC2012 \ 67 | --dataset ${dataset} --arch ${arch} \ 68 | --save_path ${SAVED} \ 69 | --grad_clip 5 \ 70 | --init_channels ${channels} --layers ${layers} \ 71 | --model_config ./configs/nas-imagenet-${BATCH}.config \ 72 | --manualSeed ${seed} \ 73 | --print_freq 200 --workers 20 74 | -------------------------------------------------------------------------------- /scripts-rnn/train-PTB.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | if [ "$#" -ne 1 ] ;then 3 | echo "Input illegal number of parameters " $# 4 | echo "Need 1 parameters for the GPU and the architecture" 5 | exit 1 6 | fi 7 | 8 | arch=$1 9 | SAVED=./output/NAS-RNN/Search-${arch}-PTB 10 | PY_C="./env/bin/python" 11 | 12 | if [ ! -f ${PY_C} ]; then 13 | echo "Local Run with Python: "`which python` 14 | PY_C="python" 15 | else 16 | echo "Cluster Run with Python: "${PY_C} 17 | fi 18 | 19 | ${PY_C} --version 20 | 21 | ${PY_C} ./exps-rnn/train_rnn_base.py \ 22 | --arch ${arch} \ 23 | --save_path ${SAVED} \ 24 | --config_path ./configs/NAS-PTB-BASE.config \ 25 | --print_freq 200 26 | -------------------------------------------------------------------------------- /scripts-rnn/train-WT2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$#" -ne 1 ] ;then 3 | echo "Input illegal number of parameters " $# 4 | echo "Need 1 parameters for the architectures" 5 | exit 1 6 | fi 7 | 8 | arch=$1 9 | SAVED=./output/NAS-RNN/Search-${arch}-WT2 10 | PY_C="./env/bin/python" 11 | 12 | if [ ! -f ${PY_C} ]; then 13 | echo "Local Run with Python: "`which python` 14 | PY_C="python" 15 | else 16 | echo "Cluster Run with Python: "${PY_C} 17 | fi 18 | 19 | ${PY_C} --version 20 | 21 | ${PY_C} ./exps-rnn/train_rnn_base.py \ 22 | --arch ${arch} \ 23 | --save_path ${SAVED} \ 24 | --config_path ./configs/NAS-WT2-BASE.config \ 25 | --print_freq 300 26 | --------------------------------------------------------------------------------