├── logs └── .gitkeep ├── data ├── test │ └── .gitkeep ├── train │ └── .gitkeep └── valid │ └── .gitkeep ├── model └── .gitkeep ├── images ├── test │ └── .gitkeep ├── train │ └── .gitkeep └── valid │ └── .gitkeep ├── .dockerignore ├── util ├── log_accuracy.sh ├── num_layers.sh ├── export_tmpl │ ├── signature.mo │ ├── mms_app_gpu.conf.mo │ ├── mms_app_cpu.conf.mo │ ├── mxnet_vision_service.mo │ └── mxnet_vision_service_center_crop.mo ├── functions.py ├── compose-template-nvidia-docker2.mo ├── compare_results.py ├── slack_file_upload.py ├── compose-template.mo ├── move_images.sh ├── counter.sh ├── train_imagenet.py ├── classification_report.py ├── gen_test.sh ├── caltech101_prepare.sh ├── export_model.sh ├── confusion_matrix.py ├── save_model.sh ├── test.sh ├── ensemble.sh ├── gen_train.sh ├── sample_config.yml ├── predict.py ├── train_loss.py ├── fine-tune.py ├── ensemble.py ├── train_accuracy.py ├── functions ├── finetune.sh └── vendor │ └── mo ├── .gitignore ├── docs ├── README.md ├── setup.md ├── use_se_resnext.md ├── use_densenet.md ├── train_from_scratch.md ├── freeze_layers.md ├── benchmark.md └── pretrained_models.md ├── Makefile ├── docker-entrypoint.sh ├── Dockerfile ├── setup.sh ├── common ├── util.py ├── modelzoo.py └── fit.py └── README.md /logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/test/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/train/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/valid/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/test/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/train/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/valid/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !Dockerfile 3 | !common 4 | !docker-entrypoint.sh 5 | !util 6 | -------------------------------------------------------------------------------- /util/log_accuracy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | LOG="$1" 4 | 5 | cat "$LOG" | grep Validation-acc | sort -t'=' -k2 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docker-compose.yml 2 | docker-compose.yml-e 3 | config.yml 4 | config.yml-e 5 | data/train/ 6 | data/valid/ 7 | data/test/ 8 | images/train/ 9 | images/valid/ 10 | images/test/ 11 | logs/ 12 | model/ 13 | classify_example/ 14 | example_images/ 15 | 101_ObjectCategories.tar.gz 16 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # mxnet-finetuner documentation 2 | 3 | Contents 4 | -------- 5 | * [Available pretrained models](pretrained_models.md) 6 | * [Benchmark](benchmark.md) 7 | * [How to freeze layers during fine-tuning](freeze_layers.md) 8 | * [Training from scratch](train_from_scratch.md) 9 | * [Use DenseNet](use_densenet.md) 10 | * [Use SE-ResNext](use_se_resnext.md) 11 | -------------------------------------------------------------------------------- /util/num_layers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/functions" 7 | 8 | MODEL="$1" 9 | LAYER_BEFORE_FULLC=$(get_layer_before_fullc "$MODEL") 10 | python3 util/fine-tune.py --pretrained-model $MODEL --layer-before-fullc $LAYER_BEFORE_FULLC --num-classes 2 --print-layers-and-exit 11 | -------------------------------------------------------------------------------- /util/export_tmpl/signature.mo: -------------------------------------------------------------------------------- 1 | { 2 | "inputs": [ 3 | { 4 | "data_name": "data", 5 | "data_shape": [0, 3, {{MODEL_IMAGE_SIZE}}, {{MODEL_IMAGE_SIZE}}], 6 | "rgb_mean": [{{RGB_MEAN}}], 7 | "rgb_std": [{{RGB_STD}}] 8 | } 9 | ], 10 | "input_type": "image/jpeg", 11 | "outputs": [ 12 | { 13 | "data_name": "softmax", 14 | "data_shape": [0, {{NUM_CLASSES}}] 15 | } 16 | ], 17 | "output_type": "application/json" 18 | } 19 | -------------------------------------------------------------------------------- /util/functions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Utility functions 5 | 6 | def get_image_size(model_prefix): 7 | if 'caffenet' in model_prefix: 8 | return 227 9 | elif 'squeezenet' in model_prefix: 10 | return 227 11 | elif 'alexnet' in model_prefix: 12 | return 227 13 | elif 'googlenet' in model_prefix: 14 | return 299 15 | elif 'inception-v3' in model_prefix: 16 | return 299 17 | elif 'inception-v4' in model_prefix: 18 | return 299 19 | elif 'inception-resnet-v2' in model_prefix: 20 | return 299 21 | else: 22 | return 224 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DOCKER_IMAGE_VERSION=gpu 2 | DOCKER_IMAGE_NAME=knjcode/mxnet-finetuner 3 | DOCKER_IMAGE_TAGNAME=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) 4 | 5 | default: build 6 | 7 | build: 8 | docker pull nvidia/cuda:9.0-cudnn7-devel 9 | docker build -t $(DOCKER_IMAGE_TAGNAME) . 10 | docker tag $(DOCKER_IMAGE_TAGNAME) $(DOCKER_IMAGE_NAME):latest 11 | 12 | rebuild: 13 | docker pull nvidia/cuda:9.0-cudnn7-devel 14 | docker build --no-cache -t $(DOCKER_IMAGE_TAGNAME) . 15 | docker tag $(DOCKER_IMAGE_TAGNAME) $(DOCKER_IMAGE_NAME):latest 16 | 17 | push: 18 | docker push $(DOCKER_IMAGE_TAGNAME) 19 | docker push $(DOCKER_IMAGE_NAME):latest 20 | 21 | test: 22 | docker-compose run finetuner help 23 | -------------------------------------------------------------------------------- /util/export_tmpl/mms_app_gpu.conf.mo: -------------------------------------------------------------------------------- 1 | [MMS Arguments] 2 | --models 3 | {{MODEL_NAME}}=/model/{{MODEL_FILE}} 4 | 5 | --service 6 | optional 7 | 8 | --gen-api 9 | optional 10 | 11 | --log-file 12 | optional 13 | 14 | --log-rotation-time 15 | optional 16 | 17 | --log-level 18 | optional 19 | 20 | --metrics-write-to 21 | optional 22 | 23 | --num-gpu 24 | optional 25 | 26 | [Gunicorn Arguments] 27 | 28 | --bind 29 | unix:/tmp/mms_app.sock 30 | 31 | --workers 32 | 1 33 | 34 | 35 | --worker-class 36 | gevent 37 | 38 | --limit-request-line 39 | 0 40 | 41 | [Nginx Configurations] 42 | server { 43 | listen 8080; 44 | 45 | location / { 46 | proxy_pass http://unix:/tmp/mms_app.sock; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util/export_tmpl/mms_app_cpu.conf.mo: -------------------------------------------------------------------------------- 1 | [MMS Arguments] 2 | --models 3 | {{MODEL_NAME}}=/model/{{MODEL_FILE}} 4 | 5 | --service 6 | optional 7 | 8 | --gen-api 9 | optional 10 | 11 | --log-file 12 | optional 13 | 14 | --log-rotation-time 15 | optional 16 | 17 | --log-level 18 | optional 19 | 20 | --metrics-write-to 21 | optional 22 | 23 | [Gunicorn Arguments] 24 | 25 | --bind 26 | unix:/tmp/mms_app.sock 27 | 28 | --workers 29 | 1 30 | 31 | --worker-class 32 | gevent 33 | 34 | --limit-request-line 35 | 0 36 | 37 | [Nginx Configurations] 38 | server { 39 | listen 8080; 40 | 41 | location / { 42 | proxy_pass http://unix:/tmp/mms_app.sock; 43 | } 44 | } 45 | 46 | [MXNet Environment Variables] 47 | OMP_NUM_THREADS=1 48 | -------------------------------------------------------------------------------- /docs/setup.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | ``` 4 | $ git clone https://github.com/knjcode/mxnet-finetuner 5 | $ cd mxnet-finetuner 6 | $ bash setup.sh 7 | ``` 8 | 9 | `setup.sh` will automatically generate` docker-compose.yml` and `config.yml` which are necessary for executing this tool according to your environment such as existence of the GPU. 10 | 11 | 12 | ## After updating the GPU driver of the host machine 13 | 14 | When updating the GPU driver of the host machine, it is necessary to stop all GPU containers and delete the volume. 15 | 16 | ``` 17 | $ docker volume ls -q -f driver=nvidia-docker | xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f 18 | ``` 19 | 20 | To create a volume, execute the `nvidia-docker` command once. 21 | 22 | ``` 23 | $ nvidia-docker run --rm nvidia/cuda nvidia-smi 24 | ``` 25 | 26 | And then, re-running `bash setup.sh`. 27 | -------------------------------------------------------------------------------- /util/compose-template-nvidia-docker2.mo: -------------------------------------------------------------------------------- 1 | version: "2.3" 2 | 3 | services: 4 | finetuner: 5 | image: knjcode/mxnet-finetuner 6 | runtime: nvidia 7 | environment: 8 | - SLACK_API_TOKEN 9 | - MXNET_CUDNN_AUTOTUNE_DEFAULT 10 | ports: 11 | - "8888:8888" 12 | volumes: 13 | - "$PWD:/config:ro" 14 | - "$PWD/images:/images:rw" 15 | - "$PWD/data:/data:rw" 16 | - "$PWD/model:/mxnet/example/image-classification/model:rw" 17 | - "$PWD/logs:/mxnet/example/image-classification/logs:rw" 18 | - "$PWD/classify_example:/mxnet/example/image-classification/classify_example:rw" 19 | 20 | mms: 21 | image: awsdeeplearningteam/mms_gpu 22 | runtime: nvidia 23 | command: "mxnet-model-server start --mms-config /model/mms_app_gpu.conf" 24 | ports: 25 | - "8080:8080" 26 | volumes: 27 | - "$PWD/model:/model" 28 | -------------------------------------------------------------------------------- /docs/use_se_resnext.md: -------------------------------------------------------------------------------- 1 | # Use SE-ResNeXt 2 | 3 | ImageNet pretrained SE-ResNeXt-50 model is introduced on [SENet.mxnet]. 4 | 5 | To use SE-ResNeXt-50 pretrained models, specify the `imagenet1k-se-resext-50` in `config.yml`. 6 | 7 | `mxnet-finetuner` is designed to download the SE-ResNeXt-50 model automatically, 8 | but if it is not downloaded automatically, you can use this model as below. 9 | 10 | ## Download parameter and symbol files 11 | 12 | se-resnext-imagenet-50-0-0125.params 13 | https://drive.google.com/uc?id=0B_M7XF_l0CzXOHNybXVWLWZteEE 14 | 15 | se-resnext-imagenet-50-0-symbol.json 16 | https://raw.githubusercontent.com/bruinxiong/SENet.mxnet/master/se-resnext-imagenet-50-0-symbol.json 17 | 18 | ## Rearrange downloaded files 19 | 20 | Change the name of the downloaded files and store it as below. 21 | 22 | ``` 23 | model/imagenet1k-se-resnext-50-0000.params 24 | model/imagenet1k-se-resnext-50-symbol.json 25 | ``` 26 | 27 | 28 | [SENet.mxnet]: https://github.com/bruinxiong/SENet.mxnet 29 | -------------------------------------------------------------------------------- /util/compare_results.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Compare prediction results 5 | # 6 | # Usage: 7 | # $ util/compare_results.py 8 | # $ util/compare_results.py logs/predict_results1.txt logs/predict_results2.txt 9 | # 10 | 11 | import sys 12 | import pandas as pd 13 | 14 | results_info_rows=3 15 | 16 | results_file1 = sys.argv[1] 17 | results_file2 = sys.argv[2] 18 | 19 | print("results_file1:", results_file1) 20 | print("results_file2:", results_file2) 21 | 22 | df1 = pd.read_csv(results_file1, skiprows=results_info_rows, sep=" ", header=None) 23 | df2 = pd.read_csv(results_file2, skiprows=results_info_rows, sep=" ", header=None) 24 | 25 | if df1.iloc[:,0:2].equals(df2.iloc[:,0:2]) != True: 26 | print("Test files in predicting results do not match.") 27 | sys.exit(1) 28 | 29 | diff = df1[2] == df2[2] 30 | 31 | count = len(diff) 32 | match = len(diff[diff == True]) 33 | match_rate = match / count 34 | 35 | print("test count:", count) 36 | print("match rate:", match_rate) 37 | -------------------------------------------------------------------------------- /docs/use_densenet.md: -------------------------------------------------------------------------------- 1 | # Use DenseNet 2 | 3 | ImageNet pretrained DenseNet-169 model is introduced on [A MXNet implementation of DenseNet with BC structure]. 4 | 5 | To use DenseNet-169 pretrained models, specify the `imagenet1k-densenet-169` in `config.yml`. 6 | 7 | `mxnet-finetuner` is designed to download the DenseNet-169 model automatically, 8 | but if it is not downloaded automatically, you can use this model as below. 9 | 10 | ## Download parameter and symbol files 11 | 12 | densenet-imagenet-169-0-0125.params 13 | https://drive.google.com/open?id=0B_M7XF_l0CzXX3V3WXJoUnNKZFE 14 | 15 | densenet-imagenet-169-0-symbol.json 16 | https://raw.githubusercontent.com/bruinxiong/densenet.mxnet/master/densenet-imagenet-169-0-symbol.json 17 | 18 | ## Rearrange downloaded files 19 | 20 | Change the name of the downloaded files and store it as below. 21 | 22 | ``` 23 | model/imagenet1k-densenet-169-0000.params 24 | model/imagenet1k-densenet-169-symbol.json 25 | ``` 26 | 27 | 28 | [A MXNet implementation of DenseNet with BC structure]: https://github.com/bruinxiong/densenet.mxnet 29 | -------------------------------------------------------------------------------- /docs/train_from_scratch.md: -------------------------------------------------------------------------------- 1 | # Training from scratch 2 | 3 | Edit `config.yml` as below. 4 | 5 | ``` 6 | finetune: 7 | models: 8 | - scratch-alexnet 9 | ``` 10 | 11 | You can also run fine-tuning and training from scratch together. 12 | 13 | ``` 14 | finetune: 15 | models: 16 | - imagenet1k-inception-v3 17 | - scratch-inception-v3 18 | ``` 19 | 20 | ## Available models traininig from scratch 21 | 22 | |from scratch model name | 23 | |:------------------------------| 24 | |scratch-alexnet | 25 | |scratch-googlenet | 26 | |scratch-inception-bn | 27 | |scratch-inception-resnet-v2 | 28 | |scratch-inception-v3 | 29 | |scratch-inception-v4 | 30 | |scratch-lenet | 31 | |scratch-mlp | 32 | |scratch-mobilenet | 33 | |scratch-resnet-N | 34 | |scratch-resnext-N | 35 | |scratch-vgg-N | 36 | 37 | Specify the number of layers for N in scratch-resnet, scratch-resnext and scratch-vgg. 38 | 39 | For scratch-resnet and scrach-resnext, N can be set to 18, 34, 50, 101, 152, 200 and 269, 40 | and for scratch-vgg, N can be set to 11, 13, 16 and 19. 41 | -------------------------------------------------------------------------------- /util/slack_file_upload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Upload a file to slack 5 | # 6 | # Environments: 7 | # $ export SLACK_API_TOKEN="your-slack-api-token" 8 | # Usage: 9 | # $ ./slack_file_upload.py 10 | # $ ./slack_file_upload.py "general random" logs/$MODEL_PREFIX-$MODEL-$OPTIMIZER.png 11 | 12 | import re 13 | import os 14 | import sys 15 | from slackclient import SlackClient 16 | 17 | channels = sys.argv[1] 18 | upload_file = sys.argv[2] 19 | 20 | # "general random" or "general\nrandom" -> "#general,#random" 21 | channels = ','.join(['#'+ch for ch in re.split(r' |\n', channels)]) 22 | 23 | print("Uploading a file to Slack. channels=%s" % channels) 24 | 25 | try: 26 | slack_token = os.environ["SLACK_API_TOKEN"] 27 | except KeyError: 28 | print("Error: Set your Slack API token as SLACK_API_TOKEN environment variable.") 29 | sys.exit(1) 30 | 31 | sc = SlackClient(slack_token) 32 | 33 | response = sc.api_call( 34 | 'files.upload', 35 | channels=channels, 36 | filename=upload_file, 37 | file=open(upload_file, 'rb') 38 | ) 39 | 40 | if response['ok']: 41 | print(" Upload success.") 42 | else: 43 | print(" Upload failed.") 44 | print(response) 45 | -------------------------------------------------------------------------------- /util/compose-template.mo: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | finetuner: 5 | image: knjcode/mxnet-finetuner 6 | environment: 7 | - SLACK_API_TOKEN 8 | - MXNET_CUDNN_AUTOTUNE_DEFAULT 9 | ports: 10 | - "8888:8888" 11 | volumes: 12 | {{#VOLUMES}} 13 | - "{{.}}" 14 | {{/VOLUMES}} 15 | - "$PWD:/config:ro" 16 | - "$PWD/images:/images:rw" 17 | - "$PWD/data:/data:rw" 18 | - "$PWD/model:/mxnet/example/image-classification/model:rw" 19 | - "$PWD/logs:/mxnet/example/image-classification/logs:rw" 20 | - "$PWD/classify_example:/mxnet/example/image-classification/classify_example:rw" 21 | {{#ExistDEV}} 22 | devices: 23 | {{#DEVICES}} 24 | - "{{.}}" 25 | {{/DEVICES}} 26 | {{/ExistDEV}} 27 | 28 | mms: 29 | image: awsdeeplearningteam/mms_gpu 30 | command: "mxnet-model-server start --mms-config /model/mms_app_gpu.conf" 31 | ports: 32 | - "8080:8080" 33 | volumes: 34 | {{#VOLUMES}} 35 | - "{{.}}" 36 | {{/VOLUMES}} 37 | - "$PWD/model:/model" 38 | {{#ExistDEV}} 39 | devices: 40 | {{#DEVICES}} 41 | - "{{.}}" 42 | {{/DEVICES}} 43 | {{/ExistDEV}} 44 | {{#VOLUMES}} 45 | volumes: 46 | {{.}}FIX_VOLUME_NAME 47 | external: true 48 | {{/VOLUMES}} 49 | -------------------------------------------------------------------------------- /util/move_images.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Move the specified number of jpeg images from the target directory to the output directory 4 | # while maintaining the directory structure. 5 | # If there is no output direcotry, it will be created automatically. 6 | # 7 | # Usage: 8 | # $ util/move_images.sh 9 | # $ util/move_images.sh 20 /images/trian /images/valid 10 | 11 | set -u 12 | shopt -s expand_aliases 13 | 14 | usage_exit() { 15 | echo 'Error: Wrong number of arguments' 1>&2 16 | echo 'Usage: util/move_images.sh ' 1>&2 17 | exit 1 18 | } 19 | 20 | if [[ ! "$#" = 3 ]]; then 21 | usage_exit 22 | fi 23 | 24 | IMG_NUM="$1" 25 | TARGET_DIR="$2" 26 | OUTPUT_DIR="$3" 27 | 28 | if which shuf > /dev/null; then 29 | alias shuffle='shuf' 30 | else 31 | if which gshuf > /dev/null; then 32 | alias shuffle='gshuf' 33 | else 34 | echo 'shuf or gshuf command not found.' 1>&2 35 | exit 1 36 | fi 37 | fi 38 | 39 | mkdir -p "$OUTPUT_DIR" 40 | for i in "$TARGET_DIR"/*; do 41 | c=$(basename "$i") 42 | echo "processing $c" 43 | mkdir -p "$OUTPUT_DIR/$c" 44 | for j in $(find "$i" -type f \( -iname "*.png" -or -iname "*.jpg" -or -iname "*.jpeg" \) | shuffle | head -n "$IMG_NUM"); do 45 | mv "$j" "$OUTPUT_DIR/$c/" 46 | done 47 | done 48 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ln /dev/null /dev/raw1394 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/util/functions" 7 | 8 | TASK="$1" 9 | ARG1="$2" 10 | 11 | CONFIG_FILE="/config/config.yml" 12 | 13 | if [ ! -f "$CONFIG_FILE" ]; then 14 | echo "Error: config.yml does not exist." 15 | exit 1 16 | fi 17 | 18 | if [[ "$TASK" = 'finetune' ]]; then 19 | util/finetune.sh 20 | elif [[ "$TASK" = 'gen_train' ]]; then 21 | if [[ "$ARG1" = '' ]]; then 22 | IMAGE_SIZE=224 23 | else 24 | IMAGE_SIZE="$ARG1" 25 | fi 26 | util/gen_train.sh "$CONFIG_FILE" "$IMAGE_SIZE" 27 | elif [[ "$TASK" = 'gen_test' ]]; then 28 | if [[ "$ARG1" = '' ]]; then 29 | IMAGE_SIZE=224 30 | else 31 | IMAGE_SIZE="$ARG1" 32 | fi 33 | util/gen_test.sh "$CONFIG_FILE" "$IMAGE_SIZE" 34 | elif [[ "$TASK" = 'test' ]]; then 35 | util/test.sh 36 | elif [[ "$TASK" = 'export' ]]; then 37 | util/export_model.sh 38 | elif [[ "$TASK" = 'ensemble' ]]; then 39 | util/ensemble.sh "$ARG1" 40 | elif [[ "$TASK" = 'num_layers' ]]; then 41 | util/num_layers.sh "$ARG1" 42 | elif [[ "$TASK" = 'jupyter' ]]; then 43 | jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --allow-root 44 | elif [[ "$TASK" = 'version' ]]; then 45 | version 46 | elif [[ "$TASK" = 'help' ]]; then 47 | usage 48 | else 49 | if [[ "$TASK" = '' ]]; then 50 | util/finetune.sh 51 | else 52 | echo "mxnet-finetuner: '$TASK' is not a mxnet-finetuner command." 53 | usage 54 | fi 55 | fi 56 | -------------------------------------------------------------------------------- /util/counter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Count the number of files/directories in each subdirectory 4 | # 5 | # Usage: 6 | # $ util/counter.sh [options] 7 | # 8 | # Options: 9 | # -r Sort in descending order of the number of files 10 | # 11 | 12 | shopt -s expand_aliases 13 | 14 | usage_exit() { 15 | echo "Usage: util/counter.sh [options] " 1>&2 16 | echo "Options:" 1>&2 17 | echo " -r Sort in descending order of the number of files" 1>&2 18 | exit 1 19 | } 20 | 21 | count_files() { 22 | local DIR="$1" 23 | find "$DIR" -maxdepth 1 -mindepth 1 -type d | while read -r subdir;do 24 | echo "$(basename "$subdir"),$(echo $(ls "$subdir" | wc -l))"; 25 | done 26 | } 27 | 28 | count_directories() { 29 | local DIR="$1" 30 | echo $(find ${DIR}/* -maxdepth 0 -type d | wc -l) 31 | } 32 | 33 | # Reference: http://qiita.com/b4b4r07/items/dcd6be0bb9c9185475bb 34 | declare -i argc=0 35 | declare -a argv=() 36 | while (( $# > 0 )) 37 | do 38 | case "$1" in 39 | -*) 40 | if [[ "$1" =~ 'r' ]]; then 41 | REVERSE=1 42 | fi 43 | if [[ "$1" =~ 'h' ]]; then 44 | usage_exit 45 | fi 46 | shift 47 | ;; 48 | *) 49 | ((++argc)) 50 | argv=("${argv[@]}" "$1") 51 | shift 52 | ;; 53 | esac 54 | done 55 | 56 | if [ $argc -lt 1 ]; then 57 | usage_exit 58 | fi 59 | 60 | DIR="${argv[0]}" 61 | 62 | FILES=$(count_files "$DIR") 63 | CLASSES=$(count_directories "$DIR") 64 | RESULT=$(echo "$FILES" | sort -n -t',' -k2 | tr ',' ' ') 65 | 66 | if which tac >/dev/null; then 67 | alias tac='tac' 68 | else 69 | alias tac='tail -r' 70 | fi 71 | 72 | if [[ "$REVERSE" = 1 ]]; then 73 | RESULT=$(echo "$RESULT" | tac) 74 | fi 75 | 76 | echo "$DIR contains "$CLASSES" directories" 77 | echo "$RESULT" | column -t 78 | -------------------------------------------------------------------------------- /docs/freeze_layers.md: -------------------------------------------------------------------------------- 1 | # How to freeze layers during fine-tuning 2 | 3 | If you set the number of target layer to `finetune.num_active_layers` in `config.yml` as below, only layers whose number is not greater than the number of the specified layer will be train. 4 | 5 | ``` 6 | finetune: 7 | models: 8 | - imagenet1k-nin 9 | optimizers: 10 | - sgd 11 | num_active_layers: 6 12 | ``` 13 | 14 | The default for `finetune.num_active_layers` is 0, in which case all layers are trained. 15 | 16 | If you set `1` to `finetune.num_active_layers`, only the last fully-connected layers are trained. 17 | 18 | You can check the layer numbers of various pretrained models with `num_layers` command. 19 | 20 | ``` 21 | $ docker-compose run finetuner num_layers 22 | ``` 23 | 24 | An example of checking the layer number of `imagenet1k-caffenet` is as follows. 25 | 26 | ``` 27 | $ docker-compose run finetuner num_layers imagenet1k-caffenet 28 | (...snip...) 29 | Number of the layer of imagenet1k-caffenet 30 | 27: data 31 | 26: conv1_weight 32 | 25: conv1_bias 33 | 24: conv1_output 34 | 23: relu1_output 35 | 22: pool1_output 36 | 21: norm1_output 37 | 20: conv2_weight 38 | 19: conv2_bias 39 | 18: conv2_output 40 | 17: relu2_output 41 | 16: pool2_output 42 | 15: norm2_output 43 | 14: conv3_weight 44 | 13: conv3_bias 45 | 12: conv3_output 46 | 11: relu3_output 47 | 10: conv4_weight 48 | 9: conv4_bias 49 | 8: conv4_output 50 | 7: relu4_output 51 | 6: conv5_weight 52 | 5: conv5_bias 53 | 4: conv5_output 54 | 3: relu5_output 55 | 2: pool5_output 56 | 1: flatten_0_output 57 | If you set the number of a layer displayed above to num_active_layers in config.yml, 58 | only layers whose number is not greater than the number of the specified layer will be train. 59 | ``` 60 | -------------------------------------------------------------------------------- /util/train_imagenet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Modified from https://raw.githubusercontent.com/apache/incubator-mxnet/master/example/image-classification/train_imagenet.py 19 | 20 | import os 21 | import sys 22 | import argparse 23 | import logging 24 | logging.basicConfig(level=logging.DEBUG) 25 | sys.path.append(os.getcwd()) 26 | from common import find_mxnet, data, fit 27 | from common.util import download_file 28 | import mxnet as mx 29 | 30 | if __name__ == '__main__': 31 | # parse args 32 | parser = argparse.ArgumentParser(description="train imagenet-1k", 33 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 34 | fit.add_fit_args(parser) 35 | data.add_data_args(parser) 36 | data.add_data_aug_args(parser) 37 | args = parser.parse_args() 38 | 39 | # load network 40 | from importlib import import_module 41 | net = import_module('symbols.'+args.network) 42 | sym = net.get_symbol(**vars(args)) 43 | 44 | # train 45 | fit.fit(args, sym, data.get_rec_iter) 46 | -------------------------------------------------------------------------------- /util/classification_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Make classification report from prediction results 5 | # 6 | # Usage: 7 | # $ ./classification_report.py 8 | # $ ./classification_report.py /config/config.yml model/$MODEL-labels.txt logs/$MODEL-epoch$EPOCH-test-predict_results.txt logs/$MODEL-epoch$EPOCH-test-classification_report.txt 9 | # 10 | 11 | import sys 12 | import yaml 13 | from sklearn.metrics import classification_report 14 | 15 | try: 16 | reload(sys) 17 | sys.setdefaultencoding('utf-8') 18 | except NameError: 19 | pass 20 | 21 | config_file = sys.argv[1] 22 | labels_file = sys.argv[2] 23 | result_file = sys.argv[3] 24 | output_file = sys.argv[4] 25 | 26 | with open(config_file) as rf: 27 | config = yaml.safe_load(rf) 28 | 29 | try: 30 | config = config['test'] 31 | except AttributeError: 32 | print('Error: Missing test section at config.yml') 33 | sys.exit(1) 34 | 35 | cr_digits = config.get('classification_report_digits', 3) 36 | 37 | with open(labels_file) as sf: 38 | labels = [l.split(' ')[-1].strip() for l in sf.readlines()] 39 | 40 | with open(result_file) as rf: 41 | lines = rf.readlines() 42 | model_prefix = lines[0][14:].strip() 43 | model_epoch = int(lines[1][13:].split(',')[0].strip()) 44 | target_data = lines[2] 45 | results = [(l.split(' ')[0], l.split(' ')[1], l.split(' ')[2]) for l in lines[3:]] 46 | 47 | y_true = [labels[int(i[1])] for i in results] 48 | y_pred = [labels[int(i[2])] for i in results] 49 | 50 | digits = int(cr_digits) 51 | report = classification_report(y_true, y_pred, target_names=labels, digits=digits) 52 | with open(output_file, 'w') as f: 53 | f.write("model_prefix: %s\n" % model_prefix) 54 | f.write("model_epoch: %s\n" % model_epoch) 55 | f.write("%s\n" % target_data) 56 | f.write(report) 57 | print("Saved classification report to \"%s\"" % output_file) 58 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:9.0-cudnn7-devel 2 | 3 | RUN apt-get update \ 4 | && apt-get upgrade -y \ 5 | && apt-get install -y \ 6 | build-essential \ 7 | cmake \ 8 | font-manager \ 9 | fonts-ipaexfont \ 10 | git \ 11 | language-pack-ja \ 12 | libatlas-base-dev \ 13 | libcurl4-openssl-dev \ 14 | libgtest-dev \ 15 | libopencv-dev \ 16 | libprotoc-dev \ 17 | protobuf-compiler \ 18 | python-opencv \ 19 | python-dev \ 20 | python-numpy \ 21 | python-tk \ 22 | python3-dev \ 23 | unzip \ 24 | wget \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | RUN cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib && \ 28 | cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py 29 | 30 | RUN wget --quiet https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \ 31 | && chmod +x jq-linux64 \ 32 | && mv jq-linux64 /usr/bin/jq 33 | 34 | RUN pip3 install \ 35 | attrdict \ 36 | awscli \ 37 | jupyter \ 38 | matplotlib \ 39 | nose \ 40 | nose-timer \ 41 | numpy \ 42 | opencv-python \ 43 | pandas \ 44 | pandas_ml \ 45 | Pillow \ 46 | pylint \ 47 | pyyaml \ 48 | requests \ 49 | seaborn \ 50 | sklearn-pandas \ 51 | slackclient \ 52 | tqdm 53 | 54 | RUN git clone https://github.com/apache/incubator-mxnet.git mxnet --branch 1.3.0 \ 55 | && cd mxnet \ 56 | && git config user.email "knjcode@gmail.com" \ 57 | && git cherry-pick ceabcaac77543d99246415b2fb2d8c973a830453 58 | 59 | # install mxnet-model-server 60 | RUN git clone https://github.com/awslabs/mxnet-model-server.git --branch v0.4.0 \ 61 | && cd mxnet-model-server \ 62 | && pip3 install -e . 63 | 64 | RUN pip3 uninstall -y mxnet 65 | RUN pip3 install mxnet-cu90==1.2.1.post1 66 | 67 | RUN locale-gen en_US.UTF-8 68 | ENV LANG en_US.UTF-8 69 | ENV LANGUAGE en_US:en 70 | ENV LC_ALL en_US.UTF-8 71 | 72 | WORKDIR /mxnet/example/image-classification 73 | 74 | COPY common /mxnet/example/image-classification/common/ 75 | COPY util /mxnet/example/image-classification/util/ 76 | COPY docker-entrypoint.sh . 77 | 78 | ENTRYPOINT ["./docker-entrypoint.sh"] 79 | -------------------------------------------------------------------------------- /util/gen_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Generate test image record files. 4 | # Settings other than image_size are read from config.yml 5 | # 6 | # Usage: 7 | # $ util/gen_test.sh 8 | # $ util/gen_test.sh /config/config.yml 224 9 | 10 | set -u 11 | 12 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 13 | source "$CUR_DIR/functions" 14 | 15 | CONFIG_FILE="$1" 16 | RESIZE="$2" 17 | 18 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < "$CONFIG_FILE" > config.json 19 | config=$(jq -Mc '.' config.json) 20 | 21 | TEST="/images/test" 22 | DATA_TEST="/data/test" 23 | mkdir -p "${DATA_TEST}" 24 | 25 | NUM_THREAD=$(get_conf "$config" ".common.num_threads" "4") 26 | QUALITY=$(get_conf "$config" ".data.quality" "100") 27 | CENTER_CROP=$(get_conf "$config" ".data.test_center_crop" "0") 28 | 29 | echo "RESIZE=$RESIZE" 30 | 31 | echo "CENTER_CROP=$CENTER_CROP" 32 | if [[ $CENTER_CROP = 1 ]]; then 33 | CENTER_CROP="--center-crop" 34 | else 35 | CENTER_CROP="" 36 | fi 37 | 38 | # Generate test image list from test directory. 39 | python3 -u /mxnet/tools/im2rec.py --list --recursive --no-shuffle \ 40 | "images-test-${RESIZE}" "${TEST}/" 41 | 42 | # Check whether test images exist. 43 | TEST_IMAGES_NUM=$(echo "$(cat "images-test-${RESIZE}.lst" | wc -l)") 44 | if [[ "$TEST_IMAGES_NUM" = 0 ]]; then 45 | echo 'Error: Test images do not exist.' 1>&2 46 | echo 'Please put test images in images/test direcotory.' 1>&2 47 | exit 1 48 | fi 49 | 50 | # Generate test image record file. 51 | python3 -u /mxnet/tools/im2rec.py --resize "${RESIZE}" --quality "${QUALITY}" --no-shuffle \ 52 | --num-thread "${NUM_THREAD}" ${CENTER_CROP} \ 53 | "images-test-${RESIZE}" "${TEST}/" 54 | mv images-test* "${DATA_TEST}" 55 | 56 | # Create labels.txt 57 | find ${TEST}/* -type d | LC_ALL=C sort | awk -F/ '{print NR-1, $NF}' > ${DATA_TEST}/labels.txt 58 | 59 | # Create images.txt 60 | LC_ALL=C $CUR_DIR/counter.sh "${TEST}" | sed -e '1d' > ${DATA_TEST}/images-test-${RESIZE}.txt 61 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Generate docker-compose.yml and config.yml for mxnet-finetuner 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/util/functions" 7 | source "$CUR_DIR/util/vendor/mo" 8 | 9 | CONFIG_FILE='config.yml' 10 | DOCKER_COMPOSE_FILE='docker-compose.yml' 11 | 12 | if ! which jq > /dev/null 2>&1; then 13 | echo 'jq command not found.' 1>&2 14 | exit 1 15 | fi 16 | 17 | # check nvidia-docker version 18 | if nvidia-docker > /dev/null 2>&1; then 19 | NVIDIA_DOCKER_VERSION=$(nvidia-docker version|head -n1|cut -d' ' -f3|cut -d'.' -f1) 20 | fi 21 | 22 | ### for nvidia-docker ver 1.x 23 | if [[ $NVIDIA_DOCKER_VERSION != "2" ]]; then 24 | if which wget > /dev/null 2>&1; then 25 | config=$(wget -qO- 'http://localhost:3476/docker/cli/json') 26 | else 27 | if which curl > /dev/null 2>&1; then 28 | config=$(curl -s 'http://localhost:3476/docker/cli/json') 29 | else 30 | echo 'wget and curl commands not found.' 1>&2 31 | exit 1 32 | fi 33 | fi 34 | 35 | VOLUMES=($(echo "$config" | jq -r ".Volumes | .[]")) 36 | ExistDEV=$(echo "$config" | jq .Devices) 37 | DEVICES=($(echo "$config" | jq -r ".Devices | .[]")) 38 | fi 39 | ### for nvidia-docker ver 1.x end 40 | 41 | # Generate docker-compose.yml according to your environment 42 | if [ -e docker-compose.yml ]; then 43 | echo -n "Overwrite docker-compose.yml? (y/n [n]): " 44 | read -r ANS 45 | case $ANS in 46 | "Y" | "y" | "yes" | "Yes" | "YES" ) 47 | generate_compose "$CUR_DIR" "$DOCKER_COMPOSE_FILE" "$NVIDIA_DOCKER_VERSION" \ 48 | && update_compose "$CUR_DIR" "$DEVICES" "$DOCKER_COMPOSE_FILE" "$NVIDIA_DOCKER_VERSION" 49 | ;; 50 | * ) 51 | echo "not overwritten" 1>&2 52 | ;; 53 | esac 54 | else 55 | generate_compose "$CUR_DIR" "$DOCKER_COMPOSE_FILE" \ 56 | && update_compose "$CUR_DIR" "$DEVICES" "$DOCKER_COMPOSE_FILE" 57 | fi 58 | 59 | # Generate config.yml if it does not exist yet. 60 | if [ -e config.yml ]; then 61 | echo -n "Overwrite config.yml? (y/n [n]): " 62 | read -r ANS 63 | case $ANS in 64 | "Y" | "y" | "yes" | "Yes" | "YES" ) 65 | generate_config "$CUR_DIR" "$CONFIG_FILE" \ 66 | && update_config "$CUR_DIR" "$DEVICES" "$CONFIG_FILE" 67 | ;; 68 | * ) 69 | echo "not overwritten" 1>&2 70 | ;; 71 | esac 72 | else 73 | generate_config "$CUR_DIR" "$CONFIG_FILE" \ 74 | && update_config "$CUR_DIR" "$DEVICES" "$CONFIG_FILE" 75 | fi 76 | -------------------------------------------------------------------------------- /common/util.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Modified from https://github.com/dmlc/mxnet/blob/master/example/image-classification/common/util.py 19 | 20 | import subprocess 21 | import os 22 | import errno 23 | from tqdm import tqdm 24 | 25 | def download_file(url, local_fname=None, force_write=False): 26 | # requests is not default installed 27 | import requests 28 | if local_fname is None: 29 | local_fname = url.split('/')[-1] 30 | if not force_write and os.path.exists(local_fname): 31 | return local_fname 32 | 33 | dir_name = os.path.dirname(local_fname) 34 | 35 | if dir_name != "": 36 | if not os.path.exists(dir_name): 37 | try: # try to create the directory if it doesn't exists 38 | os.makedirs(dir_name) 39 | except OSError as exc: 40 | if exc.errno != errno.EEXIST: 41 | raise 42 | 43 | r = requests.get(url, stream=True) 44 | assert r.status_code == 200, "failed to open %s" % url 45 | file_size = int(r.headers["content-length"]) 46 | pbar = tqdm(total=file_size, unit="B", unit_scale=True) 47 | with open(local_fname, 'wb') as f: 48 | for chunk in r.iter_content(chunk_size=1024): 49 | if chunk: # filter out keep-alive new chunks 50 | f.write(chunk) 51 | pbar.update(len(chunk)) 52 | pbar.close() 53 | return local_fname 54 | 55 | def get_gpus(): 56 | """ 57 | return a list of GPUs 58 | """ 59 | try: 60 | re = subprocess.check_output(["nvidia-smi", "-L"], universal_newlines=True) 61 | except OSError: 62 | return [] 63 | return range(len([i for i in re.split('\n') if 'GPU' in i])) 64 | -------------------------------------------------------------------------------- /util/export_tmpl/mxnet_vision_service.mo: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"). 3 | # You may not use this file except in compliance with the License. 4 | # A copy of the License is located at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # or in the "license" file accompanying this file. This file is distributed 7 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 8 | # express or implied. See the License for the specific language governing 9 | # permissions and limitations under the License. 10 | 11 | # Modified from https://github.com/awslabs/mxnet-model-server/blob/master/mms/model_service/mxnet_vision_service.py 12 | 13 | """`MXNetVisionService` defines a MXNet base vision service 14 | """ 15 | 16 | from mms.model_service.mxnet_model_service import MXNetBaseService 17 | from mms.utils.mxnet import image, ndarray 18 | 19 | 20 | class MXNetVisionService(MXNetBaseService): 21 | """MXNetVisionService defines a fundamental service for image classification task. 22 | In preprocess, input image buffer is read to NDArray and resized respect to input 23 | shape in signature. 24 | In post process, top-5 labels are returned. 25 | """ 26 | def _preprocess(self, data): 27 | img_list = [] 28 | for idx, img in enumerate(data): 29 | input_shape = self.signature['inputs'][idx]['data_shape'] 30 | try: 31 | rgb_mean = self.signature['inputs'][idx]['rgb_mean'] 32 | rgb_std = self.signature['inputs'][idx]['rgb_std'] 33 | except KeyError: 34 | rgb_mean = None 35 | rgb_std = None 36 | # We are assuming input shape is NCHW 37 | [h, w] = input_shape[2:] 38 | img_arr = image.read(img) 39 | img_arr = image.resize(img_arr, w, h) 40 | if rgb_mean: 41 | img_arr = image.color_normalize(img_arr, mx.nd.array(rgb_mean), std=mx.nd.array(rgb_std)) 42 | img_arr = image.transform_shape(img_arr) 43 | img_list.append(img_arr) 44 | return img_list 45 | 46 | def _postprocess(self, data): 47 | assert hasattr(self, 'labels'), \ 48 | "Can't find labels attribute. Did you put synset.txt file into " \ 49 | "model archive or manually load class label file in __init__?" 50 | return [ndarray.top_probability(d, self.labels, top={{TOP_K}}) for d in data] 51 | 52 | -------------------------------------------------------------------------------- /util/caltech101_prepare.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This file download the caltech 101 dataset 4 | # (http://www.vision.caltech.edu/Image_Datasets/Caltech101/), and split it into 5 | # example_images directory. 6 | 7 | set -u 8 | shopt -s expand_aliases 9 | 10 | PWD=$(pwd) 11 | 12 | # number of images per class for training 13 | TRAIN_NUM=60 14 | VALID_NUM=20 15 | TEST_NUM=20 16 | 17 | # target classes (10 classes) 18 | CLASSES="airplanes Motorbikes Faces watch Leopards bonsai car_side ketch chandelier hawksbill" 19 | 20 | if [ ! -e "$PWD/101_ObjectCategories.tar.gz" ]; then 21 | if which wget > /dev/null 2>&1; then 22 | wget -O "$PWD/101_ObjectCategories.tar.gz" http://www.vision.caltech.edu/Image_Datasets/Caltech101/101_ObjectCategories.tar.gz 23 | else 24 | if which curl > /dev/null 2>&1; then 25 | curl http://www.vision.caltech.edu/Image_Datasets/Caltech101/101_ObjectCategories.tar.gz > "$PWD/101_ObjectCategories.tar.gz" 26 | else 27 | echo 'wget and curl commands not found.' 1>&2 28 | exit 1 29 | fi 30 | fi 31 | fi 32 | 33 | if which shuf > /dev/null; then 34 | alias shuffle='shuf' 35 | else 36 | if which gshuf > /dev/null; then 37 | alias shuffle='gshuf' 38 | else 39 | echo 'shuf or gshuf command not found.' 1>&2 40 | exit 1 41 | fi 42 | fi 43 | 44 | # check example_images 45 | if [ -e "$PWD/example_images" ]; then 46 | echo './example_images directory already exits. Remove it and retry.' 1>&2 47 | exit 1 48 | fi 49 | 50 | # split into train, validation and test set 51 | tar -xf "$PWD/101_ObjectCategories.tar.gz" 52 | TRAIN_DIR="$PWD/example_images/train" 53 | VALID_DIR="$PWD/example_images/valid" 54 | TEST_DIR="$PWD/example_images/test" 55 | mkdir -p "$TRAIN_DIR" "$VALID_DIR" "$TEST_DIR" 56 | for i in ${PWD}/101_ObjectCategories/*; do 57 | c=$(basename "$i") 58 | if echo "$CLASSES" | grep -q "$c" 59 | then 60 | echo "processing $c" 61 | mkdir -p "$TRAIN_DIR/$c" "$VALID_DIR/$c" "$TEST_DIR/$c" 62 | for j in $(find "$i" -name '*.jpg' | shuffle | head -n "$TRAIN_NUM"); do 63 | mv "$j" "$TRAIN_DIR/$c/" 64 | done 65 | for j in $(find "$i" -name '*.jpg' | shuffle | head -n "$VALID_NUM"); do 66 | mv "$j" "$VALID_DIR/$c/" 67 | done 68 | for j in $(find "$i" -name '*.jpg' | shuffle | head -n "$TEST_NUM"); do 69 | mv "$j" "$TEST_DIR/$c/" 70 | done 71 | fi 72 | done 73 | 74 | # touch .gitignore 75 | touch "$TRAIN_DIR/.gitkeep" "$VALID_DIR/.gitkeep" "$TEST_DIR/.gitkeep" 76 | 77 | # clean 78 | rm -rf "$PWD/101_ObjectCategories/" 79 | -------------------------------------------------------------------------------- /util/export_tmpl/mxnet_vision_service_center_crop.mo: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # Licensed under the Apache License, Version 2.0 (the "License"). 3 | # You may not use this file except in compliance with the License. 4 | # A copy of the License is located at 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # or in the "license" file accompanying this file. This file is distributed 7 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 8 | # express or implied. See the License for the specific language governing 9 | # permissions and limitations under the License. 10 | 11 | # Modified from https://github.com/awslabs/mxnet-model-server/blob/master/mms/model_service/mxnet_vision_service.py 12 | 13 | """`MXNetVisionService` defines a MXNet base vision service 14 | """ 15 | 16 | from mms.model_service.mxnet_model_service import MXNetBaseService 17 | from mms.utils.mxnet import image, ndarray 18 | import mxnet as mx 19 | 20 | class MXNetVisionService(MXNetBaseService): 21 | """MXNetVisionService defines a fundamental service for image classification task. 22 | In preprocess, input image buffer is read to NDArray and resized respect to input 23 | shape in signature. 24 | In post process, top-5 labels are returned. 25 | """ 26 | def _preprocess(self, data): 27 | img_list = [] 28 | for idx, img in enumerate(data): 29 | input_shape = self.signature['inputs'][idx]['data_shape'] 30 | try: 31 | rgb_mean = self.signature['inputs'][idx]['rgb_mean'] 32 | rgb_std = self.signature['inputs'][idx]['rgb_std'] 33 | except KeyError: 34 | rgb_mean = None 35 | rgb_std = None 36 | # We are assuming input shape is NCHW 37 | [h, w] = input_shape[2:] 38 | img_arr = image.read(img) 39 | # resize and center_crop 40 | img_arr = mx.image.resize_short(img_arr, h) 41 | img_arr = mx.image.center_crop(img_arr, (h, w))[0] 42 | if rgb_mean: 43 | img_arr = image.color_normalize(img_arr, mx.nd.array(rgb_mean), std=mx.nd.array(rgb_std)) 44 | img_arr = image.transform_shape(img_arr) 45 | img_list.append(img_arr) 46 | return img_list 47 | 48 | def _postprocess(self, data): 49 | assert hasattr(self, 'labels'), \ 50 | "Can't find labels attribute. Did you put synset.txt file into " \ 51 | "model archive or manually load class label file in __init__?" 52 | return [ndarray.top_probability(d, self.labels, top={{TOP_K}}) for d in data] 53 | 54 | -------------------------------------------------------------------------------- /util/export_model.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Generate MXNet models for mxnet-model-server 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/functions" 7 | source "$CUR_DIR/vendor/mo" 8 | 9 | CONFIG_FILE="/config/config.yml" 10 | 11 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < $CONFIG_FILE > config.json 12 | config=$(jq -Mc '.' config.json) 13 | 14 | DATA_TRAIN="/data/train" 15 | LATEST_RESULT_LOG="logs/latest_result.txt" 16 | 17 | USE_LATEST=$(get_conf "$config" ".export.use_latest" "1") 18 | TOP_K=$(get_conf "$config" ".export.top_k" "10") 19 | RGB_MEAN=$(get_conf "$config" ".export.rgb_mean" "123.68,116.779,103.939") 20 | RGB_STD=$(get_conf "$config" ".export.rgb_std" "1,1,1") 21 | CENTER_CROP=$(get_conf "$config" ".export.center_crop" "1") 22 | 23 | MODEL_NAME=$(get_conf "$config" ".export.model_name" "model") 24 | 25 | if [[ $USE_LATEST = 1 ]]; then 26 | # Check latest_result.txt 27 | MODEL=$(head -n 1 $LATEST_RESULT_LOG) 28 | EPOCH=$(tail -n 1 $LATEST_RESULT_LOG) 29 | else 30 | MODEL_AND_EPOCH=$(get_conf "$config" ".export.model" "") 31 | if [[ "$MODEL_AND_EPOCH" = "" ]]; then 32 | echo 'Error: export.model in config.yml is empty.' 1>&2 33 | exit 1 34 | fi 35 | # Get model_prefix and epoch 36 | MODEL=${MODEL_AND_EPOCH%-*} 37 | EPOCH=$(echo $MODEL_AND_EPOCH|rev|cut -d'-' -f1|rev|sed "s/0*\([0-9]*[0-9]$\)/\1/g") 38 | fi 39 | 40 | PARAMS="model/$MODEL-$(printf '%04d' $EPOCH).params" 41 | SYMBOL_JSON="model/$MODEL-symbol.json" 42 | LABELS_TXT="model/$MODEL-labels.txt" 43 | 44 | # Check existence of .params file 45 | if [ ! -e $PARAMS ]; then 46 | echo "Error: $PARAMS does not exist." 1>&2 47 | exit 1 48 | elif [ ! -e $SYMBOL_JSON ]; then 49 | echo "Error: $SYMBOL_JSON does not exist." 1>&2 50 | exit 1 51 | elif [ ! -e $LABELS_TXT ]; then 52 | echo "Error: $LABELS_TXT does not exist." 1>&2 53 | exit 1 54 | fi 55 | 56 | echo "Start generating $MODEL.model" 1>&2 57 | 58 | NUM_CLASSES=$(echo $(cat "model/$MODEL-labels.txt" | wc -l)) 59 | if [ $TOP_K -gt $NUM_CLASSES ]; then 60 | TOP_K=$NUM_CLASSES 61 | echo "INFO: TOP_K must less or equal NUM_CLASSES. Set TOP_K=$NUM_CLASSES" 1>&2 62 | fi 63 | # Determine MODEL_IMAGE_SIZE 64 | MODEL_IMAGE_SIZE=$(get_image_size "$MODEL") 65 | 66 | export_tmp_dir=$(mktemp -d tmp.XXXXXXXXXX) 67 | service_tmp_dir=$(mktemp -d tmp.XXXXXXXXXX) 68 | 69 | cp $PARAMS $export_tmp_dir \ 70 | && echo "Use $PARAMS" 1>&2 71 | 72 | cp $SYMBOL_JSON $export_tmp_dir \ 73 | && echo "Use $SYMBOL_JSON" 1>&2 74 | 75 | cat $LABELS_TXT | cut -d' ' -f2 > $export_tmp_dir/synset.txt \ 76 | && echo "Use $LABELS_TXT as synset.txt" 1>&2 77 | 78 | # save config.yml 79 | CONFIG_LOG="logs/$MODEL-$(printf "%04d" $EPOCH)-export-config.yml" 80 | cp "$CONFIG_FILE" "$CONFIG_LOG" \ 81 | && echo "Saved config file to \"$CONFIG_LOG\"" 1>&2 82 | 83 | generate_export_model_signature "$CUR_DIR" "$MODEL_IMAGE_SIZE" "$RGB_MEAN" "$RGB_STD" "$NUM_CLASSES" "$export_tmp_dir" 84 | generate_export_model_service "$CUR_DIR" "$CENTER_CROP" "$TOP_K" "$service_tmp_dir" 85 | generate_export_model_conf "$CUR_DIR" "$MODEL_NAME" "$MODEL.model" 86 | 87 | mxnet-model-export --model-name "$MODEL" --model-path "$export_tmp_dir" --service "$service_tmp_dir/mxnet_finetuner_service.py" \ 88 | && cp $MODEL.model model/ \ 89 | && echo "Saved model to \"model/$MODEL.model\"" 1>&2 90 | -------------------------------------------------------------------------------- /util/confusion_matrix.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Create an image of confusion matrix from prediction results 5 | # 6 | # Usage: 7 | # $ ./confusion_matrix.py 8 | # $ ./confusion_matrix.py /config/config.yml logs/$MODEL-PREFIX-lables.txt logs/confusion_matrix.png logs/predict_results.txt 9 | # 10 | # References 11 | # http://hayataka2049.hatenablog.jp/entry/2016/12/15/222339 12 | # http://qiita.com/hik0107/items/67ad4cfbc9e84032fc6b 13 | # http://minus9d.hatenablog.com/entry/2015/07/16/231608 14 | # 15 | 16 | import sys 17 | import yaml 18 | import matplotlib 19 | matplotlib.use('agg') 20 | import matplotlib.pyplot as plt 21 | import pandas as pd 22 | import seaborn as sns 23 | import unicodedata 24 | from pandas_ml import ConfusionMatrix 25 | from sklearn.metrics import confusion_matrix 26 | 27 | 28 | def is_japanese(string): 29 | for ch in string: 30 | name = unicodedata.name(ch) 31 | if "CJK UNIFIED" in name \ 32 | or "HIRAGANA" in name \ 33 | or "KATAKANA" in name: 34 | return True 35 | return False 36 | 37 | 38 | try: 39 | reload(sys) 40 | sys.setdefaultencoding('utf-8') 41 | except NameError: 42 | pass 43 | 44 | config_file = sys.argv[1] 45 | labels_file = sys.argv[2] 46 | output_file = sys.argv[3] 47 | result_file = sys.argv[4] 48 | 49 | with open(config_file) as rf: 50 | config = yaml.safe_load(rf) 51 | 52 | with open(labels_file) as sf: 53 | labels = [l.split(' ')[-1].strip() for l in sf.readlines()] 54 | 55 | try: 56 | cm_fontsize = config['test'].get('confusion_matrix_fontsize', 12) 57 | cm_figsize = config['test'].get('confusion_matrix_figsize', 'auto') 58 | if cm_figsize == 'auto': 59 | num_class = len(labels) 60 | if 0 < num_class <= 10: 61 | cm_figsize = '8,6' 62 | elif 10 < num_class <= 30: 63 | cm_figsize = '12,9' 64 | else: 65 | cm_figsize = '16,12' 66 | cm_figsize = tuple(float(i) for i in cm_figsize.split(',')) 67 | except AttributeError: 68 | print('Error: Missing test and/or data section at config.yml') 69 | sys.exit(1) 70 | 71 | 72 | with open(result_file) as rf: 73 | lines = rf.readlines() 74 | model_prefix = lines[0][14:].strip() 75 | model_epoch = int(lines[1][13:].split(',')[0].strip()) 76 | target_data = lines[2] 77 | results = [(l.split(' ')[0], l.split(' ')[1], l.split(' ')[2]) for l in lines[3:]] 78 | 79 | y_true = [labels[int(i[1])] for i in results] 80 | y_pred = [labels[int(i[2])] for i in results] 81 | 82 | if is_japanese(''.join(labels)): 83 | matplotlib.rcParams['font.family'] = 'IPAexGothic' 84 | sns.set(font=['IPAexGothic']) 85 | else: 86 | sns.set() 87 | 88 | fig = plt.figure(figsize = cm_figsize) 89 | plt.rcParams["font.size"] = cm_fontsize 90 | 91 | cmx_data = confusion_matrix(y_true, y_pred, labels=labels) 92 | df_cmx = pd.DataFrame(cmx_data, index=labels, columns=labels) 93 | 94 | sns.heatmap(df_cmx, annot=True, fmt='g', cmap='Blues') 95 | 96 | plt.title("Confusion matrix\n%s (%d epoch)\n%s" % (model_prefix, model_epoch, target_data)) 97 | plt.xlabel("Predict") 98 | plt.ylabel("Actual") 99 | 100 | fig.tight_layout() 101 | 102 | plt.savefig(output_file) 103 | print("Saved confusion matrix to \"%s\"" % output_file) 104 | -------------------------------------------------------------------------------- /util/save_model.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Save and/or compress specified model and logs 4 | # 5 | # Usage: 6 | # $ util/save_model.sh [options] 7 | # $ util/save_model.sh -z -f 201705292200-imagenet1k-nin-sgd-0001 my-nin-model 8 | # 9 | 10 | set -eu 11 | 12 | ### usage 13 | function usage() { 14 | local err=${1:-} 15 | local ret=0 16 | if [ "${err}" ]; then 17 | exec 1>&2 18 | echo "${err}" 19 | ret=1 20 | fi 21 | cat < 23 | Options are: 24 | -h, --help show help 25 | -f, --force overwrite dst_dir and compressed file 26 | -m, --model copy model only (config, model, valid logs (default - test logs)) 27 | -z, --compress compress dst_dir in tar.gz format 28 | Example usage: $(basename "$0") -z -f 201705292200-imagenet1k-nin-sgd-0001 my-nin-model 29 | EOF 30 | exit ${ret} 31 | } 32 | 33 | ### options 34 | opt_overwrite= 35 | opt_copy_mode="default" 36 | opt_compress= 37 | while [ ${#} -ne 0 ]; do 38 | case ${1} in 39 | -h | --help) 40 | usage 41 | ;; 42 | -f | --force | --overwrite) 43 | opt_overwrite=yes 44 | shift 45 | ;; 46 | -m | --model | --copy_model_only) 47 | opt_copy_mode="model_only" 48 | shift 49 | ;; 50 | -z | --compress) 51 | opt_compress=yes 52 | shift 53 | ;; 54 | --) 55 | break 56 | ;; 57 | -*) 58 | usage "'${1}' not supported" 59 | ;; 60 | *) 61 | break 62 | ;; 63 | esac 64 | done 65 | 66 | ### arguments 67 | if [ ${#} -eq 2 ]; then 68 | model_prefix_with_epoch=${1} 69 | dst_dir=${2} 70 | else 71 | usage 72 | fi 73 | 74 | if [ -e "${dst_dir}" ]; then 75 | if [ "${opt_overwrite}" == "yes" ]; then 76 | rm -fr "${dst_dir}" 77 | else 78 | echo "${dst_dir} is exist!" 79 | exit 1 80 | fi 81 | fi 82 | 83 | ### setup 84 | model_prefix=${model_prefix_with_epoch%-*} 85 | params="model/${model_prefix_with_epoch}.params" 86 | symbol="model/${model_prefix}-symbol.json" 87 | labels="model/${model_prefix}-labels.txt" 88 | 89 | test -e "${params}" || { echo "${params} is not exist"; exit 1; } 90 | 91 | ### copying 92 | test -d "${dst_dir}" || mkdir "${dst_dir}" 93 | 94 | # model 95 | mkdir "${dst_dir}/model" 96 | for f in "${params}" "${symbol}" "${labels}"; do 97 | cp -i "${f}" "${dst_dir}/model" 98 | done 99 | 100 | # logs 101 | mkdir "${dst_dir}/logs" 102 | find "logs" \ 103 | | grep "${model_prefix}" \ 104 | | { test "${opt_copy_mode}" == "model_only" \ 105 | && grep -v "${model_prefix}-.*-test-" \ 106 | || cat ; } \ 107 | | xargs -I{} cp -i {} "${dst_dir}/logs" 108 | 109 | # show tree 110 | if which tree >/dev/null; then 111 | echo "# tree -L 3 ${dst_dir}" 112 | tree -L 3 "${dst_dir}" 113 | fi 114 | 115 | # compress 116 | if [ "${opt_compress}" == "yes" ]; then 117 | compressed="${dst_dir}.tar.gz" 118 | if [ -e "${compressed}" ]; then 119 | if [ "${opt_overwrite}" == "yes" ]; then 120 | rm -f "${compressed}" 121 | else 122 | echo "${compressed} is exist! Compression skipped." 123 | exit 1 124 | fi 125 | fi 126 | tar czf "${dst_dir}.tar.gz" "${dst_dir}" \ 127 | && echo "compressed model and logs: ${dst_dir}.tar.gz" 128 | fi 129 | 130 | # end 131 | -------------------------------------------------------------------------------- /docs/benchmark.md: -------------------------------------------------------------------------------- 1 | # Benchmark 2 | 3 | ## Speed (images/sec) 4 | 5 | - dataset: 8000 samples 6 | - batch size: 10, 20, 30, 40 7 | - Optimizer: SGD 8 | - GPU: Maxwell TITAN X (12GiB Memory) 9 | 10 | |model |batch size 10|batch size 20|batch size 30|batch size 40| 11 | |:------------------------------|------------:|------------:|------------:|------------:| 12 | |CaffeNet |755.64 |1054.47 |1019.24 |1077.63 | 13 | |SqueezeNet |458.27 |579.37 |534.68 |549.55 | 14 | |NIN |443.88 |516.21 |612.83 |656.14 | 15 | |ResNet-18 |257.40 |308.30 |331.57 |339.09 | 16 | |ResNet-34 |149.88 |182.69 |201.75 |207.49 | 17 | |Inception-BN |147.60 |183.82 |193.74 |203.86 | 18 | |ResNet-50 |88.55 |102.44 |109.98 |111.04 | 19 | |Inception-v3 |67.11 |75.90 |80.67 |82.34 | 20 | |VGG16 |56.38 |58.01 |59.80 |59.35 | 21 | |ResNet-101 |53.42 |63.28 |68.14 |68.35 | 22 | |VGG19 |45.02 |46.88 |48.62 |48.28 | 23 | |ResNet-152 |37.88 |44.89 |48.48 |48.62 | 24 | |ResNet-200 |22.58 |25.61 |27.17 |27.32 | 25 | |ResNeXt-50 |53.30 |64.40 |71.07 |72.96 | 26 | |ResNeXt-101 |31.76 |39.56 |42.90 |43.99 | 27 | |ResNeXt-101-64x4d |18.22 |23.08 |out of memory|out of memory| 28 | 29 | 30 | ## Memory usage (MiB) 31 | 32 | - dataset: 8000 samples 33 | - batch size: 10, 20, 30, 40 34 | - Optimizer: SGD 35 | - GPU: Maxwell TITAN X (12GiB GPU Memory) 36 | 37 | |model |batch size 10|batch size 20|batch size 30|batch size 40|Reference accuracy
(imagenet1k Top-5)| 38 | |:----------------|------------:|------------:|------------:|------------:|---------------------------------------:| 39 | |CaffeNet |430 |496 |631 |716 |78.3% | 40 | |SqueezeNet |608 |937 |1331 |1672 |78.8% | 41 | |NIN |650 |902 |1062 |1222 |81.3% | 42 | |ResNet-18 |814 |1163 |1497 |1853 |88.7% | 43 | |ResNet-34 |1127 |1619 |2094 |2598 |91.0% | 44 | |Inception-BN |1007 |1569 |2212 |2772 |90.8% | 45 | |ResNet-50 |1875 |3080 |4265 |5483 |92.6% | 46 | |Inception-v3 |2075 |3509 |4944 |6383 |93.3% | 47 | |VGG16 |1738 |2960 |4751 |5977 |89.8% | 48 | |ResNet-101 |2791 |4576 |6341 |8158 |93.3% | 49 | |VGG19 |1920 |3242 |5133 |6458 |89.8% | 50 | |ResNet-152 |3790 |6296 |8777 |11330 |93.1% | 51 | |ResNet-200 |2051 |2769 |3471 |4201 |unknown | 52 | |ResNeXt-50 |2248 |3863 |5468 |7089 |93.3% | 53 | |ResNeXt-101 |3350 |5749 |8126 |10539 |94.1% | 54 | |ResNeXt-101-64x4d|5140 |8679 |out of memory|out of memory|94.3% | 55 | -------------------------------------------------------------------------------- /util/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -u 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/functions" 7 | 8 | CONFIG_FILE="/config/config.yml" 9 | 10 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < $CONFIG_FILE > config.json 11 | config=$(jq -Mc '.' config.json) 12 | 13 | TEST="/images/test" 14 | DATA_TEST="/data/test" 15 | LATEST_RESULT_LOG="logs/latest_result.txt" 16 | 17 | CONFUSION_MATRIX_OUTPUT=$(get_conf "$config" ".test.confusion_matrix_output" "1") 18 | SLACK_UPLOAD=$(get_conf "$config" ".test.confusion_matrix_slack_upload" "0") 19 | SLACK_CHANNELS=$(get_conf_array "$config" ".test.confusion_matrix_slack_channels" "general") 20 | CLASSIFICATION_REPORT_OUTPUT=$(get_conf "$config" ".test.classification_report_output" "1") 21 | 22 | USE_LATEST=$(get_conf "$config" ".test.use_latest" "1") 23 | EPOCH_UP_TO=$(get_conf "$config" ".test.model_epoch_up_to" "") 24 | 25 | if [[ $USE_LATEST = 1 ]]; then 26 | # Check latest_result.txt 27 | MODEL=$(head -n 1 $LATEST_RESULT_LOG) 28 | EPOCH=$(tail -n 1 $LATEST_RESULT_LOG) 29 | else 30 | MODEL_AND_EPOCH=$(get_conf "$config" ".test.model" "") 31 | if [[ "$MODEL_AND_EPOCH" = "" ]]; then 32 | echo 'Error: test.model in config.yml is empty.' 1>&2 33 | exit 1 34 | fi 35 | # Get model_prefix and epoch 36 | MODEL=${MODEL_AND_EPOCH%-*} 37 | EPOCH=$(echo $MODEL_AND_EPOCH|rev|cut -d'-' -f1|rev|sed "s/0*\([0-9]*[0-9]$\)/\1/g") 38 | fi 39 | 40 | # Determine MODEL_IMAGE_SIZE 41 | MODEL_IMAGE_SIZE=$(get_image_size "$MODEL") 42 | 43 | # If necessary image records do not exist, they are generated. 44 | if [ "$DATA_TEST/images-test-$MODEL_IMAGE_SIZE.rec" -ot "$TEST" ]; then 45 | echo "$DATA_TEST/images-test-$MODEL_IMAGE_SIZE.rec does not exist or is outdated." 1>&2 46 | echo "Generate image records for test." 1>&2 47 | $CUR_DIR/gen_test.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" 48 | fi 49 | 50 | # Check the number of image files. If it is different from previous one, regenerate images records 51 | diff --brief <(LC_ALL=C $CUR_DIR/counter.sh $TEST | sed -e '1d') <(cat $DATA_TEST/images-test-$MODEL_IMAGE_SIZE.txt) > /dev/null 2>&1 52 | if [ "$?" -eq 1 ]; then 53 | echo "$DATA_TEST/images-test-$MODEL_IMAGE_SIZE.rec is outdated." 1>&2 54 | echo 'Generate image records for test.' 1>&2 55 | $CUR_DIR/gen_test.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" || exit 1 56 | fi 57 | 58 | # TARGET EPOCHS 59 | if [[ "$EPOCH_UP_TO" ]]; then 60 | EPOCHS=$(seq $EPOCH $EPOCH_UP_TO) 61 | else 62 | EPOCHS="$EPOCH" 63 | fi 64 | 65 | for CUR_EPOCH in $EPOCHS; do 66 | # Predict with specified model. 67 | python3 util/predict.py "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" "test" "$MODEL" "$CUR_EPOCH" 68 | 69 | # save config.yml 70 | CONFIG_LOG="logs/$MODEL-$(printf "%04d" $CUR_EPOCH)-test-config.yml" 71 | cp "$CONFIG_FILE" "$CONFIG_LOG" \ 72 | && echo "Saved config file to \"$CONFIG_LOG\"" 1>&2 73 | 74 | LABELS="model/$MODEL-labels.txt" 75 | LABELS_TEST="$DATA_TEST/labels.txt" 76 | 77 | diff --brief "$LABELS" "$LABELS_TEST" 78 | if [[ "$?" -eq 1 ]]; then 79 | echo 'The directory structure of images/train and images/test is different.' 1>&2 80 | echo 'Skip making a confusion matrix and/or a classification report.' 1>&2 81 | else 82 | # Make a confusion matrix from prediction results. 83 | if [[ "$CONFUSION_MATRIX_OUTPUT" = 1 ]]; then 84 | PREDICT_RESULTS_LOG="logs/$MODEL-$(printf "%04d" $CUR_EPOCH)-test-results.txt" 85 | IMAGE="logs/$MODEL-$(printf "%04d" $CUR_EPOCH)-test-confusion_matrix.png" 86 | python3 util/confusion_matrix.py "$CONFIG_FILE" "$LABELS" "$IMAGE" "$PREDICT_RESULTS_LOG" 87 | if [[ "$SLACK_UPLOAD" = 1 ]]; then 88 | python3 util/slack_file_upload.py "$SLACK_CHANNELS" "$IMAGE" 89 | fi 90 | fi 91 | # Make a classification report from prediction results. 92 | if [[ "$CLASSIFICATION_REPORT_OUTPUT" = 1 ]]; then 93 | PREDICT_RESULTS_LOG="logs/$MODEL-$(printf "%04d" $CUR_EPOCH)-test-results.txt" 94 | REPORT="logs/$MODEL-$(printf "%04d" $CUR_EPOCH)-test-classification_report.txt" 95 | python3 util/classification_report.py "$CONFIG_FILE" "$LABELS" "$PREDICT_RESULTS_LOG" "$REPORT" 96 | if [[ -e "$REPORT" ]]; then 97 | print_classification_report "$REPORT" 98 | else 99 | echo 'Error: classification report does not exist.' 1>&2 100 | fi 101 | fi 102 | fi 103 | 104 | done 105 | -------------------------------------------------------------------------------- /util/ensemble.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -u 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/functions" 7 | 8 | TARGET="$1" 9 | 10 | CONFIG_FILE="/config/config.yml" 11 | 12 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < $CONFIG_FILE > config.json 13 | config=$(jq -Mc '.' config.json) 14 | 15 | VALID="/images/valid" 16 | TEST="/images/test" 17 | DATA_VALID="/data/valid" 18 | DATA_TEST="/data/test" 19 | 20 | if [[ "$TARGET" = "valid" ]]; then 21 | DATA_DIR="$DATA_VALID" 22 | IMAGE_DIR="$VALID" 23 | elif [[ "$TARGET" = "test" ]]; then 24 | DATA_DIR="$DATA_TEST" 25 | IMAGE_DIR="$TEST" 26 | else 27 | echo 'Error: Invalid target name. Please specify `test` or `valid`.' 1>&2 28 | exit 1 29 | fi 30 | 31 | CONFUSION_MATRIX_OUTPUT=$(get_conf "$config" ".ensemble.confusion_matrix_output" "1") 32 | SLACK_UPLOAD=$(get_conf "$config" ".ensemble.confusion_matrix_slack_upload" "0") 33 | SLACK_CHANNELS=$(get_conf_array "$config" ".ensemble.confusion_matrix_slack_channels" "general") 34 | CLASSIFICATION_REPORT_OUTPUT=$(get_conf "$config" ".ensemble.classification_report_output" "1") 35 | 36 | MODELS=$(get_conf_array "$config" ".ensemble.models" "") 37 | if [[ "$MODELS" = "" ]]; then 38 | echo 'Error: ensemble.models in config.yml is empty.' 1>&2 39 | exit 1 40 | fi 41 | echo MODELS=$MODELS 42 | 43 | for MODEL_AND_EPOCH in $MODELS; do 44 | # Remove epoch 45 | MODEL=${MODEL_AND_EPOCH%-*} 46 | # Determine MODEL_IMAGE_SIZE 47 | MODEL_IMAGE_SIZE=$(get_image_size "$MODEL") 48 | 49 | # If necessary image records do not exist, they are generated. 50 | if [ "$DATA_DIR/images-$TARGET-$MODEL_IMAGE_SIZE.rec" -ot "$IMAGE_DIR" ]; then 51 | echo "$DATA_DIR/images-$TARGET-$MODEL_IMAGE_SIZE.rec does not exist or is outdated." 1>&2 52 | echo "Generate image records for $TARGET." 1>&2 53 | if [[ "$TARGET" = "valid" ]]; then 54 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" 55 | else 56 | $CUR_DIR/gen_test.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" 57 | fi 58 | fi 59 | 60 | # Check the number of image files. If it is different from previous one, regenerate images records 61 | diff --brief <(LC_ALL=C $CUR_DIR/counter.sh $IMAGE_DIR | sed -e '1d') <(cat $DATA_DIR/images-$TARGET-$MODEL_IMAGE_SIZE.txt) > /dev/null 2>&1 62 | if [ "$?" -eq 1 ]; then 63 | echo "$DATA_DIR/images-$TARGET-$MODEL_IMAGE_SIZE.rec is outdated." 1>&2 64 | echo "Generate image records for $TARGET." 1>&2 65 | if [[ "$TARGET" = "valid" ]]; then 66 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" || exit 1 67 | else 68 | $CUR_DIR/gen_test.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" || exit 1 69 | fi 70 | fi 71 | done 72 | 73 | # Predict with specified model. 74 | MODEL_PREFIX="$(date +%Y%m%d%H%M%S)-ensemble" 75 | python3 util/ensemble.py "$CONFIG_FILE" "$TARGET" "$MODEL_PREFIX" 76 | 77 | # save config.yml 78 | CONFIG_LOG="logs/$MODEL_PREFIX-$TARGET-config.yml" 79 | cp "$CONFIG_FILE" "$CONFIG_LOG" \ 80 | && echo "Saved config file to \"$CONFIG_LOG\"" 1>&2 81 | 82 | LABELS="model/$MODEL-labels.txt" 83 | LABELS_TEST="$DATA_DIR/labels.txt" 84 | 85 | diff --brief "$LABELS" "$LABELS_TEST" 86 | if [[ "$?" -eq 1 ]]; then 87 | echo 'The directory structure of images/train and images/test is different.' 1>&2 88 | echo 'Skip making a confusion matrix and/or a classification report.' 1>&2 89 | else 90 | # Make a confusion matrix from prediction results. 91 | if [[ "$CONFUSION_MATRIX_OUTPUT" = 1 ]]; then 92 | PREDICT_RESULTS_LOG="logs/$MODEL_PREFIX-$TARGET-results.txt" 93 | IMAGE="logs/$MODEL_PREFIX-$TARGET-confusion_matrix.png" 94 | python3 util/confusion_matrix.py "$CONFIG_FILE" "$LABELS" "$IMAGE" "$PREDICT_RESULTS_LOG" 95 | if [[ "$SLACK_UPLOAD" = 1 ]]; then 96 | python3 util/slack_file_upload.py "$SLACK_CHANNELS" "$IMAGE" 97 | fi 98 | fi 99 | # Make a classification report from prediction results. 100 | if [[ "$CLASSIFICATION_REPORT_OUTPUT" = 1 ]]; then 101 | PREDICT_RESULTS_LOG="logs/$MODEL_PREFIX-$TARGET-results.txt" 102 | REPORT="logs/$MODEL_PREFIX-$TARGET-classification_report.txt" 103 | python3 util/classification_report.py "$CONFIG_FILE" "$LABELS" "$PREDICT_RESULTS_LOG" "$REPORT" 104 | if [[ -e "$REPORT" ]]; then 105 | print_classification_report "$REPORT" 106 | else 107 | echo 'Error: classification report does not exist.' 1>&2 108 | fi 109 | fi 110 | fi 111 | -------------------------------------------------------------------------------- /util/gen_train.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Generate train and validation image record files. 4 | # Settings other than image_size are read from config.yml 5 | # 6 | # Usage: 7 | # $ util/gen_train.sh 8 | # $ util/gen_train.sh /config/config.yml 224 9 | 10 | set -u 11 | 12 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 13 | source "$CUR_DIR/functions" 14 | 15 | CONFIG_FILE="$1" 16 | RESIZE="$2" 17 | 18 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < "$CONFIG_FILE" > config.json 19 | config=$(jq -Mc '.' config.json) 20 | 21 | TRAIN="/images/train" 22 | VALID="/images/valid" 23 | DATA_TRAIN="/data/train" 24 | DATA_VALID="/data/valid" 25 | mkdir -p $DATA_TRAIN $DATA_VALID 26 | 27 | NUM_THREAD=$(get_conf "$config" ".common.num_threads" "4") 28 | TRAIN_RATIO=$(get_conf "$config" ".data.train_ratio" "1") 29 | QUALITY=$(get_conf "$config" ".data.quality" "100") 30 | SHUFFLE=$(get_conf "$config" ".data.shuffle" "1") 31 | CENTER_CROP=$(get_conf "$config" ".data.center_crop" "0") 32 | 33 | echo "TRAIN_RATIO=$TRAIN_RATIO" 34 | echo "RESIZE=$RESIZE" 35 | 36 | echo "CENTER_CROP=$CENTER_CROP" 37 | if [[ $CENTER_CROP = 1 ]]; then 38 | CENTER_CROP="--center-crop" 39 | else 40 | CENTER_CROP="" 41 | fi 42 | 43 | echo "SHUFFLE=$SHUFFLE" 44 | if [[ $SHUFFLE = 1 ]]; then 45 | SHUFFLE="" 46 | else 47 | SHUFFLE="--no-shuffle" 48 | fi 49 | 50 | 51 | # Generate train image list from train directory. 52 | python3 -u /mxnet/tools/im2rec.py --list --recursive \ 53 | ${SHUFFLE} --train-ratio "${TRAIN_RATIO}" \ 54 | "images-train-${RESIZE}" "${TRAIN}/" 55 | 56 | if [[ "$TRAIN_RATIO" != "1" ]]; then 57 | # TRAIN_RATIO < 1.0 58 | # Generate validation image record file from train directory. 59 | mv "images-train-${RESIZE}_train.lst" "images-train-${RESIZE}.lst" 60 | mv "images-train-${RESIZE}_val.lst" "images-valid-${RESIZE}.lst" 61 | python3 -u /mxnet/tools/im2rec.py --resize "${RESIZE}" --quality "${QUALITY}" ${SHUFFLE} \ 62 | --num-thread "${NUM_THREAD}" ${CENTER_CROP} \ 63 | "images-valid-${RESIZE}" "${TRAIN}/" 64 | mv images-valid* "${DATA_VALID}" 65 | 66 | # Create valid labels.txt from train directory 67 | find ${TRAIN}/* -type d | LC_ALL=C sort | awk -F/ '{print NR-1, $NF}' > ${DATA_VALID}/labels.txt 68 | 69 | # Create valid images.txt from train directory 70 | LC_ALL=C $CUR_DIR/counter.sh "${TRAIN}" | sed -e '1d' > ${DATA_VALID}/images-valid-${RESIZE}.txt 71 | else 72 | # TRAIN_RATIO = 1.0 73 | # Generate validation image list from valid directory. 74 | python3 -u /mxnet/tools/im2rec.py --list --recursive \ 75 | ${SHUFFLE} --train-ratio 1.0 \ 76 | "images-valid-${RESIZE}" "${VALID}/" 77 | 78 | # Check whether validation images exist. 79 | VALID_IMAGES_NUM=$(echo $(cat "images-valid-${RESIZE}.lst" | wc -l)) 80 | if [[ "$VALID_IMAGES_NUM" = 0 ]]; then 81 | echo 'Error: Validation images do not exist.' 1>&2 82 | echo 'Please put validation images in images/valid direcotory.' 1>&2 83 | echo 'or' 1>&2 84 | echo 'Set train_ratio in config.yml smaller than 1.0 to use part of train images for validation.' 1>&2 85 | exit 1 86 | else 87 | # Generate validation image record file. 88 | python3 -u /mxnet/tools/im2rec.py --resize "${RESIZE}" --quality "${QUALITY}" ${SHUFFLE} \ 89 | --num-thread "${NUM_THREAD}" ${CENTER_CROP} \ 90 | "images-valid-${RESIZE}" ${VALID}/ 91 | mv images-valid* "${DATA_VALID}" 92 | fi 93 | 94 | # Create valid labels.txt 95 | find ${VALID}/* -type d | LC_ALL=C sort | awk -F/ '{print NR-1, $NF}' > ${DATA_VALID}/labels.txt 96 | 97 | # Create valid images.txt 98 | LC_ALL=C $CUR_DIR/counter.sh "${VALID}" | sed -e '1d' > ${DATA_VALID}/images-valid-${RESIZE}.txt 99 | fi 100 | 101 | # Check wheter train images exist. 102 | TRAIN_IMAGES_NUM=$(echo $(cat "images-train-${RESIZE}.lst" | wc -l)) 103 | if [[ $TRAIN_IMAGES_NUM = 0 ]]; then 104 | echo 'Error: Train images do not exist.' 1>&2 105 | echo 'Please put train images in images/train direcotory.' 1>&2 106 | exit 1 107 | fi 108 | 109 | # Generate train image record file. 110 | python3 -u /mxnet/tools/im2rec.py --resize "${RESIZE}" --quality "${QUALITY}" ${SHUFFLE} \ 111 | --num-thread "${NUM_THREAD}" ${CENTER_CROP} \ 112 | "images-train-${RESIZE}" ${TRAIN}/ 113 | mv images-train* "${DATA_TRAIN}" 114 | 115 | # Create train labels.txt 116 | find ${TRAIN}/* -type d | LC_ALL=C sort | awk -F/ '{print NR-1, $NF}' > ${DATA_TRAIN}/labels.txt 117 | 118 | # Create train images.txt 119 | LC_ALL=C $CUR_DIR/counter.sh "${TRAIN}" | sed -e '1d' > ${DATA_TRAIN}/images-train-${RESIZE}.txt 120 | -------------------------------------------------------------------------------- /docs/pretrained_models.md: -------------------------------------------------------------------------------- 1 | # Available pretrained models 2 | 3 | Classification accuracy of available pretrained models. 4 | (Datasets are ImageNet1K, ImageNet11K and Place365 Challenge) 5 | 6 | *Please note that the following results are calculated with different datasets.* 7 | - ResNet accuracy from [Reproduce ResNet-v2 using MXNet] 8 | - DenseNet-169 accuracy from [A MXNet implementation of DenseNet with BC structure] 9 | - SE-ResNeXt-50 accuracy from [SENet.mxnet] 10 | - Other accuracy from [MXNet model gallery] and [MXNet - Image Classification - Pre-trained Models] 11 | 12 | |model |Top-1 Accuracy|Top-5 Accuracy|download size|model size (MXNet)|dataset |image shape| 13 | |:------------------------------|-------------:|-------------:|------------:|-----------------:|----------:|----------:| 14 | |CaffeNet |54.5% |78.3% |233MB |9.3MB |ImageNet1K |227x227 | 15 | |SqueezeNet |55.4% |78.8% |4.8MB |4.8MB |ImageNet1K |227x227 | 16 | |NIN |58.8% |81.3% |30MB |30MB |ImageNet1K |224x224 | 17 | |ResNet-18 |69.5% |89.1% |45MB |43MB |ImageNet1K |224x224 | 18 | |VGG16 |71.0% |89.8% |528MB |58MB |ImageNet1K |224x224 | 19 | |VGG19 |71.0% |89.8% |549MB |78MB |ImageNet1K |224x224 | 20 | |Inception-BN |72.5% |90.8% |44MB |40MB |ImageNet1K |224x224 | 21 | |ResNet-34 |72.8% |91.1% |84MB |82MB |ImageNet1K |224x224 | 22 | |ResNet-50 |75.6% |92.8% |98MB |90MB |ImageNet1K |224x224 | 23 | |ResNet-101 |77.3% |93.4% |171MB |163MB |ImageNet1K |224x224 | 24 | |ResNet-152 |77.8% |93.6% |231MB |223MB |ImageNet1K |224x224 | 25 | |ResNet-200 |77.9% |93.8% |248MB |240MB |ImageNet1K |224x224 | 26 | |Inception-v3 |76.9% |93.3% |92MB |84MB |ImageNet1K |299x299 | 27 | |ResNeXt-50 |76.9% |93.3% |96MB |89MB |ImageNet1K |224x224 | 28 | |ResNeXt-101 |78.3% |94.1% |170MB |162MB |ImageNet1K |224x224 | 29 | |ResNeXt-101-64x4d |79.1% |94.3% |320MB |312MB |ImageNet1K |224x224 | 30 | |ResNet-152 (imagenet11k) |41.6% |- |311MB |223MB |ImageNet11K|224x224 | 31 | |ResNet-50 (Place365 Challenge) |31.1% |- |181MB |90MB |Place365ch |224x224 | 32 | |ResNet-152 (Place365 Challenge)|33.6% |- |313MB |223MB |Place365ch |224x224 | 33 | |DenseNet-169 |75.3% |92.8% |55MB |48MB |ImageNet1K |224x224 | 34 | |SE-ResNeXt-50 |76.7% |93.4% |103MB |95MB |ImageNet1K |224x224 | 35 | 36 | - The `download size` is the file size when first downloading pretrained model. 37 | - The `model size` is the file size to be saved after fine-tuning. 38 | 39 | 40 | To use these pretrained models, specify the following pretrained model name in `config.yml`. 41 | 42 | |model |pretrained model name | 43 | |:------------------------------|:--------------------------------| 44 | |CaffeNet |imagenet1k-caffenet | 45 | |SqueezeNet |imagenet1k-squeezenet | 46 | |NIN |imagenet1k-nin | 47 | |VGG16 |imagenet1k-vgg16 | 48 | |VGG19 |imagenet1k-vgg19 | 49 | |Inception-BN |imagenet1k-inception-bn | 50 | |ResNet-18 |imagenet1k-resnet-18 | 51 | |ResNet-34 |imagenet1k-resnet-34 | 52 | |ResNet-50 |imagenet1k-resnet-50 | 53 | |ResNet-101 |imagenet1k-resnet-101 | 54 | |ResNet-152 |imagenet1k-resnet-152 | 55 | |ResNet-152 (imagenet11k) |imagenet11k-resnet-152 | 56 | |ResNet-200 |imagenet1k-resnet-200 | 57 | |Inception-v3 |imagenet1k-inception-v3 | 58 | |ResNeXt-50 |imagenet1k-resnext-50 | 59 | |ResNeXt-101 |imagenet1k-resnext-101 | 60 | |ResNeXt-101-64x4d |imagenet1k-resnext-101-64x4d | 61 | |ResNet-50 (Place365 Challenge) |imagenet11k-place365ch-resnet-50 | 62 | |ResNet-152 (Place365 Challenge)|imagenet11k-place365ch-resnet-152| 63 | |DenseNet-169 |imagenet1k-densenet-169 | 64 | |SE-ResNeXt-50 |imagenet1k-se-resnext-50 | 65 | 66 | 67 | [Reproduce ResNet-v2 using MXNet]: https://github.com/tornadomeet/ResNet 68 | [A MXNet implementation of DenseNet with BC structure]: https://github.com/bruinxiong/densenet.mxnet 69 | [SENet.mxnet]: https://github.com/bruinxiong/SENet.mxnet 70 | [MXNet model gallery]: https://github.com/dmlc/mxnet-model-gallery 71 | [MXNet - Image Classification - Pre-trained Models]: https://github.com/apache/incubator-mxnet/tree/master/example/image-classification#pre-trained-models 72 | -------------------------------------------------------------------------------- /util/sample_config.yml: -------------------------------------------------------------------------------- 1 | # mxnet-finetuner settings 2 | 3 | # common settngs 4 | common: 5 | num_threads: 4 6 | # gpus: 0 # list of gpus to run, e.g. 0 or 0,2,5. 7 | 8 | # train, validation and test RecordIO data generation settings 9 | data: 10 | quality: 100 11 | shuffle: 1 12 | center_crop: 0 13 | # test_center_crop: 1 14 | # resize_short: 256 15 | # train_ratio: 1.0 16 | 17 | # finetune settings 18 | finetune: 19 | models: # specify models to use 20 | - imagenet1k-nin 21 | # - imagenet1k-inception-v3 22 | # - imagenet1k-vgg16 23 | # - imagenet1k-resnet-50 24 | # - imagenet11k-resnet-152 25 | # - imagenet1k-resnext-101 26 | # - imagenet1k-se-resnext-50 27 | # etc 28 | optimizers: # specify optimizers to use 29 | - sgd 30 | # optimizers: sgd, nag, rmsprop, adam, adagrad, adadelta, adamax, nadam, dcasgd, signum, etc. 31 | # num_active_layers: 1 # train last n-layers without last fully-connected layer 32 | num_epochs: 10 # max num of epochs 33 | # load_epoch: 0 # specify when using user fine-tuned model 34 | lr: 0.0001 # initial learning rate 35 | lr_factor: 0.1 # the ratio to reduce lr on each step 36 | lr_step_epochs: 10 # the epochs to reduce the lr, e.g. 30,60 37 | mom: 0.9 # momentum for sgd 38 | wd: 0.0001 # weight decay for sgd 39 | batch_size: 10 # the batch size 40 | disp_batches: 10 # show progress for every n batches 41 | # top_k: 0 # report the top-k accuracy. 0 means no report. 42 | # data_aug_level: 3 # preset data augumentation level 43 | # random_crop: 0 # if or not randomly crop the image 44 | # random_mirror: 0 # if or not randomly flip horizontally 45 | # max_random_h: 0 # max change of hue, whose range is [0, 180] 46 | # max_random_s: 0 # max change of saturation, whose range is [0, 255] 47 | # max_random_l: 0 # max change of intensity, whose range is [0, 255] 48 | # max_random_aspect_ratio: 0 # max value of aspect ratio. 49 | # min_random_aspect_ratio: 0 # min value of aspect ratio, whose value is either None or a positive value. 50 | # max_random_rotate_angle: 0 # max angle to rotate, whose range is [0, 360] 51 | # max_random_shear_ratio: 0 # max ratio to shear, whose range is [0, 1] 52 | # max_random_scale: 1 # max ratio to scale 53 | # min_random_scale: 1 # min ratio to scale, should >= img_size/input_shape. otherwise use --pad-size 54 | # max_random_area: 1 # max area to crop in random resized crop, whose range is [0, 1] 55 | # min_random_area: 1 # min area to crop in random resized crop, whose range is [0, 1] 56 | # max_crop_size: -1 # Crop both width and height into a random size in [min_crop_size, max_crop_size] 57 | # min_crop_size: -1 # Crop both width and height into a random size in [min_crop_size, max_crop_size] 58 | # brightness: 0 # brightness jittering, whose range is [0, 1] 59 | # contrast: 0 # contrast jittering, whose range is [0, 1] 60 | # saturation: 0 # saturation jittering, whose range is [0, 1] 61 | # pca_noise: 0 # pca noise, whose range is [0, 1] 62 | # random_resized_crop: 0 # whether to use random resized crop 63 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 64 | # rgb_std: '1,1,1' # a tuple of size 3 for the std rgb 65 | # monitor: 0 # log network parameters every N iters if larger than 0 66 | # pad_size: 0 # padding the input image 67 | auto_test: 1 # if or not test with validation data after fine-tuneing is completed 68 | train_accuracy_graph_output: 1 69 | # train_accuracy_graph_fontsize: 12 70 | # train_accuracy_graph_figsize: 8,6 71 | # train_accuracy_graph_slack_upload: 1 72 | # train_accuracy_graph_slack_channels: 73 | # - general 74 | train_loss_graph_output: 1 75 | # train_loss_graph_fontsize: 12 76 | # train_loss_graph_figsize: 8,6 77 | # train_loss_graph_slack_upload: 1 78 | # train_loss_graph_slack_channels: 79 | # - general 80 | 81 | # test settings 82 | test: 83 | use_latest: 1 # Use last trained model. If set this option, model is ignored 84 | # model: 201705292200-imagenet1k-nin-sgd-0001 85 | # model_epoch_up_to: 10 # # test from epoch of model to model_epoch_up_to respectively 86 | test_batch_size: 10 87 | # top_k: 10 88 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 89 | # rgb_std: '1,1,1' # a tuple of size 3 for the std rgb 90 | classification_report_output: 1 91 | # classification_report_digits: 3 92 | confusion_matrix_output: 1 93 | # confusion_matrix_fontsize: 12 94 | # confusion_matrix_figsize: 16,12 95 | # confusion_matrix_slack_upload: 1 96 | # confusion_matrix_slack_channels: 97 | # - general 98 | 99 | # export settings 100 | export: 101 | use_latest: 1 # Use last trained model. If set this option, model is ignored 102 | # model: 201705292200-imagenet1k-nin-sgd-0001 103 | # top_k: 10 # report the top-k accuracy 104 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 105 | # rgb_std: '1,1,1' # a tuple of size 3 for the std rgb 106 | # center_crop: 1 # if or not center crop at image preprocessing 107 | # model_name: model 108 | 109 | # ensemble settings 110 | ensemble: 111 | models: 112 | - 20180130074818-imagenet1k-nin-nadam-0003 113 | - 20180130075252-imagenet1k-squeezenet-nadam-0003 114 | - 20180131105109-imagenet1k-caffenet-nadam-0003 115 | # weights: 1,1,1 116 | ensemble_batch_size: 10 117 | # top_k: 10 118 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 119 | # rgb_std: '1,1,1' # a tuple of size 3 for the std rgb 120 | classification_report_output: 1 121 | # classification_report_digits: 3 122 | confusion_matrix_output: 1 123 | # confusion_matrix_fontsize: 12 124 | # confusion_matrix_figsize: 16,12 125 | # confusion_matrix_slack_upload: 1 126 | # confusion_matrix_slack_channels: 127 | # - general 128 | -------------------------------------------------------------------------------- /util/predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Predict with specified model and generate predict_results.txt 5 | # Settings other than image_size are read from config.yml 6 | # 7 | # Usage: 8 | # $ ./predict.py <"test" or "valid"> 9 | # $ ./predict.py /config/config.yml 224 test 201705292200-imagenet1k-nin-sgd 3 10 | 11 | import os 12 | import cv2 13 | import heapq 14 | import numpy as np 15 | import sys 16 | import yaml 17 | sys.path.append(os.getcwd()) 18 | from common import find_mxnet 19 | import mxnet as mx 20 | from collections import namedtuple 21 | 22 | config_file = sys.argv[1] 23 | size = int(sys.argv[2]) 24 | target = sys.argv[3] 25 | model_prefix = sys.argv[4] 26 | model_epoch = int(sys.argv[5]) 27 | 28 | with open(config_file) as conf: 29 | config = yaml.safe_load(conf) 30 | 31 | try: 32 | use_latest = config['test'].get('use_latest', 1) 33 | batch_size = config['test'].get('test_batch_size', 10) 34 | top_k = config['test'].get('top_k', 10) 35 | rgb_mean = config['test'].get('rgb_mean', '123.68,116.779,103.939') 36 | rgb_mean = [float(mean) for mean in rgb_mean.split(',')] 37 | rgb_std = config['test'].get('rgb_std', '1,1,1') 38 | rgb_std = [float(std) for std in rgb_std.split(',')] 39 | except AttributeError: 40 | print('Error: Missing test section at config.yml') 41 | sys.exit(1) 42 | 43 | 44 | if top_k < 1: 45 | print('Error top_k must bigger than 0') 46 | sys.exit(1) 47 | 48 | data_shape = (3,size,size) 49 | try: 50 | gpus = str(config['common'].get('gpus', '')) 51 | except AttributeError: 52 | gpus = '' 53 | 54 | data_train="/data/train" 55 | data_valid="/data/valid" 56 | data_test="/data/test" 57 | latest_result_log="logs/latest_result.txt" 58 | 59 | Batch = namedtuple('Batch', ['data']) 60 | 61 | if use_latest: 62 | with open(latest_result_log) as r: 63 | model_prefix, model_epoch = r.read().splitlines() 64 | model_epoch = int(model_epoch) 65 | 66 | if target == 'test': 67 | data_dir = data_test 68 | elif target == 'valid': 69 | data_dir = data_valid 70 | # if target is 'valid' use latest fine-tuned model 71 | # Overwrite model and epoch from latest_result_log 72 | with open(latest_result_log) as r: 73 | model_prefix, model_epoch = r.read().splitlines() 74 | model_epoch = int(model_epoch) 75 | else: 76 | print('Error: Invalid target name. Please specify `test` or `valid`.') 77 | sys.exit(1) 78 | 79 | print("model_prefix: %s" % model_prefix) 80 | print("model_epoch: %s" % model_epoch) 81 | 82 | 83 | def load_model(model_prefix, model_epoch, batch_size, size, gpus): 84 | sym, arg_params, aux_params = mx.model.load_checkpoint('model/' + model_prefix, model_epoch) 85 | # devices for training 86 | devs = mx.cpu() if gpus is None or gpus is '' else [mx.gpu(int(i)) for i in gpus.split(',')] 87 | mod = mx.mod.Module(symbol=sym, context=devs, label_names=['softmax_label']) 88 | mod.bind(for_training=False, 89 | data_shapes=[('data', (batch_size,3,size,size))], 90 | label_shapes=[('softmax_label', (batch_size,))]) 91 | mod.set_params(arg_params, aux_params, allow_missing=True) 92 | return mod 93 | 94 | 95 | def load_image_record(imgrec, batch_size, data_shape): 96 | rec = mx.io.ImageRecordIter( 97 | path_imgrec = imgrec, 98 | label_width = 1, 99 | mean_r = rgb_mean[0], 100 | mean_g = rgb_mean[1], 101 | mean_b = rgb_mean[2], 102 | std_r = rgb_std[0], 103 | std_g = rgb_std[1], 104 | std_b = rgb_std[2], 105 | data_name = 'data', 106 | label_name = 'softmax_label', 107 | batch_size = batch_size, 108 | data_shape = data_shape, 109 | rand_crop = False, 110 | rand_mirror = False, 111 | num_parts = 1, 112 | part_index = 0) 113 | return rec 114 | 115 | 116 | def make_predict_results(imgrec, batch_size, data_shape, imglst, labels_txt, results_log, top_k, gpus): 117 | test_rec = load_image_record(imgrec, batch_size, data_shape) 118 | 119 | with open(imglst) as lst: 120 | test_list = [(l.split('\t')[1].strip(), l.split('\t')[2].strip().replace(' ', '_')) for l in lst.readlines()] 121 | 122 | with open(labels_txt) as syn: 123 | labels = [l.split(' ')[-1].strip() for l in syn.readlines()] 124 | 125 | with open(results_log, 'w') as result: 126 | result.write("model_prefix: %s\n" % model_prefix) 127 | result.write("model_epoch: %s\n" % model_epoch) 128 | result.write("data: %s\n" % imgrec) 129 | 130 | mod = load_model(model_prefix, model_epoch, batch_size, size, gpus) 131 | for preds, i_batch, batch in mod.iter_predict(test_rec, reset=False): 132 | for batch_index, (pred, label) in enumerate(zip(preds[0].asnumpy(), batch.label[0].asnumpy())): 133 | sorted_pred = heapq.nlargest(top_k, enumerate(pred), key=lambda x: x[1]) 134 | results = [] 135 | for sorted_index, value in sorted_pred: 136 | results.append("%s %s" % (sorted_index, value)) 137 | list_index = i_batch * batch_size + batch_index 138 | result.write("%s %s %s\n" % (test_list[list_index][1], int(float(test_list[list_index][0])), ' '.join(results))) 139 | 140 | 141 | imgrec = "%s/images-%s-%d.rec" % (data_dir, target, size) 142 | imglst = "%s/images-%s-%d.lst" % (data_dir, target, size) 143 | labels_txt = "model/%s-labels.txt" % model_prefix 144 | results_log = "logs/%s-%04d-%s-results.txt" % (model_prefix, model_epoch, target) 145 | 146 | make_predict_results(imgrec, batch_size, data_shape, imglst, labels_txt, results_log, top_k, gpus) 147 | print("Saved predict results to \"%s\"" % results_log) 148 | -------------------------------------------------------------------------------- /util/train_loss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Create a train loss graph 5 | # 6 | # Usage: 7 | # $ ./train_loss.py ... 8 | # $ ./train_loss.py /config/config.yml logs/$MODEL_PREFIX-$MODEL-$OPTIMIZER.png logs/$MODEL_PREFIX-$MODEL-$OPTIMIZER.log ... 9 | 10 | import re 11 | import os 12 | import sys 13 | import yaml 14 | import math 15 | import matplotlib 16 | matplotlib.use('agg') 17 | import matplotlib.pyplot as plt 18 | import numpy as np 19 | import seaborn as sns 20 | 21 | config_file = sys.argv[1] 22 | output_file = sys.argv[2] 23 | train_logs = sys.argv[3:] 24 | 25 | with open(config_file) as rf: 26 | config = yaml.safe_load(rf) 27 | 28 | try: 29 | config = config['finetune'] 30 | except AttributeError: 31 | print('Error: Missing finetune section at config.yml') 32 | sys.exit(1) 33 | 34 | tg_fontsize = config.get('train_loss_graph_fontsize', 12) 35 | tg_figsize = config.get('train_loss_graph_figsize', '8,6') 36 | tg_figsize = tuple(float(i) for i in tg_figsize.split(',')) 37 | 38 | train_log_filename = os.path.basename(train_logs[0]) 39 | model_prefix = os.path.splitext(train_log_filename)[0] 40 | 41 | model_regex = re.compile(r'pretrained_model=\'(\S+)\'') 42 | model_regex_scratch = re.compile(r'network=\'(\S+)\'') 43 | lr_regex = re.compile(r'lr=(\S+),') 44 | 45 | 46 | gpus_regex = re.compile(r'gpus=(\S+),') 47 | load_epoch_regex = re.compile(r'load_epoch=(\S+),') 48 | batch_size_regex = re.compile(r'batch_size=(\d+),') 49 | num_of_image_regex = re.compile(r'num_examples=(\d+),') 50 | optimizer_regex = re.compile(r'optimizer=\'(\S+)\'') 51 | top_k_regex = re.compile(r'top_k=(\S+),') 52 | 53 | with open(train_logs[0], 'r') as l: 54 | data = l.read() 55 | try: 56 | model = re.search(model_regex, data).groups()[0] 57 | except AttributeError: 58 | model = re.search(model_regex_scratch, data).groups()[0] 59 | lr = float(re.search(lr_regex, data).groups()[0]) 60 | try: 61 | load_epoch = int(re.search(load_epoch_regex, data).groups()[0]) 62 | except ValueError: 63 | load_epoch = 0 64 | batch_size = re.search(batch_size_regex, data).groups()[0] 65 | num_of_image = num_of_image_regex.search(data).groups()[0] 66 | optimizer = optimizer_regex.search(data).groups()[0] 67 | top_k = int(re.search(top_k_regex, data).groups()[0]) 68 | 69 | batch_per_epoch = math.ceil(float(num_of_image)/float(batch_size)) 70 | 71 | # train cross-entropy per batch 72 | # ex. Epoch[1] Batch [200] Speed: 1693.44 samples/sec accuracy=0.109375 cross-entropy=3.198293 73 | train_loss_batch = re.compile(r'Epoch\[(\d+)\] Batch \[(\d+)\]\s+Speed:\s+(\S+)\s+samples/sec\s+.*\s+cross-entropy=(\S+)') 74 | 75 | # train cross-entropy per epoch 76 | # ex Epoch[7] Train-cross-entropy=1.577511 77 | train_loss_epoch = re.compile(r'Epoch\[(\d+)\] Train-cross-entropy=(\S+)') 78 | 79 | # validation cross-entropy per epoch 80 | # ex. Epoch[9] Validation-cross-entropy=1.339509 81 | val_loss_epoch = re.compile(r'Epoch\[(\d+)\] Validation-cross-entropy=(\S+)') 82 | 83 | train_loss_x = [] 84 | train_loss_y = [] 85 | train_speed = [] 86 | val_loss_x = [] 87 | val_loss_y = [] 88 | 89 | for train_log in train_logs: 90 | with open(train_log, 'r') as l: 91 | lines = l.readlines() 92 | 93 | for line in lines: 94 | line = line.strip() 95 | 96 | match = train_loss_batch.search(line) 97 | if match: 98 | items = match.groups() 99 | epoch = int(items[0]) 100 | batch = int(items[1]) 101 | speed = float(items[2]) 102 | ce = float(items[3]) 103 | iteration = epoch * batch_per_epoch + batch 104 | train_loss_x.append(iteration) 105 | train_loss_y.append(ce) 106 | train_speed.append(speed) 107 | 108 | match = train_loss_epoch.search(line) 109 | if match: 110 | items = match.groups() 111 | epoch = int(items[0]) 112 | ce = float(items[1]) 113 | iteration = epoch * batch_per_epoch + batch_per_epoch 114 | train_loss_x.append(iteration) 115 | train_loss_y.append(ce) 116 | 117 | match = val_loss_epoch.search(line) 118 | if match: 119 | items = match.groups() 120 | epoch = int(items[0]) 121 | ce = float(items[1]) 122 | iteration = epoch * batch_per_epoch + batch_per_epoch 123 | val_loss_x.append(iteration) 124 | val_loss_y.append(ce) 125 | 126 | min_loss_index = np.argmin(val_loss_y) 127 | min_loss = val_loss_y[min_loss_index] 128 | 129 | sns.set() 130 | fig = plt.figure(figsize = tg_figsize) 131 | plt.rcParams["font.size"] = tg_fontsize 132 | 133 | plt.plot(train_loss_x, train_loss_y, "r", label="train loss") 134 | plt.plot(val_loss_x, val_loss_y, "b", label="validation loss") 135 | 136 | plt_x = train_loss_x[0] + train_loss_x[-1] * 0.1 137 | plt_y = max(max(train_loss_y), max(val_loss_y)) 138 | plt_diff = plt_y * 0.05 139 | plt.text(plt_x, plt_y - plt_diff * 1, ' model used: %s' % model) 140 | plt.text(plt_x, plt_y - plt_diff * 2, ' optimizer: %s' % optimizer) 141 | plt.text(plt_x, plt_y - plt_diff * 3, ' learning rate: %s' % lr) 142 | plt.text(plt_x, plt_y - plt_diff * 4, ' result val-loss: %s (%s epoch)' % (val_loss_y[-1], len(val_loss_y)+load_epoch)) 143 | plt.text(plt_x, plt_y - plt_diff * 5, ' best val-loss: %s (%s epoch)' % (min_loss, min_loss_index+1+load_epoch)) 144 | plt.text(plt_x, plt_y - plt_diff * 6, ' train speed: %.2f (samples/sec) (batch size: %s)' % (np.mean(train_speed), batch_size)) 145 | 146 | plt.title('model loss\n%s' % model_prefix) 147 | plt.xlabel("Iterations") 148 | plt.ylabel("loss") 149 | 150 | plt.ylim(bottom=0.0) 151 | plt.legend(loc=1) 152 | 153 | fig.tight_layout() 154 | 155 | plt.savefig(output_file) 156 | print("Saved train loss graph to \"%s\"" % output_file) 157 | -------------------------------------------------------------------------------- /common/modelzoo.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Modified from https://github.com/dmlc/mxnet/blob/master/example/image-classification/common/modelzoo.py 19 | 20 | import os 21 | from common.util import download_file 22 | 23 | _base_model_url = 'http://data.mxnet.io/models/' 24 | _dummy_model_url = 'http://example.com/' 25 | _default_model_info = { 26 | 'imagenet1k-caffenet': {'symbol':_base_model_url+'imagenet/caffenet/caffenet-symbol.json', 27 | 'params':_base_model_url+'imagenet/caffenet/caffenet-0000.params'}, 28 | 'imagenet1k-squeezenet': {'symbol':_base_model_url+'imagenet/squeezenet/squeezenet_v1.1-symbol.json', 29 | 'params':_base_model_url+'imagenet/squeezenet/squeezenet_v1.1-0000.params'}, 30 | 'imagenet1k-nin': {'symbol':_base_model_url+'imagenet/nin/nin-symbol.json', 31 | 'params':_base_model_url+'imagenet/nin/nin-0000.params'}, 32 | 'imagenet1k-vgg16': {'symbol':_base_model_url+'imagenet/vgg/vgg16-symbol.json', 33 | 'params':_base_model_url+'imagenet/vgg/vgg16-0000.params'}, 34 | 'imagenet1k-vgg19': {'symbol':_base_model_url+'imagenet/vgg/vgg19-symbol.json', 35 | 'params':_base_model_url+'imagenet/vgg/vgg19-0000.params'}, 36 | 'imagenet1k-inception-bn': {'symbol':_base_model_url+'imagenet/inception-bn/Inception-BN-symbol.json', 37 | 'params':_base_model_url+'imagenet/inception-bn/Inception-BN-0126.params'}, 38 | 'imagenet1k-inception-v3': {'symbol':_dummy_model_url+'inception-v3-symbol.json', 39 | 'params':_dummy_model_url+'inception-v3-0000.params'}, 40 | 'imagenet1k-resnet-18': {'symbol':_base_model_url+'imagenet/resnet/18-layers/resnet-18-symbol.json', 41 | 'params':_base_model_url+'imagenet/resnet/18-layers/resnet-18-0000.params'}, 42 | 'imagenet1k-resnet-34': {'symbol':_base_model_url+'imagenet/resnet/34-layers/resnet-34-symbol.json', 43 | 'params':_base_model_url+'imagenet/resnet/34-layers/resnet-34-0000.params'}, 44 | 'imagenet1k-resnet-50': {'symbol':_base_model_url+'imagenet/resnet/50-layers/resnet-50-symbol.json', 45 | 'params':_base_model_url+'imagenet/resnet/50-layers/resnet-50-0000.params'}, 46 | 'imagenet1k-resnet-101': {'symbol':_base_model_url+'imagenet/resnet/101-layers/resnet-101-symbol.json', 47 | 'params':_base_model_url+'imagenet/resnet/101-layers/resnet-101-0000.params'}, 48 | 'imagenet1k-resnet-152': {'symbol':_base_model_url+'imagenet/resnet/152-layers/resnet-152-symbol.json', 49 | 'params':_base_model_url+'imagenet/resnet/152-layers/resnet-152-0000.params'}, 50 | 'imagenet1k-resnet-200': {'symbol':_base_model_url+'imagenet/resnet/200-layers/resnet-200-symbol.json', 51 | 'params':_base_model_url+'imagenet/resnet/200-layers/resnet-200-0000.params'}, 52 | 'imagenet1k-densenet-169': {'symbol':_dummy_model_url+'imagenet1k-densenet-169-symbol.json', 53 | 'params':_dummy_model_url+'imagenet1k-densenet-169-0000.params'}, 54 | 'imagenet1k-se-resnext-50': {'symbol':_dummy_model_url+'imagenet1k-se-resnext-50-symbol.json', 55 | 'params':_dummy_model_url+'imagenet1k-se-resnext-50-0000.params'}, 56 | 'imagenet1k-resnext-50': {'symbol':_base_model_url+'imagenet/resnext/50-layers/resnext-50-symbol.json', 57 | 'params':_base_model_url+'imagenet/resnext/50-layers/resnext-50-0000.params'}, 58 | 'imagenet1k-resnext-101': {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-symbol.json', 59 | 'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-0000.params'}, 60 | 'imagenet1k-resnext-101-64x4d': {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-symbol.json', 61 | 'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-0000.params'}, 62 | 'imagenet11k-resnet-152': {'symbol':_base_model_url+'imagenet-11k/resnet-152/resnet-152-symbol.json', 63 | 'params':_base_model_url+'imagenet-11k/resnet-152/resnet-152-0000.params'}, 64 | 'imagenet11k-place365ch-resnet-152': {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-152-symbol.json', 65 | 'params':_base_model_url+'imagenet-11k-place365-ch/resnet-152-0000.params'}, 66 | 'imagenet11k-place365ch-resnet-50': {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-50-symbol.json', 67 | 'params':_base_model_url+'imagenet-11k-place365-ch/resnet-50-0000.params'}, 68 | } 69 | 70 | def download_model(model_name, dst_dir='./', meta_info=None): 71 | if meta_info is None: 72 | meta_info = _default_model_info 73 | meta_info = dict(meta_info) 74 | if model_name not in meta_info: 75 | return (None, 0) 76 | if not os.path.isdir(dst_dir): 77 | os.mkdir(dst_dir) 78 | meta = dict(meta_info[model_name]) 79 | assert 'symbol' in meta, "missing symbol url" 80 | model_name = os.path.join(dst_dir, model_name) 81 | download_file(meta['symbol'], model_name+'-symbol.json') 82 | assert 'params' in meta, "mssing parameter file url" 83 | download_file(meta['params'], model_name+'-0000.params') 84 | return (model_name, 0) 85 | 86 | 87 | if __name__ == '__main__': 88 | import sys 89 | if len(sys.argv) == 3: 90 | model_name = sys.argv[1] 91 | dst_dir = sys.argv[2] 92 | download_model(model_name, dst_dir) 93 | -------------------------------------------------------------------------------- /util/fine-tune.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Modified from https://github.com/dmlc/mxnet/blob/master/example/image-classification/fine-tune.py 19 | 20 | import os 21 | import sys 22 | import argparse 23 | import logging 24 | logging.basicConfig(level=logging.DEBUG) 25 | sys.path.append(os.getcwd()) 26 | from common import find_mxnet 27 | from common import data, fit, modelzoo 28 | import mxnet as mx 29 | 30 | def get_fine_tune_model(symbol, arg_params, num_classes, layer_name): 31 | """ 32 | symbol: the pre-trained network symbol 33 | arg_params: the argument parameters of the pre-trained model 34 | num_classes: the number of classes for the fine-tune datasets 35 | layer_name: the layer name before the last fully-connected layer 36 | """ 37 | all_layers = symbol.get_internals() 38 | net = all_layers[layer_name+'_output'] 39 | net = mx.symbol.FullyConnected(data=net, num_hidden=num_classes, name='fc') 40 | net = mx.symbol.SoftmaxOutput(data=net, name='softmax') 41 | new_args = dict({k:arg_params[k] for k in arg_params if 'fc' not in k}) 42 | return (net, new_args) 43 | 44 | 45 | if __name__ == "__main__": 46 | # parse args 47 | parser = argparse.ArgumentParser(description="fine-tune a dataset", 48 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 49 | train = fit.add_fit_args(parser) 50 | data.add_data_args(parser) 51 | aug = data.add_data_aug_args(parser) 52 | parser.add_argument('--pretrained-model', type=str, 53 | help='the pre-trained model') 54 | parser.add_argument('--layer-before-fullc', type=str, default='flatten0', 55 | help='the name of the layer before the last fullc layer') 56 | parser.add_argument('--num-active-layers', type=int, default=0, 57 | help='num of last N-layers to train. if 0 specified, train all layers') 58 | parser.add_argument('--print-layers-and-exit', action='store_true', 59 | help='print the number of layers before the last fully connected layer and exit') 60 | # use less augmentations for fine-tune 61 | # data.set_data_aug_level(parser, 1) 62 | # use a small learning rate and less regularizations 63 | parser.set_defaults(image_shape='3,224,224', num_epochs=30, 64 | lr=.01, lr_step_epochs='20') 65 | 66 | args = parser.parse_args() 67 | 68 | is_user_model = False 69 | # load pretrained model 70 | dir_path = os.path.dirname(os.path.realpath(__file__)) 71 | (prefix, epoch) = modelzoo.download_model( 72 | # args.pretrained_model, os.path.join(dir_path, 'model')) 73 | args.pretrained_model, os.path.join(dir_path, '../model')) 74 | 75 | # load user fine-tuned model 76 | if prefix is None: 77 | is_user_model = True 78 | (prefix, epoch) = (os.path.join(dir_path, '../model', args.pretrained_model), args.load_epoch) 79 | # load_epoch='15' lr_step_epoch='10,20,30' -> lr_step_epoch='25,35,45' 80 | args.lr_step_epochs = ','.join(map(str, [int(args.load_epoch) + int(ep) for ep in args.lr_step_epochs.split(',')])) 81 | else: 82 | args.load_epoch = 0 83 | 84 | sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) 85 | 86 | if is_user_model: 87 | # not remove the last fullc layer 88 | (new_sym, new_args) = (sym, arg_params) 89 | else: 90 | # remove the last fullc layer 91 | (new_sym, new_args) = get_fine_tune_model( 92 | sym, arg_params, args.num_classes, args.layer_before_fullc) 93 | 94 | if args.print_layers_and_exit: 95 | print("Number of the layer of {0}".format(args.pretrained_model)) 96 | print_layers = new_sym.get_internals().list_outputs()[:-5] 97 | for index, layer in enumerate(print_layers): 98 | print("{0:5d}: {1}".format(len(print_layers)-index, layer)) 99 | print("If you set the number of a layer displayed above to num_active_layers in config.yml,") 100 | print("only layers whose number is not greater than the number of the specified layer will be train.") 101 | sys.exit(0) 102 | 103 | # freeze layers 104 | fixed_params = [] 105 | train_params = [] 106 | if args.num_active_layers > 0: 107 | print('--------------------------------------') 108 | active_layer_num = args.num_active_layers + 4 # add the last fully-connected layers 109 | all_layers = new_sym.get_internals() 110 | 111 | freeze_layers = all_layers.list_outputs()[0:-active_layer_num-1] 112 | active_layers = all_layers.list_outputs()[-active_layer_num-1:-1] 113 | 114 | if len(freeze_layers) > 15: 115 | print('...(snip)...') 116 | for layer in freeze_layers[-15:]: 117 | print(layer) 118 | else: 119 | for layer in freeze_layers: 120 | print(layer) 121 | 122 | print('----- train the following layers -----') 123 | 124 | if len(active_layers) > 15: 125 | for layer in active_layers[:15]: 126 | print(layer) 127 | print('...(snip)...') 128 | else: 129 | for layer in active_layers: 130 | print(layer) 131 | 132 | for k in new_args: 133 | is_active = False 134 | for a in active_layers: 135 | if k == a: 136 | is_active = True 137 | train_params.append(k) 138 | if not is_active: 139 | fixed_params.append(k) 140 | 141 | print('--------------------------------------') 142 | print("Train the last fc layers and the following layers: %s" % ', '.join(train_params)) 143 | else: 144 | print("Train all the layers") 145 | 146 | # train 147 | fit.fit(args = args, 148 | network = new_sym, 149 | data_loader = data.get_rec_iter, 150 | arg_params = new_args, 151 | aux_params = aux_params, 152 | fixed_params_names = fixed_params) 153 | -------------------------------------------------------------------------------- /util/ensemble.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Predict with ensemble of specified models and generate predict_results.txt 5 | # 6 | # Usage: 7 | # $ ./ensemble.py <"test" or "valid"> 8 | # $ ./ensemble.py /config/config.yml test ensemble 9 | 10 | import os 11 | import re 12 | import cv2 13 | import heapq 14 | import numpy as np 15 | import sys 16 | import yaml 17 | sys.path.append(os.getcwd()) 18 | from common import find_mxnet 19 | import mxnet as mx 20 | from collections import namedtuple 21 | import functions 22 | 23 | config_file = sys.argv[1] 24 | target = sys.argv[2] 25 | model_prefix = sys.argv[3] 26 | 27 | with open(config_file) as conf: 28 | config = yaml.safe_load(conf) 29 | 30 | models = config['ensemble'].get('models') 31 | 32 | try: 33 | weights = config['ensemble'].get('weights', False) 34 | if weights: 35 | weights = [float(weight) for weight in weights.split(',')] 36 | batch_size = config['ensemble'].get('ensemble_batch_size', 10) 37 | top_k = config['ensemble'].get('top_k', 10) 38 | rgb_mean = config['ensemble'].get('rgb_mean', '123.68,116.779,103.939') 39 | rgb_mean = [float(mean) for mean in rgb_mean.split(',')] 40 | rgb_std = config['ensemble'].get('rgb_std', '1,1,1') 41 | rgb_std = [float(std) for std in rgb_std.split(',')] 42 | except AttributeError: 43 | print('Error: Missing ensemble section at config.yml') 44 | sys.exit(1) 45 | 46 | 47 | if top_k < 1: 48 | print('Error top_k must bigger than 0') 49 | sys.exit(1) 50 | 51 | try: 52 | gpus = str(config['common'].get('gpus', '')) 53 | except AttributeError: 54 | gpus = '' 55 | 56 | data_train="/data/train" 57 | data_valid="/data/valid" 58 | data_test="/data/test" 59 | latest_result_log="logs/latest_result.txt" 60 | 61 | if target == 'test': 62 | data_dir = data_test 63 | elif target == 'valid': 64 | data_dir = data_valid 65 | else: 66 | print('Error: Invalid target name. Please specify `test` or `valid`.') 67 | sys.exit(1) 68 | 69 | 70 | def load_model(model_prefix, model_epoch, batch_size, size, gpus): 71 | sym, arg_params, aux_params = mx.model.load_checkpoint('model/' + model_prefix, model_epoch) 72 | # devices for training 73 | devs = mx.cpu() if gpus is None or gpus is '' else [mx.gpu(int(i)) for i in gpus.split(',')] 74 | mod = mx.mod.Module(symbol=sym, context=devs, label_names=['softmax_label']) 75 | mod.bind(for_training=False, 76 | data_shapes=[('data', (batch_size,3,size,size))], 77 | label_shapes=[('softmax_label', (batch_size,))]) 78 | mod.set_params(arg_params, aux_params, allow_missing=True) 79 | return mod 80 | 81 | 82 | def load_image_record(imgrec, batch_size, data_shape): 83 | rec = mx.io.ImageRecordIter( 84 | path_imgrec = imgrec, 85 | label_width = 1, 86 | mean_r = rgb_mean[0], 87 | mean_g = rgb_mean[1], 88 | mean_b = rgb_mean[2], 89 | std_r = rgb_std[0], 90 | std_g = rgb_std[1], 91 | std_b = rgb_std[2], 92 | data_name = 'data', 93 | label_name = 'softmax_label', 94 | batch_size = batch_size, 95 | data_shape = data_shape, 96 | rand_crop = False, 97 | rand_mirror = False, 98 | num_parts = 1, 99 | part_index = 0) 100 | return rec 101 | 102 | 103 | def make_ensemble_predict_results(data_dir, target, batch_size, imglst, labels_txt, results_log, top_k, gpus): 104 | with open(imglst) as lst: 105 | test_list = [(l.split('\t')[1].strip(), l.split('\t')[2].strip()) for l in lst.readlines()] 106 | 107 | with open(labels_txt) as syn: 108 | labels = [l.split(' ')[-1].strip() for l in syn.readlines()] 109 | 110 | model_size_array = [] 111 | imgrec_array = [] 112 | for model_prefix in model_prefix_array: 113 | size = functions.get_image_size(model_prefix) 114 | model_size_array.append(size) 115 | imgrec_array.append("%s/images-%s-%d.rec" % (data_dir, target, size)) 116 | print(model_size_array) 117 | print(imgrec_array) 118 | 119 | with open(results_log, 'w') as result: 120 | result.write("model_prefix: %s\n" % ','.join(model_prefix_array)) 121 | result.write("model_epoch: %s\n" % ','.join([str(epoch) for epoch in model_epoch_array])) 122 | result.write("data: %s\n" % ','.join(imgrec_array)) 123 | 124 | pred_arrays = [] 125 | for model_prefix, model_epoch, model_size in zip(model_prefix_array, model_epoch_array, model_size_array): 126 | 127 | data_shape = (3,model_size,model_size) 128 | imgrec = "%s/images-%s-%d.rec" % (data_dir, target, model_size) 129 | 130 | mod = load_model(model_prefix, model_epoch, batch_size, model_size, gpus) 131 | test_rec = load_image_record(imgrec, batch_size, data_shape) 132 | 133 | pred_array = [] 134 | for preds, i_batch, batch in mod.iter_predict(test_rec, reset=False): 135 | for batch_index, (pred, label) in enumerate(zip(preds[0].asnumpy(), batch.label[0].asnumpy())): 136 | pred_array.append(pred) 137 | pred_arrays.append(pred_array) 138 | del mod 139 | del test_rec 140 | 141 | if weights: 142 | w = np.array(weights) 143 | w = w / np.sum(w) 144 | try: 145 | preds = np.average(pred_arrays, axis=0, weights=w) 146 | except ValueError: 147 | print('Length of weights not compatible with number of models.') 148 | sys.exit(1) 149 | else: 150 | preds = np.average(pred_arrays, axis=0) 151 | 152 | for i in range(len(preds)): 153 | sorted_pred = heapq.nlargest(top_k, enumerate(preds[i]), key=lambda x: x[1]) 154 | results = [] 155 | for sorted_index, value in sorted_pred: 156 | results.append("%s %s" % (sorted_index, value)) 157 | result.write("%s %s %s\n" % (test_list[i][1], int(float(test_list[i][0])), ' '.join(results))) 158 | 159 | 160 | model_prefix_array = [re.sub(r'-\d+$','',model) for model in models] 161 | print(model_prefix_array) 162 | model_epoch_array = [int(re.sub(r'^[\w\-\.]+-', '', model)) for model in models] 163 | print(model_epoch_array) 164 | 165 | labels_txt = "model/%s-labels.txt" % model_prefix_array[0] # use first labels.txt in models 166 | size = functions.get_image_size(model_prefix_array[0]) 167 | imglst = "%s/images-%s-%d.lst" % (data_dir, target, size) 168 | results_log = "logs/%s-%s-results.txt" % (model_prefix, target) 169 | 170 | make_ensemble_predict_results(data_dir, target, batch_size, imglst, labels_txt, results_log, top_k, gpus) 171 | print("Saved predict results to \"%s\"" % results_log) 172 | -------------------------------------------------------------------------------- /util/train_accuracy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | # Create a train accuracy graph 5 | # 6 | # Usage: 7 | # $ ./train_accuracy.py ... 8 | # $ ./train_accuracy.py /config/config.yml logs/$MODEL_PREFIX-$MODEL-$OPTIMIZER.png logs/$MODEL_PREFIX-$MODEL-$OPTIMIZER.log ... 9 | 10 | import re 11 | import os 12 | import sys 13 | import yaml 14 | import math 15 | import matplotlib 16 | matplotlib.use('agg') 17 | import matplotlib.pyplot as plt 18 | import numpy as np 19 | import seaborn as sns 20 | 21 | config_file = sys.argv[1] 22 | output_file = sys.argv[2] 23 | train_logs = sys.argv[3:] 24 | 25 | with open(config_file) as rf: 26 | config = yaml.safe_load(rf) 27 | 28 | try: 29 | config = config['finetune'] 30 | except AttributeError: 31 | print('Error: Missing finetune section at config.yml') 32 | sys.exit(1) 33 | 34 | tg_fontsize = config.get('train_accuracy_graph_fontsize', 12) 35 | tg_figsize = config.get('train_accuracy_graph_figsize', '8,6') 36 | tg_figsize = tuple(float(i) for i in tg_figsize.split(',')) 37 | 38 | train_log_filename = os.path.basename(train_logs[0]) 39 | model_prefix = os.path.splitext(train_log_filename)[0] 40 | 41 | model_regex = re.compile(r'pretrained_model=\'(\S+)\'') 42 | model_regex_scratch = re.compile(r'network=\'(\S+)\'') 43 | lr_regex = re.compile(r'lr=(\S+),') 44 | 45 | 46 | gpus_regex = re.compile(r'gpus=(\S+),') 47 | load_epoch_regex = re.compile(r'load_epoch=(\S+),') 48 | batch_size_regex = re.compile(r'batch_size=(\d+),') 49 | num_of_image_regex = re.compile(r'num_examples=(\d+),') 50 | optimizer_regex = re.compile(r'optimizer=\'(\S+)\'') 51 | top_k_regex = re.compile(r'top_k=(\S+),') 52 | 53 | with open(train_logs[0], 'r') as l: 54 | data = l.read() 55 | try: 56 | model = re.search(model_regex, data).groups()[0] 57 | except AttributeError: 58 | model = re.search(model_regex_scratch, data).groups()[0] 59 | lr = float(re.search(lr_regex, data).groups()[0]) 60 | try: 61 | load_epoch = int(re.search(load_epoch_regex, data).groups()[0]) 62 | except ValueError: 63 | load_epoch = 0 64 | batch_size = re.search(batch_size_regex, data).groups()[0] 65 | num_of_image = num_of_image_regex.search(data).groups()[0] 66 | optimizer = optimizer_regex.search(data).groups()[0] 67 | top_k = int(re.search(top_k_regex, data).groups()[0]) 68 | 69 | batch_per_epoch = math.ceil(float(num_of_image)/float(batch_size)) 70 | 71 | # train accuracy per batch 72 | # ex. Epoch[1] Batch [10] Speed: 20.16 samples/sec accuracy=0.648148 73 | train_acc_batch = re.compile(r'Epoch\[(\d+)\] Batch \[(\d+)\]\s+Speed:\s+(\S+)\s+samples/sec\s+accuracy=(\S+)') 74 | 75 | # train accuracy per epoch 76 | # ex. Epoch[2] Train-accuracy=0.916667 77 | train_acc_epoch = re.compile(r'Epoch\[(\d+)\] Train-accuracy=(\S+)') 78 | 79 | # validation accuracy per epoch 80 | # ex. Epoch[3] Validation-accuracy=1.000000 81 | val_acc_epoch = re.compile(r'Epoch\[(\d+)\] Validation-accuracy=(\S+)') 82 | 83 | # top-k accuracy per epoch 84 | # ex. Epoch[4] Validation-top_k_accuracy_5=1.000000 85 | top_k_acc_epoch = re.compile(r'Epoch\[(\d+)\] Validation-top_k_accuracy_\d=(\S+)') 86 | 87 | train_acc_x = [] 88 | train_acc_y = [] 89 | train_speed = [] 90 | val_acc_x = [] 91 | val_acc_y = [] 92 | val_acc_x_top_k = [] 93 | val_acc_y_top_k = [] 94 | 95 | for train_log in train_logs: 96 | with open(train_log, 'r') as l: 97 | lines = l.readlines() 98 | 99 | for line in lines: 100 | line = line.strip() 101 | 102 | match = train_acc_batch.search(line) 103 | if match: 104 | items = match.groups() 105 | epoch = int(items[0]) 106 | batch = int(items[1]) 107 | speed = float(items[2]) 108 | acc = float(items[3]) 109 | iteration = epoch * batch_per_epoch + batch 110 | train_acc_x.append(iteration) 111 | train_acc_y.append(acc) 112 | train_speed.append(speed) 113 | 114 | match = train_acc_epoch.search(line) 115 | if match: 116 | items = match.groups() 117 | epoch = int(items[0]) 118 | acc = float(items[1]) 119 | iteration = epoch * batch_per_epoch + batch_per_epoch 120 | train_acc_x.append(iteration) 121 | train_acc_y.append(acc) 122 | 123 | match = val_acc_epoch.search(line) 124 | if match: 125 | items = match.groups() 126 | epoch = int(items[0]) 127 | acc = float(items[1]) 128 | iteration = epoch * batch_per_epoch + batch_per_epoch 129 | val_acc_x.append(iteration) 130 | val_acc_y.append(acc) 131 | 132 | if top_k > 0: 133 | match = top_k_acc_epoch.search(line) 134 | if match: 135 | items = match.groups() 136 | epoch = int(items[0]) 137 | acc = float(items[1]) 138 | iteration = epoch * batch_per_epoch + batch_per_epoch 139 | val_acc_x_top_k.append(iteration) 140 | val_acc_y_top_k.append(acc) 141 | 142 | # print train_acc_x, train_acc_y 143 | # print val_acc_x, val_acc_y 144 | # print val_acc_x_top_k, val_acc_y_top_k 145 | 146 | max_acc_index = np.argmax(val_acc_y) 147 | max_acc = val_acc_y[max_acc_index] 148 | if top_k > 0: 149 | max_top_k_acc_index = np.argmax(val_acc_y_top_k) 150 | max_top_k_acc = val_acc_y_top_k[max_top_k_acc_index] 151 | 152 | sns.set() 153 | fig = plt.figure(figsize = tg_figsize) 154 | plt.rcParams["font.size"] = tg_fontsize 155 | 156 | plt.plot(train_acc_x, train_acc_y, "r", label="train accuracy") 157 | plt.plot(val_acc_x, val_acc_y, "b", label="validation accuracy") 158 | if top_k > 0: 159 | plt.plot(val_acc_x_top_k, val_acc_y_top_k, "g", label="top-" + str(top_k) + " val-acc") 160 | 161 | plt_x = train_acc_x[0] 162 | plt.text(plt_x, 0.35, ' model used: %s' % model) 163 | plt.text(plt_x, 0.30, ' optimizer: %s' % optimizer) 164 | plt.text(plt_x, 0.25, ' learning rate: %s' % lr) 165 | plt.text(plt_x, 0.20, ' result val-acc: %s (%s epoch)' % (val_acc_y[-1], len(val_acc_y)+load_epoch)) 166 | plt.text(plt_x, 0.15, ' best val-acc: %s (%s epoch)' % (max_acc, max_acc_index+1+load_epoch)) 167 | plt.text(plt_x, 0.10, ' train speed: %.2f (samples/sec) (batch size: %s)' % (np.mean(train_speed), batch_size)) 168 | if top_k > 0: 169 | plt.text(plt_x, 0.05, ' best top-%s val-acc: %s (%s epoch)' % (top_k, max_top_k_acc, max_top_k_acc_index+1)) 170 | 171 | plt.title('model accuracy\n%s' % model_prefix) 172 | plt.xlabel("Iterations") 173 | plt.ylabel("Accuracy") 174 | 175 | plt.ylim(0,1) 176 | plt.legend(loc=4) 177 | 178 | fig.tight_layout() 179 | 180 | plt.savefig(output_file) 181 | print("Saved train accuracy graph to \"%s\"" % output_file) 182 | -------------------------------------------------------------------------------- /util/functions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | usage() { 4 | cat <<- EOS 5 | Usage: docker-compose run finetuner 6 | Commands: (default is finetune) 7 | finetune: Do fine-tuning 8 | gen_train : Generate train and validation image recordIO files. 9 | gen_test : Generate test image recordIO files. 10 | test: test with specified model. 11 | ensemble : averaging ensemble test with validation or test dataset. 12 | export: Generate MXNet models for mxnet-model-server. 13 | num_layers : Print the number of layers before the last fullc layer. 14 | jupyter: Launch jupyter notebook. Add --service-port option 15 | when executing command. 16 | docker-commpose run --service-port finetuner jupyter 17 | version: Show the mxnet-finetuner version information. 18 | EOS 19 | } 20 | 21 | version() { 22 | echo 'mxnet-finetuner version 0.0.12' 23 | } 24 | 25 | generate_compose() { 26 | local cur_dir="$1" 27 | local docker_compose_file="$2" 28 | local nvidia_docker_version="$3" 29 | if [[ $nvidia_docker_version = "2" ]]; then 30 | mo < "$cur_dir/util/compose-template-nvidia-docker2.mo" > "$cur_dir/$docker_compose_file" \ 31 | && echo "Generate $dockr_compose_file" 32 | else 33 | mo < "$cur_dir/util/compose-template.mo" > "$cur_dir/$docker_compose_file" \ 34 | && echo "Generate $dockr_compose_file" 35 | fi 36 | } 37 | 38 | update_compose() { 39 | local cur_dir="$1" 40 | local DEVICES="$2" 41 | local docker_compose_file="$3" 42 | local nvidia_docker_version="$4" 43 | if [[ $nvidia_docker_version != "2" ]]; then 44 | if [[ "$DEVICES" = "" ]]; then 45 | sed -i -e 's/knjcode\/mxnet-finetuner/knjcode\/mxnet-finetuner:cpu/' "$cur_dir/$docker_compose_file" \ 46 | && echo "Set use cpu docker image (knjcode/mxnet-finetuner:cpu)" 47 | sed -i -e 's/awsdeeplearningteam\/mms_gpu/awsdeeplearningteam\/mms_cpu/' "$cur_dir/$docker_compose_file" \ 48 | && echo "Set use cpu docker image (awsdeeplearningteam/mms_cpu)" 49 | sed -i -e 's/mms_app_gpu.conf/mms_app_cpu.conf/' "$cur_dir/$docker_compose_file" \ 50 | && echo "Set use cpu config file for mxnet-model-server (model/mms_app_cpu.conf)" 51 | else 52 | echo "Set use gpu docker image (knjcode/mxnet-finetuner)" 53 | fi 54 | sed -i -e 's/\/usr\/local\/nvidia:roFIX_VOLUME_NAME$//' "$cur_dir/$docker_compose_file" 55 | fi 56 | } 57 | 58 | generate_config() { 59 | local cur_dir="$1" 60 | local config_file="$2" 61 | cp "$cur_dir/util/sample_config.yml" "$cur_dir/$config_file" \ 62 | && echo "Generate $config_file" 63 | } 64 | 65 | update_config() { 66 | local cur_dir="$1" 67 | local DEVICES="$2" 68 | local config_file="$3" 69 | if [[ ! "$DEVICES" = "" ]]; then 70 | sed -i -e 's/# gpus/gpus/g' "$cur_dir/$config_file" \ 71 | && echo "Detect GPUs. Activate common.gpus option in $config_file" 72 | fi 73 | } 74 | 75 | generate_export_model_signature() { 76 | local cur_dir="$1" 77 | local MODEL_IMAGE_SIZE="$2" 78 | local RGB_MEAN="$3" 79 | local RGB_STD="$4" 80 | local NUM_CLASSES="$5" 81 | local EXPORT_TMP_DIR="$6" 82 | mo < "$cur_dir/export_tmpl/signature.mo" > "$EXPORT_TMP_DIR/signature.json" \ 83 | && echo "Generate signature.json" 84 | } 85 | 86 | generate_export_model_service() { 87 | local cur_dir="$1" 88 | local CENTER_CROP="$2" 89 | local TOP_K="$3" 90 | local SERVICE_TMP_DIR="$4" 91 | if [[ "$CENTER_CROP" = '1' ]]; then 92 | mo < "$cur_dir/export_tmpl/mxnet_vision_service_center_crop.mo" > "$SERVICE_TMP_DIR/mxnet_finetuner_service.py" \ 93 | && echo "Generate mxnet_finetuner_service.py" 94 | else 95 | mo < "$cur_dir/export_tmpl/mxnet_vision_service.mo" > "$SERVICE_TMP_DIR/mxnet_finetuner_service.py" \ 96 | && echo "Generate mxnet_finetuner_service.py" 97 | fi 98 | } 99 | 100 | generate_export_model_conf() { 101 | local cur_dir="$1" 102 | local MODEL_NAME="$2" 103 | local MODEL_FILE="$3" 104 | mo < "$cur_dir/export_tmpl/mms_app_cpu.conf.mo" > model/mms_app_cpu.conf \ 105 | && echo "Saved mms config for cpu \"model/mms_app_cpu.conf\"" 106 | mo < "$cur_dir/export_tmpl/mms_app_gpu.conf.mo" > model/mms_app_gpu.conf \ 107 | && echo "Saved mms config for gpu \"model/mms_app_gpu.conf\"" 108 | } 109 | 110 | check_from_scratch() { 111 | local model="$1" 112 | if [[ "$model" = scratch-* ]]; then 113 | echo 0 114 | else 115 | echo 1 116 | fi 117 | } 118 | 119 | trim_scratch() { 120 | local model="$1" 121 | if [[ $(check_from_scratch "$model") -eq 0 ]]; then 122 | echo "$model" | sed -e 's/^scratch-//' 123 | else 124 | echo "$momdel" 125 | fi 126 | } 127 | 128 | check_has_num_layers() { 129 | local model="$1" 130 | if [[ "$model" = *resnet-v1-* ]]; then 131 | echo 0 132 | elif [[ "$model" = *resnet-* ]]; then 133 | echo 0 134 | elif [[ "$model" = *resnext-* ]]; then 135 | echo 0 136 | elif [[ "$model" = *vgg-* ]]; then 137 | echo 0 138 | else 139 | echo 1 140 | fi 141 | } 142 | 143 | check_resnet_num_layers() { 144 | # Specify the number of layers for N in scratch-resnet-v1, scratch-resnet and scratch-resnext. 145 | # N can be set to 18, 34, 50, 101, 152, 200 and 269. 146 | local model="$1" 147 | if [[ $(check_has_num_layers "$model") -eq 0 ]]; then 148 | # num-layers 149 | if [[ "$model" =~ 18|34|50|101|152|200|269 ]]; then 150 | echo 0 151 | else 152 | echo 1 153 | fi 154 | else 155 | # do not have num-layers 156 | echo 1 157 | fi 158 | } 159 | 160 | check_vgg_num_layers() { 161 | # Specify the number of layers for N in scratch-vgg. 162 | # N can be set to 11, 13, 16 and 19. 163 | local model="$1" 164 | if [[ $(check_has_num_layers "$model") -eq 0 ]]; then 165 | # num-layers 166 | if [[ "$model" =~ 11|13|16|19 ]]; then 167 | echo 0 168 | else 169 | echo 1 170 | fi 171 | else 172 | # do not have num-layers 173 | echo 1 174 | fi 175 | } 176 | 177 | get_resnet_num_layers() { 178 | local model="$1" 179 | if [[ $(check_resnet_num_layers "$model") -eq 0 ]]; then 180 | echo "$model" | awk -F - '{ print $NF }' 181 | else 182 | echo 'null' 183 | fi 184 | } 185 | 186 | get_vgg_num_layers() { 187 | local model="$1" 188 | if [[ $(check_vgg_num_layers "$model") -eq 0 ]]; then 189 | echo "$model" | awk -F - '{ print $NF }' 190 | else 191 | echo 'null' 192 | fi 193 | } 194 | 195 | get_num_layers() { 196 | local model="$1" 197 | 198 | if [[ "$model" = *vgg-* ]]; then 199 | get_vgg_num_layers "$model" 200 | elif [[ "$model" = *resnet-v1-* ]]; then 201 | get_resnet_num_layers "$model" 202 | elif [[ "$model" = *resnet-* ]]; then 203 | get_resnet_num_layers "$model" 204 | elif [[ "$model" = *resnext-* ]]; then 205 | get_resnet_num_layers "$model" 206 | else 207 | echo 'null' 208 | fi 209 | } 210 | 211 | trim_num_layers() { 212 | local model="$1" 213 | echo "$model" | sed -e 's/-[0-9]*$//' 214 | } 215 | 216 | get_conf() { 217 | local config="$1" 218 | local param="$2" 219 | local default="$3" 220 | local value 221 | value=$(echo "$config" | jq -r "$param") 222 | if [[ "$value" = 'null' ]]; then 223 | value="$default" 224 | fi 225 | echo "$value" 226 | } 227 | 228 | get_conf_array() { 229 | local config="$1" 230 | local param="$2" 231 | local default="$3" 232 | local value 233 | value=$(echo "$config" | jq -r "$param") 234 | if [[ "$value" = 'null' ]]; then 235 | value="$default" 236 | else 237 | value=$(echo "$config" | jq -r "$param | .[]") 238 | fi 239 | echo "$value" 240 | } 241 | 242 | get_image_size() { 243 | local MODEL="$1" 244 | if [[ "$MODEL" = *caffenet* ]]; then 245 | IMAGE_SIZE=227 246 | elif [[ "$MODEL" = *squeezenet* ]]; then 247 | IMAGE_SIZE=227 248 | elif [[ "$MODEL" = *alexnet* ]]; then 249 | IMAGE_SIZE=227 250 | elif [[ "$MODEL" = *googlenet* ]]; then 251 | IMAGE_SIZE=299 252 | elif [[ "$MODEL" = *inception-v3* ]]; then 253 | IMAGE_SIZE=299 254 | elif [[ "$MODEL" = *inception-v4* ]]; then 255 | IMAGE_SIZE=299 256 | elif [[ "$MODEL" = *inception-resnet-v2* ]]; then 257 | IMAGE_SIZE=299 258 | else 259 | IMAGE_SIZE=224 260 | fi 261 | echo "$IMAGE_SIZE" 262 | } 263 | 264 | download_inception_v3_model() { 265 | if [ ! -e inception-v3.tar.gz ]; then 266 | wget http://data.dmlc.ml/models/imagenet/inception-v3.tar.gz 267 | fi 268 | tar xf inception-v3.tar.gz 269 | mv model/Inception-7-0001.params model/imagenet1k-inception-v3-0000.params 270 | mv model/Inception-7-symbol.json model/imagenet1k-inception-v3-symbol.json 271 | } 272 | 273 | check_inception_v3_model() { 274 | if [ ! -e "/mxnet/example/image-classification/model/imagenet1k-inception-v3-0000.params" ]; then 275 | download_inception_v3_model 276 | fi 277 | } 278 | 279 | gdrive_download () { 280 | local FILEID="$1" 281 | local FILENAME="$2" 282 | CONFIRM=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate \ 283 | "https://docs.google.com/uc?export=download&id=$1" -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p') 284 | wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$CONFIRM&id=$1" -O $2 285 | rm -rf /tmp/cookies.txt 286 | } 287 | 288 | download_se_resnext_50_model() { 289 | gdrive_download "0B_M7XF_l0CzXOHNybXVWLWZteEE" "model/imagenet1k-se-resnext-50-0000.params" 290 | wget "https://raw.githubusercontent.com/bruinxiong/SENet.mxnet/master/se-resnext-imagenet-50-0-symbol.json" \ 291 | -O "model/imagenet1k-se-resnext-50-symbol.json" 292 | } 293 | 294 | check_se_resnext_50_model() { 295 | if [ ! -e "/mxnet/example/image-classification/model/imagenet1k-se-resnext-50-0000.params" ]; then 296 | download_se_resnext_50_model 297 | fi 298 | } 299 | 300 | download_densenet_169_model() { 301 | gdrive_download "0B_M7XF_l0CzXX3V3WXJoUnNKZFE" "model/imagenet1k-densenet-169-0000.params" 302 | wget "https://raw.githubusercontent.com/bruinxiong/densenet.mxnet/master/densenet-imagenet-169-0-symbol.json" \ 303 | -O "model/imagenet1k-densenet-169-symbol.json" 304 | } 305 | 306 | check_densenet_169_model() { 307 | if [ ! -e "/mxnet/example/image-classification/model/imagenet1k-densenet-169-0000.params" ]; then 308 | download_densenet_169_model 309 | fi 310 | } 311 | 312 | get_layer_before_fullc() { 313 | local MODEL="$1" 314 | if [[ $MODEL = *caffenet* ]]; then 315 | LAYER_BEFORE_FULLC="flatten_0" 316 | elif [[ $MODEL = *vgg* ]]; then 317 | LAYER_BEFORE_FULLC="flatten_0" 318 | elif [[ $MODEL = *nin* ]]; then 319 | LAYER_BEFORE_FULLC="flatten" 320 | elif [[ $MODEL = *squeezenet* ]]; then 321 | LAYER_BEFORE_FULLC="flatten" 322 | elif [[ $MODEL = *inception-v3* ]]; then 323 | LAYER_BEFORE_FULLC="flatten" 324 | check_inception_v3_model 325 | elif [[ $MODEL = *inception* ]]; then 326 | LAYER_BEFORE_FULLC="flatten" 327 | elif [[ $MODEL = *resnet* ]]; then 328 | LAYER_BEFORE_FULLC="flatten0" 329 | elif [[ $MODEL = *se-resnext-50* ]]; then 330 | LAYER_BEFORE_FULLC="flatten0" 331 | check_se_resnext_50_model 332 | elif [[ $MODEL = *resnext* ]]; then 333 | LAYER_BEFORE_FULLC="flatten0" 334 | elif [[ $MODEL = *densenet* ]]; then 335 | LAYER_BEFORE_FULLC="flatten0" 336 | check_densenet_169_model 337 | else 338 | LAYER_BEFORE_FULLC="flatten_0" 339 | fi 340 | echo "$LAYER_BEFORE_FULLC" 341 | } 342 | 343 | print_classification_report() { 344 | local report="$1" 345 | local body 346 | 347 | body=$(cat "$report" | tail -n +4) 348 | # body=$(cat "$report" | tail -n +4 | \ 349 | # sed -e 's/precision/a precision/' -e 's/avg \/ total/avg\/total/' | \ 350 | # column -t | sed -e 's/^a / /' | sed -e '2i \ ' | sed -e '$ i \ ') 351 | echo "$body" 352 | } 353 | -------------------------------------------------------------------------------- /common/fit.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Modified from https://raw.githubusercontent.com/apache/incubator-mxnet/master/example/image-classification/common/fit.py 19 | 20 | """ example train fit utility """ 21 | import logging 22 | import os 23 | import time 24 | import re 25 | import math 26 | import mxnet as mx 27 | 28 | 29 | def _get_lr_scheduler(args, kv): 30 | if 'lr_factor' not in args or args.lr_factor >= 1: 31 | return (args.lr, None) 32 | epoch_size = args.num_examples / args.batch_size 33 | if 'dist' in args.kv_store: 34 | epoch_size /= kv.num_workers 35 | begin_epoch = args.load_epoch if args.load_epoch else 0 36 | if 'pow' in args.lr_step_epochs: 37 | lr = args.lr 38 | max_up = args.num_epochs * epoch_size 39 | pwr = float(re.sub('pow[- ]*', '', args.lr_step_epochs)) 40 | poly_sched = mx.lr_scheduler.PolyScheduler(max_up, lr, pwr) 41 | return (lr, poly_sched) 42 | step_epochs = [int(l) for l in args.lr_step_epochs.split(',')] 43 | lr = args.lr 44 | for s in step_epochs: 45 | if begin_epoch >= s: 46 | lr *= args.lr_factor 47 | if lr != args.lr: 48 | logging.info('Adjust learning rate to %e for epoch %d', 49 | lr, begin_epoch) 50 | 51 | steps = [epoch_size * (x - begin_epoch) 52 | for x in step_epochs if x - begin_epoch > 0] 53 | return (lr, mx.lr_scheduler.MultiFactorScheduler(step=steps, factor=args.lr_factor)) 54 | 55 | 56 | def _load_model(args, rank=0): 57 | if 'load_epoch' not in args or args.load_epoch is None: 58 | return (None, None, None) 59 | assert args.model_prefix is not None 60 | model_prefix = args.model_prefix 61 | if rank > 0 and os.path.exists("%s-%d-symbol.json" % (model_prefix, rank)): 62 | model_prefix += "-%d" % (rank) 63 | sym, arg_params, aux_params = mx.model.load_checkpoint( 64 | model_prefix, args.load_epoch) 65 | logging.info('Loaded model %s_%04d.params', model_prefix, args.load_epoch) 66 | return (sym, arg_params, aux_params) 67 | 68 | 69 | def _save_model(args, rank=0): 70 | if args.model_prefix is None: 71 | return None 72 | return mx.callback.do_checkpoint(args.model_prefix if rank == 0 else "%s-%d" % ( 73 | args.model_prefix, rank), period=args.save_period) 74 | 75 | 76 | def add_fit_args(parser): 77 | """ 78 | parser : argparse.ArgumentParser 79 | return a parser added with args required by fit 80 | """ 81 | train = parser.add_argument_group('Training', 'model training') 82 | train.add_argument('--network', type=str, 83 | help='the neural network to use') 84 | train.add_argument('--num-layers', type=int, 85 | help='number of layers in the neural network, \ 86 | required by some networks such as resnet') 87 | train.add_argument('--gpus', type=str, 88 | help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu') 89 | train.add_argument('--kv-store', type=str, default='device', 90 | help='key-value store type') 91 | train.add_argument('--num-epochs', type=int, default=100, 92 | help='max num of epochs') 93 | train.add_argument('--lr', type=float, default=0.1, 94 | help='initial learning rate') 95 | train.add_argument('--lr-factor', type=float, default=0.1, 96 | help='the ratio to reduce lr on each step') 97 | train.add_argument('--lr-step-epochs', type=str, 98 | help='the epochs to reduce the lr, e.g. 30,60') 99 | train.add_argument('--initializer', type=str, default='default', 100 | help='the initializer type') 101 | train.add_argument('--optimizer', type=str, default='sgd', 102 | help='the optimizer type') 103 | train.add_argument('--mom', type=float, default=0.9, 104 | help='momentum for sgd') 105 | train.add_argument('--wd', type=float, default=0.0001, 106 | help='weight decay for sgd') 107 | train.add_argument('--batch-size', type=int, default=128, 108 | help='the batch size') 109 | train.add_argument('--disp-batches', type=int, default=20, 110 | help='show progress for every n batches') 111 | train.add_argument('--model-prefix', type=str, 112 | help='model prefix') 113 | train.add_argument('--save-period', type=int, default=1, help='params saving period') 114 | parser.add_argument('--monitor', dest='monitor', type=int, default=0, 115 | help='log network parameters every N iters if larger than 0') 116 | train.add_argument('--load-epoch', type=int, 117 | help='load the model on an epoch using the model-load-prefix') 118 | train.add_argument('--top-k', type=int, default=0, 119 | help='report the top-k accuracy. 0 means no report.') 120 | train.add_argument('--loss', type=str, default='', 121 | help='show the cross-entropy or nll loss. ce strands for cross-entropy, nll-loss stands for likelihood loss') 122 | train.add_argument('--test-io', type=int, default=0, 123 | help='1 means test reading speed without training') 124 | train.add_argument('--dtype', type=str, default='float32', 125 | help='precision: float32 or float16') 126 | train.add_argument('--gc-type', type=str, default='none', 127 | help='type of gradient compression to use, \ 128 | takes `2bit` or `none` for now') 129 | train.add_argument('--gc-threshold', type=float, default=0.5, 130 | help='threshold for 2bit gradient compression') 131 | # additional parameters for large batch sgd 132 | train.add_argument('--macrobatch-size', type=int, default=0, 133 | help='distributed effective batch size') 134 | train.add_argument('--warmup-epochs', type=int, default=5, 135 | help='the epochs to ramp-up lr to scaled large-batch value') 136 | train.add_argument('--warmup-strategy', type=str, default='linear', 137 | help='the ramping-up strategy for large batch sgd') 138 | return train 139 | 140 | 141 | def fit(args, network, data_loader, **kwargs): 142 | """ 143 | train a model 144 | args : argparse returns 145 | network : the symbol definition of the nerual network 146 | data_loader : function that returns the train and val data iterators 147 | """ 148 | # kvstore 149 | kv = mx.kvstore.create(args.kv_store) 150 | if args.gc_type != 'none': 151 | kv.set_gradient_compression({'type': args.gc_type, 152 | 'threshold': args.gc_threshold}) 153 | 154 | # logging 155 | head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s' 156 | logging.basicConfig(level=logging.DEBUG, format=head) 157 | logging.info('start with arguments %s', args) 158 | 159 | # data iterators 160 | (train, val) = data_loader(args, kv) 161 | if args.test_io: 162 | tic = time.time() 163 | for i, batch in enumerate(train): 164 | for j in batch.data: 165 | j.wait_to_read() 166 | if (i + 1) % args.disp_batches == 0: 167 | logging.info('Batch [%d]\tSpeed: %.2f samples/sec', i, 168 | args.disp_batches * args.batch_size / (time.time() - tic)) 169 | tic = time.time() 170 | 171 | return 172 | 173 | # load model 174 | if 'arg_params' in kwargs and 'aux_params' in kwargs: 175 | arg_params = kwargs['arg_params'] 176 | aux_params = kwargs['aux_params'] 177 | else: 178 | sym, arg_params, aux_params = _load_model(args, kv.rank) 179 | if sym is not None: 180 | assert sym.tojson() == network.tojson() 181 | 182 | # save model 183 | checkpoint = _save_model(args, kv.rank) 184 | 185 | # devices for training 186 | devs = mx.cpu() if args.gpus is None or args.gpus == "" else [ 187 | mx.gpu(int(i)) for i in args.gpus.split(',')] 188 | 189 | # learning rate 190 | lr, lr_scheduler = _get_lr_scheduler(args, kv) 191 | 192 | # create model 193 | model = mx.mod.Module( 194 | context=devs, 195 | symbol=network 196 | ) 197 | 198 | lr_scheduler = lr_scheduler 199 | optimizer_params = { 200 | 'learning_rate': lr, 201 | 'wd': args.wd, 202 | 'lr_scheduler': lr_scheduler, 203 | 'multi_precision': True} 204 | 205 | # Only a limited number of optimizers have 'momentum' property 206 | has_momentum = {'sgd', 'dcasgd', 'nag'} 207 | if args.optimizer in has_momentum: 208 | optimizer_params['momentum'] = args.mom 209 | 210 | monitor = mx.mon.Monitor( 211 | args.monitor, pattern=".*") if args.monitor > 0 else None 212 | 213 | # A limited number of optimizers have a warmup period 214 | has_warmup = {'lbsgd', 'lbnag'} 215 | if args.optimizer in has_warmup: 216 | if 'dist' in args.kv_store: 217 | nworkers = kv.num_workers 218 | else: 219 | nworkers = 1 220 | epoch_size = args.num_examples / args.batch_size / nworkers 221 | if epoch_size < 1: 222 | epoch_size = 1 223 | macrobatch_size = args.macrobatch_size 224 | if macrobatch_size < args.batch_size * nworkers: 225 | macrobatch_size = args.batch_size * nworkers 226 | #batch_scale = round(float(macrobatch_size) / args.batch_size / nworkers +0.4999) 227 | batch_scale = math.ceil( 228 | float(macrobatch_size) / args.batch_size / nworkers) 229 | optimizer_params['updates_per_epoch'] = epoch_size 230 | optimizer_params['begin_epoch'] = args.load_epoch if args.load_epoch else 0 231 | optimizer_params['batch_scale'] = batch_scale 232 | optimizer_params['warmup_strategy'] = args.warmup_strategy 233 | optimizer_params['warmup_epochs'] = args.warmup_epochs 234 | optimizer_params['num_epochs'] = args.num_epochs 235 | 236 | if args.initializer == 'default': 237 | if args.network == 'alexnet': 238 | # AlexNet will not converge using Xavier 239 | initializer = mx.init.Normal() 240 | # VGG will not trend to converge using Xavier-Gaussian 241 | elif args.network and 'vgg' in args.network: 242 | initializer = mx.init.Xavier() 243 | else: 244 | initializer = mx.init.Xavier( 245 | rnd_type='gaussian', factor_type="in", magnitude=2) 246 | # initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), 247 | elif args.initializer == 'xavier': 248 | initializer = mx.init.Xavier() 249 | elif args.initializer == 'msra': 250 | initializer = mx.init.MSRAPrelu() 251 | elif args.initializer == 'orthogonal': 252 | initializer = mx.init.Orthogonal() 253 | elif args.initializer == 'normal': 254 | initializer = mx.init.Normal() 255 | elif args.initializer == 'uniform': 256 | initializer = mx.init.Uniform() 257 | elif args.initializer == 'one': 258 | initializer = mx.init.One() 259 | elif args.initializer == 'zero': 260 | initializer = mx.init.Zero() 261 | 262 | # evaluation metrices 263 | eval_metrics = ['accuracy'] 264 | if args.top_k > 0: 265 | eval_metrics.append(mx.metric.create( 266 | 'top_k_accuracy', top_k=args.top_k)) 267 | 268 | supported_loss = ['ce', 'nll_loss'] 269 | if len(args.loss) > 0: 270 | # ce or nll loss is only applicable to softmax output 271 | loss_type_list = args.loss.split(',') 272 | if 'softmax_output' in network.list_outputs(): 273 | for loss_type in loss_type_list: 274 | loss_type = loss_type.strip() 275 | if loss_type == 'nll': 276 | loss_type = 'nll_loss' 277 | if loss_type not in supported_loss: 278 | logging.warning(loss_type + ' is not an valid loss type, only cross-entropy or ' \ 279 | 'negative likelihood loss is supported!') 280 | else: 281 | eval_metrics.append(mx.metric.create(loss_type)) 282 | else: 283 | logging.warning("The output is not softmax_output, loss argument will be skipped!") 284 | 285 | # callbacks that run after each batch 286 | batch_end_callbacks = [mx.callback.Speedometer( 287 | args.batch_size, args.disp_batches)] 288 | if 'batch_end_callback' in kwargs: 289 | cbs = kwargs['batch_end_callback'] 290 | batch_end_callbacks += cbs if isinstance(cbs, list) else [cbs] 291 | 292 | # run 293 | model.fit(train, 294 | begin_epoch=args.load_epoch if args.load_epoch else 0, 295 | num_epoch=args.num_epochs, 296 | eval_data=val, 297 | eval_metric=eval_metrics, 298 | kvstore=kv, 299 | optimizer=args.optimizer, 300 | optimizer_params=optimizer_params, 301 | initializer=initializer, 302 | arg_params=arg_params, 303 | aux_params=aux_params, 304 | batch_end_callback=batch_end_callbacks, 305 | epoch_end_callback=checkpoint, 306 | allow_missing=True, 307 | monitor=monitor) 308 | -------------------------------------------------------------------------------- /util/finetune.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -u 4 | 5 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | source "$CUR_DIR/functions" 7 | 8 | CONFIG_FILE="/config/config.yml" 9 | 10 | python3 -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)' < $CONFIG_FILE > config.json 11 | config=$(jq -Mc '.' config.json) 12 | 13 | TRAIN="/images/train" 14 | VALID="/images/valid" 15 | DATA_TRAIN="/data/train" 16 | DATA_VALID="/data/valid" 17 | 18 | DATA_NTHREADS=$(get_conf "$config" ".common.num_threads" "4") 19 | GPUS=$(get_conf "$config" ".common.gpus" "") 20 | if [[ ! $GPUS = "" ]]; then 21 | GPU_OPTION="--gpus $GPUS" 22 | else 23 | GPU_OPTION="" 24 | fi 25 | echo "GPU_OPTION=$GPU_OPTION" 26 | 27 | MODELS=$(get_conf_array "$config" ".finetune.models" "") 28 | if [[ "$MODELS" = "" ]]; then 29 | MODELS=$(get_conf_array "$config" ".finetune.pretrained_models" "imagenet1k-nin") 30 | fi 31 | echo "MODELS=$MODELS" 32 | OPTIMIZERS=$(get_conf_array "$config" ".finetune.optimizers" "sgd") 33 | echo "OPTIMIZERS=$OPTIMIZERS" 34 | NUM_EPOCHS=$(get_conf "$config" ".finetune.num_epochs" "10") 35 | LOAD_EPOCH=$(get_conf "$config" ".finetune.load_epoch" "0") 36 | if [[ ! $NUM_EPOCHS -gt $LOAD_EPOCH ]]; then 37 | echo 'Error: num_epochs must be bigger than load_epoch' 1>&2 38 | exit 1 39 | fi 40 | if [[ ! $LOAD_EPOCH = "0" ]]; then 41 | LOAD_EPOCH_OPTION="--load-epoch $LOAD_EPOCH" 42 | else 43 | LOAD_EPOCH_OPTION="" 44 | fi 45 | echo "LOAD_EPOCH_OPTION=$LOAD_EPOCH_OPTION" 46 | LR=$(get_conf "$config" ".finetune.lr" "0.00001") 47 | LR_FACTOR=$(get_conf "$config" ".finetune.lr_factor" "0.1") 48 | LR_STEP_EPOCHS=$(get_conf "$config" ".finetune.lr_step_epochs" "10") 49 | MOM=$(get_conf "$config" ".finetune.mom" "0.9") 50 | WD=$(get_conf "$config" ".finetune.wd" "0.00001") 51 | BATCH_SIZE=$(get_conf "$config" ".finetune.batch_size" "16") 52 | DISP_BATCHES=$(get_conf "$config" ".finetune.disp_batches" "20") 53 | TOP_K=$(get_conf "$config" ".finetune.top_k" "0") 54 | LOSS=$(get_conf "$config" ".finetune.loss" "ce") 55 | DATA_AUG_LEVEL=$(get_conf "$config" ".finetune.data_aug_level" "0") 56 | RANDOM_CROP=$(get_conf "$config" ".finetune.random_crop" "0") 57 | RANDOM_MIRROR=$(get_conf "$config" ".finetune.random_mirror" "0") 58 | MAX_RANDOM_H=$(get_conf "$config" ".finetune.max_random_h" "0") 59 | MAX_RANDOM_S=$(get_conf "$config" ".finetune.max_random_s" "0") 60 | MAX_RANDOM_L=$(get_conf "$config" ".finetune.max_random_l" "0") 61 | MAX_RANDOM_ASPECT_RATIO=$(get_conf "$config" ".finetune.max_random_aspect_ratio" "0") 62 | MIN_RANDOM_ASPECT_RATIO=$(get_conf "$config" ".finetune.min_random_aspect_ratio" "0") 63 | MAX_RANDOM_ROTATE_ANGLE=$(get_conf "$config" ".finetune.max_random_rotate_angle" "0") 64 | MAX_RANDOM_SHEAR_RATIO=$(get_conf "$config" ".finetune.max_random_shear_ratio" "0") 65 | MAX_RANDOM_SCALE=$(get_conf "$config" ".finetune.max_random_scale" "1") 66 | MIN_RANDOM_SCALE=$(get_conf "$config" ".finetune.min_random_scale" "1") 67 | MAX_RANDOM_AREA=$(get_conf "$config" ".finetune.max_random_area" "1") 68 | MIN_RANDOM_AREA=$(get_conf "$config" ".finetune.min_random_area" "1") 69 | MAX_CROP_SIZE=$(get_conf "$config" ".finetune.max_crop_size" "-1") 70 | MIN_CROP_SIZE=$(get_conf "$config" ".finetune.min_crop_size" "-1") 71 | BRIGHTNESS=$(get_conf "$config" ".finetune.brightness" "0") 72 | CONTRAST=$(get_conf "$config" ".finetune.contrast" "0") 73 | SATURATION=$(get_conf "$config" ".finetune.saturation" "0") 74 | PCA_NOISE=$(get_conf "$config" ".finetune.pca_noise" "0") 75 | RANDOM_RESIZED_CROP=$(get_conf "$config" ".finetune.random_resized_crop" "0") 76 | RGB_MEAN=$(get_conf "$config" ".finetune.rgb_mean" "123.68,116.779,103.939") 77 | RGB_STD=$(get_conf "$config" ".finetune.rgb_std" "1,1,1") 78 | MONITOR=$(get_conf "$config" ".finetune.monitor" "0") 79 | PAD_SIZE=$(get_conf "$config" ".finetune.pad_size" "0") 80 | NUM_ACTIVE_LAYERS=$(get_conf "$config" ".finetune.num_active_layers" "0") 81 | AUTO_TEST=$(get_conf "$config" ".finetune.auto_test" "1") 82 | TRAIN_ACCURACY_GRAPH_OUTPUT=$(get_conf "$config" ".finetune.train_accuracy_graph_output" "1") 83 | TRAIN_ACCURACY_SLACK_UPLOAD=$(get_conf "$config" ".finetune.train_accuracy_graph_slack_upload" "0") 84 | TRAIN_ACCURACY_SLACK_CHANNELS=$(get_conf_array "$config" ".finetune.train_accuracy_graph_slack_channels" "general") 85 | TRAIN_LOSS_GRAPH_OUTPUT=$(get_conf "$config" ".finetune.train_loss_graph_output" "1") 86 | TRAIN_LOSS_SLACK_UPLOAD=$(get_conf "$config" ".finetune.train_loss_graph_slack_upload" "0") 87 | TRAIN_LOSS_SLACK_CHANNELS=$(get_conf_array "$config" ".finetune.train_loss_graph_slack_channels" "general") 88 | 89 | CONFUSION_MATRIX_OUTPUT=$(get_conf "$config" ".test.confusion_matrix_output" "1") 90 | TEST_SLACK_UPLOAD=$(get_conf "$config" ".test.confusion_matrix_slack_upload" "0") 91 | TEST_SLACK_CHANNELS=$(get_conf_array "$config" ".test.confusion_matrix_slack_channels" "general") 92 | CLASSIFICATION_REPORT_OUTPUT=$(get_conf "$config" ".test.classification_report_output" "1") 93 | 94 | # data_aug_level 95 | echo "DATA_AUG_LEVEL=$DATA_AUG_LEVEL" 96 | if [[ $DATA_AUG_LEVEL -ge 1 ]]; then 97 | RANDOM_CROP="1" 98 | RANDOM_MIRROR="1" 99 | fi 100 | if [[ $DATA_AUG_LEVEL -ge 2 ]]; then 101 | MAX_RANDOM_H="36" 102 | MAX_RANDOM_S="50" 103 | MAX_RANDOM_L="50" 104 | fi 105 | if [[ $DATA_AUG_LEVEL -ge 3 ]]; then 106 | MAX_RANDOM_ASPECT_RATIO="0.25" 107 | MAX_RANDOM_ROTATE_ANGLE="10" 108 | MAX_RANDOM_SHEAR_RATIO="0.1" 109 | fi 110 | 111 | for MODEL in $MODELS; do 112 | # Determine IMAGE_SIZE 113 | IMAGE_SIZE=$(get_conf "$config" ".data.resize_short" "0") 114 | MODEL_IMAGE_SIZE=$(get_image_size "$MODEL") 115 | if [[ $IMAGE_SIZE -eq 0 ]]; then 116 | IMAGE_SIZE=$MODEL_IMAGE_SIZE 117 | fi 118 | if [[ $IMAGE_SIZE -lt $MODEL_IMAGE_SIZE ]]; then 119 | echo 'Error: The shorter edge after resizing must be grater than or equal to the input size of the model.' 1>&2 120 | echo 'Check data.resize_short at config.yml.' 1>&2 121 | echo "When using $MODEL model, data.resize_short must be $MODEL_IMAGE_SIZE or more." 122 | exit 1 123 | fi 124 | IMAGE_SHAPE="3,$MODEL_IMAGE_SIZE,$MODEL_IMAGE_SIZE" 125 | 126 | # Construct train commnd 127 | if [[ $(check_from_scratch "$MODEL") -eq 0 ]]; then 128 | # from scratch 129 | echo "Training from scratch $MODEL" 130 | MODEL=$(trim_scratch "$MODEL") 131 | NUM_LAYERS=$(get_num_layers "$MODEL") 132 | if [[ "$NUM_LAYERS" = 'null' ]]; then 133 | # do not have num-layers 134 | TRAIN_COMMAND="python3 util/train_imagenet.py --network $MODEL" 135 | else 136 | # num-layers 137 | TRIMED_MODEL=$(trim_num_layers "$MODEL") 138 | TRAIN_COMMAND="python3 util/train_imagenet.py --network $TRIMED_MODEL --num-layers $NUM_LAYERS" 139 | fi 140 | else 141 | # fine-tuning 142 | # Determine LAYER_BEFORE_FULLC and IMAGE_SIZE 143 | LAYER_BEFORE_FULLC=$(get_layer_before_fullc "$MODEL") 144 | TRAIN_COMMAND="python3 util/fine-tune.py --pretrained-model $MODEL --layer-before-fullc $LAYER_BEFORE_FULLC --num-active-layers $NUM_ACTIVE_LAYERS" 145 | fi 146 | echo "TRAIN_COMMAND=$TRAIN_COMMAND" 147 | 148 | # If necessary image records do not exist, they are generated. 149 | if [ "$DATA_TRAIN/images-train-$IMAGE_SIZE.rec" -ot "$TRAIN" ]; then 150 | echo "$DATA_TRAIN/images-train-$IMAGE_SIZE.rec does not exist or is outdated." 1>&2 151 | echo 'Generate image records for fine-tuning.' 1>&2 152 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$IMAGE_SIZE" || exit 1 153 | fi 154 | if [ "$DATA_VALID/images-valid-$IMAGE_SIZE.rec" -ot "$VALID" ]; then 155 | echo "$DATA_VALID/images-valid-$IMAGE_SIZE.rec does not exist or is outdated." 1>&2 156 | echo 'Generate validation image records for fine-tuning.' 1>&2 157 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$IMAGE_SIZE" || exit 1 158 | fi 159 | if [ "$DATA_VALID/images-valid-$MODEL_IMAGE_SIZE.rec" -ot "$VALID" ]; then 160 | echo "$DATA_VALID/images-valid-$MODEL_IMAGE_SIZE.rec does not exist or is outdated." 1>&2 161 | echo 'Generate validation image records for fine-tuning.' 1>&2 162 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" || exit 1 163 | fi 164 | 165 | # Check the number of image files. If it is different from previous one, regenerate images records 166 | diff --brief <(LC_ALL=C $CUR_DIR/counter.sh $TRAIN | sed -e '1d') <(cat $DATA_TRAIN/images-train-$IMAGE_SIZE.txt) > /dev/null 2>&1 167 | if [ "$?" -eq 1 ]; then 168 | echo "$DATA_TRAIN/images-train-$IMAGE_SIZE.rec is outdated." 1>&2 169 | echo 'Generate image records for fine-tuning.' 1>&2 170 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$IMAGE_SIZE" || exit 1 171 | fi 172 | diff --brief <(LC_ALL=C $CUR_DIR/counter.sh $VALID | sed -e '1d') <(cat $DATA_VALID/images-valid-$IMAGE_SIZE.txt) > /dev/null 2>&1 173 | if [ "$?" -eq 1 ]; then 174 | echo "$DATA_VALID/images-valid-$IMAGE_SIZE.rec is outdated." 1>&2 175 | echo 'Generate validation image records for fine-tuning.' 1>&2 176 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$IMAGE_SIZE" || exit 1 177 | fi 178 | diff --brief <(LC_ALL=C $CUR_DIR/counter.sh $VALID | sed -e '1d') <(cat $DATA_VALID/images-valid-$MODEL_IMAGE_SIZE.txt) > /dev/null 2>&1 179 | if [ "$?" -eq 1 ]; then 180 | echo "$DATA_VALID/images-valid-$MODEL_IMAGE_SIZE.rec is outdated." 1>&2 181 | echo 'Generate validation image records for fine-tuning.' 1>&2 182 | $CUR_DIR/gen_train.sh "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" || exit 1 183 | fi 184 | 185 | 186 | LABELS_TRAIN="$DATA_TRAIN/labels.txt" 187 | LABELS_VALID="$DATA_VALID/labels.txt" 188 | diff --brief "$LABELS_TRAIN" "$LABELS_VALID" > /dev/null 189 | if [ "$?" -eq 1 ]; then 190 | echo 'Error: The directory structure of images/train and images/valid is different.' 1>&2 191 | echo 'Check your train and validation images.' 1>&2 192 | exit 1 193 | fi 194 | 195 | NUM_CLASSES=$(echo $(cat "$DATA_TRAIN/labels.txt" | wc -l)) 196 | NUM_EXAMPLES=$(echo $(cat "$DATA_TRAIN/images-train-$IMAGE_SIZE.lst" | wc -l)) 197 | 198 | for OPTIMIZER in $OPTIMIZERS; do 199 | MODEL_PREFIX="$(date +%Y%m%d%H%M%S)-$MODEL-$OPTIMIZER" 200 | LOGS="logs/$MODEL_PREFIX.log" 201 | CONFIG_LOG="logs/$MODEL_PREFIX-config.yml" 202 | 203 | # save config.yml 204 | cp "$CONFIG_FILE" "$CONFIG_LOG" 205 | 206 | # copy labels.txt 207 | LABELS="model/$MODEL_PREFIX-labels.txt" 208 | cp "$DATA_TRAIN/labels.txt" "$LABELS" 209 | 210 | $TRAIN_COMMAND \ 211 | --data-train "$DATA_TRAIN/images-train-${IMAGE_SIZE}.rec" \ 212 | --data-val "$DATA_VALID/images-valid-${IMAGE_SIZE}.rec" \ 213 | $GPU_OPTION \ 214 | --num-epochs "$NUM_EPOCHS" \ 215 | $LOAD_EPOCH_OPTION \ 216 | --lr "$LR" \ 217 | --lr-factor "$LR_FACTOR" \ 218 | --lr-step-epochs "$LR_STEP_EPOCHS" \ 219 | --optimizer "$OPTIMIZER" \ 220 | --mom "$MOM" --wd "$WD" \ 221 | --batch-size "$BATCH_SIZE" \ 222 | --disp-batches "$DISP_BATCHES" \ 223 | --top-k "$TOP_K" \ 224 | --loss "$LOSS" \ 225 | --data-nthreads "$DATA_NTHREADS" \ 226 | --random-crop "$RANDOM_CROP" \ 227 | --random-mirror "$RANDOM_MIRROR" \ 228 | --max-random-h "$MAX_RANDOM_H" \ 229 | --max-random-s "$MAX_RANDOM_S" \ 230 | --max-random-l "$MAX_RANDOM_L" \ 231 | --max-random-aspect-ratio "$MAX_RANDOM_ASPECT_RATIO" \ 232 | --min-random-aspect-ratio "$MIN_RANDOM_ASPECT_RATIO" \ 233 | --max-random-rotate-angle "$MAX_RANDOM_ROTATE_ANGLE" \ 234 | --max-random-shear-ratio "$MAX_RANDOM_SHEAR_RATIO" \ 235 | --max-random-scale "$MAX_RANDOM_SCALE" \ 236 | --min-random-scale "$MIN_RANDOM_SCALE" \ 237 | --max-random-area "$MAX_RANDOM_AREA" \ 238 | --min-random-area "$MIN_RANDOM_AREA" \ 239 | --max-crop-size "$MAX_CROP_SIZE" \ 240 | --min-crop-size "$MIN_CROP_SIZE" \ 241 | --brightness "$BRIGHTNESS" \ 242 | --contrast "$CONTRAST" \ 243 | --saturation "$SATURATION" \ 244 | --pca-noise "$PCA_NOISE" \ 245 | --random-resized-crop "$RANDOM_RESIZED_CROP" \ 246 | --rgb-mean "$RGB_MEAN" \ 247 | --rgb-std "$RGB_STD" \ 248 | --monitor "$MONITOR" \ 249 | --pad-size "$PAD_SIZE" \ 250 | --image-shape "$IMAGE_SHAPE" \ 251 | --num-classes "$NUM_CLASSES" \ 252 | --num-examples "$NUM_EXAMPLES" \ 253 | --model-prefix "model/$MODEL_PREFIX" 2>&1 | tee "$LOGS" 254 | 255 | if [ "${PIPESTATUS[0]}" -eq 0 ]; then 256 | # Record model_prefix and best validation accuracy epoch 257 | echo "$MODEL_PREFIX" > logs/latest_result.txt 258 | COUNT=$(grep 'Validation-acc' "logs/$MODEL_PREFIX.log" | sort -t'=' -k2 | tail -n 1 | cut -d'[' -f2 | cut -d']' -f1) 259 | MODEL_EPOCH=$((COUNT + 1)) 260 | echo "$MODEL_EPOCH" >> logs/latest_result.txt 261 | 262 | if [[ $TRAIN_ACCURACY_GRAPH_OUTPUT = 1 ]]; then 263 | TRAIN_ACCURACY_IMAGE="logs/$MODEL_PREFIX-train_accuracy.png" 264 | python3 util/train_accuracy.py "$CONFIG_FILE" "$TRAIN_ACCURACY_IMAGE" "$LOGS" 265 | if [[ $TRAIN_ACCURACY_SLACK_UPLOAD = 1 ]]; then 266 | python3 util/slack_file_upload.py "$TRAIN_SLACK_CHANNELS" "$TRAIN_ACCURACY_IMAGE" 267 | fi 268 | fi 269 | 270 | if [[ $TRAIN_LOSS_GRAPH_OUTPUT = 1 ]]; then 271 | TRAIN_LOSS_IMAGE="logs/$MODEL_PREFIX-train_loss.png" 272 | python3 util/train_loss.py "$CONFIG_FILE" "$TRAIN_LOSS_IMAGE" "$LOGS" 273 | if [[ $TRAIN_LOSS_SLACK_UPLOAD = 1 ]]; then 274 | python3 util/slack_file_upload.py "$TRAIN_SLACK_CHANNELS" "$TRAIN_LOSS_IMAGE" 275 | fi 276 | fi 277 | 278 | if [[ $AUTO_TEST = 1 ]]; then 279 | echo 'Start auto test using fine-tuned model with validation data' 280 | LABELS="model/$MODEL_PREFIX-labels.txt" 281 | 282 | python3 util/predict.py "$CONFIG_FILE" "$MODEL_IMAGE_SIZE" "valid" "$MODEL_PREFIX" "$MODEL_EPOCH" 283 | 284 | # save config.yml 285 | CONFIG_LOG="logs/$MODEL_PREFIX-$(printf "%04d" $MODEL_EPOCH)-valid-config.yml" 286 | cp "$CONFIG_FILE" "$CONFIG_LOG" \ 287 | && echo "Saved config file to \"$CONFIG_LOG\"" 1>&2 288 | 289 | # Make a confusion matrix from prediction results. 290 | if [[ $CONFUSION_MATRIX_OUTPUT = 1 ]]; then 291 | PREDICT_RESULTS_LOG="logs/$MODEL_PREFIX-$(printf "%04d" $MODEL_EPOCH)-valid-results.txt" 292 | IMAGE="logs/$MODEL_PREFIX-$(printf "%04d" $MODEL_EPOCH)-valid-confusion_matrix.png" 293 | python3 util/confusion_matrix.py "$CONFIG_FILE" "$LABELS" "$IMAGE" "$PREDICT_RESULTS_LOG" 294 | 295 | if [[ $TEST_SLACK_UPLOAD = 1 ]]; then 296 | python3 util/slack_file_upload.py "$TEST_SLACK_CHANNELS" "$IMAGE" 297 | fi 298 | fi 299 | # Make a classification report from prediction results. 300 | if [[ $CLASSIFICATION_REPORT_OUTPUT = 1 ]]; then 301 | PREDICT_RESULTS_LOG="logs/$MODEL_PREFIX-$(printf "%04d" $MODEL_EPOCH)-valid-results.txt" 302 | REPORT="logs/$MODEL_PREFIX-$(printf "%04d" $MODEL_EPOCH)-valid-classification_report.txt" 303 | python3 util/classification_report.py "$CONFIG_FILE" "$LABELS" "$PREDICT_RESULTS_LOG" "$REPORT" 304 | if [[ -e "$REPORT" ]]; then 305 | print_classification_report "$REPORT" 306 | else 307 | echo 'Error: classification report does not exist.' 1>&2 308 | fi 309 | fi 310 | fi 311 | else 312 | echo "Error(Exit code ${PIPESTATUS[0]}): Failed to fine-tune: $MODEL_PREFIX" 1>&2 313 | fi 314 | 315 | done 316 | done 317 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mxnet-finetuner 2 | 3 | An all-in-one Deep Learning toolkit for image classification to fine-tuning pretrained models using MXNet. 4 | 5 | 6 | ## Prerequisites 7 | 8 | - docker 9 | - docker-compose 10 | - jq 11 | - wget or curl 12 | 13 | When using NVIDIA GPUs 14 | 15 | - nvidia-docker (Both version 1.0 and 2.0 are acceptable) 16 | 17 | If you are using nvidia-docker version 1.0 and have never been running the `nvidia-docker` command after installing it, run the following command at least once to create the volume for GPU container. 18 | 19 | ``` 20 | $ nvidia-docker run --rm nvidia/cuda nvidia-smi 21 | ``` 22 | 23 | ## Setup 24 | 25 | ``` 26 | $ git clone https://github.com/knjcode/mxnet-finetuner 27 | $ cd mxnet-finetuner 28 | $ bash setup.sh 29 | ``` 30 | 31 | `setup.sh` will automatically generate` docker-compose.yml` and `config.yml` which are necessary for executing this tool according to your environment such as existence of the GPU. Please see [common settings](#common-settings) on how to run with GPU image on NVIDIA GPUs. 32 | 33 | When updating the GPU driver of the host machine, re-running `setup.sh`. Please see [After updating the GPU driver of the host machine](docs/setup.md#after-updating-the-gpu-driver-of-the-host-machine) for details. 34 | 35 | ## Example usage 36 | 37 | 38 | ### 1. Arrange images into their respective directories 39 | 40 | A training data directory (`images/train`), validation data directory (`images/valid`), and test data directory (`images/test`) should containing one subdirectory per image class. 41 | 42 | For example, arrange training, validation, and test data as follows. 43 | 44 | ``` 45 | images/ 46 | train/ 47 | airplanes/ 48 | airplane001.jpg 49 | airplane002.jpg 50 | ... 51 | watch/ 52 | watch001.jpg 53 | watch002.jpg 54 | ... 55 | valid/ 56 | airplanes/ 57 | airplane101.jpg 58 | airplane102.jpg 59 | ... 60 | watch/ 61 | watch101.jpg 62 | watch102.jpg 63 | ... 64 | test/ 65 | airplanes/ 66 | airplane201.jpg 67 | airplane202.jpg 68 | ... 69 | watch/ 70 | watch201.jpg 71 | watch202.jpg 72 | ... 73 | ``` 74 | 75 | 76 | ### 2. Edit config.yml 77 | 78 | Edit `config.yml` as you like. 79 | 80 | For example 81 | ``` 82 | common: 83 | num_threads: 4 84 | gpus: 0 85 | 86 | data: 87 | quality: 100 88 | shuffle: 1 89 | center_crop: 0 90 | 91 | finetune: 92 | models: 93 | - imagenet1k-resnet-50 94 | optimizers: 95 | - sgd 96 | num_epochs: 30 97 | lr: 0.0001 98 | lr_factor: 0.1 99 | lr_step_epochs: 10,20 100 | mom: 0.9 101 | wd: 0.00001 102 | batch_size: 10 103 | ``` 104 | 105 | Please see [common settings](#common-settings) on how to run with GPU image on NVIDIA GPUs. 106 | 107 | ### 3. Do Fine-tuning 108 | 109 | ``` 110 | $ docker-compose run finetuner 111 | ``` 112 | 113 | mxnet-finetuner will automatically execute the followings according to `config.yml`. 114 | 115 | - Create RecordIO data from images 116 | - Download pretrained models 117 | - Replace the last fully-connected layer with a new one that outputs the desired number of classes 118 | - Data augumentaion 119 | - Do Fine-tuning 120 | - Make training accuracy/loss graph 121 | - Make confusion matrix 122 | - Upload training accuracy/loss graph and confusion matrix to Slack 123 | 124 | Training accuracy/loss graph and/or confusion matrix are save at `logs/` directory. 125 | Trained models are save at `model/` directory. 126 | 127 | Trained models are saved with the following file name for each epoch. 128 | ``` 129 | model/201705292200-imagenet1k-nin-sgd-0000.params 130 | ``` 131 | 132 | If you want to upload results to Slack, set `SLACK_API_TOKEN` environment variable and edit `config.yml` as below. 133 | ``` 134 | finetune: 135 | train_accuracy_graph_slack_upload: 1 136 | train_loss_graph_slack_upload: 1 137 | test: 138 | confusion_matrix_slack_upload: 1 139 | ``` 140 | 141 | 142 | ### 4. Predict with trained models 143 | 144 | Select the trained model and epoch you want to use for testing and edit `config.yml` 145 | 146 | If you want to use `model/201705292200-imagenet1k-nin-sgd-0001.params`, edit `config.yml` as blow. 147 | 148 | ``` 149 | test: 150 | model: 201705292200-imagenet1k-nin-sgd-0001 151 | ``` 152 | 153 | When you want to use the latest highest validation accuracy trained model, edit `config.yml` as below. 154 | 155 | ``` 156 | test: 157 | use_latest: 1 158 | ``` 159 | 160 | If set this option, `model` is ignored. 161 | 162 | When you are done, you can predict with the following command 163 | 164 | ``` 165 | $ docker-compose run finetuner test 166 | ``` 167 | 168 | Predict result and classification report and/or confusion matrix are save at `logs/` directory. 169 | 170 | 171 | ## Available pretrained models 172 | 173 | |model |pretrained model name | 174 | |:------------------------------|:--------------------------------| 175 | |CaffeNet |imagenet1k-caffenet | 176 | |SqueezeNet |imagenet1k-squeezenet | 177 | |NIN |imagenet1k-nin | 178 | |VGG16 |imagenet1k-vgg16 | 179 | |Inception-BN |imagenet1k-inception-bn | 180 | |ResNet-50 |imagenet1k-resnet-50 | 181 | |ResNet-152 |imagenet1k-resnet-152 | 182 | |Inception-v3 |imagenet1k-inception-v3 | 183 | |DenseNet-169 |imagenet1k-densenet-169 | 184 | |SE-ResNeXt-50 |imagenet1k-se-resnext-50 | 185 | 186 | To use these pretrained models, specify the following pretrained model name in `config.yml`. 187 | 188 | For details, please check [Available pretrained models](docs/pretrained_models.md) 189 | 190 | 191 | ## Available optimizers 192 | 193 | - SGD 194 | - NAG 195 | - RMSProp 196 | - Adam 197 | - AdaGrad 198 | - AdaDelta 199 | - Adamax 200 | - Nadam 201 | - DCASGD 202 | - SGLD 203 | - Signum 204 | - FTML 205 | - Ftrl 206 | 207 | To use these optimizers, specify the optimizer name in lowercase in `config.yml`. 208 | 209 | 210 | ## Benchmark (Speed and Memory Footprint) 211 | 212 | Single TITAN X (Maxwell) with batch size 40 213 | 214 | |Model |speed (images/sec)|memory (MiB)| 215 | |:-----------|:-----------------|:-----------| 216 | |CaffeNet |1077.63 |716 | 217 | |ResNet-50 |111.04 |5483 | 218 | |Inception-V3|82.34 |6383 | 219 | |ResNet-152 |48.28 |11330 | 220 | 221 | For details, please check [Benchmark](docs/benchmark.md) 222 | 223 | 224 | ## Utilities 225 | 226 | ### util/counter.sh 227 | 228 | Count the number of files in each subdirectory. 229 | 230 | ``` 231 | $ util/counter.sh testdir 232 | testdir contains 4 directories 233 | Leopards 197 234 | Motorbikes 198 235 | airplanes 199 236 | watch 200 237 | ``` 238 | 239 | ### util/move_images.sh 240 | 241 | Move the specified number of jpeg images from the target directory to the output directory while maintaining the directory structure. 242 | 243 | ``` 244 | $ util/move_images.sh 20 testdir newdir 245 | processing Leopards 246 | processing Motorbikes 247 | processing airplanes 248 | processing watch 249 | $ util/counter.sh newdir 250 | newdir contains 4 directories 251 | Leopards 20 252 | Motorbikes 20 253 | airplanes 20 254 | watch 20 255 | ``` 256 | 257 | ### Prepare sample images for fine-tuning 258 | 259 | Download [Caltech 101] dataset, and split part of it into the `example_images` directory. 260 | 261 | ``` 262 | $ util/caltech101_prepare.sh 263 | ``` 264 | 265 | - `example_images/train` is train set of 60 images for each classes 266 | - `example_images/valid` is validation set of 20 images for each classes 267 | - `example_imags/test` is test set of 20 images for each classes 268 | 269 | ``` 270 | $ util/counter.sh example_images/train 271 | example_images/train contains 10 directories 272 | Faces 60 273 | Leopards 60 274 | Motorbikes 60 275 | airplanes 60 276 | bonsai 60 277 | car_side 60 278 | chandelier 60 279 | hawksbill 60 280 | ketch 60 281 | watch 60 282 | ``` 283 | 284 | With this data you can immediately try fine-tuning. 285 | 286 | ``` 287 | $ util/caltech101_prepare.sh 288 | $ rm -rf images 289 | $ mv exmaple_images images 290 | $ docker-compose run finetuner 291 | ``` 292 | 293 | 294 | 295 | ## Misc 296 | 297 | ### How to freeze layers during fine-tuning 298 | 299 | If you set the number of target layer to `finetune.num_active_layers` in `config.yml` as below, only layers whose number is not greater than the number of the specified layer will be train. 300 | 301 | ``` 302 | finetune: 303 | models: 304 | - imagenet1k-nin 305 | optimizers: 306 | - sgd 307 | num_active_layers: 6 308 | ``` 309 | 310 | The default for `finetune.num_active_layers` is `0`, in which case all layers are trained. 311 | 312 | If you set `1` to `finetune.num_active_layers`, only the last fully-connected layers are trained. 313 | 314 | You can check the layer numbers of various pretrained models with `num_layers` command. 315 | 316 | ``` 317 | $ docker-compose run finetuner num_layers 318 | ``` 319 | 320 | For details, please check [How to freeze layers during fine-tuning](docs/freeze_layers.md) 321 | 322 | 323 | ### Training from scratch 324 | 325 | Edit `config.yml` as below. 326 | 327 | ``` 328 | finetune: 329 | models: 330 | - scratch-alexnet 331 | ``` 332 | 333 | You can also run fine-tuning and training from scratch together. 334 | 335 | ``` 336 | finetune: 337 | models: 338 | - imagenet1k-inception-v3 339 | - scratch-inception-v3 340 | ``` 341 | 342 | For details, please check [Available models training from scratch](docs/train_from_scratch.md) 343 | 344 | 345 | ## Averaging ensemble test with trained models 346 | 347 | You can do averaging ensemble test using multiple trained models. 348 | 349 | If you want to use the following the three trained models, 350 | 351 | - `model/20180130074818-imagenet1k-nin-nadam-0003.params` 352 | - `model/20180130075252-imagenet1k-squeezenet-nadam-0003.params` 353 | - `model/20180131105109-imagenet1k-caffenet-nadam-0003.params` 354 | 355 | edit `config.yml` as blow. 356 | 357 | ``` 358 | ensemble: 359 | models: 360 | - 20180130074818-imagenet1k-nin-nadam-0003 361 | - 20180130075252-imagenet1k-squeezenet-nadam-0003 362 | - 20180131105109-imagenet1k-caffenet-nadam-0003 363 | ``` 364 | 365 | 366 | When you are done, you can do averaging ensemble test with the following command. 367 | 368 | ``` 369 | $ docker-compose run finetuner ensemble test 370 | ``` 371 | 372 | If you want to use validation dataset, do as follows. 373 | 374 | ``` 375 | $ docker-compose run finetuner ensemble valid 376 | ``` 377 | 378 | Averaging ensemble test result and classification report and/or confusion matrix are save at `logs/` directory. 379 | 380 | 381 | ## Export your trained model 382 | 383 | You can export your trained model in a format that can be used with [Model Server for Apache MXNet] as follows. 384 | 385 | ``` 386 | $ docker-compose run finetuner export 387 | ``` 388 | 389 | The exported file (extension is .model) is saved at `model/` directory. 390 | 391 | Please check [export settings](#export-settings) for export settings. 392 | 393 | 394 | ## Serve your exported model 395 | 396 | You can serve your exported model as API server. 397 | 398 | With the following command, launch the API server with the last exported model using pre-configured Docker image of [Model Server for Apache MXNet]. 399 | 400 | ``` 401 | $ docker-compose up -d mms 402 | ``` 403 | 404 | The API server is started at port 8080 of your local host. 405 | 406 | Then you will `curl` a `POST` to the MMS predict endpoint with the test image. (For exmple, use `airlane.jpg`). 407 | 408 | ``` 409 | $ curl -X POST http://127.0.0.1:8080/model/predict -F "data=@airplane.jpg" 410 | ``` 411 | 412 | The predict endpoint will return a prediction response in JSON. It will look something like the following result: 413 | 414 | ``` 415 | { 416 | "prediction": [ 417 | [ 418 | { 419 | "class": "airplane", 420 | "probability": 0.9950716495513916 421 | }, 422 | { 423 | "class": "watch", 424 | "probability": 0.004928381647914648 425 | } 426 | ] 427 | ] 428 | } 429 | ``` 430 | 431 | 432 | ## Try image classification with jupyter notebook 433 | 434 | ``` 435 | $ docker-compose run --service-ports finetuner jupyter 436 | ``` 437 | *Please note that the `--service-port` option is required* 438 | 439 | Replace the IP address of the displayed URL with the IP address of the host machine and access it from the browser. 440 | 441 | Open the `classify_example/classify_example.ipynb` and try out the image classification sample using the VGG-16 pretrained model pretrained with ImageNet. 442 | 443 | 444 | 445 | ## config.yml options 446 | 447 | ### common settings 448 | 449 | ``` 450 | common: 451 | num_threads: 4 452 | gpus: 0 # list of gpus to run, e.g. 0 or 0,2,5. 453 | ``` 454 | 455 | If a machine has one or more GPU cards installed, then each card is labeled by a number starting from 0. 456 | To use GPU for training or inference, specify GPU number in common.gpus. 457 | 458 | If you do not use the GPU or you can not use it, please comment out common.gpus. 459 | 460 | In the environment where GPU can not be used, `common.gpus` in `config.yml` generated by `setup.sh` is automatically commented out. 461 | 462 | ### data settings 463 | 464 | train, validation and test RecordIO data generation settings. 465 | 466 | mxnet-finetuner resize and pack the image files into a recordIO file for increased performance. 467 | 468 | By setting the `resize_short`, you can resize shorter edge of images to that size. 469 | 470 | If `resize_short` is not specified, it is automatically determined according to the model you are using. 471 | 472 | ``` 473 | data: 474 | quality: 100 475 | shuffle: 1 476 | center_crop: 0 477 | # test_center_crop: 1 478 | # resize_short: 256 479 | ``` 480 | 481 | ## finetune settings 482 | 483 | ``` 484 | finetune: 485 | models: # specify models to use 486 | - imagenet1k-nin 487 | # - imagenet1k-inception-v3 488 | # - imagenet1k-vgg16 489 | # - imagenet1k-resnet-50 490 | # - imagenet11k-resnet-152 491 | # - imagenet1k-resnext-101 492 | # - imagenet1k-se-resnext-50 493 | # etc 494 | optimizers: # specify optimizers to use 495 | - sgd 496 | # optimizers: sgd, nag, rmsprop, adam, adagrad, adadelta, adamax, nadam, dcasgd, signum, etc. 497 | # num_active_layers: 1 # train last n-layers without last fully-connected layer 498 | num_epochs: 10 # max num of epochs 499 | # load_epoch: 0 # specify when using user fine-tuned model 500 | lr: 0.0001 # initial learning rate 501 | lr_factor: 0.1 # the ratio to reduce lr on each step 502 | lr_step_epochs: 10 # the epochs to reduce the lr, e.g. 30,60 503 | mom: 0.9 # momentum for sgd 504 | wd: 0.00001 # weight decay for sgd 505 | batch_size: 10 # the batch size 506 | disp_batches: 10 # show progress for every n batches 507 | # top_k: 0 # report the top-k accuracy. 0 means no report. 508 | # data_aug_level: 3 # preset data augumentation level 509 | # random_crop: 0 # if or not randomly crop the image 510 | # random_mirror: 0 # if or not randomly flip horizontally 511 | # max_random_h: 0 # max change of hue, whose range is [0, 180] 512 | # max_random_s: 0 # max change of saturation, whose range is [0, 255] 513 | # max_random_l: 0 # max change of intensity, whose range is [0, 255] 514 | # max_random_aspect_ratio: 0 # max change of aspect ratio, whose range is [0, 1] 515 | # max_random_rotate_angle: 0 # max angle to rotate, whose range is [0, 360] 516 | # max_random_shear_ratio: 0 # max ratio to shear, whose range is [0, 1] 517 | # max_random_scale: 1 # max ratio to scale 518 | # min_random_scale: 1 # min ratio to scale, should >= img_size/input_shape. otherwise use --pad-size 519 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 520 | # monitor: 0 # log network parameters every N iters if larger than 0 521 | # pad_size: 0 # padding the input image 522 | auto_test: 1 # if or not test with validation data after fine-tuneing is completed 523 | train_accuracy_graph_output: 1 524 | # train_accuracy_graph_fontsize: 12 525 | # train_accuracy_graph_figsize: 8,6 526 | # train_accuracy_graph_slack_upload: 1 527 | # train_accuracy_graph_slack_channels: 528 | # - general 529 | train_loss_graph_output: 1 530 | # train_loss_graph_fontsize: 12 531 | # train_loss_graph_figsize: 8,6 532 | # train_loss_graph_slack_upload: 1 533 | # train_loss_graph_slack_channels: 534 | # - general 535 | ``` 536 | 537 | #### data_aug_level 538 | 539 | By setting the `data_aug_level` parameter, you can set the data augumentation settings collectively. 540 | 541 | |Level |settings | 542 | |:------|:---------------------------------| 543 | |Level 1|random_crop: 1
random_mirror: 1| 544 | |Level 2|max_random_h: 36
max_random_s: 50
max_random_l: 50
+ Level 1| 545 | |Level 3|max_random_aspect_ratio: 0.25
max_random_rotate_angle: 10
max_random_shear_ratio: 0.1
+ Level 2| 546 | 547 | If `data_aug_level` is set, parameters related to data augumentation will be overwritten. 548 | 549 | 550 | ### test settings 551 | 552 | ``` 553 | test: 554 | use_latest: 1 # Use last trained model. If set this option, model is ignored 555 | model: 201705292200-imagenet1k-nin-sgd-0001 556 | # model_epoch_up_to: 10 # test from epoch of model to model_epoch_up_to respectively 557 | test_batch_size: 10 558 | # top_k: 10 559 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 560 | classification_report_output: 1 561 | # classification_report_digits: 3 562 | confusion_matrix_output: 1 563 | # confusion_matrix_fontsize: 12 564 | # confusion_matrix_figsize: 16,12 565 | # confusion_matrix_slack_upload: 1 566 | # confusion_matrix_slack_channels: 567 | # - general 568 | ``` 569 | 570 | 571 | ### ensemble settings 572 | 573 | ``` 574 | # ensemble settings 575 | ensemble: 576 | models: 577 | - 20180130074818-imagenet1k-nin-nadam-0003 578 | - 20180130075252-imagenet1k-squeezenet-nadam-0003 579 | - 20180131105109-imagenet1k-caffenet-nadam-0003 580 | # weights: 1,1,1 581 | ensemble_batch_size: 10 582 | # top_k: 10 583 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 584 | classification_report_output: 1 585 | # classification_report_digits: 3 586 | confusion_matrix_output: 1 587 | # confusion_matrix_fontsize: 12 588 | # confusion_matrix_figsize: 16,12 589 | # confusion_matrix_slack_upload: 1 590 | # confusion_matrix_slack_channels: 591 | # - general 592 | ``` 593 | 594 | 595 | ### export settings 596 | 597 | ``` 598 | # export settings 599 | export: 600 | use_latest: 1 # Use last trained model. If set this option, model is ignored 601 | model: 201705292200-imagenet1k-nin-sgd-0001 602 | # top_k: 10 # report the top-k accuracy 603 | # rgb_mean: '123.68,116.779,103.939' # a tuple of size 3 for the mean rgb 604 | # center_crop: 1 # if or not center crop at image preprocessing 605 | # model_name: model 606 | ``` 607 | 608 | 609 | # Acknowledgement 610 | 611 | - [MXNet] 612 | - [Mo - Mustache Templates in Bash] 613 | - [A MXNet implementation of DenseNet with BC structure] 614 | - [SENet.mxnet] 615 | - [Model Server for Apache MXNet] 616 | 617 | # Licnese 618 | 619 | [Apache-2.0] license. 620 | 621 | 622 | [Apache-2.0]: https://github.com/dmlc/mxnet/blob/master/LICENSE 623 | [MXNet]: https://github.com/apache/incubator-mxnet 624 | [Mo - Mustache Templates in Bash]: https://github.com/tests-always-included/mo 625 | [A MXNet implementation of DenseNet with BC structure]: https://github.com/bruinxiong/densenet.mxnet 626 | [Caltech 101]: http://www.vision.caltech.edu/Image_Datasets/Caltech101/ 627 | [SENet.mxnet]: https://github.com/bruinxiong/SENet.mxnet 628 | [Model Server for Apache MXNet]: https://github.com/awslabs/mxnet-model-server 629 | -------------------------------------------------------------------------------- /util/vendor/mo: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | #/ Mo is a mustache template rendering software written in bash. It inserts 4 | #/ environment variables into templates. 5 | #/ 6 | #/ Simply put, mo will change {{VARIABLE}} into the value of that 7 | #/ environment variable. You can use {{#VARIABLE}}content{{/VARIABLE}} to 8 | #/ conditionally display content or iterate over the values of an array. 9 | #/ 10 | #/ Learn more about mustache templates at https://mustache.github.io/ 11 | #/ 12 | #/ Simple usage: 13 | #/ 14 | #/ mo [--false] [--help] [--source=FILE] filenames... 15 | #/ 16 | #/ --fail-not-set - Fail upon expansion of an unset variable. 17 | #/ --false - Treat the string "false" as empty for conditionals. 18 | #/ --help - This message. 19 | #/ --source=FILE - Load FILE into the environment before processing templates. 20 | # 21 | # Mo is under a MIT style licence with an additional non-advertising clause. 22 | # See LICENSE.md for the full text. 23 | # 24 | # This is open source! Please feel free to contribute. 25 | # 26 | # https://github.com/tests-always-included/mo 27 | 28 | 29 | # Public: Template parser function. Writes templates to stdout. 30 | # 31 | # $0 - Name of the mo file, used for getting the help message. 32 | # --fail-not-set - Fail upon expansion of an unset variable. Default behavior 33 | # is to silently ignore and expand into empty string. 34 | # --false - Treat "false" as an empty value. You may set the 35 | # MO_FALSE_IS_EMPTY environment variable instead to a non-empty 36 | # value to enable this behavior. 37 | # --help - Display a help message. 38 | # --source=FILE - Source a file into the environment before processint 39 | # template files. 40 | # -- - Used to indicate the end of options. You may optionally 41 | # use this when filenames may start with two hyphens. 42 | # $@ - Filenames to parse. 43 | # 44 | # Mo uses the following environment variables: 45 | # 46 | # MO_FAIL_ON_UNSET - When set to a non-empty value, expansion of an unset 47 | # env variable will be aborted with an error. 48 | # MO_FALSE_IS_EMPTY - When set to a non-empty value, the string "false" 49 | # will be treated as an empty value for the purposes 50 | # of conditionals. 51 | # MO_ORIGINAL_COMMAND - Used to find the `mo` program in order to generate 52 | # a help message. 53 | # 54 | # Returns nothing. 55 | mo() ( 56 | # This function executes in a subshell so IFS is reset. 57 | # Namespace this variable so we don't conflict with desired values. 58 | local moContent f2source files doubleHyphens 59 | 60 | IFS=$' \n\t' 61 | files=() 62 | doubleHyphens=false 63 | 64 | if [[ $# -gt 0 ]]; then 65 | for arg in "$@"; do 66 | if $doubleHyphens; then 67 | # After we encounter two hyphens together, all the rest 68 | # of the arguments are files. 69 | files=("${files[@]}" "$arg") 70 | else 71 | case "$arg" in 72 | -h|--h|--he|--hel|--help|-\?) 73 | moUsage "$0" 74 | exit 0 75 | ;; 76 | 77 | --fail-not-set) 78 | # shellcheck disable=SC2030 79 | MO_FAIL_ON_UNSET=true 80 | ;; 81 | 82 | --false) 83 | # shellcheck disable=SC2030 84 | MO_FALSE_IS_EMPTY=true 85 | ;; 86 | 87 | --source=*) 88 | f2source="${arg#--source=}" 89 | 90 | if [[ -f "$f2source" ]]; then 91 | # shellcheck disable=SC1090 92 | . "$f2source" 93 | else 94 | echo "No such file: $f2source" >&2 95 | exit 1 96 | fi 97 | ;; 98 | 99 | --) 100 | # Set a flag indicating we've encountered double hyphens 101 | doubleHyphens=true 102 | ;; 103 | 104 | *) 105 | # Every arg that is not a flag or a option should be a file 106 | files=(${files[@]+"${files[@]}"} "$arg") 107 | ;; 108 | esac 109 | fi 110 | done 111 | fi 112 | 113 | moGetContent moContent "${files[@]}" || return 1 114 | moParse "$moContent" "" true 115 | ) 116 | 117 | 118 | # Internal: Scan content until the right end tag is found. Creates an array 119 | # with the following members: 120 | # 121 | # [0] = Content before end tag 122 | # [1] = End tag (complete tag) 123 | # [2] = Content after end tag 124 | # 125 | # Everything using this function uses the "standalone tags" logic. 126 | # 127 | # $1 - Name of variable for the array 128 | # $2 - Content 129 | # $3 - Name of end tag 130 | # $4 - If -z, do standalone tag processing before finishing 131 | # 132 | # Returns nothing. 133 | moFindEndTag() { 134 | local content remaining scanned standaloneBytes tag 135 | 136 | #: Find open tags 137 | scanned="" 138 | moSplit content "$2" '{{' '}}' 139 | 140 | while [[ "${#content[@]}" -gt 1 ]]; do 141 | moTrimWhitespace tag "${content[1]}" 142 | 143 | #: Restore content[1] before we start using it 144 | content[1]='{{'"${content[1]}"'}}' 145 | 146 | case $tag in 147 | '#'* | '^'*) 148 | #: Start another block 149 | scanned="${scanned}${content[0]}${content[1]}" 150 | moTrimWhitespace tag "${tag:1}" 151 | moFindEndTag content "${content[2]}" "$tag" "loop" 152 | scanned="${scanned}${content[0]}${content[1]}" 153 | remaining=${content[2]} 154 | ;; 155 | 156 | '/'*) 157 | #: End a block - could be ours 158 | moTrimWhitespace tag "${tag:1}" 159 | scanned="$scanned${content[0]}" 160 | 161 | if [[ "$tag" == "$3" ]]; then 162 | #: Found our end tag 163 | if [[ -z "${4-}" ]] && moIsStandalone standaloneBytes "$scanned" "${content[2]}" true; then 164 | #: This is also a standalone tag - clean up whitespace 165 | #: and move those whitespace bytes to the "tag" element 166 | standaloneBytes=( $standaloneBytes ) 167 | content[1]="${scanned:${standaloneBytes[0]}}${content[1]}${content[2]:0:${standaloneBytes[1]}}" 168 | scanned="${scanned:0:${standaloneBytes[0]}}" 169 | content[2]="${content[2]:${standaloneBytes[1]}}" 170 | fi 171 | 172 | local "$1" && moIndirectArray "$1" "$scanned" "${content[1]}" "${content[2]}" 173 | return 0 174 | fi 175 | 176 | scanned="$scanned${content[1]}" 177 | remaining=${content[2]} 178 | ;; 179 | 180 | *) 181 | #: Ignore all other tags 182 | scanned="${scanned}${content[0]}${content[1]}" 183 | remaining=${content[2]} 184 | ;; 185 | esac 186 | 187 | moSplit content "$remaining" '{{' '}}' 188 | done 189 | 190 | #: Did not find our closing tag 191 | scanned="$scanned${content[0]}" 192 | local "$1" && moIndirectArray "$1" "${scanned}" "" "" 193 | } 194 | 195 | 196 | # Internal: Find the first index of a substring. If not found, sets the 197 | # index to -1. 198 | # 199 | # $1 - Destination variable for the index 200 | # $2 - Haystack 201 | # $3 - Needle 202 | # 203 | # Returns nothing. 204 | moFindString() { 205 | local pos string 206 | 207 | string=${2%%$3*} 208 | [[ "$string" == "$2" ]] && pos=-1 || pos=${#string} 209 | local "$1" && moIndirect "$1" "$pos" 210 | } 211 | 212 | 213 | # Internal: Generate a dotted name based on current context and target name. 214 | # 215 | # $1 - Target variable to store results 216 | # $2 - Context name 217 | # $3 - Desired variable name 218 | # 219 | # Returns nothing. 220 | moFullTagName() { 221 | if [[ -z "${2-}" ]] || [[ "$2" == *.* ]]; then 222 | local "$1" && moIndirect "$1" "$3" 223 | else 224 | local "$1" && moIndirect "$1" "${2}.${3}" 225 | fi 226 | } 227 | 228 | 229 | # Internal: Fetches the content to parse into a variable. Can be a list of 230 | # partials for files or the content from stdin. 231 | # 232 | # $1 - Variable name to assign this content back as 233 | # $2-@ - File names (optional) 234 | # 235 | # Returns nothing. 236 | moGetContent() { 237 | local content filename target 238 | 239 | target=$1 240 | shift 241 | if [[ "${#@}" -gt 0 ]]; then 242 | content="" 243 | 244 | for filename in "$@"; do 245 | #: This is so relative paths work from inside template files 246 | content="$content"'{{>'"$filename"'}}' 247 | done 248 | else 249 | moLoadFile content /dev/stdin || return 1 250 | fi 251 | 252 | local "$target" && moIndirect "$target" "$content" 253 | } 254 | 255 | 256 | # Internal: Indent a string, placing the indent at the beginning of every 257 | # line that has any content. 258 | # 259 | # $1 - Name of destination variable to get an array of lines 260 | # $2 - The indent string 261 | # $3 - The string to reindent 262 | # 263 | # Returns nothing. 264 | moIndentLines() { 265 | local content fragment len posN posR result trimmed 266 | 267 | result="" 268 | 269 | #: Remove the period from the end of the string. 270 | len=$((${#3} - 1)) 271 | content=${3:0:$len} 272 | 273 | if [[ -z "${2-}" ]]; then 274 | local "$1" && moIndirect "$1" "$content" 275 | 276 | return 0 277 | fi 278 | 279 | moFindString posN "$content" $'\n' 280 | moFindString posR "$content" $'\r' 281 | 282 | while [[ "$posN" -gt -1 ]] || [[ "$posR" -gt -1 ]]; do 283 | if [[ "$posN" -gt -1 ]]; then 284 | fragment="${content:0:$posN + 1}" 285 | content=${content:$posN + 1} 286 | else 287 | fragment="${content:0:$posR + 1}" 288 | content=${content:$posR + 1} 289 | fi 290 | 291 | moTrimChars trimmed "$fragment" false true " " $'\t' $'\n' $'\r' 292 | 293 | if [[ -n "$trimmed" ]]; then 294 | fragment="$2$fragment" 295 | fi 296 | 297 | result="$result$fragment" 298 | 299 | moFindString posN "$content" $'\n' 300 | moFindString posR "$content" $'\r' 301 | 302 | # If the content ends in a newline, do not indent. 303 | if [[ "$posN" -eq ${#content} ]]; then 304 | # Special clause for \r\n 305 | if [[ "$posR" -eq "$((posN - 1))" ]]; then 306 | posR=-1 307 | fi 308 | 309 | posN=-1 310 | fi 311 | 312 | if [[ "$posR" -eq ${#content} ]]; then 313 | posR=-1 314 | fi 315 | done 316 | 317 | moTrimChars trimmed "$content" false true " " $'\t' 318 | 319 | if [[ -n "$trimmed" ]]; then 320 | content="$2$content" 321 | fi 322 | 323 | result="$result$content" 324 | 325 | local "$1" && moIndirect "$1" "$result" 326 | } 327 | 328 | 329 | # Internal: Send a variable up to the parent of the caller of this function. 330 | # 331 | # $1 - Variable name 332 | # $2 - Value 333 | # 334 | # Examples 335 | # 336 | # callFunc () { 337 | # local "$1" && moIndirect "$1" "the value" 338 | # } 339 | # callFunc dest 340 | # echo "$dest" # writes "the value" 341 | # 342 | # Returns nothing. 343 | moIndirect() { 344 | unset -v "$1" 345 | printf -v "$1" '%s' "$2" 346 | } 347 | 348 | 349 | # Internal: Send an array as a variable up to caller of a function 350 | # 351 | # $1 - Variable name 352 | # $2-@ - Array elements 353 | # 354 | # Examples 355 | # 356 | # callFunc () { 357 | # local myArray=(one two three) 358 | # local "$1" && moIndirectArray "$1" "${myArray[@]}" 359 | # } 360 | # callFunc dest 361 | # echo "${dest[@]}" # writes "one two three" 362 | # 363 | # Returns nothing. 364 | moIndirectArray() { 365 | unset -v "$1" 366 | 367 | # IFS must be set to a string containing space or unset in order for 368 | # the array slicing to work regardless of the current IFS setting on 369 | # bash 3. This is detailed further at 370 | # https://github.com/fidian/gg-core/pull/7 371 | eval "$(printf "IFS= %s=(\"\${@:2}\") IFS=%q" "$1" "$IFS")" 372 | } 373 | 374 | 375 | # Internal: Determine if a given environment variable exists and if it is 376 | # an array. 377 | # 378 | # $1 - Name of environment variable 379 | # 380 | # Be extremely careful. Even if strict mode is enabled, it is not honored 381 | # in newer versions of Bash. Any errors that crop up here will not be 382 | # caught automatically. 383 | # 384 | # Examples 385 | # 386 | # var=(abc) 387 | # if moIsArray var; then 388 | # echo "This is an array" 389 | # echo "Make sure you don't accidentally use \$var" 390 | # fi 391 | # 392 | # Returns 0 if the name is not empty, 1 otherwise. 393 | moIsArray() { 394 | # Namespace this variable so we don't conflict with what we're testing. 395 | local moTestResult 396 | 397 | moTestResult=$(declare -p "$1" 2>/dev/null) || return 1 398 | [[ "${moTestResult:0:10}" == "declare -a" ]] && return 0 399 | [[ "${moTestResult:0:10}" == "declare -A" ]] && return 0 400 | 401 | return 1 402 | } 403 | 404 | 405 | # Internal: Determine if the given name is a defined function. 406 | # 407 | # $1 - Function name to check 408 | # 409 | # Be extremely careful. Even if strict mode is enabled, it is not honored 410 | # in newer versions of Bash. Any errors that crop up here will not be 411 | # caught automatically. 412 | # 413 | # Examples 414 | # 415 | # moo () { 416 | # echo "This is a function" 417 | # } 418 | # if moIsFunction moo; then 419 | # echo "moo is a defined function" 420 | # fi 421 | # 422 | # Returns 0 if the name is a function, 1 otherwise. 423 | moIsFunction() { 424 | local functionList functionName 425 | 426 | functionList=$(declare -F) 427 | functionList=( ${functionList//declare -f /} ) 428 | 429 | for functionName in "${functionList[@]}"; do 430 | if [[ "$functionName" == "$1" ]]; then 431 | return 0 432 | fi 433 | done 434 | 435 | return 1 436 | } 437 | 438 | 439 | # Internal: Determine if the tag is a standalone tag based on whitespace 440 | # before and after the tag. 441 | # 442 | # Passes back a string containing two numbers in the format "BEFORE AFTER" 443 | # like "27 10". It indicates the number of bytes remaining in the "before" 444 | # string (27) and the number of bytes to trim in the "after" string (10). 445 | # Useful for string manipulation: 446 | # 447 | # $1 - Variable to set for passing data back 448 | # $2 - Content before the tag 449 | # $3 - Content after the tag 450 | # $4 - true/false: is this the beginning of the content? 451 | # 452 | # Examples 453 | # 454 | # moIsStandalone RESULT "$before" "$after" false || return 0 455 | # RESULT_ARRAY=( $RESULT ) 456 | # echo "${before:0:${RESULT_ARRAY[0]}}...${after:${RESULT_ARRAY[1]}}" 457 | # 458 | # Returns nothing. 459 | moIsStandalone() { 460 | local afterTrimmed beforeTrimmed char 461 | 462 | moTrimChars beforeTrimmed "$2" false true " " $'\t' 463 | moTrimChars afterTrimmed "$3" true false " " $'\t' 464 | char=$((${#beforeTrimmed} - 1)) 465 | char=${beforeTrimmed:$char} 466 | 467 | # If the content before didn't end in a newline 468 | if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]]; then 469 | # and there was content or this didn't start the file 470 | if [[ -n "$char" ]] || ! $4; then 471 | # then this is not a standalone tag. 472 | return 1 473 | fi 474 | fi 475 | 476 | char=${afterTrimmed:0:1} 477 | 478 | # If the content after doesn't start with a newline and it is something 479 | if [[ "$char" != $'\n' ]] && [[ "$char" != $'\r' ]] && [[ -n "$char" ]]; then 480 | # then this is not a standalone tag. 481 | return 2 482 | fi 483 | 484 | if [[ "$char" == $'\r' ]] && [[ "${afterTrimmed:1:1}" == $'\n' ]]; then 485 | char="$char"$'\n' 486 | fi 487 | 488 | local "$1" && moIndirect "$1" "$((${#beforeTrimmed})) $((${#3} + ${#char} - ${#afterTrimmed}))" 489 | } 490 | 491 | 492 | # Internal: Join / implode an array 493 | # 494 | # $1 - Variable name to receive the joined content 495 | # $2 - Joiner 496 | # $3-$* - Elements to join 497 | # 498 | # Returns nothing. 499 | moJoin() { 500 | local joiner part result target 501 | 502 | target=$1 503 | joiner=$2 504 | result=$3 505 | shift 3 506 | 507 | for part in "$@"; do 508 | result="$result$joiner$part" 509 | done 510 | 511 | local "$target" && moIndirect "$target" "$result" 512 | } 513 | 514 | 515 | # Internal: Read a file into a variable. 516 | # 517 | # $1 - Variable name to receive the file's content 518 | # $2 - Filename to load 519 | # 520 | # Returns nothing. 521 | moLoadFile() { 522 | local content len 523 | 524 | # The subshell removes any trailing newlines. We forcibly add 525 | # a dot to the content to preserve all newlines. 526 | # As a future optimization, it would be worth considering removing 527 | # cat and replacing this with a read loop. 528 | 529 | content=$(cat -- "$2" && echo '.') || return 1 530 | len=$((${#content} - 1)) 531 | content=${content:0:$len} # Remove last dot 532 | 533 | local "$1" && moIndirect "$1" "$content" 534 | } 535 | 536 | 537 | # Internal: Process a chunk of content some number of times. Writes output 538 | # to stdout. 539 | # 540 | # $1 - Content to parse repeatedly 541 | # $2 - Tag prefix (context name) 542 | # $3-@ - Names to insert into the parsed content 543 | # 544 | # Returns nothing. 545 | moLoop() { 546 | local content context contextBase 547 | 548 | content=$1 549 | contextBase=$2 550 | shift 2 551 | 552 | while [[ "${#@}" -gt 0 ]]; do 553 | moFullTagName context "$contextBase" "$1" 554 | moParse "$content" "$context" false 555 | shift 556 | done 557 | } 558 | 559 | 560 | # Internal: Parse a block of text, writing the result to stdout. 561 | # 562 | # $1 - Block of text to change 563 | # $2 - Current name (the variable NAME for what {{.}} means) 564 | # $3 - true when no content before this, false otherwise 565 | # 566 | # Returns nothing. 567 | moParse() { 568 | # Keep naming variables mo* here to not overwrite needed variables 569 | # used in the string replacements 570 | local moBlock moContent moCurrent moIsBeginning moNextIsBeginning moTag 571 | 572 | moCurrent=$2 573 | moIsBeginning=$3 574 | 575 | # Find open tags 576 | moSplit moContent "$1" '{{' '}}' 577 | 578 | while [[ "${#moContent[@]}" -gt 1 ]]; do 579 | moTrimWhitespace moTag "${moContent[1]}" 580 | moNextIsBeginning=false 581 | 582 | case $moTag in 583 | '#'*) 584 | # Loop, if/then, or pass content through function 585 | # Sets context 586 | moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" 587 | moTrimWhitespace moTag "${moTag:1}" 588 | moFindEndTag moBlock "$moContent" "$moTag" 589 | moFullTagName moTag "$moCurrent" "$moTag" 590 | 591 | if moTest "$moTag"; then 592 | # Show / loop / pass through function 593 | if moIsFunction "$moTag"; then 594 | #: Consider piping the output to moGetContent 595 | #: so the lambda does not execute in a subshell? 596 | moContent=$($moTag "${moBlock[0]}") 597 | moParse "$moContent" "$moCurrent" false 598 | moContent="${moBlock[2]}" 599 | elif moIsArray "$moTag"; then 600 | eval "moLoop \"\${moBlock[0]}\" \"$moTag\" \"\${!${moTag}[@]}\"" 601 | else 602 | moParse "${moBlock[0]}" "$moCurrent" false 603 | fi 604 | fi 605 | 606 | moContent="${moBlock[2]}" 607 | ;; 608 | 609 | '>'*) 610 | # Load partial - get name of file relative to cwd 611 | moPartial moContent "${moContent[@]}" "$moIsBeginning" "$moCurrent" 612 | moNextIsBeginning=${moContent[1]} 613 | moContent=${moContent[0]} 614 | ;; 615 | 616 | '/'*) 617 | # Closing tag - If hit in this loop, we simply ignore 618 | # Matching tags are found in moFindEndTag 619 | moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" 620 | ;; 621 | 622 | '^'*) 623 | # Display section if named thing does not exist 624 | moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" 625 | moTrimWhitespace moTag "${moTag:1}" 626 | moFindEndTag moBlock "$moContent" "$moTag" 627 | moFullTagName moTag "$moCurrent" "$moTag" 628 | 629 | if ! moTest "$moTag"; then 630 | moParse "${moBlock[0]}" "$moCurrent" false "$moCurrent" 631 | fi 632 | 633 | moContent="${moBlock[2]}" 634 | ;; 635 | 636 | '!'*) 637 | # Comment - ignore the tag content entirely 638 | # Trim spaces/tabs before the comment 639 | moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" 640 | ;; 641 | 642 | .) 643 | # Current content (environment variable or function) 644 | moStandaloneDenied moContent "${moContent[@]}" 645 | moShow "$moCurrent" "$moCurrent" 646 | ;; 647 | 648 | '=') 649 | # Change delimiters 650 | # Any two non-whitespace sequences separated by whitespace. 651 | # This tag is ignored. 652 | moStandaloneAllowed moContent "${moContent[@]}" "$moIsBeginning" 653 | ;; 654 | 655 | '{'*) 656 | # Unescaped - split on }}} not }} 657 | moStandaloneDenied moContent "${moContent[@]}" 658 | moContent="${moTag:1}"'}}'"$moContent" 659 | moSplit moContent "$moContent" '}}}' 660 | moTrimWhitespace moTag "${moContent[0]}" 661 | moFullTagName moTag "$moCurrent" "$moTag" 662 | moContent=${moContent[1]} 663 | 664 | # Now show the value 665 | moShow "$moTag" "$moCurrent" 666 | ;; 667 | 668 | '&'*) 669 | # Unescaped 670 | moStandaloneDenied moContent "${moContent[@]}" 671 | moTrimWhitespace moTag "${moTag:1}" 672 | moFullTagName moTag "$moCurrent" "$moTag" 673 | moShow "$moTag" "$moCurrent" 674 | ;; 675 | 676 | *) 677 | # Normal environment variable or function call 678 | moStandaloneDenied moContent "${moContent[@]}" 679 | moFullTagName moTag "$moCurrent" "$moTag" 680 | moShow "$moTag" "$moCurrent" 681 | ;; 682 | esac 683 | 684 | moIsBeginning=$moNextIsBeginning 685 | moSplit moContent "$moContent" '{{' '}}' 686 | done 687 | 688 | echo -n "${moContent[0]}" 689 | } 690 | 691 | 692 | # Internal: Process a partial. 693 | # 694 | # Indentation should be applied to the entire partial. 695 | # 696 | # This sends back the "is beginning" flag because the newline after a 697 | # standalone partial is consumed. That newline is very important in the middle 698 | # of content. We send back this flag to reset the processing loop's 699 | # `moIsBeginning` variable, so the software thinks we are back at the 700 | # beginning of a file and standalone processing continues to work. 701 | # 702 | # Prefix all variables. 703 | # 704 | # $1 - Name of destination variable. Element [0] is the content, [1] is the 705 | # true/false flag indicating if we are at the beginning of content. 706 | # $2 - Content before the tag that was not yet written 707 | # $3 - Tag content 708 | # $4 - Content after the tag 709 | # $5 - true/false: is this the beginning of the content? 710 | # $6 - Current context name 711 | # 712 | # Returns nothing. 713 | moPartial() { 714 | # Namespace variables here to prevent conflicts. 715 | local moContent moFilename moIndent moIsBeginning moPartial moStandalone moUnindented 716 | 717 | if moIsStandalone moStandalone "$2" "$4" "$5"; then 718 | moStandalone=( $moStandalone ) 719 | echo -n "${2:0:${moStandalone[0]}}" 720 | moIndent=${2:${moStandalone[0]}} 721 | moContent=${4:${moStandalone[1]}} 722 | moIsBeginning=true 723 | else 724 | moIndent="" 725 | echo -n "$2" 726 | moContent=$4 727 | moIsBeginning=$5 728 | fi 729 | 730 | moTrimWhitespace moFilename "${3:1}" 731 | 732 | # Execute in subshell to preserve current cwd and environment 733 | ( 734 | # It would be nice to remove `dirname` and use a function instead, 735 | # but that's difficult when you're only given filenames. 736 | cd "$(dirname -- "$moFilename")" || exit 1 737 | moUnindented="$( 738 | moLoadFile moPartial "${moFilename##*/}" || exit 1 739 | moParse "${moPartial}" "$6" true 740 | 741 | # Fix bash handling of subshells and keep trailing whitespace. 742 | # This is removed in moIndentLines. 743 | echo -n "." 744 | )" || exit 1 745 | moIndentLines moPartial "$moIndent" "$moUnindented" 746 | echo -n "$moPartial" 747 | ) || exit 1 748 | 749 | # If this is a standalone tag, the trailing newline after the tag is 750 | # removed and the contents of the partial are added, which typically 751 | # contain a newline. We need to send a signal back to the processing 752 | # loop that the moIsBeginning flag needs to be turned on again. 753 | # 754 | # [0] is the content, [1] is that flag. 755 | local "$1" && moIndirectArray "$1" "$moContent" "$moIsBeginning" 756 | } 757 | 758 | 759 | # Internal: Show an environment variable or the output of a function to 760 | # stdout. 761 | # 762 | # Limit/prefix any variables used. 763 | # 764 | # $1 - Name of environment variable or function 765 | # $2 - Current context 766 | # 767 | # Returns nothing. 768 | moShow() { 769 | # Namespace these variables 770 | local moJoined moNameParts 771 | 772 | if moIsFunction "$1"; then 773 | CONTENT=$($1 "") 774 | moParse "$CONTENT" "$2" false 775 | return 0 776 | fi 777 | 778 | moSplit moNameParts "$1" "." 779 | 780 | if [[ -z "${moNameParts[1]}" ]]; then 781 | if moIsArray "$1"; then 782 | eval moJoin moJoined "," "\${$1[@]}" 783 | echo -n "$moJoined" 784 | else 785 | # shellcheck disable=SC2031 786 | if [[ -z "$MO_FAIL_ON_UNSET" ]] || moTestVarSet "$1"; then 787 | echo -n "${!1}" 788 | else 789 | echo "Env variable not set: $1" >&2 790 | exit 1 791 | fi 792 | fi 793 | else 794 | # Further subindexes are disallowed 795 | eval "echo -n \"\${${moNameParts[0]}[${moNameParts[1]%%.*}]}\"" 796 | fi 797 | } 798 | 799 | 800 | # Internal: Split a larger string into an array. 801 | # 802 | # $1 - Destination variable 803 | # $2 - String to split 804 | # $3 - Starting delimiter 805 | # $4 - Ending delimiter (optional) 806 | # 807 | # Returns nothing. 808 | moSplit() { 809 | local pos result 810 | 811 | result=( "$2" ) 812 | moFindString pos "${result[0]}" "$3" 813 | 814 | if [[ "$pos" -ne -1 ]]; then 815 | # The first delimiter was found 816 | result[1]=${result[0]:$pos + ${#3}} 817 | result[0]=${result[0]:0:$pos} 818 | 819 | if [[ -n "${4-}" ]]; then 820 | moFindString pos "${result[1]}" "$4" 821 | 822 | if [[ "$pos" -ne -1 ]]; then 823 | # The second delimiter was found 824 | result[2]="${result[1]:$pos + ${#4}}" 825 | result[1]="${result[1]:0:$pos}" 826 | fi 827 | fi 828 | fi 829 | 830 | local "$1" && moIndirectArray "$1" "${result[@]}" 831 | } 832 | 833 | 834 | # Internal: Handle the content for a standalone tag. This means removing 835 | # whitespace (not newlines) before a tag and whitespace and a newline after 836 | # a tag. That is, assuming, that the line is otherwise empty. 837 | # 838 | # $1 - Name of destination "content" variable. 839 | # $2 - Content before the tag that was not yet written 840 | # $3 - Tag content (not used) 841 | # $4 - Content after the tag 842 | # $5 - true/false: is this the beginning of the content? 843 | # 844 | # Returns nothing. 845 | moStandaloneAllowed() { 846 | local bytes 847 | 848 | if moIsStandalone bytes "$2" "$4" "$5"; then 849 | bytes=( $bytes ) 850 | echo -n "${2:0:${bytes[0]}}" 851 | local "$1" && moIndirect "$1" "${4:${bytes[1]}}" 852 | else 853 | echo -n "$2" 854 | local "$1" && moIndirect "$1" "$4" 855 | fi 856 | } 857 | 858 | 859 | # Internal: Handle the content for a tag that is never "standalone". No 860 | # adjustments are made for newlines and whitespace. 861 | # 862 | # $1 - Name of destination "content" variable. 863 | # $2 - Content before the tag that was not yet written 864 | # $3 - Tag content (not used) 865 | # $4 - Content after the tag 866 | # 867 | # Returns nothing. 868 | moStandaloneDenied() { 869 | echo -n "$2" 870 | local "$1" && moIndirect "$1" "$4" 871 | } 872 | 873 | 874 | # Internal: Determines if the named thing is a function or if it is a 875 | # non-empty environment variable. When MO_FALSE_IS_EMPTY is set to a 876 | # non-empty value, then "false" is also treated is an empty value. 877 | # 878 | # Do not use variables without prefixes here if possible as this needs to 879 | # check if any name exists in the environment 880 | # 881 | # $1 - Name of environment variable or function 882 | # $2 - Current value (our context) 883 | # MO_FALSE_IS_EMPTY - When set to a non-empty value, this will say the 884 | # string value "false" is empty. 885 | # 886 | # Returns 0 if the name is not empty, 1 otherwise. When MO_FALSE_IS_EMPTY 887 | # is set, this returns 1 if the name is "false". 888 | moTest() { 889 | # Test for functions 890 | moIsFunction "$1" && return 0 891 | 892 | if moIsArray "$1"; then 893 | # Arrays must have at least 1 element 894 | eval "[[ \"\${#${1}[@]}\" -gt 0 ]]" && return 0 895 | else 896 | # If MO_FALSE_IS_EMPTY is set, then return 1 if the value of 897 | # the variable is "false". 898 | # shellcheck disable=SC2031 899 | [[ -n "${MO_FALSE_IS_EMPTY-}" ]] && [[ "${!1-}" == "false" ]] && return 1 900 | 901 | # Environment variables must not be empty 902 | [[ -n "${!1-}" ]] && return 0 903 | fi 904 | 905 | return 1 906 | } 907 | 908 | # Internal: Determine if a variable is assigned, even if it is assigned an empty 909 | # value. 910 | # 911 | # $1 - Variable name to check. 912 | # 913 | # Returns true (0) if the variable is set, 1 if the variable is unset. 914 | moTestVarSet() { 915 | [[ "${!1-a}" == "${!1-b}" ]] 916 | } 917 | 918 | 919 | # Internal: Trim the leading whitespace only. 920 | # 921 | # $1 - Name of destination variable 922 | # $2 - The string 923 | # $3 - true/false - trim front? 924 | # $4 - true/false - trim end? 925 | # $5-@ - Characters to trim 926 | # 927 | # Returns nothing. 928 | moTrimChars() { 929 | local back current front last target varName 930 | 931 | target=$1 932 | current=$2 933 | front=$3 934 | back=$4 935 | last="" 936 | shift 4 # Remove target, string, trim front flag, trim end flag 937 | 938 | while [[ "$current" != "$last" ]]; do 939 | last=$current 940 | 941 | for varName in "$@"; do 942 | $front && current="${current/#$varName}" 943 | $back && current="${current/%$varName}" 944 | done 945 | done 946 | 947 | local "$target" && moIndirect "$target" "$current" 948 | } 949 | 950 | 951 | # Internal: Trim leading and trailing whitespace from a string. 952 | # 953 | # $1 - Name of variable to store trimmed string 954 | # $2 - The string 955 | # 956 | # Returns nothing. 957 | moTrimWhitespace() { 958 | local result 959 | 960 | moTrimChars result "$2" true true $'\r' $'\n' $'\t' " " 961 | local "$1" && moIndirect "$1" "$result" 962 | } 963 | 964 | 965 | # Internal: Displays the usage for mo. Pulls this from the file that 966 | # contained the `mo` function. Can only work when the right filename 967 | # comes is the one argument, and that only happens when `mo` is called 968 | # with `$0` set to this file. 969 | # 970 | # $1 - Filename that has the help message 971 | # 972 | # Returns nothing. 973 | moUsage() { 974 | grep '^#/' "${MO_ORIGINAL_COMMAND}" | cut -c 4- 975 | } 976 | 977 | 978 | # Save the original command's path for usage later 979 | MO_ORIGINAL_COMMAND="$(cd "${BASH_SOURCE[0]%/*}" || exit 1; pwd)/${BASH_SOURCE[0]##*/}" 980 | 981 | # If sourced, load all functions. 982 | # If executed, perform the actions as expected. 983 | if [[ "$0" == "${BASH_SOURCE[0]}" ]] || [[ -z "${BASH_SOURCE[0]}" ]]; then 984 | mo "$@" 985 | fi 986 | --------------------------------------------------------------------------------