├── utils ├── __init__.py ├── errors.py ├── data_manager.py └── cfg_process.py ├── dawnbench └── __init__.py ├── reporting ├── __init__.py ├── .gitignore ├── requirements.txt ├── utils │ ├── __init__.py │ └── email.py ├── scripts │ └── update_lambda.sh ├── lambda_handler.py └── report.py ├── requirement.txt ├── image_classification ├── __init__.py ├── common │ ├── __init__.py │ ├── find_mxnet.py │ ├── util.py │ └── modelzoo.py ├── symbols │ ├── __init__.py │ ├── README.md │ ├── mlp.py │ ├── mobilenetv2.py │ ├── lenet.py │ ├── alexnet.py │ ├── vgg.py │ └── googlenet.py ├── scripts │ ├── resnet50v1-imagenet-fp32.sh │ ├── mxnet_1_1_resnet50v1-imagenet-fp32.sh │ ├── resnet50v1-imagenet-fp16.sh │ ├── mxnet_1_1_resnet50v1-imagenet-fp16.sh │ ├── test.sh │ ├── resnet50v2-imagenet-fp16.sh │ └── resnet50v2-imagenet-fp32.sh ├── train_imagenet.py ├── get_data.py └── predict_image.py ├── tensorflow ├── inception │ ├── __init__.py │ ├── inception │ │ ├── __init__.py │ │ ├── slim │ │ │ ├── __init__.py │ │ │ ├── slim.py │ │ │ └── BUILD │ │ ├── flowers_train.py │ │ ├── imagenet_train.py │ │ ├── flowers_eval.py │ │ ├── imagenet_eval.py │ │ ├── flowers_data.py │ │ ├── imagenet_data.py │ │ ├── imagenet_distributed_train.py │ │ ├── data │ │ │ ├── preprocess_imagenet_validation_data.py │ │ │ ├── download_and_preprocess_flowers.sh │ │ │ ├── download_and_preprocess_flowers_mac.sh │ │ │ └── download_and_preprocess_imagenet.sh │ │ └── dataset.py │ ├── WORKSPACE │ └── g3doc │ │ └── inception_v3_architecture.png ├── resnet │ ├── g3doc │ │ ├── cifar_resnet.gif │ │ └── cifar_resnet_legends.gif │ ├── BUILD │ └── README.md └── AUTHORS ├── word_language_model ├── __init__.py ├── get_ptb_data.sh ├── data.py └── model.py ├── .gitignore ├── djl ├── gradle │ └── wrapper │ │ ├── .gitignore │ │ ├── gradle-wrapper.properties │ │ └── GradleWrapperDownloader.java └── benchmark_training.sh ├── tensorflow_benchmark └── tf_cnn_benchmarks │ ├── models │ ├── __init__.py │ ├── trivial_model.py │ ├── lenet_model.py │ ├── overfeat_model.py │ ├── model.py │ ├── googlenet_model.py │ ├── vgg_model.py │ ├── alexnet_model.py │ ├── densenet_model.py │ └── model_config.py │ ├── benchmark_storage.py │ ├── tf_cnn_benchmarks.py │ ├── README.md │ └── cbuild_benchmark_storage.py ├── mms ├── noop.json ├── config.properties ├── lstm.json └── README.md ├── scala-mxnet ├── java-bm │ ├── .mvn │ │ └── wrapper │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── bin │ │ ├── get_resnet50_ssd_data.sh │ │ └── run_ssd.sh │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── mxnet │ │ │ └── Utils.java │ └── pom.xml ├── scala-bm │ ├── .mvn │ │ └── wrapper │ │ │ ├── maven-wrapper.jar │ │ │ └── maven-wrapper.properties │ ├── bin │ │ ├── get_resnet_model.sh │ │ ├── get_resnet18_data.sh │ │ ├── get_resnet50_ssd_data.sh │ │ ├── run_ic.sh │ │ ├── run_ssd.sh │ │ └── run_charrnn_example.sh │ └── src │ │ └── main │ │ └── scala │ │ └── mxnet │ │ ├── RnnModel.scala │ │ └── CharRnnExample.scala └── duplicate_files.sh ├── end_to_end_model_benchmark ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties ├── java-bm │ ├── bin │ │ └── build.sh │ └── pom.xml ├── scala-bm │ └── bin │ │ └── build.sh ├── get_model.sh ├── log4j.properties └── run_benchmark.sh ├── benchmark ├── tensorflow │ ├── average.py │ ├── killall.sh │ ├── runincluster.sh │ ├── runall.sh │ ├── tabulate.sh │ ├── plotgraph.py │ └── runscalabilitytest.sh └── plotgraph.py ├── ssd └── get_resnet_model.sh ├── question_answering └── scripts │ ├── run_test_1gpu.sh │ └── run_test_multigpu.sh ├── test ├── test_benchmark_compute_method.py └── test_cfg_process.py ├── DockerFiles └── Dockerfile.gpu_mxnet ├── README.md ├── onnx_benchmark ├── setup.sh └── import_benchmarkscript.py ├── benchmark_driver.py ├── benchmark_runner.py ├── dependency_update └── mlp.py └── README_BENCHMARKAI.md /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dawnbench/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /reporting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- 1 | psutil 2 | -------------------------------------------------------------------------------- /image_classification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /reporting/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /tensorflow/inception/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /word_language_model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /image_classification/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /image_classification/symbols/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/slim/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /djl/gradle/wrapper/.gitignore: -------------------------------------------------------------------------------- 1 | gradle-wrapper.jar 2 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tensorflow/inception/WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "inception") 2 | -------------------------------------------------------------------------------- /mms/noop.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input_sentence": "Hello World" 4 | } 5 | ] -------------------------------------------------------------------------------- /tensorflow/resnet/g3doc/cifar_resnet.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/tensorflow/resnet/g3doc/cifar_resnet.gif -------------------------------------------------------------------------------- /mms/config.properties: -------------------------------------------------------------------------------- 1 | inference_address=http://0.0.0.0:8080 2 | management_address=http://0.0.0.0:8081 3 | job_queue_size=1000 4 | async_logging=true 5 | -------------------------------------------------------------------------------- /tensorflow/resnet/g3doc/cifar_resnet_legends.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/tensorflow/resnet/g3doc/cifar_resnet_legends.gif -------------------------------------------------------------------------------- /mms/lstm.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "input_sentence": "on the exchange floor as soon as ual stopped trading we for a panic said one top floor trader" 4 | } 5 | ] -------------------------------------------------------------------------------- /scala-mxnet/java-bm/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/scala-mxnet/java-bm/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/scala-mxnet/scala-bm/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /scala-mxnet/java-bm/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip -------------------------------------------------------------------------------- /tensorflow/inception/g3doc/inception_v3_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/tensorflow/inception/g3doc/inception_v3_architecture.png -------------------------------------------------------------------------------- /end_to_end_model_benchmark/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/deeplearning-benchmark/HEAD/end_to_end_model_benchmark/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /end_to_end_model_benchmark/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip -------------------------------------------------------------------------------- /benchmark/tensorflow/average.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sum = 0 4 | n = 0 5 | 6 | for line in sys.stdin: 7 | sum += float(line) 8 | n += 1 9 | 10 | print(round(sum/n, 3)) 11 | -------------------------------------------------------------------------------- /utils/errors.py: -------------------------------------------------------------------------------- 1 | class CommandExecutionError(Exception): 2 | pass 3 | 4 | 5 | class MetricComputeMethodError(Exception): 6 | pass 7 | 8 | 9 | class MetricPatternError(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /scala-mxnet/duplicate_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | cd /incubator-mxnet/scala-package/examples/scripts/infer/images/ 5 | max=1000 6 | for i in `seq 2 $max` 7 | do 8 | cp $1.jpg $1$i.jpg 9 | done 10 | 11 | echo "Done copying" -------------------------------------------------------------------------------- /djl/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Jan 16 22:46:13 PST 2019 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-bin.zip 7 | -------------------------------------------------------------------------------- /tensorflow/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Google Inc. 10 | David Dao 11 | -------------------------------------------------------------------------------- /reporting/requirements.txt: -------------------------------------------------------------------------------- 1 | Babel==2.9.1 2 | boto3==1.9.31 3 | botocore==1.12.31 4 | docutils==0.14 5 | et-xmlfile==1.0.1 6 | jdcal==1.4 7 | jmespath==0.9.3 8 | openpyxl==2.5.9 9 | python-dateutil==2.7.4 10 | pytz==2018.6 11 | PyYAML>=4.2b1 12 | s3transfer==0.1.13 13 | six==1.11.0 14 | urllib3==1.26.5 15 | git+git://github.com/vishaalkapoor/xlsx2html@hyperlink#egg=xlsx2html 16 | XlsxWriter==1.1.2 17 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/java-bm/bin/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCALA_VERSION_PROFILE=2.11 3 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 4 | 5 | # build the project with maven wrapper to avoid avoid dead lock when using the apt install maven but sudo apt update still running on background 6 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$1 -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 7 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/scala-bm/bin/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SCALA_VERSION_PROFILE=2.11 3 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 4 | 5 | # build the project with maven wrapper to avoid avoid dead lock when using the apt install maven but sudo apt update still running on background 6 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$1 -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 7 | -------------------------------------------------------------------------------- /image_classification/scripts/resnet50v1-imagenet-fp32.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --gpu 0,1,2,3,4,5,6,7 --batch-size 512 --num-epochs 90 --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --disp-batches 100 --network resnet-v1 --num-layers 50 --data-nthreads 40 --min-random-scale 0.533 --lr-step-epochs pow2 --max-random-shear-ratio 0.1 --max-random-aspect-ratio 0.25 --max-random-rotate-angle 10 --max-random-h 0 --max-random-l 0 --max-random-s 0 --dtype float32 --lr 0.25 --optimizer nag -------------------------------------------------------------------------------- /image_classification/scripts/mxnet_1_1_resnet50v1-imagenet-fp32.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --gpu 0,1,2,3,4,5,6,7 --batch-size 512 --num-epochs 90 --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --disp-batches 100 --network resnet-v1 --num-layers 50 --data-nthreads 40 --min-random-scale 0.533 --lr-step-epochs 30,60,80 --max-random-shear-ratio 0.1 --max-random-aspect-ratio 0.25 --max-random-rotate-angle 10 --max-random-h 0 --max-random-l 0 --max-random-s 0 --dtype float32 --lr 0.25 --optimizer nag -------------------------------------------------------------------------------- /image_classification/scripts/resnet50v1-imagenet-fp16.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --gpu 0,1,2,3,4,5,6,7 --batch-size 512 --num-epochs 90 --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --disp-batches 100 --network resnet-v1 --dtype float16 --num-layers 50 --data-nthreads 40 --min-random-scale 0.533 --lr-step-epochs pow2 --max-random-shear-ratio 0.1 --max-random-aspect-ratio 0.25 --max-random-rotate-angle 10 --max-random-h 0 --max-random-l 0 --max-random-s 0 --dtype float32 --lr 0.25 --optimizer nag -------------------------------------------------------------------------------- /image_classification/symbols/README.md: -------------------------------------------------------------------------------- 1 | # Symbol 2 | 3 | This fold contains definition of various networks. To add a new network, please 4 | use the following format. 5 | 6 | ## Python 7 | 8 | - A file implements one network proposed in a paper, with the network name as the 9 | filename. 10 | - Mention the paper and the modifications made if any at the beginning 11 | of the file. 12 | - Indicate how to reproduce the accuracy numbers in the paper if it is not straightforward 13 | - Provide a function `get_symbol()` that return the network 14 | -------------------------------------------------------------------------------- /image_classification/scripts/mxnet_1_1_resnet50v1-imagenet-fp16.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --gpu 0,1,2,3,4,5,6,7 --batch-size 512 --num-epochs 90 --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --disp-batches 100 --network resnet-v1 --dtype float16 --num-layers 50 --data-nthreads 40 --min-random-scale 0.533 --lr-step-epochs 30,60,80 --max-random-shear-ratio 0.1 --max-random-aspect-ratio 0.25 --max-random-rotate-angle 10 --max-random-h 0 --max-random-l 0 --max-random-s 0 --dtype float32 --lr 0.25 --optimizer nag -------------------------------------------------------------------------------- /ssd/get_resnet_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | 6 | data_path=/tmp/ 7 | 8 | image_path=$data_path/images/ 9 | 10 | if [ ! -d "$data_path" ]; then 11 | mkdir -p "$data_path" 12 | fi 13 | 14 | if [ ! -d "$image_path" ]; then 15 | mkdir -p "$image_path" 16 | fi 17 | 18 | if [ ! -f "$data_path" ]; then 19 | wget https://s3.amazonaws.com/model-server/model_archive_1.0/onnx-resnet50v1.mar -P $data_path 20 | unzip /tmp/onnx-resnet50v1.mar -d /tmp/onnx-resnet50v1 21 | cd $image_path 22 | wget https://s3.amazonaws.com/model-server/inputs/kitten.jpg -O kitten.jpg 23 | fi 24 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/get_resnet_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | 6 | data_path=/tmp/ 7 | 8 | image_path=$data_path/images/ 9 | 10 | if [ ! -d "$data_path" ]; then 11 | mkdir -p "$data_path" 12 | fi 13 | 14 | if [ ! -d "$image_path" ]; then 15 | mkdir -p "$image_path" 16 | fi 17 | 18 | if [ ! -f "$data_path" ]; then 19 | wget https://s3.amazonaws.com/model-server/model_archive_1.0/onnx-resnet50v1.mar -P $data_path 20 | unzip /tmp/onnx-resnet50v1.mar -d /tmp/onnx-resnet50v1 21 | cd $image_path 22 | wget https://s3.amazonaws.com/model-server/inputs/kitten.jpg -O kitten.jpg 23 | fi 24 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/get_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 6 | 7 | if [[ $1 = e2e ]] 8 | then 9 | model_url="https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models" 10 | model_name="resnet18_v1_end_to_end" 11 | else 12 | model_url="https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models" 13 | model_name="resnet18_v1" 14 | fi 15 | model_path=models/ 16 | if [ ! -d "$model_path" ]; then 17 | mkdir -p "$model_path" 18 | fi 19 | 20 | if [ ! -f "$model_path" ]; then 21 | wget "$model_url/$model_name-symbol.json" -P $model_path 22 | wget "$model_url/$model_name-0000.params" -P $model_path 23 | fi 24 | -------------------------------------------------------------------------------- /question_answering/scripts/run_test_1gpu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | git clone https://github.com/Ishitori/gluon-nlp 5 | cd gluon-nlp/ 6 | git checkout bidaf_assembled 7 | 8 | pip3 install gluonnlp --pre --user 9 | pip3 install nltk --user 10 | python3 -m nltk.downloader punkt 11 | 12 | python3 -m scripts.question_answering.train_question_answering --train --preprocessed_dataset_path scripts/question_answering/preprocessed_dataset_q30_nltk.p --save_dir scripts/question_answering/output_1gpu/ --word_vocab_path scripts/question_answering/squad_word_vocab.p --char_vocab_path scripts/question_answering/squad_char_vocab.p --gpu 0 --early_stop 0 --log_interval 0 --terminate_training_on_reaching_F1_threshold 75 13 | 14 | cd .. 15 | rm -rf gluon-nlp/ 16 | -------------------------------------------------------------------------------- /question_answering/scripts/run_test_multigpu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exuo pipefail 3 | 4 | git clone https://github.com/Ishitori/gluon-nlp 5 | cd gluon-nlp/ 6 | git checkout bidaf_assembled 7 | 8 | pip3 install gluonnlp --pre --user 9 | pip3 install nltk --user 10 | python3 -m nltk.downloader punkt 11 | 12 | python3 -m scripts.question_answering.train_question_answering --train --evaluate --preprocessed_dataset_path scripts/question_answering/preprocessed_dataset_q30_nltk.p --save_dir scripts/question_answering/output_1gpu/ --word_vocab_path scripts/question_answering/squad_word_vocab.p --char_vocab_path scripts/question_answering/squad_char_vocab.p --lr 4 --batch_size 240 --gpu 0,1,2,3 --early_stop 0 --log_interval 0 13 | 14 | cd .. 15 | rm -rf gluon-nlp/ 16 | -------------------------------------------------------------------------------- /benchmark/tensorflow/killall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while [[ $# -gt 1 ]] 4 | do 5 | key="$1" 6 | 7 | case $key in 8 | -h|--hosts_file) 9 | HOSTS_FILE="$2" 10 | shift 11 | ;; 12 | *) 13 | # unknown option 14 | ;; 15 | esac 16 | shift # past argument or value 17 | done 18 | 19 | cat $HOSTS_FILE | 20 | while read line; do 21 | 22 | tuple=( $line ) 23 | ssh_alias=${tuple[1]} 24 | 25 | ssh -o "StrictHostKeyChecking no" -n $ssh_alias "ps -ef | grep 'ps_hosts' | grep -v grep | sed 's/ \+/ /g' | cut -d ' ' -f 2 | xargs kill -9" > /dev/null 2>&1 26 | ssh -o "StrictHostKeyChecking no" -n $ssh_alias "ps -ef | grep tail | grep worker | grep -v grep | sed 's/ \+/ /g' | cut -d ' ' -f 2 | xargs kill -9" > /dev/null 2>&1 27 | done 28 | -------------------------------------------------------------------------------- /tensorflow/resnet/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = [":internal"]) 2 | 3 | licenses(["notice"]) # Apache 2.0 4 | 5 | exports_files(["LICENSE"]) 6 | 7 | package_group( 8 | name = "internal", 9 | packages = [ 10 | "//resnet/...", 11 | ], 12 | ) 13 | 14 | filegroup( 15 | name = "py_srcs", 16 | data = glob([ 17 | "**/*.py", 18 | ]), 19 | ) 20 | 21 | py_library( 22 | name = "resnet_model", 23 | srcs = ["resnet_model.py"], 24 | ) 25 | 26 | py_binary( 27 | name = "resnet_main", 28 | srcs = [ 29 | "resnet_main.py", 30 | ], 31 | deps = [ 32 | ":cifar_input", 33 | ":resnet_model", 34 | ], 35 | ) 36 | 37 | py_library( 38 | name = "cifar_input", 39 | srcs = ["cifar_input.py"], 40 | ) 41 | -------------------------------------------------------------------------------- /benchmark/tensorflow/runincluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while [[ $# -gt 1 ]] 4 | do 5 | key="$1" 6 | 7 | case $key in 8 | -h|--hosts_file) 9 | NODES_FILE="$2" 10 | shift 11 | ;; 12 | -n|--num_nodes) 13 | NUM_NODES="$2" 14 | shift 15 | ;; 16 | -c|--command) 17 | COMMAND="$2" 18 | shift 19 | ;; 20 | *) 21 | # unknown option 22 | ;; 23 | esac 24 | shift # past argument or value 25 | done 26 | 27 | if [ -z "$NUM_NODES" ]; then 28 | NUM_NODES=`cat $NODES_FILE | grep -v '^$' | wc -l | xargs` 29 | fi 30 | 31 | head -$NUM_NODES $NODES_FILE | 32 | while read line; do 33 | 34 | tuple=( $line ) 35 | ssh_alias=${tuple[1]} 36 | 37 | ssh -o "StrictHostKeyChecking no" -n $ssh_alias ${COMMAND} 38 | 39 | done 40 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/get_resnet18_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | 6 | data_path=/tmp/resnet-18/ 7 | 8 | image_path=$data_path/images/ 9 | 10 | if [ ! -d "$data_path" ]; then 11 | mkdir -p "$data_path" 12 | fi 13 | 14 | if [ ! -d "$image_path" ]; then 15 | mkdir -p "$image_path" 16 | fi 17 | 18 | if [ ! -f "$data_path" ]; then 19 | wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/resnet-18-symbol.json -P $data_path 20 | wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/resnet-18-0000.params -P $data_path 21 | wget https://s3.us-east-2.amazonaws.com/scala-infer-models/resnet-18/synset.txt -P $data_path 22 | wget https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/resnet152/kitten.jpg -P $image_path 23 | fi 24 | -------------------------------------------------------------------------------- /scala-mxnet/java-bm/bin/get_resnet50_ssd_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | 6 | data_path=/tmp/resnet50_ssd/ 7 | 8 | image_path=$data_path/images/ 9 | 10 | if [ ! -d "$data_path" ]; then 11 | mkdir -p "$data_path" 12 | fi 13 | 14 | if [ ! -d "$image_path" ]; then 15 | mkdir -p "$image_path" 16 | fi 17 | 18 | if [ ! -f "$data_path" ]; then 19 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json -P $data_path 20 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params -P $data_path 21 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/synset.txt -P $data_path 22 | cd $image_path 23 | wget https://cloud.githubusercontent.com/assets/3307514/20012567/cbb60336-a27d-11e6-93ff-cbc3f09f5c9e.jpg -O dog.jpg 24 | fi 25 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/get_resnet50_ssd_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | 6 | data_path=/tmp/resnet50_ssd/ 7 | 8 | image_path=$data_path/images/ 9 | 10 | if [ ! -d "$data_path" ]; then 11 | mkdir -p "$data_path" 12 | fi 13 | 14 | if [ ! -d "$image_path" ]; then 15 | mkdir -p "$image_path" 16 | fi 17 | 18 | if [ ! -f "$data_path" ]; then 19 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json -P $data_path 20 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params -P $data_path 21 | wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/synset.txt -P $data_path 22 | cd $image_path 23 | wget https://cloud.githubusercontent.com/assets/3307514/20012567/cbb60336-a27d-11e6-93ff-cbc3f09f5c9e.jpg -O dog.jpg 24 | fi 25 | -------------------------------------------------------------------------------- /reporting/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /image_classification/scripts/test.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 35 --num-epochs 1 --dtype float16 --min-random-scale 0.533 --lr 0.1 --lr-step-epochs 30,60,90 --model-prefix v2fp16/ --save-final-model-only --max-random-scale 1 --max-random-shear-ratio 0.1 --max-random-rotate-angle 10 --max-random-aspect-ratio 0.25 --optimizer nag 2 | python image_classification/train_imagenet.py --data-train ~/data/train-256px-q95.rec --data-val ~/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 35 --num-epochs 2 --dtype float16 --lr 0.1 --lr-step-epochs 30,60,90 --model-prefix v2fp16/ --save-final-model-only --optimizer nag --max-random-h 0 --max-random-s 0 --max-random-l 0 --max-random-rotate-angle 0 --max-random-shear-ratio 0 --max-random-aspect-ratio 0 --load-epoch 1 -------------------------------------------------------------------------------- /reporting/scripts/update_lambda.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Updates the Lambda code in AWS to generate the report daily. 3 | # Ensure AWS environment variables 4 | # The AWS CLI requires the following environment variables to be set accordingly. 5 | # * AWS_ACCESS_KEY_ID 6 | # * AWS_SECRET_ACCESS_KEY 7 | # * AWS_DEFAULT_REGION 8 | # Run this from the main directory, e.g. scripts/update_lambda.sh 9 | 10 | set -exuo pipefail 11 | 12 | # Create a temporary directory and copy the code to that directory. 13 | TMP_DIR=$(mktemp -d) 14 | cp -r . ${TMP_DIR} 15 | 16 | # Prepare the zip file. 17 | pushd ${TMP_DIR} 18 | pip3 install -r requirements.txt -t ./ 19 | ZIP_PACKAGE='bai-report-lambda.zip' 20 | zip -r ${ZIP_PACKAGE} . 21 | 22 | # Upload the zip 23 | echo "Updating Lambda with package: ${TMP_DIR}/${ZIP_PACKAGE}" 24 | aws lambda update-function-code --function-name benchmark-ai-report --zip fileb://${ZIP_PACKAGE} 25 | popd 26 | 27 | -------------------------------------------------------------------------------- /scala-mxnet/java-bm/bin/run_ssd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | echo $OSTYPE 6 | 7 | hw_type=cpu 8 | if [[ $1 = gpu ]] 9 | then 10 | hw_type=gpu 11 | fi 12 | 13 | 14 | SCALA_VERSION_PROFILE=2.11 15 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 16 | 17 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 18 | 19 | echo $CURR_DIR 20 | 21 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$hw_type -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 22 | 23 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 24 | 25 | CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* 26 | 27 | # model dir 28 | MODEL_PATH_PREFIX=$2 29 | # input image 30 | INPUT_IMG=$3 31 | 32 | BATCHSIZE=$4 33 | 34 | RUNS=$5 35 | 36 | java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASSPATH mxnet.SSDObjectDetection \ 37 | --modelPathPrefix $MODEL_PATH_PREFIX \ 38 | --inputImagePath $INPUT_IMG \ 39 | --batchSize $BATCHSIZE \ 40 | --times $RUNS 41 | -------------------------------------------------------------------------------- /image_classification/scripts/resnet50v2-imagenet-fp16.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --data-train ~/data/train-480px-q95.rec --data-val ~/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 35 --num-epochs 85 --dtype float16 --min-random-scale 0.533 --lr 0.1 --lr-step-epochs 30,60 --model-prefix v2fp16/ --save-final-model-only --max-random-scale 1 --max-random-shear-ratio 0.1 --max-random-rotate-angle 10 --max-random-aspect-ratio 0.25 --optimizer nag 2 | python image_classification/train_imagenet.py --data-train ~/data/train-256px-q95.rec --data-val ~/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 35 --num-epochs 90 --dtype float16 --lr 0.1 --lr-step-epochs 30,60 --model-prefix v2fp16/ --save-final-model-only --optimizer nag --max-random-h 0 --max-random-s 0 --max-random-l 0 --max-random-rotate-angle 0 --max-random-shear-ratio 0 --max-random-aspect-ratio 0 --load-epoch 85 3 | # above gets to 75% with 512 and 0.1 4 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/run_ic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | echo $OSTYPE 6 | 7 | hw_type=cpu 8 | if [[ $1 = gpu ]] 9 | then 10 | hw_type=gpu 11 | fi 12 | 13 | SCALA_VERSION_PROFILE=2.11 14 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 15 | 16 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 17 | 18 | echo $CURR_DIR 19 | 20 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$hw_type -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 21 | 22 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 23 | 24 | CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* 25 | 26 | # model dir 27 | MODEL_PATH_PREFIX=$2 28 | # input image 29 | INPUT_IMG=$3 30 | 31 | BATCHSIZE=$4 32 | 33 | RUNS=$5 34 | 35 | java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASSPATH mxnet.ImageClassification \ 36 | --modelPathPrefix $MODEL_PATH_PREFIX \ 37 | --inputImagePath $INPUT_IMG \ 38 | --batchSize $BATCHSIZE \ 39 | --times $RUNS \ 40 | --context $hw_type 41 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/run_ssd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | echo $OSTYPE 6 | 7 | hw_type=cpu 8 | if [[ $1 = gpu ]] 9 | then 10 | hw_type=gpu 11 | fi 12 | 13 | 14 | SCALA_VERSION_PROFILE=2.11 15 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 16 | 17 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 18 | 19 | echo $CURR_DIR 20 | 21 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$hw_type -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 22 | 23 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 24 | 25 | CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* 26 | 27 | # model dir 28 | MODEL_PATH_PREFIX=$2 29 | # input image 30 | INPUT_IMG=$3 31 | 32 | BATCHSIZE=$4 33 | 34 | RUNS=$5 35 | 36 | java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASSPATH mxnet.SSDObjectDetection \ 37 | --modelPathPrefix $MODEL_PATH_PREFIX \ 38 | --inputImagePath $INPUT_IMG \ 39 | --batchSize $BATCHSIZE \ 40 | --times $RUNS \ 41 | --context $hw_type 42 | -------------------------------------------------------------------------------- /benchmark/tensorflow/runall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while [[ $# -gt 1 ]] 4 | do 5 | key="$1" 6 | 7 | case $key in 8 | -m|--models) 9 | MODELS="$2" 10 | shift # past argument 11 | ;; 12 | -h|--hosts_list) 13 | HOSTS_LIST="$2" 14 | shift # past argument 15 | ;; 16 | -g|--gpu_per_host) 17 | GPU_PER_HOST="$2" 18 | shift # past argument 19 | ;; 20 | -r|--remote_dir) 21 | REMOTE_DIR="$2" 22 | shift # past argument 23 | ;; 24 | -x|--max_gpus) 25 | MAX_GPUS="$2" 26 | shift # past argument 27 | ;; 28 | *) 29 | # unknown option 30 | ;; 31 | esac 32 | shift # past argument or value 33 | done 34 | 35 | for model in `echo $MODELS | tr ',' ' '`; do 36 | model=`echo $model | tr ':' ' ' | tr '[:upper:]' '[:lower:]'` 37 | arr=( $model ) 38 | model_name=${arr[0]} 39 | batch_size=${arr[1]} 40 | 41 | bash runscalabilitytest.sh -h $HOSTS_LIST -m $model_name -g $GPU_PER_HOST -b $batch_size -r $REMOTE_DIR -x $MAX_GPUS 42 | done 43 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/bin/run_charrnn_example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | echo $OSTYPE 6 | 7 | hw_type=cpu 8 | if [[ $1 = gpu ]] 9 | then 10 | hw_type=gpu 11 | fi 12 | 13 | SCALA_VERSION_PROFILE=2.11 14 | MXNET_VERSION="[1.5.0-SNAPSHOT,)" 15 | 16 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 17 | 18 | echo $CURR_DIR 19 | 20 | ./mvnw clean install dependency:copy-dependencies package -Dmxnet.hw_type=$hw_type -Dmxnet.scalaprofile=$SCALA_VERSION_PROFILE -Dmxnet.version=$MXNET_VERSION 21 | 22 | CURR_DIR=$(cd $(dirname $0)/../; pwd) 23 | 24 | CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/* 25 | 26 | # model dir 27 | MODEL_PATH_PREFIX=$2 28 | # input image 29 | DATA_PATH=$3 30 | 31 | #Starter Sentence 32 | STARTER_SENTENCE=$4 33 | 34 | RUNS=$5 35 | 36 | java -Xmx8G -Dmxnet.traceLeakedObjects=true -cp $CLASSPATH mxnet.CharRnnExample \ 37 | --modelPathPrefix $MODEL_PATH_PREFIX \ 38 | --data-path $DATA_PATH \ 39 | --starter-sentence "$STARTER_SENTENCE" \ 40 | --times $RUNS \ 41 | --context $hw_type 42 | -------------------------------------------------------------------------------- /test/test_benchmark_compute_method.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append("..") 3 | 4 | from benchmark_driver import BenchmarkMetricComputeMethod 5 | import unittest 6 | 7 | 8 | class MetricComputeTest(unittest.TestCase): 9 | def test_metric_compute_average(self): 10 | metric = [1, 2, 3] 11 | metric_compute_method = 'average' 12 | self.assertEqual(BenchmarkMetricComputeMethod.compute(metric_compute_method, metric), 2) 13 | 14 | def test_metric_compute_total(self): 15 | metric = [1, 2, 3] 16 | metric_compute_method = 'total' 17 | self.assertEqual(BenchmarkMetricComputeMethod.compute(metric_compute_method, metric), 6) 18 | 19 | def test_metric_compute_last(self): 20 | metric = [1, 2, 3] 21 | metric_compute_method = 'last' 22 | self.assertEqual(BenchmarkMetricComputeMethod.compute(metric_compute_method, metric), 3) 23 | 24 | 25 | if __name__ == '__main__': 26 | suite = unittest.TestLoader().loadTestsFromTestCase(MetricComputeTest) 27 | unittest.TextTestRunner(verbosity=2).run(suite) -------------------------------------------------------------------------------- /image_classification/common/find_mxnet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os, sys 19 | try: 20 | import mxnet as mx 21 | except ImportError: 22 | curr_path = os.path.abspath(os.path.dirname(__file__)) 23 | sys.path.append(os.path.join(curr_path, "../../../python")) 24 | import mxnet as mx 25 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/slim/slim.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """TF-Slim grouped API. Please see README.md for details and usage.""" 16 | # pylint: disable=unused-import 17 | 18 | # Collapse tf-slim into a single namespace. 19 | from inception.slim import inception_model as inception 20 | from inception.slim import losses 21 | from inception.slim import ops 22 | from inception.slim import scopes 23 | from inception.slim import variables 24 | from inception.slim.scopes import arg_scope 25 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/trivial_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Trivial model configuration.""" 16 | 17 | from models import model 18 | 19 | 20 | class TrivialModel(model.Model): 21 | """Trivial model configuration.""" 22 | 23 | def __init__(self): 24 | super(TrivialModel, self).__init__('trivial', 224 + 3, 32, 0.005) 25 | 26 | def add_inference(self, cnn): 27 | cnn.reshape([-1, 227 * 227 * 3]) 28 | cnn.affine(1) 29 | cnn.affine(4096) 30 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | log4j.rootLogger = debug, stdout 19 | 20 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 21 | log4j.appender.stdout.Target = System.out 22 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 23 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c] [%p] - %m%n 24 | -------------------------------------------------------------------------------- /benchmark/tensorflow/tabulate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ADD_IDEAL=0 4 | 5 | while [[ $# -gt 1 ]] 6 | do 7 | key="$1" 8 | 9 | case $key in 10 | -m|--model) 11 | MODEL="$2" 12 | shift # past argument 13 | ;; 14 | -b|--batch_size) 15 | BATCH_SIZE="$2" 16 | shift # past argument 17 | ;; 18 | -l|--log_dir) 19 | LOG_DIR="$2" 20 | shift # past argument 21 | ;; 22 | -g|--gpu_list) 23 | GPU_LIST="$2" 24 | shift # past argument 25 | ;; 26 | -f|--table_file) 27 | TABLE_FILE="$2" 28 | shift # past argument 29 | ;; 30 | -i|--add_ideal) 31 | ADD_IDEAL=1 32 | shift # past argument 33 | ;; 34 | *) 35 | # unknown option 36 | ;; 37 | esac 38 | shift # past argument or value 39 | done 40 | 41 | echo "Writing table to $TABLE_FILE" 42 | IDEAL_FILE=$TABLE_FILE.ideal 43 | rm -f $TABLE_FILE 44 | rm -f $IDEAL_FILE 45 | 46 | images_one_gpu=`cat ${LOG_DIR}/${MODEL}_b${BATCH_SIZE}_g1/imagespersec` 47 | 48 | for gpu in $(echo $GPU_LIST | sed "s/,/ /g") 49 | do 50 | images_per_gpu=`cat ${LOG_DIR}/${MODEL}_b${BATCH_SIZE}_g${gpu}/imagespersec` 51 | images_per_cluster=`echo $images_per_gpu*$gpu | bc` 52 | echo ${gpu},${images_per_cluster} >> $TABLE_FILE 53 | ideal=`echo $images_one_gpu*$gpu | bc` 54 | echo ${gpu},$ideal >> $IDEAL_FILE 55 | done 56 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/lenet_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Lenet model configuration. 17 | 18 | References: 19 | LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner 20 | Gradient-based learning applied to document recognition 21 | Proceedings of the IEEE (1998) 22 | """ 23 | 24 | from models import model 25 | 26 | 27 | class Lenet5Model(model.Model): 28 | 29 | def __init__(self): 30 | super(Lenet5Model, self).__init__('lenet5', 28, 32, 0.005) 31 | 32 | def add_inference(self, cnn): 33 | # Note: This matches TF's MNIST tutorial model 34 | cnn.conv(32, 5, 5) 35 | cnn.mpool(2, 2) 36 | cnn.conv(64, 5, 5) 37 | cnn.mpool(2, 2) 38 | cnn.reshape([-1, 64 * 7 * 7]) 39 | cnn.affine(512) 40 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/flowers_train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to train Inception on the flowers data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | import tensorflow as tf 24 | 25 | from inception import inception_train 26 | from inception.flowers_data import FlowersData 27 | 28 | FLAGS = tf.app.flags.FLAGS 29 | 30 | 31 | def main(_): 32 | dataset = FlowersData(subset=FLAGS.subset) 33 | assert dataset.data_files() 34 | if tf.gfile.Exists(FLAGS.train_dir): 35 | tf.gfile.DeleteRecursively(FLAGS.train_dir) 36 | tf.gfile.MakeDirs(FLAGS.train_dir) 37 | inception_train.train(dataset) 38 | 39 | 40 | if __name__ == '__main__': 41 | tf.app.run() 42 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/imagenet_train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to train Inception on the ImageNet data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | import tensorflow as tf 24 | 25 | from inception import inception_train 26 | from inception.imagenet_data import ImagenetData 27 | 28 | FLAGS = tf.app.flags.FLAGS 29 | 30 | 31 | def main(_): 32 | dataset = ImagenetData(subset=FLAGS.subset) 33 | #assert dataset.data_files() 34 | if tf.gfile.Exists(FLAGS.train_dir): 35 | tf.gfile.DeleteRecursively(FLAGS.train_dir) 36 | tf.gfile.MakeDirs(FLAGS.train_dir) 37 | inception_train.train(dataset) 38 | 39 | 40 | if __name__ == '__main__': 41 | tf.app.run() 42 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/flowers_eval.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to evaluate Inception on the flowers data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | import tensorflow as tf 23 | 24 | from inception import inception_eval 25 | from inception.flowers_data import FlowersData 26 | 27 | FLAGS = tf.app.flags.FLAGS 28 | 29 | 30 | def main(unused_argv=None): 31 | dataset = FlowersData(subset=FLAGS.subset) 32 | assert dataset.data_files() 33 | if tf.gfile.Exists(FLAGS.eval_dir): 34 | tf.gfile.DeleteRecursively(FLAGS.eval_dir) 35 | tf.gfile.MakeDirs(FLAGS.eval_dir) 36 | inception_eval.evaluate(dataset) 37 | 38 | 39 | if __name__ == '__main__': 40 | tf.app.run() 41 | -------------------------------------------------------------------------------- /word_language_model/get_ptb_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | 21 | RNN_DIR=$(cd `dirname $0`; pwd) 22 | DATA_DIR="${RNN_DIR}/data/" 23 | 24 | if [[ ! -d "${DATA_DIR}" ]]; then 25 | echo "${DATA_DIR} doesn't exist, will create one"; 26 | mkdir -p ${DATA_DIR} 27 | fi 28 | 29 | wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.train.txt; 30 | wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.valid.txt; 31 | wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.test.txt; 32 | wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt; 33 | -------------------------------------------------------------------------------- /test/test_cfg_process.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append("..") 3 | from utils import cfg_process 4 | import unittest 5 | import tempfile 6 | 7 | try: 8 | import ConfigParser 9 | config = ConfigParser.ConfigParser() 10 | except ImportError: 11 | import configparser 12 | config = configparser.ConfigParser() 13 | 14 | 15 | CONFIG_TEMPLATE_DIR = '../task_config_template.cfg' 16 | 17 | 18 | class ConfigTest(unittest.TestCase): 19 | def test_change_num_gpu_and_epochs(self): 20 | args = { 21 | "task_name": "resnet50_cifar10_symbolic", 22 | "num_gpus": 3, 23 | "epochs": 3, 24 | "metrics_postfix": "daily", 25 | } 26 | tmp = os.path.join(tempfile.gettempdir(), "task_config.cfg") 27 | cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, tmp, **args) 28 | config.read(tmp) 29 | self.assertEqual(int(config.get(args["task_name"], "num_gpus")), args["num_gpus"]) 30 | cmd = config.get(args["task_name"], "command_to_execute") 31 | self.assertEqual(cmd.strip(), "python image_classification/image_classification.py " 32 | "--model resnet50_v1 --dataset cifar10" 33 | " --mode symbolic --gpus {0} --epochs {1} " 34 | "--log-interval 50".format(args["num_gpus"], args["epochs"])) 35 | os.remove(tmp) 36 | 37 | if __name__ == '__main__': 38 | suite = unittest.TestLoader().loadTestsFromTestCase(ConfigTest) 39 | unittest.TextTestRunner(verbosity=2).run(suite) -------------------------------------------------------------------------------- /image_classification/symbols/mlp.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | a simple multilayer perceptron 20 | """ 21 | import mxnet as mx 22 | 23 | def get_symbol(num_classes=10, **kwargs): 24 | data = mx.symbol.Variable('data') 25 | data = mx.sym.Flatten(data=data) 26 | fc1 = mx.symbol.FullyConnected(data = data, name='fc1', num_hidden=128) 27 | act1 = mx.symbol.Activation(data = fc1, name='relu1', act_type="relu") 28 | fc2 = mx.symbol.FullyConnected(data = act1, name = 'fc2', num_hidden = 64) 29 | act2 = mx.symbol.Activation(data = fc2, name='relu2', act_type="relu") 30 | fc3 = mx.symbol.FullyConnected(data = act2, name='fc3', num_hidden=num_classes) 31 | mlp = mx.symbol.SoftmaxOutput(data = fc3, name = 'softmax') 32 | return mlp 33 | -------------------------------------------------------------------------------- /benchmark/tensorflow/plotgraph.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import getopt 4 | import csv 5 | import sys 6 | 7 | def main(argv): 8 | 9 | label_list = None 10 | csv_list = None 11 | 12 | try: 13 | opts, args = getopt.getopt(argv, "", ["labels=","csv=","file="]) 14 | except getopt.GetoptError: 15 | print("Incorrect args") 16 | sys.exit(2) 17 | 18 | for opt, arg in opts: 19 | if opt == "--labels": 20 | label_list = arg 21 | elif opt == "--csv": 22 | csv_list = arg 23 | elif opt == "--file": 24 | out_file = arg 25 | 26 | 27 | if(label_list == None or csv_list == None or out_file == None): 28 | print("Incorrect args") 29 | sys.exit(2) 30 | 31 | labels = label_list.split(",") 32 | map(str.strip, labels) 33 | 34 | csv_files = csv_list.split(",") 35 | map(str.strip, csv_files) 36 | 37 | index = 0 38 | for csv_file in csv_files: 39 | with open(csv_file, mode='r') as infile: 40 | xval = np.array([0]) 41 | yval = np.array([0.0]) 42 | reader = csv.reader(infile, delimiter=',') 43 | for row in reader: 44 | if(len(row) == 2): 45 | xval = np.append(xval, row[0]) 46 | yval = np.append(yval, row[1]) 47 | plt.plot(xval, yval, linestyle='-', marker='+', label=labels[index]) 48 | index += 1 49 | 50 | plt.xlabel("Number of GPUs") 51 | plt.ylabel("Images/sec") 52 | plt.legend(loc="upper left") 53 | 54 | plt.savefig(out_file) 55 | 56 | 57 | if __name__ == "__main__": 58 | main(sys.argv[1:]) 59 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/imagenet_eval.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A binary to evaluate Inception on the flowers data set. 16 | 17 | Note that using the supplied pre-trained inception checkpoint, the eval should 18 | achieve: 19 | precision @ 1 = 0.7874 recall @ 5 = 0.9436 [50000 examples] 20 | 21 | See the README.md for more details. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | 28 | import tensorflow as tf 29 | 30 | from inception import inception_eval 31 | from inception.imagenet_data import ImagenetData 32 | 33 | FLAGS = tf.app.flags.FLAGS 34 | 35 | 36 | def main(unused_argv=None): 37 | dataset = ImagenetData(subset=FLAGS.subset) 38 | assert dataset.data_files() 39 | if tf.gfile.Exists(FLAGS.eval_dir): 40 | tf.gfile.DeleteRecursively(FLAGS.eval_dir) 41 | tf.gfile.MakeDirs(FLAGS.eval_dir) 42 | inception_eval.evaluate(dataset) 43 | 44 | 45 | if __name__ == '__main__': 46 | tf.app.run() 47 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/overfeat_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Overfeat model configuration. 17 | 18 | References: 19 | OverFeat: Integrated Recognition, Localization and Detection using 20 | Convolutional Networks 21 | Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus, 22 | Yann LeCun, 2014 23 | http://arxiv.org/abs/1312.6229 24 | """ 25 | 26 | from models import model 27 | 28 | 29 | class OverfeatModel(model.Model): 30 | 31 | def __init__(self): 32 | super(OverfeatModel, self).__init__('overfeat', 231, 32, 0.005) 33 | 34 | def add_inference(self, cnn): 35 | # Note: VALID requires padding the images by 3 in width and height 36 | cnn.conv(96, 11, 11, 4, 4, mode='VALID') 37 | cnn.mpool(2, 2) 38 | cnn.conv(256, 5, 5, 1, 1, mode='VALID') 39 | cnn.mpool(2, 2) 40 | cnn.conv(512, 3, 3) 41 | cnn.conv(1024, 3, 3) 42 | cnn.conv(1024, 3, 3) 43 | cnn.mpool(2, 2) 44 | cnn.reshape([-1, 1024 * 6 * 6]) 45 | cnn.affine(3072) 46 | cnn.affine(4096) 47 | -------------------------------------------------------------------------------- /image_classification/scripts/resnet50v2-imagenet-fp32.sh: -------------------------------------------------------------------------------- 1 | python image_classification/train_imagenet.py --data-train ~/data/train-480px-q95.rec --data-val /home/ubuntu/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 35 --num-epochs 78 --dtype float32 --min-random-scale 0.533 --lr 0.1 --lr-step-epochs 30,60 --model-prefix v2fp32/ --save-final-model-only --max-random-scale 1 --max-random-shear-ratio 0.1 --max-random-rotate-angle 10 --max-random-aspect-ratio 0.25 --optimizer nag 2 | python image_classification/train_imagenet.py --data-train /home/ubuntu/data/train-256px-q95.rec --data-val /home/ubuntu/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --save-final-model-only --batch-size 512 --data-nthreads 35 --num-epochs 85 --dtype float32 --lr 0.1 --lr-step-epochs 30,60 --model-prefix v2fp32/ --optimizer nag --load-epoch 78 --max-random-h 0 --max-random-s 0 --max-random-l 0 --max-random-rotate-angle 0 --max-random-shear-ratio 0 --max-random-aspect-ratio 0 3 | #above works till 74.7 #30 4 | #python train_imagenet.py --data-train /home/ubuntu/data/imagenet_train.rec --data-val /home/ubuntu/data/val-256px-q95.rec --gpus 0,1,2,3,4,5,6,7 --batch-size 960 --data-nthreads 35 --num-epochs 80 --dtype float32 --min-random-scale 0.533 --lr 0.25 --lr-step-epochs 30,55,75 --model-prefix v2fp32/ --max-random-scale 1 --max-random-shear-ratio 0.1 --max-random-rotate-angle 10 --max-random-aspect-ratio 0.25 --optimizer nag 2>&1 | tee ~/efs/benchmark-imagenet-logs/v2fp32.log 5 | #python train_imagenet.py --data-train /home/ubuntu/data/train-256px-q95.rec --data-val /home/ubuntu/data/val-256px-q95.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 960 --data-nthreads 35 --num-epochs 120 --dtype float32 --lr 0.25 --lr-step-epochs 30,55,75 --model-prefix v2fp32/ --optimizer nag --load-epoch 80 ~//efs/benchmark-imagenet-logs/v2fp32-2.log 6 | 7 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/benchmark_storage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Provides ways to store benchmark output.""" 16 | 17 | 18 | def store_benchmark(data, storage_type=None): 19 | """Store benchmark data. 20 | 21 | Args: 22 | data: Dictionary mapping from string benchmark name to 23 | numeric benchmark value. 24 | storage_type: (string) Specifies where to store benchmark 25 | result. If storage_type is 26 | 'cbuild_benchmark_datastore': store outputs in our continuous 27 | build datastore. gcloud must be setup in current environment 28 | pointing to the project where data will be added. 29 | """ 30 | if storage_type == 'cbuild_benchmark_datastore': 31 | try: 32 | # pylint: disable=g-import-not-at-top 33 | import cbuild_benchmark_storage 34 | # pylint: enable=g-import-not-at-top 35 | except ImportError: 36 | raise ImportError( 37 | 'Missing cbuild_benchmark_storage.py required for ' 38 | 'benchmark_cloud_datastore option') 39 | cbuild_benchmark_storage.upload_to_benchmark_datastore(data) 40 | else: 41 | assert False, 'unknown storage_type: ' + storage_type 42 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Benchmark script for TensorFlow. 17 | 18 | See the README for more information. 19 | """ 20 | 21 | from __future__ import print_function 22 | 23 | 24 | import tensorflow as tf 25 | 26 | import benchmark_cnn 27 | import cnn_util 28 | from cnn_util import log_fn 29 | 30 | benchmark_cnn.define_flags() 31 | 32 | 33 | def main(extra_flags): 34 | # extra_flags is a list of command line arguments, excluding those defined 35 | # in tf.flags.FLAGS. extra_flags[0] is always the program name. It is an error 36 | # to supply flags not defined with tf.flags.FLAGS, so we raise an ValueError 37 | # in that case. 38 | assert len(extra_flags) >= 1 39 | if len(extra_flags) > 1: 40 | raise ValueError('Received unknown flags: %s' % extra_flags[1:]) 41 | 42 | params = benchmark_cnn.make_params_from_flags() 43 | benchmark_cnn.setup(params) 44 | bench = benchmark_cnn.BenchmarkCNN(params) 45 | 46 | tfversion = cnn_util.tensorflow_version_tuple() 47 | log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) 48 | 49 | bench.print_info() 50 | bench.run() 51 | 52 | 53 | if __name__ == '__main__': 54 | tf.app.run() 55 | -------------------------------------------------------------------------------- /benchmark/tensorflow/runscalabilitytest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while [[ $# -gt 1 ]] 4 | do 5 | key="$1" 6 | 7 | case $key in 8 | -h|--nodes_file) 9 | NODES_FILE="$2" 10 | shift # past argument 11 | ;; 12 | -m|--model) 13 | MODEL="$2" 14 | shift # past argument 15 | ;; 16 | -g|--gpu_per_machine) 17 | MAX_GPU_PER_MACHINE="$2" 18 | shift # past argument 19 | ;; 20 | -b|--batch_size) 21 | BATCH_SIZE="$2" 22 | shift # past argument 23 | ;; 24 | -r|--remote_dir) 25 | REMOTE_DIR="$2" 26 | shift # past argument 27 | ;; 28 | -x|--max_gpu) 29 | MAX_GPUS="$2" 30 | shift # past argument 31 | ;; 32 | *) 33 | # unknown option 34 | ;; 35 | esac 36 | shift # past argument or value 37 | done 38 | 39 | 40 | ngpu=1 41 | gpu_list="" 42 | while [ "$ngpu" -le "$MAX_GPUS" ]; do 43 | echo "Running with $ngpu GPUs" 44 | 45 | num_machines=$(($ngpu/$MAX_GPU_PER_MACHINE)) 46 | if (($ngpu % $MAX_GPU_PER_MACHINE)); then 47 | num_machines=$((num_machines + 1)) 48 | fi 49 | 50 | gpu_per_machine=$MAX_GPU_PER_MACHINE 51 | if ((ngpu < MAX_GPU_PER_MACHINE)); then 52 | gpu_per_machine=$ngpu 53 | fi 54 | 55 | bash runtest.sh -m $MODEL -h $NODES_FILE -r $REMOTE_DIR -n $num_machines -g $gpu_per_machine -b $BATCH_SIZE 56 | 57 | gpu_list=${gpu_list}${ngpu}, 58 | 59 | if [ "$ngpu" -eq "$MAX_GPUS" ] ; then 60 | break 61 | fi 62 | 63 | ngpu=$(($ngpu * 2)) 64 | ngpu=$(( $ngpu < $MAX_GPUS ? $ngpu : $MAX_GPUS )) 65 | done 66 | 67 | gpu_list=`echo $gpu_list | rev | cut -c 2- | rev` 68 | csv_file=logs/${MODEL}_b${BATCH_SIZE}.csv 69 | graph_file=logs/${MODEL}_b${BATCH_SIZE}.svg 70 | ideal_csv_file=$csv_file.ideal 71 | 72 | bash tabulate.sh -m $MODEL -b $BATCH_SIZE -l logs -g "$gpu_list" -f $csv_file -i 73 | 74 | #python plotgraph.py --labels="TensorFlow,Ideal" --csv="${csv_file},${ideal_csv_file}" --file=$graph_file 75 | 76 | -------------------------------------------------------------------------------- /utils/data_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def getImagenetData(dataset): 4 | if dataset == 'imagenet' or dataset == 'imagenet-256px-q90': 5 | os.system('mkdir -p data') 6 | if not os.path.exists('./data/imagenet1k-train.rec'): 7 | os.system("aws s3 cp s3://imagenet-rec/imagenet1k-train.rec data/") 8 | if not os.path.exists('./data/imagenet1k-val.rec'): 9 | os.system("aws s3 cp s3://imagenet-rec/imagenet1k-val.rec data/") 10 | elif dataset == 'imagenet-480px-q95': 11 | if not os.path.exists(os.path.expanduser('~/data/train-480px-q95.rec')): 12 | os.system("wget -q https://s3.amazonaws.com/aws-ml-platform-datasets/imagenet/480px-q95/train-480px-q95.rec -P ~/data/") 13 | if not os.path.exists(os.path.expanduser('~/data/val-256px-q95.rec')): 14 | os.system("wget -q https://s3.amazonaws.com/aws-ml-platform-datasets/imagenet/256px-q95/val-256px-q95.rec -P ~/data/") 15 | elif dataset == 'imagenet-480px-256px-q95': 16 | if not os.path.exists(os.path.expanduser('~/data/train-480px-q95.rec')): 17 | os.system("wget -q https://s3.amazonaws.com/aws-ml-platform-datasets/imagenet/480px-q95/train-480px-q95.rec -P ~/data/") 18 | if not os.path.exists(os.path.expanduser('~/data/val-256px-q95.rec')): 19 | os.system("wget -q https://s3.amazonaws.com/aws-ml-platform-datasets/imagenet/256px-q95/val-256px-q95.rec -P ~/data/") 20 | if not os.path.exists(os.path.expanduser('~/data/train-256px-q95.rec')): 21 | os.system("wget -q https://s3.amazonaws.com/aws-ml-platform-datasets/imagenet/256px-q95/train-256px-q95.rec -P ~/data/") 22 | elif dataset == 'imagenet-ebs': 23 | if not os.path.exists(os.path.expanduser('~/data/')): 24 | os.system('mkdir -p ~/data/') 25 | os.system("aws s3 sync s3://aws-ml-platform-datasets/imagenet/pass-through/ ~/data/ --exclude \"train*\"") 26 | 27 | else: 28 | raise ValueError('Unknown dataset') 29 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # tf_cnn_benchmarks: High performance benchmarks 2 | 3 | tf_cnn_benchmarks contains implementations of several popular convolutional 4 | models, and is designed to be as fast as possible. tf_cnn_benchmarks supports 5 | both running on a single machine or running in distributed mode across multiple 6 | hosts. See the [High-Performance models 7 | guide](https://www.tensorflow.org/performance/performance_models) for more 8 | information. 9 | 10 | These models utilize many of the strategies in the [TensorFlow Performance 11 | Guide](https://www.tensorflow.org/performance/performance_guide). Benchmark 12 | results can be found [here](https://www.tensorflow.org/performance/benchmarks). 13 | 14 | These models are designed for performance. For models that have clean and 15 | easy-to-read implementations, see the [TensorFlow Official 16 | Models](https://github.com/tensorflow/models/tree/master/official). 17 | 18 | ## Getting Started 19 | 20 | To run ResNet50 with synthetic data without distortions with a single GPU, run 21 | 22 | ``` 23 | python tf_cnn_benchmarks.py --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=parameter_server 24 | ``` 25 | 26 | Some important flags are 27 | 28 | * model: Model to use, e.g. resnet50, inception3, vgg16, and alexnet. 29 | * num_gpus: Number of GPUs to use. 30 | * data_dir: Path to data to process. If not set, synthetic data is used. To 31 | use Imagenet data use these 32 | [instructions](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) 33 | as a starting point. 34 | * batch_size: Batch size for each GPU. 35 | * variable_update: The method for managing variables: parameter_server 36 | ,replicated, distributed_replicated, independent 37 | * local_parameter_device: Device to use as parameter server: cpu or gpu. 38 | 39 | See 40 | [benchmark_cnn.py](https://github.com/tensorflow/benchmarks/blob/master/scripts/tf_cnn_benchmarks/benchmark_cnn.py) 41 | for the full list of flags. The `_DEFAULT_PARAMS` dict in that file contains the 42 | flags. 43 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/flowers_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to the flowers data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | from inception.dataset import Dataset 24 | 25 | 26 | class FlowersData(Dataset): 27 | """Flowers data set.""" 28 | 29 | def __init__(self, subset): 30 | super(FlowersData, self).__init__('Flowers', subset) 31 | 32 | def num_classes(self): 33 | """Returns the number of classes in the data set.""" 34 | return 5 35 | 36 | def num_examples_per_epoch(self): 37 | """Returns the number of examples in the data subset.""" 38 | if self.subset == 'train': 39 | return 3170 40 | if self.subset == 'validation': 41 | return 500 42 | 43 | def download_message(self): 44 | """Instruction to download and extract the tarball from Flowers website.""" 45 | 46 | print('Failed to find any Flowers %s files'% self.subset) 47 | print('') 48 | print('If you have already downloaded and processed the data, then make ' 49 | 'sure to set --data_dir to point to the directory containing the ' 50 | 'location of the sharded TFRecords.\n') 51 | print('Please see README.md for instructions on how to build ' 52 | 'the flowers dataset using download_and_preprocess_flowers.\n') 53 | -------------------------------------------------------------------------------- /DockerFiles/Dockerfile.gpu_mxnet: -------------------------------------------------------------------------------- 1 | # Use local version of image built from Dockerfile.gpu in /docker/1.6.0/base 2 | #MAINTAINER Amazon AI 3 | FROM nvidia/cuda:9.0-base-ubuntu16.04 4 | 5 | RUN apt-get update && apt-get install -y --no-install-recommends \ 6 | build-essential \ 7 | cuda-command-line-tools-9-0 \ 8 | cuda-cublas-dev-9-0 \ 9 | cuda-cudart-dev-9-0 \ 10 | cuda-cufft-dev-9-0 \ 11 | cuda-curand-dev-9-0 \ 12 | cuda-cusolver-dev-9-0 \ 13 | cuda-cusparse-dev-9-0 \ 14 | curl \ 15 | git \ 16 | libcudnn7=7.0.5.15-1+cuda9.0 \ 17 | libcudnn7-dev=7.0.5.15-1+cuda9.0 \ 18 | libcurl3-dev \ 19 | libfreetype6-dev \ 20 | libpng12-dev \ 21 | libzmq3-dev \ 22 | pkg-config \ 23 | python-dev \ 24 | rsync \ 25 | software-properties-common \ 26 | unzip \ 27 | zip \ 28 | zlib1g-dev \ 29 | wget \ 30 | vim \ 31 | nginx \ 32 | iputils-ping \ 33 | libjemalloc-dev \ 34 | && \ 35 | rm -rf /var/lib/apt/lists/* && \ 36 | find /usr/local/cuda-9.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \ 37 | rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a 38 | 39 | RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ 40 | python get-pip.py && \ 41 | rm get-pip.py 42 | 43 | RUN pip --no-cache-dir install \ 44 | numpy \ 45 | scipy \ 46 | sklearn \ 47 | pandas \ 48 | h5py \ 49 | psutil \ 50 | memory_profiler \ 51 | opencv-python \ 52 | boto3 \ 53 | awscli 54 | 55 | 56 | #RUN pip install numpy tensorflow-serving-api==1.5 57 | 58 | 59 | ARG framework_installable 60 | 61 | WORKDIR /root 62 | 63 | # Will install from pypi once packages are released there. For now, copy from local file system. 64 | RUN echo "Creating the new docker image" 65 | RUN framework_installable_local=$(basename $framework_installable) && \ 66 | \ 67 | pip install $framework_installable_local --pre && \ 68 | \ 69 | echo "DONE" 70 | 71 | 72 | -------------------------------------------------------------------------------- /scala-mxnet/java-bm/src/main/java/mxnet/Utils.java: -------------------------------------------------------------------------------- 1 | package mxnet; 2 | 3 | import org.apache.mxnet.javaapi.Context; 4 | import org.kohsuke.args4j.CmdLineParser; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | public class Utils { 11 | 12 | private static boolean runBatch = false; 13 | 14 | public static List getContext() { 15 | 16 | List context = new ArrayList(); 17 | if (System.getenv().containsKey("SCALA_TEST_ON_GPU") && 18 | Integer.parseInt(System.getenv("SCALA_TEST_ON_GPU")) == 1) { 19 | context.add(Context.gpu()); 20 | } else { 21 | context.add(Context.cpu()); 22 | } 23 | return context; 24 | } 25 | 26 | public static CmdLineParser parse(Object inst, String[] args) { 27 | CmdLineParser parser = new CmdLineParser(inst); 28 | try { 29 | parser.parseArgument(args); 30 | } catch (Exception e) { 31 | System.err.println(e.getMessage() + e); 32 | parser.printUsage(System.err); 33 | System.exit(1); 34 | } 35 | return parser; 36 | } 37 | 38 | private static double percentile(int p, double[] seq) { 39 | Arrays.sort(seq); 40 | int k = (int) Math.ceil((seq.length - 1) * (p / 100.0)); 41 | return seq[k]; 42 | } 43 | 44 | public static void printStatistics(double[] inferenceTimes, String metricsPrefix) { 45 | 46 | double[] times = inferenceTimes; 47 | double p50 = percentile(50, times); 48 | double p99 = percentile(99, times); 49 | double p90 = percentile(90, times); 50 | 51 | double sum = 0; 52 | 53 | for (double i : times) 54 | sum += i; 55 | double average = sum / (times.length * 1.0); 56 | 57 | System.out.printf("\n%s_p99 %1.2f, %s_p90 %1.2f, %s_p50 %1.2f, %s_average %1.2f\n", metricsPrefix, 58 | p99, metricsPrefix, p90, metricsPrefix, p50, metricsPrefix, average); 59 | 60 | } 61 | } -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/src/main/scala/mxnet/RnnModel.scala: -------------------------------------------------------------------------------- 1 | package mxnet 2 | 3 | import org.apache.mxnet.{Context, NDArray, Shape} 4 | 5 | object RnnModel { 6 | class LSTMInferenceModel(numLstmLayer: Int, inputSize: Int, numHidden: Int, 7 | numEmbed: Int, numLabel: Int, argParams: Map[String, NDArray], 8 | ctx: Context = Context.cpu(), dropout: Float = 0f) { 9 | private val sym = Lstm.lstmInferenceSymbol(numLstmLayer, 10 | inputSize, 11 | numHidden, 12 | numEmbed, 13 | numLabel, 14 | dropout) 15 | private val batchSize = 1 16 | private val initC = (for (l <- 0 until numLstmLayer) 17 | yield (s"l${l}_init_c_beta" -> Shape(batchSize, numHidden))).toMap 18 | private val initH = (for (l <- 0 until numLstmLayer) 19 | yield (s"l${l}_init_h_beta" -> Shape(batchSize, numHidden))).toMap 20 | private val dataShape = Map("data" -> Shape(batchSize)) 21 | private val inputShape = initC ++ initH ++ dataShape 22 | private val executor = sym.simpleBind(ctx = ctx, shapeDict = inputShape) 23 | 24 | for (key <- this.executor.argDict.keys) { 25 | if (!inputShape.contains(key) && argParams.contains(key) && key != "softmax_label") { 26 | argParams(key).copyTo(this.executor.argDict(key)) 27 | } 28 | } 29 | 30 | private var stateName = (Array[String]() /: (0 until numLstmLayer)) { (acc, i) => 31 | acc :+ s"l${i}_init_c_beta" :+ s"l${i}_init_h_beta" 32 | } 33 | 34 | private val statesDict = stateName.zip(this.executor.outputs.drop(1)).toMap 35 | private val inputArr = NDArray.zeros(dataShape("data")) 36 | 37 | def forward(inputData: NDArray, newSeq: Boolean = false): Array[Float] = { 38 | if (newSeq == true) { 39 | for (key <- this.statesDict.keys) { 40 | this.executor.argDict(key).set(0f) 41 | } 42 | } 43 | inputData.copyTo(this.executor.argDict("data")) 44 | this.executor.forward() 45 | for (key <- this.statesDict.keys) { 46 | this.statesDict(key).copyTo(this.executor.argDict(key)) 47 | } 48 | val prob = this.executor.outputs(0).toArray 49 | prob 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /image_classification/common/util.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import subprocess 19 | import os 20 | import errno 21 | 22 | def download_file(url, local_fname=None, force_write=False): 23 | # requests is not default installed 24 | import requests 25 | if local_fname is None: 26 | local_fname = url.split('/')[-1] 27 | if not force_write and os.path.exists(local_fname): 28 | return local_fname 29 | 30 | dir_name = os.path.dirname(local_fname) 31 | 32 | if dir_name != "": 33 | if not os.path.exists(dir_name): 34 | try: # try to create the directory if it doesn't exists 35 | os.makedirs(dir_name) 36 | except OSError as exc: 37 | if exc.errno != errno.EEXIST: 38 | raise 39 | 40 | r = requests.get(url, stream=True) 41 | assert r.status_code == 200, "failed to open %s" % url 42 | with open(local_fname, 'wb') as f: 43 | for chunk in r.iter_content(chunk_size=1024): 44 | if chunk: # filter out keep-alive new chunks 45 | f.write(chunk) 46 | return local_fname 47 | 48 | def get_gpus(): 49 | """ 50 | return a list of GPUs 51 | """ 52 | try: 53 | re = subprocess.check_output(["nvidia-smi", "-L"], universal_newlines=True) 54 | except OSError: 55 | return [] 56 | return range(len([i for i in re.split('\n') if 'GPU' in i])) 57 | -------------------------------------------------------------------------------- /mms/README.md: -------------------------------------------------------------------------------- 1 | # MXNet Model Server Benchmarking 2 | 3 | The benchmarks measure the performance of MMS on various models and benchmarks. 4 | It supports passed in a URL to the .mar file. 5 | It also runs various benchmarks using these models (see benchmarks section below). 6 | MMS is run on the same machine in a docker instance against MMS nightly docker image from docker hub. 7 | 8 | ## Installation 9 | 10 | The benchmarking script requires the following tools to run: 11 | - Docker-ce with the current user added to the docker group 12 | - Nvidia-docker (for GPU) 13 | - ab: Apache bench to run bench mark 14 | - bc: for metric percentile calculation 15 | - aws cli: for upload output to S3 bucket 16 | 17 | ## Benchmarks 18 | 19 | We support several basic benchmarks: 20 | - MMS throughput 21 | - MMS latency P50 22 | - MMS latency P90 23 | - MMS latency P99 24 | - MMS latency mean 25 | - MMS HTTP error rate 26 | - Model latency P50 27 | - Model latency P90 28 | - Model latency P99 29 | 30 | ## Usage 31 | The benchmarking script will automatically detect GPU/CPU based on docker runtime. 32 | If `nvidia-docker` is installed, the benchmark will be run against GPU instance. 33 | 34 | ## Examples 35 | 36 | Run benchmark test on resnet-50v1 model. 37 | It use kitten.jpg image as input from: https://s3.amazonaws.com/model-server/inputs/kitten.jpg 38 | ```bash 39 | ./benchmark.sh -u https://s3.amazonaws.com/model-server/model_archive_1.0/onnx-resnet50v1.mar 40 | ``` 41 | 42 | Run benchmark test on lstm_ptb model with json input 43 | ```bash 44 | benchmark.sh -i lstm.json -u https://s3.amazonaws.com/model-server/model_archive_1.0/lstm_ptb.mar 45 | ``` 46 | 47 | By default, the script will use 100 concurrency and run 1000 requests. to change concurrent: 48 | ```bash 49 | ./benchmark.sh -c 200 -n 2000 -u https://s3.amazonaws.com/model-server/model_archive_1.0/noop-v1.0.mar 50 | ``` 51 | 52 | You can pass `-s` parameter to upload results to S3: 53 | ```bash 54 | ./benchmark.sh -s -u https://s3.amazonaws.com/model-server/model_archive_1.0/noop-v1.0.mar 55 | ``` 56 | 57 | You can also choose your local docker image to run benchmark 58 | ```bash 59 | ./benchmark.sh -d mms-cpu-local -u https://s3.amazonaws.com/model-server/model_archive_1.0/noop-v1.0.mar 60 | ``` 61 | -------------------------------------------------------------------------------- /utils/cfg_process.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | try: 5 | import ConfigParser 6 | config = ConfigParser.ConfigParser() 7 | new_config = ConfigParser.RawConfigParser() 8 | except ImportError: 9 | import configparser 10 | config = configparser.ConfigParser() 11 | new_config = configparser.RawConfigParser() 12 | 13 | 14 | def generate_cfg(cfg_template, cfg_path, **infra_spec): 15 | """ Read infrastructure specific parameters and genreate a new task config 16 | 17 | :param: cfg_template string 18 | file path to the cfg template 19 | :param: cfg_path string 20 | file path of generated cfg file 21 | :param: infra_spec dict 22 | dictionary containing infrastructure specific parameters 23 | """ 24 | if os.path.isfile(cfg_path): 25 | os.remove(cfg_path) 26 | config.read(cfg_template) 27 | selected_task = infra_spec['task_name'] 28 | new_config.add_section(selected_task) 29 | for name, value in config.items(selected_task): 30 | if name != "num_gpus" and name != "command_to_execute": 31 | new_config.set(selected_task, name, config.get(selected_task, name)) 32 | elif "num_gpus" in infra_spec and name == "num_gpus": 33 | new_config.set(selected_task, name, infra_spec[name]) 34 | # check for overrides, if any 35 | elif name == "command_to_execute": 36 | cmd = config.get(selected_task, name) 37 | if "num_gpus" in infra_spec and infra_spec["num_gpus"] > 0: 38 | cmd = re.sub("--gpus \d", "--gpus %d" % infra_spec["num_gpus"], cmd) 39 | elif "num_gpus" in infra_spec: 40 | cmd = re.sub("--gpus \d", "", cmd) 41 | if "epochs" in infra_spec and infra_spec["epochs"] > 0: 42 | cmd = re.sub("--epochs \d+", "--epochs %d" % infra_spec["epochs"], cmd) 43 | if "kvstore" in infra_spec: 44 | cmd = re.sub("--kvstore device", "--kvstore %s" % infra_spec["kvstore"], cmd) 45 | if "dtype" in infra_spec: 46 | cmd = re.sub("--dtype float32", "--dtype %s" % infra_spec["dtype"], cmd) 47 | 48 | 49 | new_config.set(selected_task, name, cmd) 50 | with open(cfg_path, 'w') as cfg: 51 | new_config.write(cfg) 52 | -------------------------------------------------------------------------------- /image_classification/train_imagenet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | import argparse 20 | import logging 21 | logging.basicConfig(level=logging.DEBUG) 22 | from common import find_mxnet, data, fit 23 | from common.util import download_file 24 | import mxnet as mx 25 | 26 | if __name__ == '__main__': 27 | # parse args 28 | parser = argparse.ArgumentParser(description="train imagenet-1k", 29 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 30 | fit.add_fit_args(parser) 31 | data.add_data_args(parser) 32 | data.add_data_aug_args(parser) 33 | # use a large aug level 34 | data.set_data_aug_level(parser, 3) 35 | parser.set_defaults( 36 | # network 37 | network = 'resnet', 38 | num_layers = 50, 39 | # data 40 | num_classes = 1000, 41 | num_examples = 1281167, 42 | image_shape = '3,224,224', 43 | min_random_scale = 1, # if input image has min size k, suggest to use 44 | # 256.0/x, e.g. 0.533 for 480 45 | # train 46 | num_epochs = 80, 47 | lr_step_epochs = '30,60', 48 | dtype = 'float32' 49 | ) 50 | args = parser.parse_args() 51 | 52 | # load network 53 | from importlib import import_module 54 | net = import_module('symbols.'+args.network) 55 | sym = net.get_symbol(**vars(args)) 56 | 57 | # train 58 | fit.fit(args, sym, data.get_rec_iter) 59 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Base model configuration for CNN benchmarks.""" 16 | 17 | 18 | class Model(object): 19 | """Base model configuration for CNN benchmarks.""" 20 | 21 | def __init__(self, 22 | model, 23 | image_size, 24 | batch_size, 25 | learning_rate, 26 | layer_counts=None, 27 | fp16_loss_scale=128): 28 | self.model = model 29 | self.image_size = image_size 30 | self.batch_size = batch_size 31 | self.default_batch_size = batch_size 32 | self.learning_rate = learning_rate 33 | self.layer_counts = layer_counts 34 | # TODO(reedwm) Set custom loss scales for each model instead of using the 35 | # default of 128. 36 | self.fp16_loss_scale = fp16_loss_scale 37 | 38 | def get_model(self): 39 | return self.model 40 | 41 | def get_image_size(self): 42 | return self.image_size 43 | 44 | def get_batch_size(self): 45 | return self.batch_size 46 | 47 | def set_batch_size(self, batch_size): 48 | self.batch_size = batch_size 49 | 50 | def get_default_batch_size(self): 51 | return self.default_batch_size 52 | 53 | def get_layer_counts(self): 54 | return self.layer_counts 55 | 56 | def get_fp16_loss_scale(self): 57 | return self.fp16_loss_scale 58 | 59 | def get_learning_rate(self, global_step, batch_size): 60 | del global_step 61 | del batch_size 62 | return self.learning_rate 63 | 64 | def add_inference(self, unused_cnn): 65 | raise ValueError('Must be implemented in derived classes') 66 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/slim/BUILD: -------------------------------------------------------------------------------- 1 | # Description: 2 | # Contains the operations and nets for building TensorFlow-Slim models. 3 | 4 | package(default_visibility = ["//inception:internal"]) 5 | 6 | licenses(["notice"]) # Apache 2.0 7 | 8 | exports_files(["LICENSE"]) 9 | 10 | py_library( 11 | name = "scopes", 12 | srcs = ["scopes.py"], 13 | ) 14 | 15 | py_test( 16 | name = "scopes_test", 17 | size = "small", 18 | srcs = ["scopes_test.py"], 19 | deps = [ 20 | ":scopes", 21 | ], 22 | ) 23 | 24 | py_library( 25 | name = "variables", 26 | srcs = ["variables.py"], 27 | deps = [ 28 | ":scopes", 29 | ], 30 | ) 31 | 32 | py_test( 33 | name = "variables_test", 34 | size = "small", 35 | srcs = ["variables_test.py"], 36 | deps = [ 37 | ":variables", 38 | ], 39 | ) 40 | 41 | py_library( 42 | name = "losses", 43 | srcs = ["losses.py"], 44 | ) 45 | 46 | py_test( 47 | name = "losses_test", 48 | size = "small", 49 | srcs = ["losses_test.py"], 50 | deps = [ 51 | ":losses", 52 | ], 53 | ) 54 | 55 | py_library( 56 | name = "ops", 57 | srcs = ["ops.py"], 58 | deps = [ 59 | ":losses", 60 | ":scopes", 61 | ":variables", 62 | ], 63 | ) 64 | 65 | py_test( 66 | name = "ops_test", 67 | size = "small", 68 | srcs = ["ops_test.py"], 69 | deps = [ 70 | ":ops", 71 | ":variables", 72 | ], 73 | ) 74 | 75 | py_library( 76 | name = "inception", 77 | srcs = ["inception_model.py"], 78 | deps = [ 79 | ":ops", 80 | ":scopes", 81 | ], 82 | ) 83 | 84 | py_test( 85 | name = "inception_test", 86 | size = "medium", 87 | srcs = ["inception_test.py"], 88 | deps = [ 89 | ":inception", 90 | ], 91 | ) 92 | 93 | py_library( 94 | name = "slim", 95 | srcs = ["slim.py"], 96 | deps = [ 97 | ":inception", 98 | ":losses", 99 | ":ops", 100 | ":scopes", 101 | ":variables", 102 | ], 103 | ) 104 | 105 | py_test( 106 | name = "collections_test", 107 | size = "small", 108 | srcs = ["collections_test.py"], 109 | deps = [ 110 | ":slim", 111 | ], 112 | ) 113 | -------------------------------------------------------------------------------- /benchmark/plotgraph.py: -------------------------------------------------------------------------------- 1 | import pygal 2 | import numpy as np 3 | import sys 4 | import getopt 5 | import csv 6 | import math 7 | 8 | def main(argv): 9 | 10 | label_list = None 11 | csv_list = None 12 | 13 | try: 14 | opts, args = getopt.getopt(argv, "", ["labels=","csv=","file=","maxgpu="]) 15 | except getopt.GetoptError: 16 | print("Incorrect args") 17 | sys.exit(2) 18 | 19 | for opt, arg in opts: 20 | if opt == "--labels": 21 | label_list = arg 22 | elif opt == "--csv": 23 | csv_list = arg 24 | elif opt == "--file": 25 | out_file = arg 26 | elif opt == "--maxgpu": 27 | max_gpu = int(arg) 28 | 29 | if(label_list == None or csv_list == None or out_file == None or max_gpu == None): 30 | print("Incorrect args") 31 | sys.exit(2) 32 | 33 | labels = label_list.split(",") 34 | map(str.strip, labels) 35 | 36 | csv_files = csv_list.split(",") 37 | map(str.strip, csv_files) 38 | 39 | line_chart = pygal.Line(logarithmic=True, truncate_legend=100, legend_at_bottom=True) 40 | line_chart.title = "Deep Learning Frameworks - Performance Comparison" 41 | 42 | num_runs = math.ceil(math.log(max_gpu,2)) + 1 43 | x = np.arange(0,num_runs) 44 | x = np.power(2,x) 45 | x[-1] = max_gpu 46 | 47 | line_chart.x_labels = map(str, x.tolist()) 48 | 49 | # Add ideal plot 50 | ideal = np.copy(x) 51 | line_chart.add('Ideal', ideal.tolist() ) 52 | 53 | index = 0 54 | for csv_file in csv_files: 55 | with open(csv_file, mode='r') as infile: 56 | reader = csv.reader(infile, delimiter=',') 57 | baseline = 0 58 | yval = np.empty([0]) 59 | for row in reader: 60 | if(len(row) == 2): 61 | if baseline == 0: 62 | baseline = float(row[1]) 63 | yval = np.append(yval, float(row[1])/baseline) 64 | line_chart.add(labels[index], yval.tolist(), formatter= lambda speedup, images_per_gpu=baseline: 'Speedup: %0.2f, Images/Sec: %0.2f' % (speedup, images_per_gpu*speedup)) 65 | index += 1 66 | 67 | 68 | line_chart.render_to_file(out_file) 69 | 70 | 71 | if __name__ == "__main__": 72 | main(sys.argv[1:]) 73 | -------------------------------------------------------------------------------- /reporting/utils/email.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # -*- coding: utf-8 -*- 18 | """ 19 | Methods to send a Benchmark.AI e-mail report. 20 | """ 21 | import boto3 22 | import datetime 23 | import logging 24 | 25 | REPORT_EMAIL_FROM = 'benchmark-ai@amazon.com' 26 | REPORT_MAILLIST = 'benchmark-ai-reports@amazon.com' 27 | 28 | def email_report(report_html, email_addr=REPORT_MAILLIST, footnotes=''): 29 | """ 30 | Send a Benchmark.AI report to an e-mail address 31 | TODO(vishaalk): Validate the e-mail address format. 32 | 33 | Parameters 34 | ---------- 35 | report_html : str 36 | the report formatted in HTML 37 | email_addr : str 38 | the e-mail address to send the report to 39 | footnotes : str 40 | any additional information to append to the information (e.g. non-public links). This should 41 | be formatted with HTML tags. 42 | """ 43 | 44 | html = "{}

{}".format(report_html, footnotes) 45 | logging.info("Sending e-mail report.") 46 | ses = boto3.client('ses') 47 | response = ses.send_email( 48 | Source=REPORT_EMAIL_FROM, 49 | Destination={ 50 | 'ToAddresses': [email_addr] 51 | }, 52 | Message={ 53 | 'Subject': { 54 | 'Data': 'Benchmark.AI Report' 55 | }, 56 | 'Body': { 57 | 'Html': { 58 | 'Data': html 59 | } 60 | } 61 | } 62 | ) 63 | logging.info("Response: {}".format(response)) 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scalability Comparison Scripts for Deep Learning Frameworks 2 | 3 | This repository contains scripts that compares the scalability of deep learning frameworks. 4 | 5 | The scripts train Inception v3 and AlexNet using synchronous stochastic gradient descent (SGD). To run the comparison in reasonable time, we run few tens of iterations of SGD and compute the throughput as images processed per second. 6 | 7 | Comparisons can be done on clusters created with AWS CloudFormation using the Amazon Deep Learning AMI. 8 | 9 | ###To run comparisons in a deep learning cluster created with CloudFormation 10 | 11 | Step 1: [Create a deep learning cluster using CloudFormation](https://github.com/dmlc/mxnet/tree/master/tools/cfn). 12 | 13 | Step 2: Log in to the master instance using SSH, including the -A option to enable SSH agent forwarding. Example: `ssh -A masternode` 14 | 15 | Step 3: Run the following command: 16 | `git clone https://github.com/awslabs/deeplearning-benchmark.git && cd deeplearning-benchmark/benchmark/ && bash runscalabilitytest.sh` 17 | 18 | The runscalabilitytest.sh script runs scalability tests and records the throughput as images/sec in CSV files under 'csv_*' directories. Each line in the CSV file contains a key-value pair, where the key is the number of GPUs the test was run on and the value is the images processed per second. The script also plots this data in a SVG file named comparison_graph.svg. 19 | 20 | Note: The following mini-batch sizes are used by default: 21 | 22 | | | P2 Instance | G2 Instance | 23 | |--------------|------|------| 24 | | Inception v3 | 32 | 8 | 25 | | Alexnet | 512 | 128 | 26 | 27 | Mini-batch size can be changed using the --models switch. For example to run Inception-v3 with a batch size of 16 and AlexNet with a batch size of 256, run the following: 28 | `bash runscalabilitytest.sh --models "Inceptionv3:16,Alexnet:256"`. 29 | 30 | To run training across multiple machines, the scripts use parameter servers to update parameters. It is possible to get better performance on a single machine by not using the parameter servers. For simplicity, these scripts don't run different code optimized for a single machine for tests that run on single machine, given that we are interested only in distributed performance across multiple machines. This should not affect the results for distributed training. 31 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/run_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [[ $3 = gpu ]] 5 | then 6 | hw_type=gpu 7 | use_gpu="--use-gpu" 8 | else 9 | hw_type=cpu 10 | use_gpu="" 11 | fi 12 | 13 | if [[ $2 = e2e ]] 14 | then 15 | model_path="../models/resnet18_v1_end_to_end" 16 | end_to_end="--end-to-end" 17 | else 18 | model_path="../models/resnet18_v1" 19 | end_to_end="" 20 | fi 21 | # copy the maven wrapper related script into scala or java folder and 22 | # enter either java or scala folder 23 | if [[ $1 = scala ]] 24 | then 25 | cp -r ./.mvn scala-bm 26 | cp -r ./mvnw* scala-bm 27 | cd scala-bm 28 | else 29 | cp -r ./.mvn java-bm 30 | cp -r ./mvnw* java-bm 31 | cd java-bm 32 | fi 33 | # build the project 34 | bash bin/build.sh $hw_type 35 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.2/lib64/ 36 | CURR_DIR=$(pwd) 37 | CLASSPATH=$CLASSPATH:$CURR_DIR/target/*:$CLASSPATH:$CURR_DIR/target/dependency/*:$CLASSPATH:$CURR_DIR/target/classes/lib/* 38 | # run single inference 39 | output_single=$(java -Xmx8G \ 40 | -Dlog4j.configuration=file:/home/ubuntu/benchmarkai/end_to_end_model_benchmark/log4j.properties \ 41 | -cp $CLASSPATH mxnet.EndToEndModelWoPreprocessing \ 42 | --model-path-prefix $model_path \ 43 | --num-runs $4 \ 44 | --batchsize 1 \ 45 | --warm-up 5 \ 46 | $end_to_end \ 47 | $use_gpu) 48 | 49 | sum=0.0 50 | # the defualt value is 25 so tha we have enough CPU and GPU memory 51 | num_runs=25 52 | num_iter=$(($4 / $num_runs)) 53 | if (( $4 < $num_runs )); then num_runs=$4; fi 54 | if (( $num_iter == 0 )); then num_iter=1; fi 55 | for n in `seq 1 $num_iter` 56 | do 57 | output_batch=$(java -Xmx8G \ 58 | -Dlog4j.configuration=file:/home/ubuntu/benchmarkai/end_to_end_model_benchmark/log4j.properties \ 59 | -cp $CLASSPATH mxnet.EndToEndModelWoPreprocessing \ 60 | --model-path-prefix $model_path \ 61 | --num-runs $num_runs \ 62 | --batchsize 25 \ 63 | --warm-up 1 \ 64 | $end_to_end \ 65 | $use_gpu) 66 | value=$(echo $output_batch | grep -oP '(E2E|Non E2E) (single|batch)_inference_average \K(\d+.\d+)(?=ms)') 67 | # use awk to support float calculation 68 | sum=$(awk "BEGIN {print $sum+$value}") 69 | done 70 | 71 | metrix=$(echo $output_batch | grep -oE '(single|batch)_inference_average') 72 | echo "$output_single $metrix $(awk "BEGIN {print $sum / $num_iter}")ms" 73 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/googlenet_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Googlenet model configuration. 17 | 18 | References: 19 | Szegedy, Christian, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, 20 | Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich 21 | Going deeper with convolutions 22 | arXiv preprint arXiv:1409.4842 (2014) 23 | """ 24 | 25 | from models import model 26 | 27 | 28 | class GooglenetModel(model.Model): 29 | 30 | def __init__(self): 31 | super(GooglenetModel, self).__init__('googlenet', 224, 32, 0.005) 32 | 33 | def add_inference(self, cnn): 34 | def inception_v1(cnn, k, l, m, n, p, q): 35 | cols = [[('conv', k, 1, 1)], [('conv', l, 1, 1), ('conv', m, 3, 3)], 36 | [('conv', n, 1, 1), ('conv', p, 5, 5)], 37 | [('mpool', 3, 3, 1, 1, 'SAME'), ('conv', q, 1, 1)]] 38 | cnn.inception_module('incept_v1', cols) 39 | 40 | cnn.conv(64, 7, 7, 2, 2) 41 | cnn.mpool(3, 3, 2, 2, mode='SAME') 42 | cnn.conv(64, 1, 1) 43 | cnn.conv(192, 3, 3) 44 | cnn.mpool(3, 3, 2, 2, mode='SAME') 45 | inception_v1(cnn, 64, 96, 128, 16, 32, 32) 46 | inception_v1(cnn, 128, 128, 192, 32, 96, 64) 47 | cnn.mpool(3, 3, 2, 2, mode='SAME') 48 | inception_v1(cnn, 192, 96, 208, 16, 48, 64) 49 | inception_v1(cnn, 160, 112, 224, 24, 64, 64) 50 | inception_v1(cnn, 128, 128, 256, 24, 64, 64) 51 | inception_v1(cnn, 112, 144, 288, 32, 64, 64) 52 | inception_v1(cnn, 256, 160, 320, 32, 128, 128) 53 | cnn.mpool(3, 3, 2, 2, mode='SAME') 54 | inception_v1(cnn, 256, 160, 320, 32, 128, 128) 55 | inception_v1(cnn, 384, 192, 384, 48, 128, 128) 56 | cnn.apool(7, 7, 1, 1, mode='VALID') 57 | cnn.reshape([-1, 1024]) 58 | -------------------------------------------------------------------------------- /image_classification/get_data.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: skip-file 19 | import os, gzip 20 | import pickle as pickle 21 | import sys 22 | from zipfile import ZipFile 23 | 24 | # download mnist.pkl.gz 25 | def GetMNIST_pkl(): 26 | if not os.path.isdir("data/"): 27 | os.system("mkdir data/") 28 | if not os.path.exists('data/mnist.pkl.gz'): 29 | os.system("wget -q http://deeplearning.net/data/mnist/mnist.pkl.gz -P data/") 30 | 31 | # download ubyte version of mnist and untar 32 | def GetMNIST_ubyte(): 33 | if not os.path.isdir("data/"): 34 | os.system("mkdir data/") 35 | if (not os.path.exists('data/train-images-idx3-ubyte')) or \ 36 | (not os.path.exists('data/train-labels-idx1-ubyte')) or \ 37 | (not os.path.exists('data/t10k-images-idx3-ubyte')) or \ 38 | (not os.path.exists('data/t10k-labels-idx1-ubyte')): 39 | os.system("wget -q http://data.mxnet.io/mxnet/data/mnist.zip -P data/") 40 | os.chdir("./data") 41 | with ZipFile('mnist.zip', 'r') as zipObj: 42 | zipObj.extractall() 43 | os.chdir("..") 44 | 45 | # download cifar 46 | def GetCifar10(): 47 | if not os.path.isdir("data/"): 48 | os.system("mkdir data/") 49 | if (not os.path.exists('data/cifar/train.rec')) or \ 50 | (not os.path.exists('data/cifar/test.rec')) or \ 51 | (not os.path.exists('data/cifar/train.lst')) or \ 52 | (not os.path.exists('data/cifar/test.lst')): 53 | os.system("wget -q http://data.mxnet.io/mxnet/data/cifar10.zip -P data/") 54 | os.chdir("./data") 55 | with ZipFile('cifar10.zip', 'r') as zipObj: 56 | zipObj.extractall() 57 | os.chdir("..") 58 | -------------------------------------------------------------------------------- /reporting/lambda_handler.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # -*- coding: utf-8 -*- 18 | """ 19 | The AWS Lambda handler to generate benchmark reports for Benchmark.AI. 20 | 21 | The following environment variables will normally be set by AWS Lambda. Boto3 requires them to be 22 | set accordingly. 23 | 24 | * AWS_ACCESS_KEY_ID 25 | * AWS_SECRET_ACCESS_KEY 26 | * AWS_DEFAULT_REGION 27 | """ 28 | import io 29 | import json 30 | import logging 31 | import os 32 | 33 | from utils.benchmarks import Benchmarks 34 | from utils.email import email_report 35 | from utils.report_generation import HTML_EXTENSION 36 | from utils.report_generation import generate_report 37 | 38 | 39 | logging.getLogger().setLevel(logging.INFO) 40 | logging.getLogger('boto3').setLevel(logging.CRITICAL) 41 | logging.getLogger('botocore').setLevel(logging.CRITICAL) 42 | 43 | 44 | def lambda_handler(event, context): 45 | """ 46 | Main entry point for AWS Lambda. 47 | The EMAIL environment variable is required to be set. 48 | """ 49 | logging.info("Reading configuration and fetching metrics from Cloudwatch.") 50 | benchmarks = Benchmarks() 51 | 52 | logging.info('Generating report.') 53 | REPORT_FILE = '/tmp/bai-report' 54 | generate_report(REPORT_FILE, benchmarks) 55 | report_html = io.open(REPORT_FILE + HTML_EXTENSION, mode='r', encoding='utf-8').read() 56 | 57 | logging.info('Emailing report.') 58 | EMAIL = os.environ['EMAIL'] 59 | FOOTNOTES = os.environ['FOOTNOTES'] # allows passing in non-public information. 60 | email_report(report_html, EMAIL, footnotes=FOOTNOTES) 61 | 62 | return { 63 | "statusCode": 200, 64 | "body": json.dumps('Report successfully generated and e-mailed.') 65 | } 66 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/imagenet_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to the ImageNet data set. 16 | """ 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | 23 | from inception.dataset import Dataset 24 | 25 | 26 | class ImagenetData(Dataset): 27 | """ImageNet data set.""" 28 | 29 | def __init__(self, subset): 30 | super(ImagenetData, self).__init__('ImageNet', subset) 31 | 32 | def num_classes(self): 33 | """Returns the number of classes in the data set.""" 34 | return 1000 35 | 36 | def num_examples_per_epoch(self): 37 | """Returns the number of examples in the data set.""" 38 | # Bounding box data consists of 615299 bounding boxes for 544546 images. 39 | if self.subset == 'train': 40 | return 1281167 41 | if self.subset == 'validation': 42 | return 50000 43 | 44 | def download_message(self): 45 | """Instruction to download and extract the tarball from Flowers website.""" 46 | 47 | print('Failed to find any ImageNet %s files'% self.subset) 48 | print('') 49 | print('If you have already downloaded and processed the data, then make ' 50 | 'sure to set --data_dir to point to the directory containing the ' 51 | 'location of the sharded TFRecords.\n') 52 | print('If you have not downloaded and prepared the ImageNet data in the ' 53 | 'TFRecord format, you will need to do this at least once. This ' 54 | 'process could take several hours depending on the speed of your ' 55 | 'computer and network connection\n') 56 | print('Please see README.md for instructions on how to build ' 57 | 'the ImageNet dataset using download_and_preprocess_imagenet.\n') 58 | print('Note that the raw data size is 300 GB and the processed data size ' 59 | 'is 150 GB. Please ensure you have at least 500GB disk space.') 60 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/vgg_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Vgg model configuration. 17 | 18 | Includes multiple models: vgg11, vgg16, vgg19, corresponding to 19 | model A, D, and E in Table 1 of [1]. 20 | 21 | References: 22 | [1] Simonyan, Karen, Andrew Zisserman 23 | Very Deep Convolutional Networks for Large-Scale Image Recognition 24 | arXiv:1409.1556 (2014) 25 | """ 26 | 27 | from six.moves import xrange # pylint: disable=redefined-builtin 28 | from models import model 29 | 30 | 31 | def _construct_vgg(cnn, num_conv_layers): 32 | """Build vgg architecture from blocks.""" 33 | assert len(num_conv_layers) == 5 34 | for _ in xrange(num_conv_layers[0]): 35 | cnn.conv(64, 3, 3) 36 | cnn.mpool(2, 2) 37 | for _ in xrange(num_conv_layers[1]): 38 | cnn.conv(128, 3, 3) 39 | cnn.mpool(2, 2) 40 | for _ in xrange(num_conv_layers[2]): 41 | cnn.conv(256, 3, 3) 42 | cnn.mpool(2, 2) 43 | for _ in xrange(num_conv_layers[3]): 44 | cnn.conv(512, 3, 3) 45 | cnn.mpool(2, 2) 46 | for _ in xrange(num_conv_layers[4]): 47 | cnn.conv(512, 3, 3) 48 | cnn.mpool(2, 2) 49 | cnn.reshape([-1, 512 * 7 * 7]) 50 | cnn.affine(4096) 51 | cnn.dropout() 52 | cnn.affine(4096) 53 | cnn.dropout() 54 | 55 | 56 | class Vgg11Model(model.Model): 57 | 58 | def __init__(self): 59 | super(Vgg11Model, self).__init__('vgg11', 224, 64, 0.005) 60 | 61 | def add_inference(self, cnn): 62 | _construct_vgg(cnn, [1, 1, 2, 2, 2]) 63 | 64 | 65 | class Vgg16Model(model.Model): 66 | 67 | def __init__(self): 68 | super(Vgg16Model, self).__init__('vgg16', 224, 64, 0.005) 69 | 70 | def add_inference(self, cnn): 71 | _construct_vgg(cnn, [2, 2, 3, 3, 3]) 72 | 73 | 74 | class Vgg19Model(model.Model): 75 | 76 | def __init__(self): 77 | super(Vgg19Model, self).__init__('vgg19', 224, 64, 0.005) 78 | 79 | def add_inference(self, cnn): 80 | _construct_vgg(cnn, [2, 2, 4, 4, 4]) 81 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/imagenet_distributed_train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | # pylint: disable=line-too-long 16 | """A binary to train Inception in a distributed manner using multiple systems. 17 | 18 | Please see accompanying README.md for details and instructions. 19 | """ 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | import tensorflow as tf 25 | 26 | from inception import inception_distributed_train 27 | from inception.imagenet_data import ImagenetData 28 | 29 | FLAGS = tf.app.flags.FLAGS 30 | 31 | 32 | def main(unused_args): 33 | assert FLAGS.job_name in ['ps', 'worker'], 'job_name must be ps or worker' 34 | 35 | # Extract all the hostnames for the ps and worker jobs to construct the 36 | # cluster spec. 37 | ps_hosts = FLAGS.ps_hosts.split(',') 38 | worker_hosts = FLAGS.worker_hosts.split(',') 39 | tf.logging.info('PS hosts are: %s' % ps_hosts) 40 | tf.logging.info('Worker hosts are: %s' % worker_hosts) 41 | 42 | cluster_spec = tf.train.ClusterSpec({'ps': ps_hosts, 43 | 'worker': worker_hosts}) 44 | server = tf.train.Server( 45 | {'ps': ps_hosts, 46 | 'worker': worker_hosts}, 47 | job_name=FLAGS.job_name, 48 | task_index=FLAGS.task_id) 49 | 50 | if FLAGS.job_name == 'ps': 51 | # `ps` jobs wait for incoming connections from the workers. 52 | server.join() 53 | else: 54 | # `worker` jobs will actually do the work. 55 | dataset = ImagenetData(subset=FLAGS.subset) 56 | #assert dataset.data_files() 57 | # Only the chief checks for or creates train_dir. 58 | if FLAGS.task_id == 0: 59 | if not tf.gfile.Exists(FLAGS.train_dir): 60 | tf.gfile.MakeDirs(FLAGS.train_dir) 61 | inception_distributed_train.train(server.target, dataset, cluster_spec) 62 | 63 | if __name__ == '__main__': 64 | tf.logging.set_verbosity(tf.logging.INFO) 65 | tf.app.run() 66 | -------------------------------------------------------------------------------- /word_language_model/data.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | import sys 20 | import numpy as np 21 | import mxnet as mx 22 | import subprocess 23 | 24 | 25 | class Dictionary(object): 26 | def __init__(self): 27 | self.word2idx = {} 28 | self.idx2word = [] 29 | 30 | def add_word(self, word): 31 | if word not in self.word2idx: 32 | self.idx2word.append(word) 33 | self.word2idx[word] = len(self.idx2word) - 1 34 | return self.word2idx[word] 35 | 36 | def __len__(self): 37 | return len(self.idx2word) 38 | 39 | 40 | class Corpus(object): 41 | def __init__(self, path): 42 | self.dictionary = Dictionary() 43 | self.train = self.tokenize(path + 'train.txt') 44 | self.valid = self.tokenize(path + 'valid.txt') 45 | self.test = self.tokenize(path + 'test.txt') 46 | 47 | def tokenize(self, path): 48 | """Tokenizes a text file.""" 49 | try: 50 | assert os.path.exists(path) 51 | except AssertionError: 52 | subprocess.call(['{}/get_ptb_data.sh'.format(os.path.dirname(__file__))]) 53 | assert os.path.exists(path) 54 | 55 | # Add words to the dictionary 56 | with open(path, 'r') as f: 57 | tokens = 0 58 | for line in f: 59 | words = line.split() + [''] 60 | tokens += len(words) 61 | for word in words: 62 | self.dictionary.add_word(word) 63 | 64 | # Tokenize file content 65 | with open(path, 'r') as f: 66 | ids = np.zeros((tokens,), dtype='int32') 67 | token = 0 68 | for line in f: 69 | words = line.split() + [''] 70 | for word in words: 71 | ids[token] = self.dictionary.word2idx[word] 72 | token += 1 73 | 74 | return mx.nd.array(ids, dtype='int32') 75 | -------------------------------------------------------------------------------- /image_classification/symbols/mobilenetv2.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # -*- coding:utf-8 -*- 19 | ''' 20 | MobileNetV2, implemented in Gluon. 21 | 22 | Reference: 23 | Inverted Residuals and Linear Bottlenecks: 24 | Mobile Networks for Classification, Detection and Segmentation 25 | https://arxiv.org/abs/1801.04381 26 | ''' 27 | __author__ = 'dwSun' 28 | __date__ = '18/1/31' 29 | 30 | import mxnet as mx 31 | 32 | from mxnet.gluon.model_zoo.vision.mobilenet import MobileNetV2 33 | 34 | 35 | __all__ = ['MobileNetV2', 'get_symbol'] 36 | 37 | 38 | def get_symbol(num_classes=1000, multiplier=1.0, ctx=mx.cpu(), **kwargs): 39 | r"""MobileNetV2 model from the 40 | `"Inverted Residuals and Linear Bottlenecks: 41 | Mobile Networks for Classification, Detection and Segmentation" 42 | `_ paper. 43 | 44 | Parameters 45 | ---------- 46 | num_classes : int, default 1000 47 | Number of classes for the output layer. 48 | multiplier : float, default 1.0 49 | The width multiplier for controling the model size. The actual number of channels 50 | is equal to the original channel size multiplied by this multiplier. 51 | ctx : Context, default CPU 52 | The context in which to initialize the model weights. 53 | """ 54 | net = MobileNetV2(multiplier=multiplier, classes=num_classes, **kwargs) 55 | net.initialize(ctx=ctx, init=mx.init.Xavier()) 56 | net.hybridize() 57 | 58 | data = mx.sym.var('data') 59 | out = net(data) 60 | sym = mx.sym.SoftmaxOutput(out, name='softmax') 61 | return sym 62 | 63 | 64 | def plot_net(): 65 | """ 66 | Visualize the network. 67 | """ 68 | sym = get_symbol(1000, prefix='mob_') 69 | 70 | # plot network graph 71 | mx.viz.plot_network(sym, shape={'data': (8, 3, 224, 224)}, 72 | node_attrs={'shape': 'oval', 'fixedsize': 'fasl==false'}).view() 73 | 74 | 75 | if __name__ == '__main__': 76 | plot_net() 77 | -------------------------------------------------------------------------------- /onnx_benchmark/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | install_mxnet() 4 | { 5 | if [ $1 == "cpu" ]; then 6 | sudo pip uninstall --yes mxnet-cu90 7 | echo "Installing mxnet cpu ......" 8 | sudo pip install mxnet --pre 9 | fi 10 | if [ $1 == "gpu" ]; then 11 | sudo pip uninstall --yes mxnet 12 | echo "Installing mxnet gpu ......" 13 | sudo pip install mxnet-cu90 --pre 14 | fi 15 | } 16 | 17 | install_onnx() 18 | { 19 | echo "Installing protobuf ......." 20 | sudo apt-get -y install protobuf-compiler libprotoc-dev 21 | echo "Installing ONNX version 1.3.0 ........" 22 | sudo pip install protobuf==3.5.2 onnx==1.3.0 23 | } 24 | 25 | 26 | get_models() 27 | { 28 | if [ ! -d "models" ]; then 29 | mkdir models 30 | fi 31 | if [ ! -f "models/bvlc_alexnet.tar.gz" ]; then 32 | curl -o models/bvlc_alexnet.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/bvlc_alexnet.tar.gz 33 | fi 34 | if [ ! -f "models/bvlc_googlenet.tar.gz" ]; then 35 | curl -o models/bvlc_googlenet.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/bvlc_googlenet.tar.gz 36 | fi 37 | if [ ! -f "models/bvlc_reference_caffenet.tar.gz" ]; then 38 | curl -o models/bvlc_reference_caffenet.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/bvlc_reference_caffenet.tar.gz 39 | fi 40 | if [ ! -f "models/bvlc_reference_rcnn_ilsvrc13.tar.gz" ]; then 41 | curl -o models/bvlc_reference_rcnn_ilsvrc13.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/bvlc_reference_rcnn_ilsvrc13.tar.gz 42 | fi 43 | if [ ! -f "models/densenet121.tar.gz" ]; then 44 | curl -o models/densenet121.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/densenet121.tar.gz 45 | fi 46 | if [ ! -f "models/resnet50.tar.gz" ]; then 47 | curl -o models/resnet50.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/resnet50.tar.gz 48 | fi 49 | if [ ! -f "models/shufflenet.tar.gz" ]; then 50 | curl -o models/shufflenet.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/shufflenet.tar.gz 51 | fi 52 | if [ ! -f "models/squeezenet.tar.gz" ]; then 53 | curl -o models/squeezenet.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/squeezenet.tar.gz 54 | fi 55 | if [ ! -f "models/vgg19.tar.gz" ]; then 56 | curl -o models/vgg19.tar.gz https://s3.amazonaws.com/download.onnx/models/opset_3/vgg19.tar.gz 57 | fi 58 | 59 | for f in models/*.tar.gz; do tar xzf "$f" -C models/; done 60 | } 61 | 62 | main() { 63 | get_models 64 | install_onnx 65 | if [ $1 == "cpu" ]; then 66 | install_mxnet "cpu" 67 | fi 68 | if [ $1 == "gpu" ]; then 69 | install_mxnet "gpu" 70 | fi 71 | } 72 | 73 | main $1 74 | -------------------------------------------------------------------------------- /benchmark_driver.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import argparse 3 | import os 4 | 5 | from ast import literal_eval 6 | import logging 7 | 8 | try: 9 | import ConfigParser 10 | config = ConfigParser.ConfigParser() 11 | except ImportError: 12 | import configparser 13 | config = configparser.ConfigParser() 14 | 15 | from utils import cfg_process, metrics_manager 16 | 17 | CONFIG_TEMPLATE_DIR = './task_config_template.cfg' 18 | CONFIG_DIR = './task_config.cfg' 19 | 20 | 21 | 22 | 23 | 24 | def run_benchmark(args): 25 | # modify the template config file and generate the user defined config file. 26 | cfg_process.generate_cfg(CONFIG_TEMPLATE_DIR, CONFIG_DIR, **vars(args)) 27 | config.read(CONFIG_DIR) 28 | 29 | # the user defined config file should only have one task 30 | selected_task = config.sections()[0] 31 | metric_patterns = literal_eval(config.get(selected_task, "patterns")) 32 | metric_names = literal_eval(config.get(selected_task, "metrics")) 33 | metric_compute_methods = literal_eval(config.get(selected_task, "compute_method")) 34 | command_to_execute = config.get(selected_task, "command_to_execute") 35 | num_gpus = int(config.get(selected_task, "num_gpus")) 36 | 37 | metrics_manager.benchmark( 38 | command_to_execute=command_to_execute, 39 | metric_patterns=metric_patterns, 40 | metric_names=metric_names, 41 | metric_compute_methods=metric_compute_methods, 42 | num_gpus=num_gpus, 43 | task_name=selected_task, 44 | suffix=args.metrics_suffix, 45 | framework=args.framework 46 | ) 47 | 48 | # clean up 49 | os.remove(CONFIG_DIR) 50 | 51 | if __name__ == '__main__': 52 | parser = argparse.ArgumentParser(description="Run a benchmark task.") 53 | parser.add_argument('--framework', type=str, help='Framework name e.g. mxnet') 54 | parser.add_argument('--task-name', type=str, help='Task Name e.g. resnet50_cifar10_symbolic.') 55 | parser.add_argument('--num-gpus', type=int, help='Numbers of gpus. e.g. --num-gpus 8') 56 | parser.add_argument('--epochs', type=int, help='Numbers of epochs for training. e.g. --epochs 20') 57 | parser.add_argument('--metrics-suffix', type=str, help='Metrics suffix e.g. --metrics-suffix daily') 58 | parser.add_argument('--kvstore', type=str, default='device',help='kvstore to use for trainer/module.') 59 | parser.add_argument('--dtype', type=str, default='float32',help='floating point precision to use') 60 | 61 | args = parser.parse_args() 62 | 63 | log_file_location = args.task_name + ".log" 64 | logging.basicConfig(filename=log_file_location,level=logging.DEBUG) 65 | 66 | try: 67 | run_benchmark(args) 68 | except Exception: 69 | logging.exception("Fatal error in run_benchmark") 70 | exit() 71 | 72 | -------------------------------------------------------------------------------- /benchmark_runner.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import print_function 3 | import argparse 4 | from ast import literal_eval 5 | 6 | import logging 7 | 8 | from utils import metrics_manager 9 | from utils import data_manager 10 | try: 11 | import ConfigParser 12 | config = ConfigParser.ConfigParser() 13 | except ImportError: 14 | import configparser 15 | config = configparser.ConfigParser() 16 | 17 | 18 | 19 | # --metrics-policy metrics_parameters_images --task-name custom.p316xlarge.fp32.bs32 --metrics-suffix nightly --num-gpus 8 --command-to-execute \"Hello world\" 20 | CONFIG_TEMPLATE = './task_config_template.cfg' 21 | 22 | 23 | 24 | 25 | def run_benchmark(args): 26 | if 'imagenet' in args.data_set: 27 | data_manager.getImagenetData(args.data_set) 28 | 29 | config.read(args.metrics_template) 30 | 31 | for name, value in config.items(args.metrics_policy): 32 | if(name == 'patterns'): 33 | metric_patterns = literal_eval(value) 34 | elif(name == 'metrics'): 35 | metric_names= literal_eval(value) 36 | else: 37 | metric_compute_methods = literal_eval(value) 38 | 39 | metrics_manager.BenchmarkResultManager.uptime() 40 | 41 | metrics_manager.benchmark( 42 | command_to_execute=args.command_to_execute, 43 | metric_patterns=metric_patterns, 44 | metric_names=metric_names, 45 | metric_compute_methods=metric_compute_methods, 46 | num_gpus=args.num_gpus, 47 | task_name=args.task_name, 48 | suffix=args.metrics_suffix, 49 | framework=args.framework 50 | ) 51 | 52 | if __name__ == '__main__': 53 | parser = argparse.ArgumentParser(description="Run a benchmark task.") 54 | parser.add_argument('--framework', type=str, help='Framework eg. mxnet') 55 | parser.add_argument('--metrics-policy', type=str, help='Metrics policy section name e.g. metrics_paramaters_images') 56 | parser.add_argument('--task-name', type=str, help='Task Name e.g. resnet50_cifar10_symbolic.') 57 | parser.add_argument('--metrics-suffix', type=str, help='Metrics suffix e.g. --metrics-suffix daily') 58 | parser.add_argument('--num-gpus', type=int, help='Numbers of gpus. e.g. --num-gpus 8') 59 | parser.add_argument('--command-to-execute', type=str, help='The script command that performs benchmarking') 60 | parser.add_argument('--data-set', type=str, help='The data set to use for benchmarking, eg. imagenet, imagenet-480px-256px-q95') 61 | parser.add_argument('--metrics-template', type=str, help='The template file to use for metrics pattern', default=CONFIG_TEMPLATE) 62 | 63 | args = parser.parse_args() 64 | 65 | log_file_location = args.task_name + ".log" 66 | logging.basicConfig(filename=log_file_location,level=logging.DEBUG) 67 | 68 | try: 69 | run_benchmark(args) 70 | except Exception: 71 | logging.exception("Fatal error in run_benchmark") 72 | exit() 73 | 74 | -------------------------------------------------------------------------------- /word_language_model/model.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import mxnet as mx 19 | from mxnet import gluon 20 | from mxnet.gluon import nn, rnn 21 | 22 | class RNNModel(gluon.Block): 23 | """A model with an encoder, recurrent layer, and a decoder.""" 24 | 25 | def __init__(self, mode, vocab_size, num_embed, num_hidden, 26 | num_layers, dropout=0.5, tie_weights=False, **kwargs): 27 | super(RNNModel, self).__init__(**kwargs) 28 | with self.name_scope(): 29 | self.drop = nn.Dropout(dropout) 30 | self.encoder = nn.Embedding(vocab_size, num_embed, 31 | weight_initializer=mx.init.Uniform(0.1)) 32 | if mode == 'rnn_relu': 33 | self.rnn = rnn.RNN(num_hidden, 'relu', num_layers, dropout=dropout, 34 | input_size=num_embed) 35 | elif mode == 'rnn_tanh': 36 | self.rnn = rnn.RNN(num_hidden, num_layers, dropout=dropout, 37 | input_size=num_embed) 38 | elif mode == 'lstm': 39 | self.rnn = rnn.LSTM(num_hidden, num_layers, dropout=dropout, 40 | input_size=num_embed) 41 | elif mode == 'gru': 42 | self.rnn = rnn.GRU(num_hidden, num_layers, dropout=dropout, 43 | input_size=num_embed) 44 | else: 45 | raise ValueError("Invalid mode %s. Options are rnn_relu, " 46 | "rnn_tanh, lstm, and gru"%mode) 47 | 48 | if tie_weights: 49 | self.decoder = nn.Dense(vocab_size, in_units=num_hidden, 50 | params=self.encoder.params) 51 | else: 52 | self.decoder = nn.Dense(vocab_size, in_units=num_hidden) 53 | 54 | self.num_hidden = num_hidden 55 | 56 | def forward(self, inputs, hidden): 57 | emb = self.drop(self.encoder(inputs)) 58 | output, hidden = self.rnn(emb, hidden) 59 | output = self.drop(output) 60 | decoded = self.decoder(output.reshape((-1, self.num_hidden))) 61 | return decoded, hidden 62 | 63 | def begin_state(self, *args, **kwargs): 64 | return self.rnn.begin_state(*args, **kwargs) 65 | -------------------------------------------------------------------------------- /djl/benchmark_training.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # prepare CUDA 10.1 env 6 | sudo rm -rf /usr/local/cuda 7 | sudo ln -s /usr/local/cuda-10.1 /usr/local/cuda 8 | export PATH=/usr/local/cuda-10.1/bin${PATH:+:${PATH}}$ 9 | export LD_LIBRARY_PATH=/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} 10 | 11 | sudo pip3 install mxnet-cu101mkl 12 | 13 | rm -rf /tmp/benchmark 14 | mkdir -p /tmp/benchmark 15 | 16 | python3 trainCiFar10.py --epochs $1 --model resnet50_v1 --mode hybrid --lr 0.001 --wd 0.001 --gpus 0 --use_thumbnail 17 | mv image-classification.log /tmp/benchmark/python_res50_cifar10_sym.log 18 | python3 trainCiFar10.py --epochs $1 --model resnet50_v1 --mode imperative --lr 0.001 --wd 0.001 --lr-steps 20,60,90,120,180 --lr-factor 0.31623 --gpus 0 --use_thumbnail 19 | mv image-classification.log /tmp/benchmark/python_res50_cifar10_imp.log 20 | 21 | sudo pip3 uninstall -y mxnet-cu101mkl 22 | 23 | git clone https://github.com/awslabs/djl.git 24 | 25 | cd djl/examples 26 | 27 | echo 'Running training Res50...' 28 | ./gradlew run -Dmain=ai.djl.examples.training.transferlearning.TrainResnetWithCifar10 --args="-g 1 -b 32 -e ${1} -o logs/" > /tmp/benchmark/djl_res50_cifar10_imp.log 2>&1 29 | echo 'Running training Res50 Symbolic mode...' 30 | ./gradlew run -Dmain=ai.djl.examples.training.transferlearning.TrainResnetWithCifar10 --args="-g 1 -b 32 -e ${1} -o logs/ -s" > /tmp/benchmark/djl_res50_cifar10_sym.log 2>&1 31 | echo 'Running training Res50 Symbolic mode with Pretrained Model ...' 32 | ./gradlew run -Dmain=ai.djl.examples.training.transferlearning.TrainResnetWithCifar10 --args="-g 1 -b 32 -e ${1} -o logs/ -s -p" > /tmp/benchmark/djl_res50_cifar10_sym_pretrain.log 2>&1 33 | 34 | declare -a python_models=("python_res50_cifar10_sym" "python_res50_cifar10_imp") 35 | declare -a djl_models=("djl_res50_cifar10_imp" "djl_res50_cifar10_sym" "djl_res50_cifar10_sym_pretrain") 36 | 37 | speed_file="/tmp/benchmark/speed.txt" 38 | 39 | { 40 | printf "Python Training Result\n" 41 | for model in "${python_models[@]}"; do 42 | printf "======================================\n" 43 | # delete speed.txt if it exists 44 | if [ -f $speed_file ]; then 45 | rm $speed_file 46 | fi 47 | grep 'Speed' /tmp/benchmark/"${model}".log | awk '{ print ($8) }' | sort -g > $speed_file 48 | line50=$(($(wc -l < $speed_file) / 2)) 49 | printf "%s speed P50: %s\n" "$model" "$(sed -n "${line50}p" $speed_file)" 50 | printf "%s accuracy: %s\n" "$model" "$(grep -oP "training: accuracy=\K(\d+.\d+)" /tmp/benchmark/"${model}".log | tail -1)" 51 | done 52 | printf "\nDJL Training Result\n" 53 | for djl_model in "${djl_models[@]}"; do 54 | printf "======================================\n" 55 | printf "%s speed P50: %s\n" "$djl_model" "$(grep "train P50:" /tmp/benchmark/"${djl_model}".log | awk '{ print 32 / $6 * 1000 }')" 56 | printf "%s accuracy: %s\n" "$djl_model" "$(grep -oP "train accuracy: \K(\d+.\d+)" /tmp/benchmark/"${djl_model}".log | tail -1)" 57 | done 58 | 59 | } >> /tmp/benchmark/report.txt 60 | 61 | cat /tmp/benchmark/report.txt 62 | -------------------------------------------------------------------------------- /djl/gradle/wrapper/GradleWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, 13 | software distributed under the License is distributed on an 14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | KIND, either express or implied. See the License for the 16 | specific language governing permissions and limitations 17 | under the License. 18 | */ 19 | 20 | import java.net.*; 21 | import java.io.*; 22 | import java.nio.channels.*; 23 | import java.util.Properties; 24 | 25 | public class GradleWrapperDownloader { 26 | 27 | /** 28 | * URL to download the gradle-wrapper.jar from. 29 | */ 30 | private static final String DEFAULT_DOWNLOAD_URL = 31 | "https://raw.githubusercontent.com/gradle/gradle/master/gradle/wrapper/gradle-wrapper.jar"; 32 | 33 | /** 34 | * Path where the gradle-wrapper.jar will be saved to. 35 | */ 36 | private static final String GRADLE_WRAPPER_JAR_PATH = 37 | "gradle/wrapper/gradle-wrapper.jar"; 38 | 39 | public static void main(String args[]) { 40 | System.out.println("- Downloader started"); 41 | File baseDirectory = new File(args[0]); 42 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 43 | 44 | String url = DEFAULT_DOWNLOAD_URL; 45 | System.out.println("- Downloading from: : " + url); 46 | 47 | File outputFile = new File(baseDirectory.getAbsolutePath(), GRADLE_WRAPPER_JAR_PATH); 48 | if(!outputFile.getParentFile().exists()) { 49 | if(!outputFile.getParentFile().mkdirs()) { 50 | System.out.println( 51 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 52 | } 53 | } 54 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 55 | try { 56 | downloadFileFromURL(url, outputFile); 57 | System.out.println("Done"); 58 | System.exit(0); 59 | } catch (Throwable e) { 60 | System.out.println("- Error downloading"); 61 | e.printStackTrace(); 62 | System.exit(1); 63 | } 64 | } 65 | 66 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 67 | URL website = new URL(urlString); 68 | ReadableByteChannel rbc; 69 | rbc = Channels.newChannel(website.openStream()); 70 | FileOutputStream fos = new FileOutputStream(destination); 71 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 72 | fos.close(); 73 | rbc.close(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /image_classification/symbols/lenet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner. 20 | Gradient-based learning applied to document recognition. 21 | Proceedings of the IEEE (1998) 22 | """ 23 | import mxnet as mx 24 | 25 | def get_loc(data, attr={'lr_mult':'0.01'}): 26 | """ 27 | the localisation network in lenet-stn, it will increase acc about more than 1%, 28 | when num-epoch >=15 29 | """ 30 | loc = mx.symbol.Convolution(data=data, num_filter=30, kernel=(5, 5), stride=(2,2)) 31 | loc = mx.symbol.Activation(data = loc, act_type='relu') 32 | loc = mx.symbol.Pooling(data=loc, kernel=(2, 2), stride=(2, 2), pool_type='max') 33 | loc = mx.symbol.Convolution(data=loc, num_filter=60, kernel=(3, 3), stride=(1,1), pad=(1, 1)) 34 | loc = mx.symbol.Activation(data = loc, act_type='relu') 35 | loc = mx.symbol.Pooling(data=loc, global_pool=True, kernel=(2, 2), pool_type='avg') 36 | loc = mx.symbol.Flatten(data=loc) 37 | loc = mx.symbol.FullyConnected(data=loc, num_hidden=6, name="stn_loc", attr=attr) 38 | return loc 39 | 40 | 41 | def get_symbol(num_classes=10, add_stn=False, **kwargs): 42 | data = mx.symbol.Variable('data') 43 | if add_stn: 44 | data = mx.sym.SpatialTransformer(data=data, loc=get_loc(data), target_shape = (28,28), 45 | transform_type="affine", sampler_type="bilinear") 46 | # first conv 47 | conv1 = mx.symbol.Convolution(data=data, kernel=(5,5), num_filter=20) 48 | tanh1 = mx.symbol.Activation(data=conv1, act_type="tanh") 49 | pool1 = mx.symbol.Pooling(data=tanh1, pool_type="max", 50 | kernel=(2,2), stride=(2,2)) 51 | # second conv 52 | conv2 = mx.symbol.Convolution(data=pool1, kernel=(5,5), num_filter=50) 53 | tanh2 = mx.symbol.Activation(data=conv2, act_type="tanh") 54 | pool2 = mx.symbol.Pooling(data=tanh2, pool_type="max", 55 | kernel=(2,2), stride=(2,2)) 56 | # first fullc 57 | flatten = mx.symbol.Flatten(data=pool2) 58 | fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=500) 59 | tanh3 = mx.symbol.Activation(data=fc1, act_type="tanh") 60 | # second fullc 61 | fc2 = mx.symbol.FullyConnected(data=tanh3, num_hidden=num_classes) 62 | # loss 63 | lenet = mx.symbol.SoftmaxOutput(data=fc2, name='softmax') 64 | return lenet 65 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/alexnet_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Alexnet model configuration. 17 | 18 | References: 19 | Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton 20 | ImageNet Classification with Deep Convolutional Neural Networks 21 | Advances in Neural Information Processing Systems. 2012 22 | """ 23 | 24 | import tensorflow as tf 25 | from models import model 26 | 27 | 28 | class AlexnetModel(model.Model): 29 | """Alexnet cnn model.""" 30 | 31 | def __init__(self): 32 | super(AlexnetModel, self).__init__('alexnet', 224 + 3, 512, 0.005) 33 | 34 | def add_inference(self, cnn): 35 | # Note: VALID requires padding the images by 3 in width and height 36 | cnn.conv(64, 11, 11, 4, 4, 'VALID') 37 | cnn.mpool(3, 3, 2, 2) 38 | cnn.conv(192, 5, 5) 39 | cnn.mpool(3, 3, 2, 2) 40 | cnn.conv(384, 3, 3) 41 | cnn.conv(384, 3, 3) 42 | cnn.conv(256, 3, 3) 43 | cnn.mpool(3, 3, 2, 2) 44 | cnn.reshape([-1, 256 * 6 * 6]) 45 | cnn.affine(4096) 46 | cnn.dropout() 47 | cnn.affine(4096) 48 | cnn.dropout() 49 | 50 | 51 | class AlexnetCifar10Model(model.Model): 52 | """Alexnet cnn model for cifar datasets. 53 | 54 | The model architecture follows the one defined in the tensorflow tutorial 55 | model. 56 | 57 | Reference model: tensorflow/models/tutorials/image/cifar10/cifar10.py 58 | Paper: http://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf 59 | """ 60 | 61 | def __init__(self): 62 | super(AlexnetCifar10Model, self).__init__('alexnet', 32, 128, 0.1) 63 | 64 | def add_inference(self, cnn): 65 | cnn.conv(64, 5, 5, 1, 1, 'SAME', stddev=5e-2) 66 | cnn.mpool(3, 3, 2, 2, mode='SAME') 67 | cnn.lrn(depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 68 | cnn.conv(64, 5, 5, 1, 1, 'SAME', bias=0.1, stddev=5e-2) 69 | cnn.lrn(depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) 70 | cnn.mpool(3, 3, 2, 2, mode='SAME') 71 | shape = cnn.top_layer.get_shape().as_list() 72 | flat_dim = shape[1] * shape[2] * shape[3] 73 | cnn.reshape([-1, flat_dim]) 74 | cnn.affine(384, stddev=0.04, bias=0.1) 75 | cnn.affine(192, stddev=0.04, bias=0.1) 76 | 77 | def get_learning_rate(self, global_step, batch_size): 78 | num_examples_per_epoch = 50000 79 | num_epochs_per_decay = 100 80 | decay_steps = int(num_epochs_per_decay * num_examples_per_epoch / 81 | batch_size) 82 | decay_factor = 0.1 83 | return tf.train.exponential_decay( 84 | self.learning_rate, global_step, decay_steps, decay_factor, 85 | staircase=True) 86 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/data/preprocess_imagenet_validation_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | """Process the ImageNet Challenge bounding boxes for TensorFlow model training. 17 | 18 | Associate the ImageNet 2012 Challenge validation data set with labels. 19 | 20 | The raw ImageNet validation data set is expected to reside in JPEG files 21 | located in the following directory structure. 22 | 23 | data_dir/ILSVRC2012_val_00000001.JPEG 24 | data_dir/ILSVRC2012_val_00000002.JPEG 25 | ... 26 | data_dir/ILSVRC2012_val_00050000.JPEG 27 | 28 | This script moves the files into a directory structure like such: 29 | data_dir/n01440764/ILSVRC2012_val_00000293.JPEG 30 | data_dir/n01440764/ILSVRC2012_val_00000543.JPEG 31 | ... 32 | where 'n01440764' is the unique synset label associated with 33 | these images. 34 | 35 | This directory reorganization requires a mapping from validation image 36 | number (i.e. suffix of the original file) to the associated label. This 37 | is provided in the ImageNet development kit via a Matlab file. 38 | 39 | In order to make life easier and divorce ourselves from Matlab, we instead 40 | supply a custom text file that provides this mapping for us. 41 | 42 | Sample usage: 43 | ./preprocess_imagenet_validation_data.py ILSVRC2012_img_val \ 44 | imagenet_2012_validation_synset_labels.txt 45 | """ 46 | 47 | from __future__ import absolute_import 48 | from __future__ import division 49 | from __future__ import print_function 50 | 51 | import os 52 | import os.path 53 | import sys 54 | 55 | 56 | if __name__ == '__main__': 57 | if len(sys.argv) < 3: 58 | print('Invalid usage\n' 59 | 'usage: preprocess_imagenet_validation_data.py ' 60 | ' ') 61 | sys.exit(-1) 62 | data_dir = sys.argv[1] 63 | validation_labels_file = sys.argv[2] 64 | 65 | # Read in the 50000 synsets associated with the validation data set. 66 | labels = [l.strip() for l in open(validation_labels_file).readlines()] 67 | unique_labels = set(labels) 68 | 69 | # Make all sub-directories in the validation data dir. 70 | for label in unique_labels: 71 | labeled_data_dir = os.path.join(data_dir, label) 72 | os.makedirs(labeled_data_dir) 73 | 74 | # Move all of the image to the appropriate sub-directory. 75 | for i in xrange(len(labels)): 76 | basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) 77 | original_filename = os.path.join(data_dir, basename) 78 | if not os.path.exists(original_filename): 79 | print('Failed to find: ' % original_filename) 80 | sys.exit(-1) 81 | new_filename = os.path.join(data_dir, labels[i], basename) 82 | os.rename(original_filename, new_filename) 83 | -------------------------------------------------------------------------------- /tensorflow/resnet/README.md: -------------------------------------------------------------------------------- 1 | Reproduced ResNet on CIFAR-10 and CIFAR-100 dataset. 2 | 3 | contact: panyx0718 (xpan@google.com) 4 | 5 | Dataset: 6 | 7 | https://www.cs.toronto.edu/~kriz/cifar.html 8 | 9 | Related papers: 10 | 11 | Identity Mappings in Deep Residual Networks 12 | 13 | https://arxiv.org/pdf/1603.05027v2.pdf 14 | 15 | Deep Residual Learning for Image Recognition 16 | 17 | https://arxiv.org/pdf/1512.03385v1.pdf 18 | 19 | Wide Residual Networks 20 | 21 | https://arxiv.org/pdf/1605.07146v1.pdf 22 | 23 | Settings: 24 | 25 | * Random split 50k training set into 45k/5k train/eval split. 26 | * Pad to 36x36 and random crop. Horizontal flip. Per-image whitenting. 27 | * Momentum optimizer 0.9. 28 | * Learning rate schedule: 0.1 (40k), 0.01 (60k), 0.001 (>60k). 29 | * L2 weight decay: 0.002. 30 | * Batch size: 128. (28-10 wide and 1001 layer bottleneck use 64) 31 | 32 | Results: 33 | 34 | 35 | ![Precisions](g3doc/cifar_resnet.gif) 36 | 37 | 38 | ![Precisions Legends](g3doc/cifar_resnet_legends.gif) 39 | 40 | 41 | 42 | CIFAR-10 Model|Best Precision|Steps 43 | --------------|--------------|------ 44 | 32 layer|92.5%|~80k 45 | 110 layer|93.6%|~80k 46 | 164 layer bottleneck|94.5%|~80k 47 | 1001 layer bottleneck|94.9%|~80k 48 | 28-10 wide|95%|~90k 49 | 50 | CIFAR-100 Model|Best Precision|Steps 51 | ---------------|--------------|----- 52 | 32 layer|68.1%|~45k 53 | 110 layer|71.3%|~60k 54 | 164 layer bottleneck|75.7%|~50k 55 | 1001 layer bottleneck|78.2%|~70k 56 | 28-10 wide|78.3%|~70k 57 | 58 | Prerequisite: 59 | 60 | 1. Install TensorFlow, Bazel. 61 | 62 | 2. Download CIFAR-10/CIFAR-100 dataset. 63 | 64 | ```shell 65 | curl -o cifar-10-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz 66 | curl -o cifar-100-binary.tar.gz https://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz 67 | ``` 68 | 69 | How to run: 70 | 71 | ```shell 72 | # cd to the your workspace. 73 | # It contains an empty WORKSPACE file, resnet codes and cifar10 dataset. 74 | # Note: User can split 5k from train set for eval set. 75 | ls -R 76 | .: 77 | cifar10 resnet WORKSPACE 78 | 79 | ./cifar10: 80 | data_batch_1.bin data_batch_2.bin data_batch_3.bin data_batch_4.bin 81 | data_batch_5.bin test_batch.bin 82 | 83 | ./resnet: 84 | BUILD cifar_input.py g3doc README.md resnet_main.py resnet_model.py 85 | 86 | # Build everything for GPU. 87 | bazel build -c opt --config=cuda resnet/... 88 | 89 | # Train the model. 90 | bazel-bin/resnet/resnet_main --train_data_path=cifar10/data_batch* \ 91 | --log_root=/tmp/resnet_model \ 92 | --train_dir=/tmp/resnet_model/train \ 93 | --dataset='cifar10' \ 94 | --num_gpus=1 95 | 96 | # Evaluate the model. 97 | # Avoid running on the same GPU as the training job at the same time, 98 | # otherwise, you might run out of memory. 99 | bazel-bin/resnet/resnet_main --eval_data_path=cifar10/test_batch.bin \ 100 | --log_root=/tmp/resnet_model \ 101 | --eval_dir=/tmp/resnet_model/test \ 102 | --mode=eval \ 103 | --dataset='cifar10' \ 104 | --num_gpus=0 105 | ``` 106 | -------------------------------------------------------------------------------- /image_classification/symbols/alexnet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | Reference: 20 | 21 | Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. "Imagenet classification with deep convolutional neural networks." Advances in neural information processing systems. 2012. 22 | """ 23 | import mxnet as mx 24 | import numpy as np 25 | 26 | def get_symbol(num_classes, dtype='float32', **kwargs): 27 | input_data = mx.sym.Variable(name="data") 28 | if dtype == 'float16': 29 | input_data = mx.sym.Cast(data=input_data, dtype=np.float16) 30 | # stage 1 31 | conv1 = mx.sym.Convolution(name='conv1', 32 | data=input_data, kernel=(11, 11), stride=(4, 4), num_filter=96) 33 | relu1 = mx.sym.Activation(data=conv1, act_type="relu") 34 | lrn1 = mx.sym.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5) 35 | pool1 = mx.sym.Pooling( 36 | data=lrn1, pool_type="max", kernel=(3, 3), stride=(2,2)) 37 | # stage 2 38 | conv2 = mx.sym.Convolution(name='conv2', 39 | data=pool1, kernel=(5, 5), pad=(2, 2), num_filter=256) 40 | relu2 = mx.sym.Activation(data=conv2, act_type="relu") 41 | lrn2 = mx.sym.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5) 42 | pool2 = mx.sym.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max") 43 | # stage 3 44 | conv3 = mx.sym.Convolution(name='conv3', 45 | data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=384) 46 | relu3 = mx.sym.Activation(data=conv3, act_type="relu") 47 | conv4 = mx.sym.Convolution(name='conv4', 48 | data=relu3, kernel=(3, 3), pad=(1, 1), num_filter=384) 49 | relu4 = mx.sym.Activation(data=conv4, act_type="relu") 50 | conv5 = mx.sym.Convolution(name='conv5', 51 | data=relu4, kernel=(3, 3), pad=(1, 1), num_filter=256) 52 | relu5 = mx.sym.Activation(data=conv5, act_type="relu") 53 | pool3 = mx.sym.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max") 54 | # stage 4 55 | flatten = mx.sym.Flatten(data=pool3) 56 | fc1 = mx.sym.FullyConnected(name='fc1', data=flatten, num_hidden=4096) 57 | relu6 = mx.sym.Activation(data=fc1, act_type="relu") 58 | dropout1 = mx.sym.Dropout(data=relu6, p=0.5) 59 | # stage 5 60 | fc2 = mx.sym.FullyConnected(name='fc2', data=dropout1, num_hidden=4096) 61 | relu7 = mx.sym.Activation(data=fc2, act_type="relu") 62 | dropout2 = mx.sym.Dropout(data=relu7, p=0.5) 63 | # stage 6 64 | fc3 = mx.sym.FullyConnected(name='fc3', data=dropout2, num_hidden=num_classes) 65 | if dtype == 'float16': 66 | fc3 = mx.sym.Cast(data=fc3, dtype=np.float32) 67 | softmax = mx.sym.SoftmaxOutput(data=fc3, name='softmax') 68 | return softmax 69 | -------------------------------------------------------------------------------- /dependency_update/mlp.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """The scirpt is used for MXNet performance benchmark""" 19 | from __future__ import print_function 20 | import numpy as np 21 | import mxnet as mx 22 | from mxnet import nd, autograd, gluon 23 | import time 24 | 25 | ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu() 26 | data_ctx = ctx 27 | model_ctx = ctx 28 | 29 | batch_size = 64 30 | num_inputs = 784 31 | num_outputs = 10 32 | num_examples = 60000 33 | def transform(data, label): 34 | return data.astype(np.float32)/255, label.astype(np.float32) 35 | 36 | train_data = mx.gluon.data.DataLoader(mx.gluon.data.vision.MNIST(train=True, transform=transform), 37 | batch_size, shuffle=True) 38 | test_data = mx.gluon.data.DataLoader(mx.gluon.data.vision.MNIST(train=False, transform=transform), 39 | batch_size, shuffle=False) 40 | num_hidden = 64 41 | net = gluon.nn.Sequential() 42 | with net.name_scope(): 43 | net.add(gluon.nn.Dense(num_hidden, activation="relu")) 44 | net.add(gluon.nn.Dense(num_hidden, activation="relu")) 45 | net.add(gluon.nn.Dense(num_outputs)) 46 | net.collect_params().initialize(mx.init.Normal(sigma=.1), ctx=model_ctx) 47 | softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() 48 | trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': .01}) 49 | 50 | def evaluate_accuracy(data_iterator, net): 51 | acc = mx.metric.Accuracy() 52 | for i, (data, label) in enumerate(data_iterator): 53 | data = data.as_in_context(model_ctx).reshape((-1, 784)) 54 | label = label.as_in_context(model_ctx) 55 | output = net(data) 56 | predictions = nd.argmax(output, axis=1) 57 | acc.update(preds=predictions, labels=label) 58 | return acc.get()[1] 59 | 60 | epochs = 10 61 | smoothing_constant = .01 62 | 63 | for e in range(epochs): 64 | cumulative_loss = 0 65 | btic = time.time() 66 | for i, (data, label) in enumerate(train_data): 67 | data = data.as_in_context(model_ctx).reshape((-1, 784)) 68 | label = label.as_in_context(model_ctx) 69 | with autograd.record(): 70 | output = net(data) 71 | loss = softmax_cross_entropy(output, label) 72 | loss.backward() 73 | trainer.step(data.shape[0]) 74 | cumulative_loss += nd.sum(loss).asscalar() 75 | print("Speed {}".format(num_examples / (time.time() - btic))) 76 | 77 | test_accuracy = evaluate_accuracy(test_data, net) 78 | train_accuracy = evaluate_accuracy(train_data, net) 79 | print("Epoch {}. Loss: {}, Train_acc {}, Test_acc {}".format(e, cumulative_loss/num_examples, train_accuracy, test_accuracy)) 80 | -------------------------------------------------------------------------------- /scala-mxnet/scala-bm/src/main/scala/mxnet/CharRnnExample.scala: -------------------------------------------------------------------------------- 1 | package mxnet 2 | 3 | import org.apache.mxnet.{Context, Model, NDArrayCollector} 4 | import org.kohsuke.args4j.{CmdLineParser, Option} 5 | 6 | import scala.collection.JavaConverters._ 7 | 8 | object CharRnnExample { 9 | 10 | private var vocab : Map[String, Int] = null 11 | 12 | class CLIParser { 13 | 14 | @Option(name = "--modelPathPrefix", usage = "the model prefix") 15 | val modelPathPrefix: String = "./model/obama" 16 | 17 | @Option(name = "--data-path", usage = "the input train data file") 18 | val dataPath: String = "./data/obama.txt" 19 | 20 | @Option(name = "--starter-sentence", usage = "the starter sentence") 21 | val starterSentence: String = "The joke" 22 | 23 | @Option(name = "--times", usage = "Number of times to run the benchmark") 24 | val times: Int = 1 25 | 26 | @Option(name = "--context", usage = "Context to run on") 27 | val context: String = "cpu" 28 | 29 | } 30 | 31 | 32 | def loadModel(modelPathPrefix: String, dataPath: String, context: Context): RnnModel.LSTMInferenceModel = { 33 | 34 | val buckets = List(129) 35 | val numHidden = 512 36 | val numEmbed = 256 37 | val numLstmLayer = 3 38 | 39 | val (_, argParams, _) = Model.loadCheckpoint(modelPathPrefix, 75) 40 | this.vocab = Utils.buildVocab(dataPath) 41 | 42 | val model = new RnnModel.LSTMInferenceModel(numLstmLayer, vocab.size + 1, numHidden = numHidden, numEmbed = numEmbed, numLabel = vocab.size + 1, argParams = argParams, dropout = 0.2f, ctx = context) 43 | 44 | model 45 | } 46 | 47 | def runSingleInference(modelPathPrefix: String, dataPath: String, starterSentence: String, context: Context, times: Int): List[Double] = { 48 | 49 | val model = loadModel(modelPathPrefix, dataPath, context) 50 | 51 | var inferenceTimes : List[Double] = List() 52 | 53 | // Warm up intervals 54 | println("Warming up the system") 55 | for (i <- 1 to 5) { 56 | NDArrayCollector.auto().withScope { 57 | val output = Utils.runPrediction(model, starterSentence, vocab) 58 | } 59 | } 60 | println("Warm up done") 61 | 62 | for (i <- 1 to times) { 63 | NDArrayCollector.auto().withScope { 64 | 65 | val startTime = System.nanoTime() 66 | val output = Utils.runPrediction(model, starterSentence, vocab) 67 | val estimatedTime = (System.nanoTime() - startTime) / (1e6 * 1.0) 68 | inferenceTimes = estimatedTime :: inferenceTimes 69 | 70 | println("Inference time at iteration: %d is : %f \n".format(i, estimatedTime)) 71 | } 72 | } 73 | 74 | inferenceTimes 75 | 76 | } 77 | 78 | 79 | def main(args: Array[String]): Unit = { 80 | 81 | val inst = new CLIParser 82 | 83 | val parser: CmdLineParser = new CmdLineParser(inst) 84 | 85 | parser.parseArgument(args.toList.asJava) 86 | 87 | var context = Utils.getContext(inst.context) 88 | 89 | val modelPathPrefix = inst.modelPathPrefix 90 | 91 | val dataPath = inst.dataPath 92 | 93 | val starterSentence = inst.starterSentence 94 | 95 | val times = inst.times 96 | 97 | NDArrayCollector.auto().withScope { 98 | println("Running single inference") 99 | val inferenceTimeSingle = runSingleInference(modelPathPrefix, dataPath, starterSentence, context, times) 100 | 101 | Utils.printStatistics(inferenceTimeSingle, "single_inference") 102 | 103 | } 104 | 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/densenet_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Densenet model configuration. 17 | 18 | References: 19 | "Densely Connected Convolutional Networks": https://arxiv.org/pdf/1608.06993 20 | """ 21 | import numpy as np 22 | from six.moves import xrange # pylint: disable=redefined-builtin 23 | import tensorflow as tf 24 | from models import model as model_lib 25 | 26 | 27 | class DensenetCifar10Model(model_lib.Model): 28 | """Densenet cnn network configuration.""" 29 | 30 | def __init__(self, model, layer_counts, growth_rate): 31 | self.growth_rate = growth_rate 32 | super(DensenetCifar10Model, self).__init__(model, 32, 64, 0.1, 33 | layer_counts=layer_counts) 34 | self.batch_norm_config = {'decay': 0.9, 'epsilon': 1e-5, 'scale': True} 35 | 36 | def dense_block(self, cnn, growth_rate): 37 | input_layer = cnn.top_layer 38 | c = cnn.batch_norm(input_layer, **self.batch_norm_config) 39 | c = tf.nn.relu(c) 40 | c = cnn.conv(growth_rate, 3, 3, 1, 1, stddev=np.sqrt(2.0/9/growth_rate), 41 | activation=None, input_layer=c) 42 | channel_index = 3 if cnn.channel_pos == 'channels_last' else 1 43 | cnn.top_layer = tf.concat([input_layer, c], channel_index) 44 | cnn.top_size += growth_rate 45 | 46 | def transition_layer(self, cnn): 47 | in_size = cnn.top_size 48 | cnn.batch_norm(**self.batch_norm_config) 49 | cnn.top_layer = tf.nn.relu(cnn.top_layer) 50 | cnn.conv(in_size, 1, 1, 1, 1, stddev=np.sqrt(2.0/9/in_size)) 51 | cnn.apool(2, 2, 2, 2) 52 | 53 | def add_inference(self, cnn): 54 | if self.layer_counts is None: 55 | raise ValueError('Layer counts not specified for %s' % self.get_model()) 56 | if self.growth_rate is None: 57 | raise ValueError('Growth rate not specified for %s' % self.get_model()) 58 | 59 | cnn.conv(16, 3, 3, 1, 1, activation=None) 60 | # Block 1 61 | for _ in xrange(self.layer_counts[0]): 62 | self.dense_block(cnn, self.growth_rate) 63 | self.transition_layer(cnn) 64 | # Block 2 65 | for _ in xrange(self.layer_counts[1]): 66 | self.dense_block(cnn, self.growth_rate) 67 | self.transition_layer(cnn) 68 | # Block 3 69 | for _ in xrange(self.layer_counts[2]): 70 | self.dense_block(cnn, self.growth_rate) 71 | cnn.batch_norm(**self.batch_norm_config) 72 | cnn.top_layer = tf.nn.relu(cnn.top_layer) 73 | channel_index = 3 if cnn.channel_pos == 'channels_last' else 1 74 | cnn.top_size = cnn.top_layer.get_shape().as_list()[channel_index] 75 | cnn.spatial_mean() 76 | 77 | def get_learning_rate(self, global_step, batch_size): 78 | num_batches_per_epoch = int(50000 / batch_size) 79 | boundaries = num_batches_per_epoch * np.array([150, 225, 300], 80 | dtype=np.int64) 81 | boundaries = [x for x in boundaries] 82 | values = [0.1, 0.01, 0.001, 0.0001] 83 | return tf.train.piecewise_constant(global_step, boundaries, values) 84 | -------------------------------------------------------------------------------- /onnx_benchmark/import_benchmarkscript.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import glob 4 | import time 5 | import numpy as np 6 | 7 | 8 | def get_model_input(model_dir): 9 | import onnx 10 | from onnx import numpy_helper 11 | 12 | model_inputs = [] 13 | for test_data_npz in glob.glob( 14 | os.path.join(model_dir, 'test_data_*.npz')): 15 | test_data = np.load(test_data_npz, encoding='bytes') 16 | model_inputs = list(test_data['inputs']) 17 | 18 | for test_data_dir in glob.glob( 19 | os.path.join(model_dir, "test_data_set*")): 20 | inputs_num = len(glob.glob(os.path.join(test_data_dir, 'input_*.pb'))) 21 | for i in range(inputs_num): 22 | input_file = os.path.join(test_data_dir, 'input_{}.pb'.format(i)) 23 | tensor = onnx.TensorProto() 24 | with open(input_file, 'rb') as f: 25 | tensor.ParseFromString(f.read()) 26 | model_inputs.append(numpy_helper.to_array(tensor)) 27 | 28 | input_shape = model_inputs[-1].shape 29 | # generating 1000 data points for inference time test 30 | for _ in range(1000 - len(model_inputs)): 31 | model_inputs.append(np.random.randn(*input_shape)) 32 | 33 | return model_inputs 34 | 35 | 36 | def profile_model(model_path, test_data, context): 37 | import mxnet as mx 38 | 39 | sym, arg_params, aux_params = mx.contrib.onnx.import_model(model_path) 40 | ctx = mx.gpu(0) if context == "gpu" else mx.cpu() 41 | data_names = [graph_input for graph_input in sym.list_inputs() 42 | if graph_input not in arg_params and graph_input not in aux_params] 43 | 44 | inference_time_list = [] 45 | data_shapes = [(data_names[0], test_data[0].shape)] 46 | 47 | # create a module 48 | mod = mx.mod.Module(symbol=sym, data_names=data_names, context=ctx, label_names=None) 49 | mod.bind(for_training=False, data_shapes=data_shapes, label_shapes=None) 50 | 51 | # initializing parameters for calculating result of each individual node 52 | if arg_params is None and aux_params is None: 53 | mod.init_params() 54 | else: 55 | mod.set_params(arg_params=arg_params, aux_params=aux_params) 56 | 57 | mx.nd.waitall() 58 | 59 | for val in test_data: 60 | data_forward = [mx.nd.array(val, ctx=ctx)] 61 | start = time.time() 62 | mod.forward(mx.io.DataBatch(data_forward)) 63 | for output in mod.get_outputs(): 64 | output.wait_to_read() 65 | total_time_in_ms = (time.time() - start) * 1000 66 | inference_time_list.append(total_time_in_ms) 67 | 68 | return inference_time_list 69 | 70 | 71 | if __name__ == '__main__': 72 | from sys import argv 73 | 74 | ctx = str(argv[1]) 75 | for directory in os.listdir("./models"): 76 | model_dir = os.path.join("./models", directory) 77 | if os.path.isdir(model_dir): 78 | model_path = os.path.join(model_dir, "model.onnx") 79 | test_data = get_model_input(model_dir) 80 | 81 | infer_time_list = profile_model(model_path, test_data, ctx) 82 | avg_infer_time = np.average(infer_time_list) 83 | p50_infer_time = np.percentile(infer_time_list, 50) 84 | p90_infer_time = np.percentile(infer_time_list, 90) 85 | p99_infer_time = np.percentile(infer_time_list, 99) 86 | 87 | print('Average_inference_time_{}_{}: {:.9f}'.format(directory, ctx, avg_infer_time)) 88 | print('P50_inference_time_{}_{}: {:.9f}'.format(directory, ctx, p50_infer_time)) 89 | print('P90_inference_time_{}_{}: {:.9f}'.format(directory, ctx, p90_infer_time)) 90 | print('P99_inference_time_{}_{}: {:.9f}'.format(directory, ctx, p99_infer_time)) 91 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Small library that points to a data set. 16 | 17 | Methods of Data class: 18 | data_files: Returns a python list of all (sharded) data set files. 19 | num_examples_per_epoch: Returns the number of examples in the data set. 20 | num_classes: Returns the number of classes in the data set. 21 | reader: Return a reader for a single entry from the data set. 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | from abc import ABCMeta 28 | from abc import abstractmethod 29 | import os 30 | 31 | 32 | import tensorflow as tf 33 | 34 | FLAGS = tf.app.flags.FLAGS 35 | 36 | # Basic model parameters. 37 | tf.app.flags.DEFINE_string('data_dir', '/tmp/mydata', 38 | """Path to the processed data, i.e. """ 39 | """TFRecord of Example protos.""") 40 | 41 | 42 | class Dataset(object): 43 | """A simple class for handling data sets.""" 44 | __metaclass__ = ABCMeta 45 | 46 | def __init__(self, name, subset): 47 | """Initialize dataset using a subset and the path to the data.""" 48 | assert subset in self.available_subsets(), self.available_subsets() 49 | self.name = name 50 | self.subset = subset 51 | 52 | @abstractmethod 53 | def num_classes(self): 54 | """Returns the number of classes in the data set.""" 55 | pass 56 | # return 10 57 | 58 | @abstractmethod 59 | def num_examples_per_epoch(self): 60 | """Returns the number of examples in the data subset.""" 61 | pass 62 | # if self.subset == 'train': 63 | # return 10000 64 | # if self.subset == 'validation': 65 | # return 1000 66 | 67 | @abstractmethod 68 | def download_message(self): 69 | """Prints a download message for the Dataset.""" 70 | pass 71 | 72 | def available_subsets(self): 73 | """Returns the list of available subsets.""" 74 | return ['train', 'validation'] 75 | 76 | def data_files(self): 77 | """Returns a python list of all (sharded) data subset files. 78 | 79 | Returns: 80 | python list of all (sharded) data set files. 81 | Raises: 82 | ValueError: if there are not data_files matching the subset. 83 | """ 84 | tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset) 85 | data_files = tf.gfile.Glob(tf_record_pattern) 86 | if not data_files: 87 | print('No files found for dataset %s/%s at %s' % (self.name, 88 | self.subset, 89 | FLAGS.data_dir)) 90 | 91 | self.download_message() 92 | exit(-1) 93 | return data_files 94 | 95 | def reader(self): 96 | """Return a reader for a single entry from the data set. 97 | 98 | See io_ops.py for details of Reader class. 99 | 100 | Returns: 101 | Reader object that reads the data set. 102 | """ 103 | return tf.TFRecordReader() 104 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/models/model_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Model configurations for CNN benchmarks. 17 | """ 18 | 19 | from models import alexnet_model 20 | from models import densenet_model 21 | from models import googlenet_model 22 | from models import inception_model 23 | from models import lenet_model 24 | from models import overfeat_model 25 | from models import resnet_model 26 | from models import trivial_model 27 | from models import vgg_model 28 | 29 | 30 | def get_model_config(model, dataset): 31 | """Map model name to model network configuration.""" 32 | if 'cifar10' == dataset.name: 33 | return get_cifar10_model_config(model) 34 | if model == 'vgg11': 35 | mc = vgg_model.Vgg11Model() 36 | elif model == 'vgg16': 37 | mc = vgg_model.Vgg16Model() 38 | elif model == 'vgg19': 39 | mc = vgg_model.Vgg19Model() 40 | elif model == 'lenet': 41 | mc = lenet_model.Lenet5Model() 42 | elif model == 'googlenet': 43 | mc = googlenet_model.GooglenetModel() 44 | elif model == 'overfeat': 45 | mc = overfeat_model.OverfeatModel() 46 | elif model == 'alexnet': 47 | mc = alexnet_model.AlexnetModel() 48 | elif model == 'trivial': 49 | mc = trivial_model.TrivialModel() 50 | elif model == 'inception3': 51 | mc = inception_model.Inceptionv3Model() 52 | elif model == 'inception4': 53 | mc = inception_model.Inceptionv4Model() 54 | elif model == 'resnet50' or model == 'resnet50_v2': 55 | mc = resnet_model.ResnetModel(model, (3, 4, 6, 3)) 56 | elif model == 'resnet101' or model == 'resnet101_v2': 57 | mc = resnet_model.ResnetModel(model, (3, 4, 23, 3)) 58 | elif model == 'resnet152' or model == 'resnet152_v2': 59 | mc = resnet_model.ResnetModel(model, (3, 8, 36, 3)) 60 | else: 61 | raise KeyError('Invalid model name \'%s\' for dataset \'%s\'' % 62 | (model, dataset.name)) 63 | return mc 64 | 65 | 66 | def get_cifar10_model_config(model): 67 | """Map model name to model network configuration for cifar10 dataset.""" 68 | if model == 'alexnet': 69 | mc = alexnet_model.AlexnetCifar10Model() 70 | elif model == 'resnet20' or model == 'resnet20_v2': 71 | mc = resnet_model.ResnetCifar10Model(model, (3, 3, 3)) 72 | elif model == 'resnet32' or model == 'resnet32_v2': 73 | mc = resnet_model.ResnetCifar10Model(model, (5, 5, 5)) 74 | elif model == 'resnet44' or model == 'resnet44_v2': 75 | mc = resnet_model.ResnetCifar10Model(model, (7, 7, 7)) 76 | elif model == 'resnet56' or model == 'resnet56_v2': 77 | mc = resnet_model.ResnetCifar10Model(model, (9, 9, 9)) 78 | elif model == 'resnet110' or model == 'resnet110_v2': 79 | mc = resnet_model.ResnetCifar10Model(model, (18, 18, 18)) 80 | elif model == 'densenet40_k12': 81 | mc = densenet_model.DensenetCifar10Model(model, (12, 12, 12), 12) 82 | elif model == 'densenet100_k12': 83 | mc = densenet_model.DensenetCifar10Model(model, (32, 32, 32), 12) 84 | elif model == 'densenet100_k24': 85 | mc = densenet_model.DensenetCifar10Model(model, (32, 32, 32), 24) 86 | else: 87 | raise KeyError('Invalid model name \'%s\' for Cifar10 DataSet.' % model) 88 | return mc 89 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/data/download_and_preprocess_flowers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "usage download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | wget -O ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | shuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/data/download_and_preprocess_flowers_mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess the flowers data set. This data set 18 | # provides a demonstration for how to perform fine-tuning (i.e. tranfer 19 | # learning) from one model to a new data set. 20 | # 21 | # This script provides a demonstration for how to prepare an arbitrary 22 | # data set for training an Inception v3 model. 23 | # 24 | # We demonstrate this with the flowers data set which consists of images 25 | # of labeled flower images from 5 classes: 26 | # 27 | # daisy, dandelion, roses, sunflowers, tulips 28 | # 29 | # The final output of this script are sharded TFRecord files containing 30 | # serialized Example protocol buffers. See build_image_data.py for 31 | # details of how the Example protocol buffer contains image data. 32 | # 33 | # usage: 34 | # ./download_and_preprocess_flowers.sh [data-dir] 35 | set -e 36 | 37 | if [ -z "$1" ]; then 38 | echo "usage download_and_preprocess_flowers.sh [data dir]" 39 | exit 40 | fi 41 | 42 | # Create the output and temporary directories. 43 | DATA_DIR="${1%/}" 44 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 45 | mkdir -p "${DATA_DIR}" 46 | mkdir -p "${SCRATCH_DIR}" 47 | WORK_DIR="$0.runfiles/inception/inception" 48 | 49 | # Download the flowers data. 50 | DATA_URL="http://download.tensorflow.org/example_images/flower_photos.tgz" 51 | CURRENT_DIR=$(pwd) 52 | cd "${DATA_DIR}" 53 | TARBALL="flower_photos.tgz" 54 | if [ ! -f ${TARBALL} ]; then 55 | echo "Downloading flower data set." 56 | wget -O ${TARBALL} "${DATA_URL}" 57 | else 58 | echo "Skipping download of flower data." 59 | fi 60 | 61 | # Note the locations of the train and validation data. 62 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 63 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 64 | 65 | # Expands the data into the flower_photos/ directory and rename it as the 66 | # train directory. 67 | tar xf flower_photos.tgz 68 | rm -rf "${TRAIN_DIRECTORY}" "${VALIDATION_DIRECTORY}" 69 | mv flower_photos "${TRAIN_DIRECTORY}" 70 | 71 | # Generate a list of 5 labels: daisy, dandelion, roses, sunflowers, tulips 72 | LABELS_FILE="${SCRATCH_DIR}/labels.txt" 73 | ls -1 "${TRAIN_DIRECTORY}" | grep -v 'LICENSE' | sed 's/\///' | sort > "${LABELS_FILE}" 74 | 75 | # Generate the validation data set. 76 | while read LABEL; do 77 | VALIDATION_DIR_FOR_LABEL="${VALIDATION_DIRECTORY}${LABEL}" 78 | TRAIN_DIR_FOR_LABEL="${TRAIN_DIRECTORY}${LABEL}" 79 | 80 | # Move the first randomly selected 100 images to the validation set. 81 | mkdir -p "${VALIDATION_DIR_FOR_LABEL}" 82 | VALIDATION_IMAGES=$(ls -1 "${TRAIN_DIR_FOR_LABEL}" | gshuf | head -100) 83 | for IMAGE in ${VALIDATION_IMAGES}; do 84 | mv -f "${TRAIN_DIRECTORY}${LABEL}/${IMAGE}" "${VALIDATION_DIR_FOR_LABEL}" 85 | done 86 | done < "${LABELS_FILE}" 87 | 88 | # Build the TFRecords version of the image data. 89 | cd "${CURRENT_DIR}" 90 | BUILD_SCRIPT="${WORK_DIR}/build_image_data" 91 | OUTPUT_DIRECTORY="${DATA_DIR}" 92 | "${BUILD_SCRIPT}" \ 93 | --train_directory="${TRAIN_DIRECTORY}" \ 94 | --validation_directory="${VALIDATION_DIRECTORY}" \ 95 | --output_directory="${OUTPUT_DIRECTORY}" \ 96 | --labels_file="${LABELS_FILE}" 97 | -------------------------------------------------------------------------------- /README_BENCHMARKAI.md: -------------------------------------------------------------------------------- 1 | # Deep Learning Benchmark Script Driver 2 | 3 | This repo holds the benchmark tasks that are executed regularly and also the driver script. 4 | It is designed to be simple and flexible. Basically it is a thin wrapper on top of your benchmark script which provides 5 | basic utilities for managing tasks, profiling gpu and cpu memories, and extract metrics from your logging. 6 | 7 | To use the benchmark driver, you just need to look at the task_config_template.cfg to see what benchmark tasks are supported 8 | and use the command `python benchmark_driver.py --task-name [TASK_NAME] --num-gpus [NUM_GPUS]` then the driver will trigger a 9 | task and write the benchmark result to a JSON file called "dlbenchmark_result.json". e.g. 10 | 11 | ``` 12 | { 13 | 'resnet50_cifar10_hybrid.total_training_time': 797.3191379999998, 14 | 'resnet50_cifar10_hybrid.gpu_memory_usage_max': 749.0, 15 | 'resnet50_cifar10_hybrid.cpu_memory_usage': 788, 16 | 'resnet50_cifar10_hybrid.validation_acc': 0.560998, 17 | 'resnet50_cifar10_hybrid.speed': 1268.8090892, 18 | 'resnet50_cifar10_hybrid.gpu_memory_usage_std': 66.27262632335068, 19 | 'resnet50_cifar10_hybrid.training_acc': 0.751256, 20 | 'resnet50_cifar10_hybrid.gpu_memory_usage_mean': 602.6772235225278 21 | } 22 | ``` 23 | 24 | There are also optional flags such as `--framework`, and `--metrics-suffix`, which are used for decorating the metrics names. 25 | 26 | To add a new task you just need to put your benchmark script or your benchmark script directory under this repo and 27 | add a section in the `task_config_template.cfg` like the following example: 28 | 29 | ``` 30 | [resnet50_cifar10_imperative] 31 | patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)'] 32 | metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time'] 33 | compute_method = ['average', 'last', 'last', 'total'] 34 | command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --gpus 8 --epochs 20 --log-interval 50 35 | num_gpus = 8 36 | ``` 37 | 38 | `patterns`, `metrics`, `compute_method` need to be placed in corresponding order so that the metrics map knows the key pair relationship. 39 | Users need to defined the logging extraction rule using python's regular expression. 40 | 41 | Typical logging looks like the following example: 42 | 43 | ``` 44 | INFO:root:Epoch[0] Batch [49] Speed: 829.012392 samples/sec accuracy=0.172578 45 | INFO:root:Epoch[0] Batch [99] Speed: 844.157227 samples/sec accuracy=0.206563 46 | INFO:root:Epoch[0] Batch [149] Speed: 835.582445 samples/sec accuracy=0.230781 47 | INFO:root:[Epoch 0] training: accuracy=0.248027 48 | INFO:root:[Epoch 0] time cost: 69.030488 49 | INFO:root:[Epoch 0] validation: accuracy=0.296484 50 | INFO:root:Epoch[1] Batch [49] Speed: 797.687061 samples/sec accuracy=0.322344 51 | INFO:root:Epoch[1] Batch [99] Speed: 821.444257 samples/sec accuracy=0.329883 52 | INFO:root:Epoch[1] Batch [149] Speed: 810.339386 samples/sec accuracy=0.342969 53 | INFO:root:[Epoch 1] training: accuracy=0.351983 54 | INFO:root:[Epoch 1] time cost: 61.266612 55 | INFO:root:[Epoch 1] validation: accuracy=0.393930 56 | ``` 57 | 58 | Here you can see the pattern `Speed: (\d+\.\d+|\d+) samples/sec` correspond to `Speed: 829.012392 samples/sec`, 59 | `Speed: 844.157227 samples/sec`, `Speed: 835.582445 samples/sec`, ... So the driver will extract these parts 60 | into a list and map a number extractor to this list. So in the end we will get `metric=[829.012392, 844.157227, 61 | 835.582445,...]`. The `compute_method` is `average`, suppose `average([829.012392, 844.157227, 62 | 835.582445,...])=825.25` ,so it will put the pair `speed: 825.25` in the final result file. 63 | 64 | The driver will redirect the logging into a logfile and will remove it after the metrics have been successfully 65 | extracted. -------------------------------------------------------------------------------- /image_classification/symbols/vgg.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """References: 19 | 20 | Simonyan, Karen, and Andrew Zisserman. "Very deep convolutional networks for 21 | large-scale image recognition." arXiv preprint arXiv:1409.1556 (2014). 22 | """ 23 | 24 | import mxnet as mx 25 | import numpy as np 26 | 27 | def get_feature(internel_layer, layers, filters, batch_norm = False, **kwargs): 28 | for i, num in enumerate(layers): 29 | for j in range(num): 30 | internel_layer = mx.sym.Convolution(data = internel_layer, kernel=(3, 3), pad=(1, 1), num_filter=filters[i], name="conv%s_%s" %(i + 1, j + 1)) 31 | if batch_norm: 32 | internel_layer = mx.symbol.BatchNorm(data=internel_layer, name="bn%s_%s" %(i + 1, j + 1)) 33 | internel_layer = mx.sym.Activation(data=internel_layer, act_type="relu", name="relu%s_%s" %(i + 1, j + 1)) 34 | internel_layer = mx.sym.Pooling(data=internel_layer, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool%s" %(i + 1)) 35 | return internel_layer 36 | 37 | def get_classifier(input_data, num_classes, **kwargs): 38 | flatten = mx.sym.Flatten(data=input_data, name="flatten") 39 | fc6 = mx.sym.FullyConnected(data=flatten, num_hidden=4096, name="fc6") 40 | relu6 = mx.sym.Activation(data=fc6, act_type="relu", name="relu6") 41 | drop6 = mx.sym.Dropout(data=relu6, p=0.5, name="drop6") 42 | fc7 = mx.sym.FullyConnected(data=drop6, num_hidden=4096, name="fc7") 43 | relu7 = mx.sym.Activation(data=fc7, act_type="relu", name="relu7") 44 | drop7 = mx.sym.Dropout(data=relu7, p=0.5, name="drop7") 45 | fc8 = mx.sym.FullyConnected(data=drop7, num_hidden=num_classes, name="fc8") 46 | return fc8 47 | 48 | def get_symbol(num_classes, num_layers=11, batch_norm=False, dtype='float32', **kwargs): 49 | """ 50 | Parameters 51 | ---------- 52 | num_classes : int, default 1000 53 | Number of classification classes. 54 | num_layers : int 55 | Number of layers for the variant of densenet. Options are 11, 13, 16, 19. 56 | batch_norm : bool, default False 57 | Use batch normalization. 58 | dtype: str, float32 or float16 59 | Data precision. 60 | """ 61 | vgg_spec = {11: ([1, 1, 2, 2, 2], [64, 128, 256, 512, 512]), 62 | 13: ([2, 2, 2, 2, 2], [64, 128, 256, 512, 512]), 63 | 16: ([2, 2, 3, 3, 3], [64, 128, 256, 512, 512]), 64 | 19: ([2, 2, 4, 4, 4], [64, 128, 256, 512, 512])} 65 | if num_layers not in vgg_spec: 66 | raise ValueError("Invalide num_layers {}. Possible choices are 11,13,16,19.".format(num_layers)) 67 | layers, filters = vgg_spec[num_layers] 68 | data = mx.sym.Variable(name="data") 69 | if dtype == 'float16': 70 | data = mx.sym.Cast(data=data, dtype=np.float16) 71 | feature = get_feature(data, layers, filters, batch_norm) 72 | classifier = get_classifier(feature, num_classes) 73 | if dtype == 'float16': 74 | classifier = mx.sym.Cast(data=classifier, dtype=np.float32) 75 | symbol = mx.sym.SoftmaxOutput(data=classifier, name='softmax') 76 | return symbol 77 | -------------------------------------------------------------------------------- /scala-mxnet/java-bm/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | mxnet 5 | javaBenchmark 6 | jar 7 | 1.0-SNAPSHOT 8 | javaBenchmark 9 | 10 | 2.11 11 | [1.5.0-SNAPSHOT,) 12 | cpu 13 | 14 | 15 | 16 | 17 | Apache Snapshot 18 | https://repository.apache.org/content/groups/snapshots 19 | 20 | 21 | 22 | 23 | 24 | osx-x86_64 25 | 26 | 27 | mac 28 | 29 | 30 | 31 | osx-x86_64-cpu 32 | 33 | 34 | 35 | linux-x86_64 36 | 37 | 38 | unix 39 | Linux 40 | 41 | 42 | 43 | linux-x86_64-${mxnet.hw_type} 44 | 45 | 46 | 47 | 48 | 49 | 50 | org.apache.mxnet 51 | mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} 52 | ${mxnet.version} 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-compiler-plugin 61 | 3.8.0 62 | 63 | 1.8 64 | 1.8 65 | 66 | 67 | 68 | maven-resources-plugin 69 | 2.7 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-dependency-plugin 74 | 2.9 75 | 76 | 77 | copy-dependencies 78 | package 79 | 80 | copy-dependencies 81 | 82 | 83 | ${project.build.outputDirectory}/lib 84 | runtime 85 | test,provided 86 | false 87 | false 88 | true 89 | 90 | 91 | 92 | 93 | 94 | org.apache.maven.plugins 95 | maven-jar-plugin 96 | 2.5 97 | 98 | 99 | package 100 | 101 | jar 102 | 103 | 104 | 105 | **/* 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /end_to_end_model_benchmark/java-bm/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | mxnet 5 | javaBenchmark 6 | jar 7 | 1.0-SNAPSHOT 8 | javaBenchmark 9 | 10 | 2.11 11 | [1.5.0-SNAPSHOT,) 12 | cpu 13 | 14 | 15 | 16 | 17 | Apache Snapshot 18 | https://repository.apache.org/content/groups/snapshots 19 | 20 | 21 | 22 | 23 | 24 | osx-x86_64 25 | 26 | 27 | mac 28 | 29 | 30 | 31 | osx-x86_64-cpu 32 | 33 | 34 | 35 | linux-x86_64 36 | 37 | 38 | unix 39 | Linux 40 | 41 | 42 | 43 | linux-x86_64-${mxnet.hw_type} 44 | 45 | 46 | 47 | 48 | 49 | 50 | org.apache.mxnet 51 | mxnet-full_${mxnet.scalaprofile}-${mxnet.profile} 52 | ${mxnet.version} 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-compiler-plugin 61 | 3.8.0 62 | 63 | 1.8 64 | 1.8 65 | 66 | 67 | 68 | maven-resources-plugin 69 | 2.7 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-dependency-plugin 74 | 2.9 75 | 76 | 77 | copy-dependencies 78 | package 79 | 80 | copy-dependencies 81 | 82 | 83 | ${project.build.outputDirectory}/lib 84 | runtime 85 | test,provided 86 | false 87 | false 88 | true 89 | 90 | 91 | 92 | 93 | 94 | org.apache.maven.plugins 95 | maven-jar-plugin 96 | 2.5 97 | 98 | 99 | package 100 | 101 | jar 102 | 103 | 104 | 105 | **/* 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /reporting/report.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | # -*- coding: utf-8 -*- 18 | """ 19 | A CLI to generate benchmark reports for Benchmark.AI. 20 | 21 | The boto3 clients require the environment variables to be set accordingly: 22 | * AWS_ACCESS_KEY_ID 23 | * AWS_SECRET_ACCESS_KEY 24 | * AWS_DEFAULT_REGION 25 | """ 26 | import argparse 27 | import io 28 | import json 29 | import logging 30 | import os 31 | import pickle 32 | import sys 33 | 34 | from utils.benchmarks import Benchmarks 35 | from utils.email import email_report 36 | from utils.report_generation import HTML_EXTENSION 37 | from utils.report_generation import generate_report 38 | 39 | 40 | logging.getLogger().setLevel(logging.INFO) 41 | logging.getLogger('boto3').setLevel(logging.CRITICAL) 42 | logging.getLogger('botocore').setLevel(logging.CRITICAL) 43 | 44 | 45 | if __name__ == '__main__': 46 | parser = argparse.ArgumentParser(description='Generate a Benchmark report.') 47 | parser.add_argument('-f', '--report-file', default='bai-report', help='the report file name \ 48 | minus the extension. An .xlsx and a .html report will be generated.') 49 | parser.add_argument('-l', '--load-benchmarks', default='', 50 | help='whether to load benchmarks from file (for debugging).') 51 | parser.add_argument('-s', '--save-benchmarks', default='', 52 | help='the file to save the benchmarks (for debugging).') 53 | parser.add_argument('-e', '--email-addr', default='', 54 | help='send a report to e-mail address.') 55 | parser.add_argument('-lm', '--list-all-metrics', default='false', 56 | help='gets all Benchmark.AI the metrics (for debugging') 57 | 58 | args = parser.parse_args() 59 | if args.list_all_metrics == 'true': 60 | benchmarks = Benchmarks(fetch_metrics=False) 61 | print('\n'.join(benchmarks.list_all_metrics())) 62 | sys.exit(0) 63 | 64 | if args.load_benchmarks: 65 | logging.info("Loading benchmarks from {}".format(args.load_benchmarks)) 66 | benchmarks = pickle.load(open(args.load_benchmarks, 'rb')) 67 | else: 68 | logging.info("Reading configuration and fetching metrics from Cloudwatch.") 69 | benchmarks = Benchmarks() 70 | 71 | if args.save_benchmarks: 72 | logging.info("Saving benchmarks to {}".format(args.save_benchmarks)) 73 | # For pickling, remove Boto client (we don't need the boto client after this point). 74 | benchmarks._cw = None 75 | pickle.dump(benchmarks, open(args.save_benchmarks, 'wb')) 76 | 77 | # TODO(vishaalk): If e-mail is requested and report file name not specified, use a temp file. 78 | if not args.report_file: 79 | logging.error('Report filename prefix required to generate report. Skipping report.') 80 | sys.exit(1) 81 | 82 | logging.info('Generating report.') 83 | generate_report(args.report_file, benchmarks) 84 | if args.email_addr: 85 | report_html = io.open(args.report_file + HTML_EXTENSION, mode='r', encoding='utf-8').read() 86 | email_report(report_html, args.email_addr) 87 | 88 | -------------------------------------------------------------------------------- /tensorflow_benchmark/tf_cnn_benchmarks/cbuild_benchmark_storage.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Provides a way to store benchmark results in GCE Datastore. 16 | 17 | Datastore client is initialized from current environment. 18 | Data is stored using the format defined in: 19 | https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/test/upload_test_benchmarks_index.yaml 20 | """ 21 | from datetime import datetime 22 | import json 23 | import os 24 | import sys 25 | from google.cloud import datastore 26 | import six 27 | 28 | 29 | _TEST_NAME_ENV_VAR = 'TF_DIST_BENCHMARK_NAME' 30 | 31 | 32 | def upload_to_benchmark_datastore(data, test_name=None, start_time=None): 33 | """Use a new datastore.Client to upload data to datastore. 34 | 35 | Create the datastore Entities from that data and upload them to the 36 | datastore in a batch using the client connection. 37 | 38 | Args: 39 | data: Map from benchmark names to values. 40 | test_name: Name of this test. If not specified, name will be set either 41 | from TF_DIST_BENCHMARK_NAME environment variable or to default name 42 | 'TestBenchmark'. 43 | start_time: (datetime) Time to record for this test. 44 | 45 | Raises: 46 | ValueError: if test_name is not passed in and TF_DIST_BENCHMARK_NAME 47 | is not set. 48 | """ 49 | client = datastore.Client() 50 | 51 | if not test_name: 52 | if _TEST_NAME_ENV_VAR in os.environ: 53 | test_name = os.environ[_TEST_NAME_ENV_VAR] 54 | else: 55 | raise ValueError( 56 | 'No test name passed in for benchmarks. ' 57 | 'Either pass a test_name to upload_to_benchmark_datastore or ' 58 | 'set %s environment variable.' % _TEST_NAME_ENV_VAR) 59 | test_name = six.text_type(test_name) 60 | 61 | if not start_time: 62 | start_time = datetime.now() 63 | 64 | # Create one Entry Entity for each benchmark entry. The wall-clock timing is 65 | # the attribute to be fetched and displayed. The full entry information is 66 | # also stored as a non-indexed JSON blob. 67 | entries = [] 68 | batch = [] 69 | for name, value in data.items(): 70 | e_key = client.key('Entry') 71 | e_val = datastore.Entity(e_key, exclude_from_indexes=['info']) 72 | entry_map = {'name': name, 'wallTime': value, 'iters': '1'} 73 | entries.append(entry_map) 74 | e_val.update({ 75 | 'test': test_name, 76 | 'start': start_time, 77 | 'entry': six.text_type(name), 78 | 'timing': value, 79 | 'info': six.text_type(json.dumps(entry_map)) 80 | }) 81 | batch.append(e_val) 82 | 83 | # Create the Test Entity containing all the test information as a 84 | # non-indexed JSON blob. 85 | test_result = json.dumps( 86 | {'name': test_name, 87 | 'startTime': (start_time - datetime(1970, 1, 1)).total_seconds(), 88 | 'entries': {'entry': entries}, 89 | 'runConfiguration': {'argument': sys.argv[1:]}}) 90 | t_key = client.key('Test') 91 | t_val = datastore.Entity(t_key, exclude_from_indexes=['info']) 92 | t_val.update({ 93 | 'test': test_name, 94 | 'start': start_time, 95 | 'info': six.text_type(test_result) 96 | }) 97 | batch.append(t_val) 98 | 99 | # Put the whole batch of Entities in the datastore. 100 | client.put_multi(batch) 101 | -------------------------------------------------------------------------------- /tensorflow/inception/inception/data/download_and_preprocess_imagenet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2016 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # ============================================================================== 16 | 17 | # Script to download and preprocess ImageNet Challenge 2012 18 | # training and validation data set. 19 | # 20 | # The final output of this script are sharded TFRecord files containing 21 | # serialized Example protocol buffers. See build_imagenet_data.py for 22 | # details of how the Example protocol buffers contain the ImageNet data. 23 | # 24 | # The final output of this script appears as such: 25 | # 26 | # data_dir/train-00000-of-01024 27 | # data_dir/train-00001-of-01024 28 | # ... 29 | # data_dir/train-00127-of-01024 30 | # 31 | # and 32 | # 33 | # data_dir/validation-00000-of-00128 34 | # data_dir/validation-00001-of-00128 35 | # ... 36 | # data_dir/validation-00127-of-00128 37 | # 38 | # Note that this script may take several hours to run to completion. The 39 | # conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending 40 | # on the speed of your machine. Please be patient. 41 | # 42 | # **IMPORTANT** 43 | # To download the raw images, the user must create an account with image-net.org 44 | # and generate a username and access_key. The latter two are required for 45 | # downloading the raw images. 46 | # 47 | # usage: 48 | # ./download_and_preprocess_imagenet.sh [data-dir] 49 | set -e 50 | 51 | if [ -z "$1" ]; then 52 | echo "usage download_and_preprocess_imagenet.sh [data dir]" 53 | exit 54 | fi 55 | 56 | # Create the output and temporary directories. 57 | DATA_DIR="${1%/}" 58 | SCRATCH_DIR="${DATA_DIR}/raw-data/" 59 | mkdir -p "${DATA_DIR}" 60 | mkdir -p "${SCRATCH_DIR}" 61 | WORK_DIR="$0.runfiles/inception/inception" 62 | 63 | # Download the ImageNet data. 64 | LABELS_FILE="${WORK_DIR}/data/imagenet_lsvrc_2015_synsets.txt" 65 | DOWNLOAD_SCRIPT="${WORK_DIR}/data/download_imagenet.sh" 66 | "${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}" 67 | 68 | # Note the locations of the train and validation data. 69 | TRAIN_DIRECTORY="${SCRATCH_DIR}train/" 70 | VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/" 71 | 72 | # Preprocess the validation data by moving the images into the appropriate 73 | # sub-directory based on the label (synset) of the image. 74 | echo "Organizing the validation data into sub-directories." 75 | PREPROCESS_VAL_SCRIPT="${WORK_DIR}/data/preprocess_imagenet_validation_data.py" 76 | VAL_LABELS_FILE="${WORK_DIR}/data/imagenet_2012_validation_synset_labels.txt" 77 | 78 | "${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}" 79 | 80 | # Convert the XML files for bounding box annotations into a single CSV. 81 | echo "Extracting bounding box information from XML." 82 | BOUNDING_BOX_SCRIPT="${WORK_DIR}/data/process_bounding_boxes.py" 83 | BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv" 84 | BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/" 85 | 86 | "${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \ 87 | | sort >"${BOUNDING_BOX_FILE}" 88 | echo "Finished downloading and preprocessing the ImageNet data." 89 | 90 | # Build the TFRecords version of the ImageNet data. 91 | BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data" 92 | OUTPUT_DIRECTORY="${DATA_DIR}" 93 | IMAGENET_METADATA_FILE="${WORK_DIR}/data/imagenet_metadata.txt" 94 | 95 | "${BUILD_SCRIPT}" \ 96 | --train_directory="${TRAIN_DIRECTORY}" \ 97 | --validation_directory="${VALIDATION_DIRECTORY}" \ 98 | --output_directory="${OUTPUT_DIRECTORY}" \ 99 | --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \ 100 | --labels_file="${LABELS_FILE}" \ 101 | --bounding_box_file="${BOUNDING_BOX_FILE}" 102 | -------------------------------------------------------------------------------- /image_classification/common/modelzoo.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os 19 | from common.util import download_file 20 | 21 | _base_model_url = 'http://data.mxnet.io/models/' 22 | _default_model_info = { 23 | 'imagenet1k-inception-bn': {'symbol':_base_model_url+'imagenet/inception-bn/Inception-BN-symbol.json', 24 | 'params':_base_model_url+'imagenet/inception-bn/Inception-BN-0126.params'}, 25 | 'imagenet1k-resnet-18': {'symbol':_base_model_url+'imagenet/resnet/18-layers/resnet-18-symbol.json', 26 | 'params':_base_model_url+'imagenet/resnet/18-layers/resnet-18-0000.params'}, 27 | 'imagenet1k-resnet-34': {'symbol':_base_model_url+'imagenet/resnet/34-layers/resnet-34-symbol.json', 28 | 'params':_base_model_url+'imagenet/resnet/34-layers/resnet-34-0000.params'}, 29 | 'imagenet1k-resnet-50': {'symbol':_base_model_url+'imagenet/resnet/50-layers/resnet-50-symbol.json', 30 | 'params':_base_model_url+'imagenet/resnet/50-layers/resnet-50-0000.params'}, 31 | 'imagenet1k-resnet-101': {'symbol':_base_model_url+'imagenet/resnet/101-layers/resnet-101-symbol.json', 32 | 'params':_base_model_url+'imagenet/resnet/101-layers/resnet-101-0000.params'}, 33 | 'imagenet1k-resnet-152': {'symbol':_base_model_url+'imagenet/resnet/152-layers/resnet-152-symbol.json', 34 | 'params':_base_model_url+'imagenet/resnet/152-layers/resnet-152-0000.params'}, 35 | 'imagenet1k-resnext-50': {'symbol':_base_model_url+'imagenet/resnext/50-layers/resnext-50-symbol.json', 36 | 'params':_base_model_url+'imagenet/resnext/50-layers/resnext-50-0000.params'}, 37 | 'imagenet1k-resnext-101': {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-symbol.json', 38 | 'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-0000.params'}, 39 | 'imagenet1k-resnext-101-64x4d': {'symbol':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-symbol.json', 40 | 'params':_base_model_url+'imagenet/resnext/101-layers/resnext-101-64x4d-0000.params'}, 41 | 'imagenet11k-resnet-152': {'symbol':_base_model_url+'imagenet-11k/resnet-152/resnet-152-symbol.json', 42 | 'params':_base_model_url+'imagenet-11k/resnet-152/resnet-152-0000.params'}, 43 | 'imagenet11k-place365ch-resnet-152': {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-152-symbol.json', 44 | 'params':_base_model_url+'imagenet-11k-place365-ch/resnet-152-0000.params'}, 45 | 'imagenet11k-place365ch-resnet-50': {'symbol':_base_model_url+'imagenet-11k-place365-ch/resnet-50-symbol.json', 46 | 'params':_base_model_url+'imagenet-11k-place365-ch/resnet-50-0000.params'}, 47 | } 48 | 49 | def download_model(model_name, dst_dir='./', meta_info=None): 50 | if meta_info is None: 51 | meta_info = _default_model_info 52 | meta_info = dict(meta_info) 53 | if model_name not in meta_info: 54 | return (None, 0) 55 | if not os.path.isdir(dst_dir): 56 | os.mkdir(dst_dir) 57 | meta = dict(meta_info[model_name]) 58 | assert 'symbol' in meta, "missing symbol url" 59 | model_name = os.path.join(dst_dir, model_name) 60 | download_file(meta['symbol'], model_name+'-symbol.json') 61 | assert 'params' in meta, "mssing parameter file url" 62 | download_file(meta['params'], model_name+'-0000.params') 63 | return (model_name, 0) 64 | -------------------------------------------------------------------------------- /image_classification/predict_image.py: -------------------------------------------------------------------------------- 1 | 2 | # # Predict with pre-trained models 3 | 4 | import argparse, time 5 | import logging 6 | logging.basicConfig(level=logging.INFO) 7 | 8 | import numpy as np 9 | import mxnet as mx 10 | import cv2 11 | from collections import namedtuple 12 | 13 | 14 | class InferenceTesting(object): 15 | def __init__(self, opt): 16 | self.Batch = namedtuple('Batch',['data']) 17 | self.model = opt.model 18 | self.iterations = opt.iterations 19 | self.model_path = opt.model_path 20 | self.url = opt.url 21 | 22 | def downloadModel(self): 23 | network = self.model.split('-')[0] 24 | layers = self.model.split('-')[1] 25 | base_path = "{}{}/{}-layers/".format(self.model_path,network, layers) 26 | print(base_path) 27 | model_json_path = "{}{}-symbol.json".format(base_path, self.model) 28 | model_params_path = "{}{}-0000.params".format(base_path, self.model) 29 | model_synset_path = "{}{}/synset.txt".format(self.model_path,network) 30 | print(model_synset_path) 31 | print(model_json_path) 32 | print(model_params_path) 33 | 34 | try: 35 | mx.test_utils.download(model_json_path) 36 | mx.test_utils.download(model_params_path) 37 | mx.test_utils.download(model_synset_path) 38 | except Exception as e: 39 | print("Error in downloading the models {}".format(e)) 40 | 41 | def __loadModel(self): 42 | sym, arg_params, aux_params = mx.model.load_checkpoint(self.model, 0) 43 | cont = mx.gpu() if opt.use_gpus > 0 else mx.cpu() 44 | self.mod = mx.mod.Module(symbol=sym, context=cont, label_names=None) 45 | self.mod.bind(for_training=False, data_shapes=[('data', (1, 3, 224, 224))], 46 | label_shapes=self.mod._label_shapes) 47 | self.mod.set_params(arg_params, aux_params, allow_missing=True) 48 | with open('synset.txt', 'r') as f: 49 | self.labels = [l.rstrip() for l in f] 50 | 51 | def __getImage(self): 52 | try: 53 | fname = mx.test_utils.download(self.url) 54 | except Exception as e: 55 | print("Error in downloading the image {}".format(e)) 56 | img = cv2.cvtColor(cv2.imread(fname), cv2.COLOR_BGR2RGB) 57 | if img is None: 58 | return None 59 | # convert into format (batch, RGB, width, height) 60 | img = cv2.resize(img, (224, 224)) 61 | img = np.swapaxes(img, 0, 2) 62 | img = np.swapaxes(img, 1, 2) 63 | img = img[np.newaxis, :] 64 | return img 65 | 66 | def predict(self): 67 | self.__loadModel() 68 | img = self.__getImage() 69 | if img is None: 70 | print("Error: Can not load the image") 71 | # compute the predict probabilities 72 | for i in range(self.iterations): 73 | tic = time.time() 74 | self.mod.forward(self.Batch([mx.nd.array(img)])) 75 | pred_time_ms = (time.time() - tic) * 1000 76 | print ("Prediction-Time: {} milliseconds".format(pred_time_ms)) 77 | 78 | if __name__ == '__main__': 79 | parser = argparse.ArgumentParser(description='Train a model for image classification.') 80 | parser.add_argument('--model', type=str, default='resnet-50', 81 | help='pre-trained model') 82 | parser.add_argument('--iterations', type=int, default=1, 83 | help='Number of times inference is generated for a single image.') 84 | parser.add_argument('--model_path', type=str, default='http://data.mxnet.io/models/imagenet/', 85 | help='Default path to load the model') 86 | parser.add_argument('--url', type=str, default='http://writm.com/wp-content/uploads/2016/08/Cat-hd-wallpapers.jpg', 87 | help='Url to the image file') 88 | parser.add_argument('--use_gpus', type=int, default=0, 89 | help="Indicate whether to use gpus") 90 | opt = parser.parse_args() 91 | 92 | print(opt) 93 | 94 | # Following sleep is added so that process runs until cpu-gpu profiler process starts. 95 | infer = InferenceTesting(opt) 96 | infer.downloadModel() 97 | infer.predict() 98 | time.sleep(10) 99 | print ("Done") 100 | exit() 101 | 102 | -------------------------------------------------------------------------------- /image_classification/symbols/googlenet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """References: 19 | 20 | Szegedy, Christian, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir 21 | Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich. "Going deeper 22 | with convolutions." arXiv preprint arXiv:1409.4842 (2014). 23 | 24 | """ 25 | 26 | import mxnet as mx 27 | 28 | def ConvFactory(data, num_filter, kernel, stride=(1,1), pad=(0, 0), name=None, suffix=''): 29 | conv = mx.symbol.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='conv_%s%s' %(name, suffix)) 30 | act = mx.symbol.Activation(data=conv, act_type='relu', name='relu_%s%s' %(name, suffix)) 31 | return act 32 | 33 | def InceptionFactory(data, num_1x1, num_3x3red, num_3x3, num_d5x5red, num_d5x5, pool, proj, name): 34 | # 1x1 35 | c1x1 = ConvFactory(data=data, num_filter=num_1x1, kernel=(1, 1), name=('%s_1x1' % name)) 36 | # 3x3 reduce + 3x3 37 | c3x3r = ConvFactory(data=data, num_filter=num_3x3red, kernel=(1, 1), name=('%s_3x3' % name), suffix='_reduce') 38 | c3x3 = ConvFactory(data=c3x3r, num_filter=num_3x3, kernel=(3, 3), pad=(1, 1), name=('%s_3x3' % name)) 39 | # double 3x3 reduce + double 3x3 40 | cd5x5r = ConvFactory(data=data, num_filter=num_d5x5red, kernel=(1, 1), name=('%s_5x5' % name), suffix='_reduce') 41 | cd5x5 = ConvFactory(data=cd5x5r, num_filter=num_d5x5, kernel=(5, 5), pad=(2, 2), name=('%s_5x5' % name)) 42 | # pool + proj 43 | pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(1, 1), pad=(1, 1), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name))) 44 | cproj = ConvFactory(data=pooling, num_filter=proj, kernel=(1, 1), name=('%s_proj' % name)) 45 | # concat 46 | concat = mx.symbol.Concat(*[c1x1, c3x3, cd5x5, cproj], name='ch_concat_%s_chconcat' % name) 47 | return concat 48 | 49 | def get_symbol(num_classes = 1000, **kwargs): 50 | data = mx.sym.Variable("data") 51 | conv1 = ConvFactory(data, 64, kernel=(7, 7), stride=(2,2), pad=(3, 3), name="conv1") 52 | pool1 = mx.sym.Pooling(conv1, kernel=(3, 3), stride=(2, 2), pool_type="max") 53 | conv2 = ConvFactory(pool1, 64, kernel=(1, 1), stride=(1,1), name="conv2") 54 | conv3 = ConvFactory(conv2, 192, kernel=(3, 3), stride=(1, 1), pad=(1,1), name="conv3") 55 | pool3 = mx.sym.Pooling(conv3, kernel=(3, 3), stride=(2, 2), pool_type="max") 56 | 57 | in3a = InceptionFactory(pool3, 64, 96, 128, 16, 32, "max", 32, name="in3a") 58 | in3b = InceptionFactory(in3a, 128, 128, 192, 32, 96, "max", 64, name="in3b") 59 | pool4 = mx.sym.Pooling(in3b, kernel=(3, 3), stride=(2, 2), pool_type="max") 60 | in4a = InceptionFactory(pool4, 192, 96, 208, 16, 48, "max", 64, name="in4a") 61 | in4b = InceptionFactory(in4a, 160, 112, 224, 24, 64, "max", 64, name="in4b") 62 | in4c = InceptionFactory(in4b, 128, 128, 256, 24, 64, "max", 64, name="in4c") 63 | in4d = InceptionFactory(in4c, 112, 144, 288, 32, 64, "max", 64, name="in4d") 64 | in4e = InceptionFactory(in4d, 256, 160, 320, 32, 128, "max", 128, name="in4e") 65 | pool5 = mx.sym.Pooling(in4e, kernel=(3, 3), stride=(2, 2), pool_type="max") 66 | in5a = InceptionFactory(pool5, 256, 160, 320, 32, 128, "max", 128, name="in5a") 67 | in5b = InceptionFactory(in5a, 384, 192, 384, 48, 128, "max", 128, name="in5b") 68 | pool6 = mx.sym.Pooling(in5b, kernel=(7, 7), stride=(1,1), global_pool=True, pool_type="avg") 69 | flatten = mx.sym.Flatten(data=pool6) 70 | fc1 = mx.sym.FullyConnected(data=flatten, num_hidden=num_classes) 71 | softmax = mx.symbol.SoftmaxOutput(data=fc1, name='softmax') 72 | return softmax 73 | --------------------------------------------------------------------------------