├── .dockerignore ├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── FAQ.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── Singularity ├── amazon-dsstne └── README.md ├── benchmarks ├── Benchmark.md ├── dsstne │ └── config.json └── tf │ └── autoencoder.py ├── docs └── getting_started │ ├── CDL.txt │ ├── DDL.txt │ ├── LDL.txt │ ├── examples.md │ ├── setup.md │ └── userguide.md ├── java ├── Makefile ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── amazon │ │ │ └── dsstne │ │ │ ├── Dim.java │ │ │ ├── Dsstne.java │ │ │ ├── NNDataSet.java │ │ │ ├── NNDataSetEnums.java │ │ │ ├── NNLayer.java │ │ │ ├── NNNetwork.java │ │ │ ├── NetworkConfig.java │ │ │ ├── TopKOutput.java │ │ │ ├── data │ │ │ ├── DenseNNDataSet.java │ │ │ ├── DenseWeightedNNDataSet.java │ │ │ ├── SparseNNDataSet.java │ │ │ └── SparseWeightedNNDataSet.java │ │ │ └── knn │ │ │ ├── AbstractKNearestNeighbors.java │ │ │ ├── BatchedKNearestNeighbors.java │ │ │ ├── DataType.java │ │ │ ├── DataUtil.java │ │ │ ├── KNearestNeighbors.java │ │ │ ├── KNearestNeighborsCuda.java │ │ │ ├── KnnResult.java │ │ │ ├── KnnSearch.java │ │ │ ├── NearestNeighbors.java │ │ │ ├── Neighbor.java │ │ │ ├── TimedBatchExecutor.java │ │ │ ├── UnbatchedKNearestNeighbors.java │ │ │ └── Vector.java │ ├── native │ │ ├── com_amazon_dsstne_Dsstne.cpp │ │ ├── com_amazon_dsstne_Dsstne.h │ │ ├── jni_util.cpp │ │ └── jni_util.h │ └── native_knn │ │ ├── com_amazon_dsstne_knn_KNearestNeighborsCuda.cpp │ │ └── com_amazon_dsstne_knn_KNearestNeighborsCuda.h │ └── test │ ├── java │ └── com │ │ └── amazon │ │ └── dsstne │ │ ├── NNLayerTest.java │ │ ├── NNNetworkTest.java │ │ ├── NetworkConfigTest.java │ │ ├── TopKOutputTest.java │ │ ├── data │ │ ├── DenseNNDataSetTest.java │ │ ├── DenseWeightedNNDataSetTest.java │ │ ├── DimTest.java │ │ └── NNDataSetEnumsTest.java │ │ ├── knn │ │ ├── DataTypeTest.java │ │ ├── KNearestNeighborsCudaTest.java │ │ └── KnnResultTest.java │ │ └── test-data │ │ ├── expected_output_1024.txt │ │ ├── index.txt │ │ ├── movielens_20m_autoencoder.nc │ │ └── movielens_input_1024.txt │ └── native │ └── main.cpp ├── python ├── CDLAccessors.h ├── Makefile ├── NNDataSetAccessors.h ├── NNLayerAccessors.h ├── NNNetworkAccessors.h ├── NNNetworkFunctions.h ├── NNWeightAccessors.h ├── README.md ├── dsstnecalculate.h ├── dsstnemodule.cc ├── dsstnemodule.h ├── encoder │ ├── README.md │ ├── embedding.json │ ├── encoder.py │ └── inference.json ├── images │ ├── README.md │ ├── checkpoint │ │ └── README.md │ ├── config.json │ ├── images.py │ ├── predict.cdl │ ├── results │ │ └── README.md │ └── train.cdl ├── setup.py └── utilities.h ├── samples ├── cifar-10 │ ├── config.json │ ├── dparse.cpp │ ├── predict.cdl │ └── train.cdl ├── movielens │ ├── config.json │ ├── convert_ratings.awk │ └── run_movielens_sample.sh └── network │ ├── config_1.json │ ├── config_2.json │ ├── config_3.json │ ├── config_4.json │ ├── config_5.json │ └── config_6.json ├── src └── amazon │ └── dsstne │ ├── Makefile │ ├── Makefile.inc │ ├── engine │ ├── GpuSort.h │ ├── GpuTypes.cpp │ ├── GpuTypes.h │ ├── Makefile │ ├── NNEnum.h │ ├── NNLayer.cpp │ ├── NNLayer.h │ ├── NNNetwork.cpp │ ├── NNNetwork.h │ ├── NNTypes.cpp │ ├── NNTypes.h │ ├── NNWeight.cpp │ ├── NNWeight.h │ ├── NcExcptionWrap.h │ ├── bitonic.h │ ├── kActivation.cu │ ├── kDelta.cu │ ├── kLoss.cu │ ├── kernels.cu │ └── kernels.h │ ├── knn │ ├── DataReader.cpp │ ├── DataReader.h │ ├── KnnData.cpp │ ├── KnnData.h │ ├── KnnExactGpu.cu │ ├── KnnExactGpu.h │ ├── Makefile │ ├── MathUtil.cu │ ├── MathUtil.h │ ├── cudautil.cpp │ ├── cudautil.h │ ├── topk.cu │ └── topk.h │ ├── runtime │ ├── DsstneContext.cpp │ ├── DsstneContext.h │ └── Makefile │ └── utils │ ├── FilterHelper.cpp │ ├── Filters.cpp │ ├── Filters.h │ ├── Makefile │ ├── NNRecsGenerator.cpp │ ├── NNRecsGenerator.h │ ├── NetCDFGenerator.cpp │ ├── NetCDFhelper.cpp │ ├── NetCDFhelper.h │ ├── Predict.cpp │ ├── Train.cpp │ ├── Utils.cpp │ ├── Utils.h │ ├── cdl.cpp │ ├── cdl.h │ └── main.cpp ├── talks ├── Data Science Summit.ppt ├── GTC2016.pdf ├── ISCA2017.pptx ├── OpenDataScienceConference2016.odp ├── QCon2017.odp └── TradeShowWest2017.odp └── tst ├── Makefile ├── amazon └── dsstne │ └── engine │ ├── TestGpuBuffer.cpp │ ├── TestNNDataSet.cpp │ └── TestNNDataSetDimensions.cpp ├── gputests ├── CMakeLists.txt ├── TestActivationFunctions.cpp ├── TestCostFunctions.cpp ├── TestDune.cpp ├── TestGpu.cpp ├── TestSort.cpp └── TestUtils.h ├── test_data ├── validate_DataScaledMarginalCrossEntropy_02.json ├── validate_L2_01.json ├── validate_L2_02.json ├── validate_L2_LRelu_01.json ├── validate_L2_LRelu_02.json ├── validate_ScaledMarginalCrossEntropy_01.json └── validate_ScaledMarginalCrossEntropy_02.json ├── unittests.cpp └── unittests ├── CMakeLists.txt ├── TestNetCDFhelper.cpp ├── TestUtils.cpp └── main.cpp /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.o 3 | /src/amazon/dsstne/bin/ 4 | /src/amazon/dsstne/lib/ 5 | /src/amazon/dsstne/utils/encoder 6 | /src/amazon/dsstne/utils/generateNetCDF 7 | /src/amazon/dsstne/utils/predict 8 | /src/amazon/dsstne/utils/train 9 | /src/amazon/dsstne/utils/TestGPU 10 | /src/amazon/dsstne/include/ 11 | /build/ 12 | /amazon-dsstne/ 13 | java/target 14 | java/target/** 15 | .gitignore 16 | 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | sudo: required 3 | dist: trusty 4 | 5 | matrix: 6 | include: 7 | - os: linux 8 | compiler: gcc 9 | addons: 10 | apt: 11 | sources: ['ubuntu-toolchain-r-test'] 12 | packages: ['g++-4.8'] 13 | env: 14 | - NETCDF_CXX4_CXXFLAGS=-Wno-write-strings 15 | - OUR_CXX=g++-4.8 16 | - OUR_CC=gcc-4.8 17 | 18 | - os: linux 19 | compiler: gcc 20 | addons: 21 | apt: 22 | sources: ['ubuntu-toolchain-r-test'] 23 | packages: ['g++-4.9'] 24 | env: 25 | - NETCDF_CXX4_CXXFLAGS=-Wno-write-strings 26 | - OUR_CXX=g++-4.9 27 | - OUR_CC=gcc-4.9 28 | 29 | - os: linux 30 | compiler: gcc 31 | addons: 32 | apt: 33 | sources: ['ubuntu-toolchain-r-test'] 34 | packages: ['g++-5'] 35 | env: 36 | - NETCDF_CXX4_CXXFLAGS=-Wno-write-strings 37 | - OUR_CXX=g++-5 38 | - OUR_CC=gcc-5 39 | 40 | - os: linux 41 | compiler: clang 42 | addons: 43 | apt: 44 | sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.7'] 45 | packages: ['clang-3.7'] 46 | env: 47 | - NETCDF_CXX4_CXXFLAGS=-Wno-c++11-compat-deprecated-writable-strings 48 | - OUR_CXX=clang++-3.7 49 | - OUR_CC=clang-3.7 50 | 51 | - os: linux 52 | compiler: clang 53 | addons: 54 | apt: 55 | sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.8'] 56 | packages: ['clang-3.8'] 57 | env: 58 | - NETCDF_CXX4_CXXFLAGS=-Wno-c++11-compat-deprecated-writable-strings 59 | - OUR_CXX=clang++-3.8 60 | - OUR_CC=clang-3.8 61 | 62 | before_install: 63 | - sudo apt-get update -qq 64 | - sudo apt-get install -qq libcppunit-dev libnetcdf-dev 65 | 66 | install: 67 | - wget http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz 68 | - tar xvf netcdf-cxx4-4.2.tar.gz 69 | - pushd netcdf-cxx4-4.2 70 | - export CC=$OUR_CC 71 | - export CXX=$OUR_CXX 72 | - CXXFLAGS=$NETCDF_CXX4_CXXFLAGS ./configure --prefix=/usr/local 73 | - make 74 | - sudo make install 75 | - popd 76 | 77 | script: 78 | - cd tst/unittests 79 | - mkdir build && cd build 80 | - cmake .. 81 | - make 82 | - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ./unittests 83 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check [existing open](https://github.com/amzn/amazon-dsstne/issues), or [recently closed](https://github.com/amzn/amazon-dsstne/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amzn/amazon-dsstne/labels/help%20wanted) issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](https://github.com/amzn/amazon-dsstne/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # VERSION 0.3 2 | # AUTHOR: DSSTNE Docker 3 | # DESCRIPTION: Docker image for Amazon DSSTNE 4 | 5 | FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 6 | 7 | # Suppress interactive prompts while installing base packages 8 | ENV DEBIAN_FRONTEND=noninteractive 9 | 10 | # Add repositories and install base packages 11 | RUN apt-get update && \ 12 | apt-get install -y build-essential libcppunit-dev libatlas-base-dev pkg-config python \ 13 | software-properties-common unzip wget && \ 14 | add-apt-repository ppa:george-edison55/cmake-3.x && \ 15 | apt-get update && \ 16 | apt-get install -y cmake && \ 17 | apt-get clean 18 | 19 | # Install OpenMPI 20 | RUN apt-get install -y libopenmpi-dev 21 | 22 | # Install JSONCPP 23 | RUN apt-get install -y libjsoncpp-dev 24 | 25 | # Install hdf5 26 | RUN apt-get install -y libhdf5-dev 27 | 28 | # Install zlib 29 | RUN apt-get install -y zlib1g-dev 30 | 31 | # Install netcdf 32 | RUN apt-get install -y libnetcdf-dev 33 | 34 | # Install netcdf-c++ 35 | RUN apt-get install -y libnetcdf-c++4-dev 36 | 37 | # Installing CUBG 38 | RUN cd /tmp && \ 39 | wget https://github.com/NVlabs/cub/archive/1.5.2.zip && \ 40 | unzip 1.5.2.zip && \ 41 | cp -rf cub-1.5.2/cub/ /usr/local/include/ && \ 42 | rm -rf /tmp/* 43 | 44 | # Ensure OpenMPI is available on path 45 | ENV PATH=/usr/local/openmpi/bin/:${PATH} \ 46 | LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/openmpi/lib/:${LD_LIBRARY_PATH} 47 | 48 | # Build latest version of DSSTNE from source 49 | COPY . /opt/amazon/dsstne 50 | RUN cd /opt/amazon/dsstne && \ 51 | make install 52 | 53 | # Cleanup 54 | RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 55 | 56 | # Add DSSTNE binaries to PATH 57 | ENV PATH=/opt/amazon/dsstne/bin/:${PATH} 58 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/bin/sh 2 | VPATH= 3 | 4 | # Prefix of the directory to install dsstne. 5 | PREFIX ?= $(shell pwd)/amazon-dsstne 6 | 7 | # Build directory. Export it for sub-makefiles to use it 8 | export BUILD_DIR ?= $(shell pwd)/build 9 | 10 | all: | engine runtime utils tests java 11 | 12 | engine: 13 | cd src/amazon/dsstne/engine && make 14 | 15 | utils: 16 | cd src/amazon/dsstne/utils && make 17 | 18 | runtime: 19 | cd src/amazon/dsstne/runtime && make 20 | 21 | tests: 22 | cd tst && make 23 | 24 | #java: | engine runtime tests 25 | # cd java && make 26 | 27 | install: all 28 | mkdir -p $(PREFIX) 29 | cp -rfp $(BUILD_DIR)/lib $(PREFIX) 30 | cp -rfp $(BUILD_DIR)/bin $(PREFIX) 31 | cp -rfp $(BUILD_DIR)/include $(PREFIX) 32 | 33 | run-tests: 34 | cd tst && make run-tests 35 | 36 | clean: 37 | cd src/amazon/dsstne/engine && make clean 38 | cd src/amazon/dsstne/utils && make clean 39 | cd tst && make clean 40 | 41 | #.PHONY: engine runtime tests java clean 42 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Amazon DSSTNE 2 | Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Amazon DSSTNE: Deep Scalable Sparse Tensor Network Engine 4 | 5 | DSSTNE (pronounced "Destiny") is an open source software library for training and deploying recommendation 6 | models with sparse inputs, fully connected hidden layers, and sparse outputs. Models with weight matrices 7 | that are too large for a single GPU can still be trained on a single host. DSSTNE has been used at Amazon 8 | to generate personalized product recommendations for our customers at Amazon's scale. It is designed for 9 | production deployment of real-world applications which need to emphasize speed and scale over experimental 10 | flexibility. 11 | 12 | DSSTNE was built with a number of features for production recommendation workloads: 13 | 14 | * **Multi-GPU Scale**: Training and prediction 15 | both scale out to use multiple GPUs, spreading out computation 16 | and storage in a model-parallel fashion for each layer. 17 | * **Large Layers**: Model-parallel scaling enables larger networks than 18 | are possible with a single GPU. 19 | * **Sparse Data**: DSSTNE is optimized for fast performance on sparse datasets, common in recommendation 20 | problems. Custom GPU kernels perform sparse computation on the GPU, without filling in lots of zeroes. 21 | 22 | ## Benchmarks 23 | * scottlegrand@ reported [near-linear scaling with multiple GPUs] on the MovieLens recommendation problem 24 | (https://medium.com/@scottlegrand/first-dsstne-benchmarks-tldr-almost-15x-faster-than-tensorflow-393dbeb80c0f#.ghe74fu1q) 25 | * Directions on how to run a benchmark can be found in [here](benchmarks/Benchmark.md) 26 | 27 | ## Scaling up 28 | * [Using Spark in AWS EMR and Dockers in AWS ECS ](http://blogs.aws.amazon.com/bigdata/post/TxGEL8IJ0CAXTK/Generating-Recommendations-at-Amazon-Scale-with-Apache-Spark-and-Amazon-DSSTNE) 29 | 30 | 31 | ## License 32 | [License](LICENSE) 33 | 34 | 35 | 36 | 37 | 38 | ## Setup 39 | * Follow [Setup](docs/getting_started/setup.md) for step by step instructions on installing and setting up DSSTNE 40 | 41 | ## User Guide 42 | * Check [User Guide](docs/getting_started/userguide.md) for detailed information about the features in DSSTNE 43 | 44 | ## Examples 45 | * Check [Examples](docs/getting_started/examples.md) to start trying your first Neural Network Modeling using DSSTNE 46 | 47 | ## Q&A 48 | [FAQ](FAQ.md) 49 | -------------------------------------------------------------------------------- /Singularity: -------------------------------------------------------------------------------- 1 | Bootstrap: docker 2 | From: nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 3 | 4 | %environment 5 | DEBIAN_FRONTEND=noninteractive 6 | LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/openmpi/lib/ 7 | PATH=/amazon-dsstne/src/amazon/dsstne/bin/:/usr/local/openmpi/bin/:${PATH} 8 | export DEBIAN_FRONTEND LD_LIBRARY_PATH PATH 9 | 10 | %setup 11 | echo $SINGULARITY_ROOTFS 12 | echo "cp -r . $SINGULARITY_ROOTFS/" 13 | cp Singularity $SINGULARITY_ROOTFS/ 14 | cp -r ./amazon-dsstne/ $SINGULARITY_ROOTFS/ 15 | 16 | %post 17 | apt-get update && \ 18 | apt-get install -y build-essential libcppunit-dev libatlas-base-dev pkg-config python \ 19 | software-properties-common unzip wget && \ 20 | add-apt-repository ppa:george-edison55/cmake-3.x && \ 21 | apt-get update && \ 22 | apt-get install -y cmake && \ 23 | apt-get clean && \ 24 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 25 | 26 | cd /tmp && \ 27 | wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz && \ 28 | tar xvfz openmpi-2.1.1.tar.gz && \ 29 | cd openmpi-2.1.1 && \ 30 | ./configure CC=gcc CXX=g++ --enable-mpi-cxx --prefix=/usr/local/openmpi && \ 31 | make -j 8 && \ 32 | sudo make install && rm -rf /tmp/* 33 | 34 | cd /tmp && \ 35 | wget https://github.com/open-source-parsers/jsoncpp/archive/svn-import.tar.gz && \ 36 | tar xvfz svn-import.tar.gz && \ 37 | cd jsoncpp-svn-import && \ 38 | mkdir -p build/release && \ 39 | cd build/release && \ 40 | cmake -DCMAKE_BUILD_TYPE=release -DJSONCPP_LIB_BUILD_SHARED=OFF -G "Unix Makefiles" ../.. && \ 41 | make -j 8 && \ 42 | make install && rm -rf /tmp/* 43 | 44 | cd /tmp && \ 45 | wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4/hdf5-1.8.9.tar.gz && \ 46 | tar xvfz hdf5-1.8.9.tar.gz && \ 47 | cd hdf5-1.8.9 && \ 48 | ./configure --prefix=/usr/local &&\ 49 | make -j 8 && \ 50 | make install && rm -rf /tmp/* 51 | 52 | cd /tmp && \ 53 | wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-4/zlib-1.2.8.tar.gz && \ 54 | tar xvf zlib-1.2.8.tar.gz && \ 55 | cd zlib-1.2.8 && \ 56 | ./configure && \ 57 | make -j 8 && \ 58 | make install && rm -rf /tmp/* 59 | 60 | cd /tmp && \ 61 | wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-4.1.3.tar.gz && \ 62 | tar xvf netcdf-4.1.3.tar.gz && \ 63 | cd netcdf-4.1.3 && \ 64 | ./configure --prefix=/usr/local && \ 65 | make -j 8 && \ 66 | make install && rm -rf /tmp/* 67 | 68 | cd /tmp && \ 69 | wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz && \ 70 | tar xvf netcdf-cxx4-4.2.tar.gz && \ 71 | cd netcdf-cxx4-4.2 && \ 72 | ./configure --prefix=/usr/local && \ 73 | make -j 8 && \ 74 | make install && rm -rf /tmp/* 75 | 76 | cd /tmp && \ 77 | wget https://github.com/NVlabs/cub/archive/1.5.2.zip && \ 78 | unzip 1.5.2.zip && \ 79 | cp -rf cub-1.5.2/cub/ /usr/local/include/ && \ 80 | rm -rf /tmp/* 81 | 82 | export LD_LIBRARY_PATH=/usr/local/lib/:/usr/local/openmpi/lib/:${LD_LIBRARY_PATH} 83 | export PATH=/amazon-dsstne/src/amazon/dsstne/bin:/usr/local/openmpi/bin:/usr/local/bin:/usr/local/include/bin:/usr/local/cuda-7.5/bin:${PATH} 84 | cd /amazon-dsstne/src/amazon/dsstne && \ 85 | make install 86 | -------------------------------------------------------------------------------- /amazon-dsstne/README.md: -------------------------------------------------------------------------------- 1 | The dsstne libraries that are created by the 'make install' command are copied from the ../build/lib/ directory and placed in the lib/ directory. -------------------------------------------------------------------------------- /benchmarks/Benchmark.md: -------------------------------------------------------------------------------- 1 | # Benchmarks for DSSTNE 2 | We ran the benchmark of DSSTNE using the Movielens data set. Training Parameter Specific 3 | * 27278 input/output dimensions 4 | * 3 Hidden Sigmoid layers with 1024 each 5 | * RMSProp learner 6 | * 256 Batchsize 7 | 8 | Time taken to run one epoch is considered for performance comparison 9 | 10 | 11 | ## DSSTNE 12 | Use the Config at [config.json](dsstne/config.json). Follow the [example](../docs/getting_started/examples.md) and change the training command 13 | ```bash 14 | train -i gl_input.nc -o gl_output.nc -d gl -c config.json -b 256 -e 20 -n gl_network.nc 15 | ``` 16 | 17 | ## TensorFlow 18 | [autoencoder.py](tf/autoencoder.py) 19 | ```bash 20 | autoencoder.py -u 1024 -b 256 -i 1082 -v54 --vocab_size 27278 -l 3 -f /input/data/ml20m-all.remotcc 21 | ``` 22 | 23 | -------------------------------------------------------------------------------- /benchmarks/dsstne/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "Network", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | 9 | "ScaledMarginalCrossEntropy" : { 10 | "oneTarget" : 1.0, 11 | "zeroTarget" : 0.0, 12 | "oneScale" : 1.0, 13 | "zeroScale" : 1.0 14 | }, 15 | "Layers" : [ 16 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "gl_input", "Sparse" : true }, 17 | { "Name" : "Hidden1", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 1024, "Activation" : "Sigmoid", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 18 | { "Name" : "Hidden2", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : ["Hidden1"], "N" : 1024, "Activation" : "Sigmoid", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 19 | { "Name" : "Hidden3", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : ["Hidden2"], "N" : 1024, "Activation" : "Sigmoid", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 20 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "gl_output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true , "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01, "Bias" : -10.2 }} 21 | ], 22 | 23 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 24 | } 25 | -------------------------------------------------------------------------------- /docs/getting_started/CDL.txt: -------------------------------------------------------------------------------- 1 | Command Definition Language (Rev 0.1): 2 | 3 | TrainingParameters - only used if Command is Train 4 | PredictionParameters - only used if Command is Predict 5 | 6 | 7 | "Network" : # NetCDF or JSON file path (default "config.json") 8 | "Command" : # (required) Command to execute "Train", "Predict", or "Validate" 9 | "RandomSeed" : # Initializes RNG for reproducible runs (default -1, sets from time of day) 10 | "Data" : [ # (required) List of data source files 11 | 12 | . 13 | . 14 | . 15 | ], 16 | "TrainingParameters" : { 17 | "Epochs" : # (required) Number of training epochs 18 | "MiniBatch" : # Mini-batch size (default 500, use 0 for entire dataset) 19 | "Alpha" : # Learning rate (default 0.1) 20 | "Lambda" : # Regularization/Weight Decay weight (default 0.001) 21 | "mu" : # Momentum update parameter (default 0.9) 22 | "AlphaInterval" : # Interval between learning rate updates (default 0: constant) 23 | "AlphaMultiplier" : # Learning rate multiplier per Interval (default: 0.5, or if lr is constant: 1.0) 24 | "Optimizer" : # Optimization method, either "SGD", "Momentum", "RMSPROP", or "Nesterov" (default "SGD") 25 | "CheckpointName" : # Location to write checkpoint information, appended with epoch number and ".nc" (default "check") 26 | "CheckpointInterval" : # Number of epochs between writing checkpoint data (default 1) 27 | "ShuffleIndices" : # Shuffle training examples once per epoch? (default false) 28 | "Results" : # Location to write results (File or S3 Object) (default "network.nc") 29 | }, 30 | "PredictionParameters" : { 31 | "MiniBatch" : # Mini-batch size (default 500, use 0 for entire dataset) 32 | }, 33 | } 34 | -------------------------------------------------------------------------------- /docs/getting_started/DDL.txt: -------------------------------------------------------------------------------- 1 | NetCDF Binary data file format 0.2 2 | 3 | NetCDF variables 4 | datasets (int) # of datasets in file 5 | 6 | Remainder of variables are formatted with the name followed by 0 to datasets - 1 for indexing 7 | 8 | nameN (string) # Name of dataset n 9 | attributesN (uint) # Attributes of dataset defined below 10 | kindN (uint) # Specifies whether data is image, numeric, or audio 11 | dimensionN (uint) # Dimensionality of data points (1-3) 12 | widthN (int) # Width of a single data point 13 | heightN (int) # Height of a single data point 14 | lengthN (int) # Length of a single data point 15 | examplesDimN (int) # Number of datapoints in dataset 16 | dataDimN (int) # Size of raw data 17 | dataN (data) # Raw data if present 18 | sparseDataDimN (int) # Total sparse datapoint count 19 | sparseStartN (uint64) # Variable containing starting offsets into sparse data per data point 20 | sparseEndN (uint64) # Variable containing end offsets per data point 21 | sparseIndexN (uint32) # Raw sparse indices 22 | sparseDataN (data) # Raw sparse values if data is not categorical 23 | 24 | 25 | JSON Image/Audio Data format 26 | { 27 | "N" : # Number of data points 28 | "Sparse" : # Indicates whether output is sparse or not 29 | "Boolean" : # Indicates whether non-zero values are 1/0 or continuous 30 | "Categorical" : # Indicates whether data is 1 of n or not 31 | "Data : 32 | [ 33 | { 34 | "File" : # Filename for source data 35 | "Output" : # Either NOutput distinct values or indices to non-zero values if sparse 36 | } 37 | . 38 | . 39 | . 40 | ] 41 | } 42 | 43 | 44 | CSV Data format 45 | Name of data set, N, width, Sparse Flag, Categorical Flag 46 | Followed by N rows of: 47 | width comma-separated data points 48 | 49 | 50 | Attribute Definitions 51 | 52 | Sparse # Signifies data is mostly 0, therefore only non-zero indices are saved 53 | Boolean # Signifies data has binary (0/1) values rather than continuous 54 | 55 | 56 | The output of any layer of any neural network can be dumped in either format 57 | 58 | The weights of a neural network can be dumped as a multi-dataset file 59 | -------------------------------------------------------------------------------- /java/Makefile: -------------------------------------------------------------------------------- 1 | MODULE_ROOT = $(shell pwd) 2 | 3 | BUILD_DIR ?= ../build 4 | 5 | DSSTNE_SRC_DIR = ../src/amazon/dsstne 6 | 7 | SOURCES_DIR = src/main/native 8 | KNN_SOURCES_DIR = src/main/native_knn 9 | 10 | TEST_DIR = src/test/native 11 | OBJECTS_DIR = target/native/build 12 | KNN_OBJECTS_DIR = target/native_knn/build 13 | LIBRARY_DIR = target/lib 14 | BIN_DIR = target/bin 15 | LIBRARY = $(LIBRARY_DIR)/libdsstne_java.so 16 | KNN_LIBRARY = $(LIBRARY_DIR)/libdsstne_knn_java.so 17 | MAIN = main.o 18 | 19 | INC = \ 20 | /usr/lib/jvm/java-1.8.0-openjdk-amd64/include \ 21 | /usr/lib/jvm/java-8-openjdk-amd64/include/linux \ 22 | /usr/local/cuda/include \ 23 | /usr/include/jsoncpp \ 24 | /usr/lib/openmpi/include \ 25 | /usr/lib/openmpi/include/openmpi \ 26 | ../src # dsstne headers 27 | 28 | INCLUDES ?= $(INC:%=-I%) 29 | 30 | LIB = \ 31 | $(BUILD_DIR)/lib \ 32 | /usr/local/cuda/lib64 \ 33 | /usr/lib/openmpi/lib 34 | 35 | LIBS ?= $(LIB:%=-L%) 36 | 37 | LLIB = \ 38 | cudnn \ 39 | curand \ 40 | cublas \ 41 | cudart \ 42 | jsoncpp \ 43 | netcdf \ 44 | netcdf_c++4 \ 45 | blas \ 46 | dl \ 47 | stdc++ \ 48 | mpi_cxx \ 49 | mpi \ 50 | dsstne_engine \ 51 | dsstne_runtime 52 | 53 | LOAD_LIBS = $(LLIB:%=-l%) 54 | 55 | SOURCES=$(shell find '$(SOURCES_DIR)' -type f -name '*.cpp') 56 | OBJECTS=$(SOURCES:$(SOURCES_DIR)/%.cpp=$(OBJECTS_DIR)/%.o) 57 | 58 | KNN_SOURCES=$(shell find '$(KNN_SOURCES_DIR)' -type f -name '*.cpp') 59 | KNN_OBJECTS=$(KNN_SOURCES:$(KNN_SOURCES_DIR)/%.cpp=$(KNN_OBJECTS_DIR)/%.o) 60 | 61 | CC = g++ 62 | CFLAGS ?=-Wall -fPIC -std=c++11 -pthread -O3 63 | LDFLAGS ?= -fPIC 64 | 65 | all: $(LIBRARY) $(KNN_LIBRARY) 66 | mkdir -p $(BUILD_DIR)/lib 67 | cp -fp $(LIBRARY) $(BUILD_DIR)/lib 68 | cp -fp $(KNN_LIBRARY) $(BUILD_DIR)/lib 69 | 70 | main: 71 | $(info ========== Building main =============) 72 | mkdir -p $(BIN_DIR) 73 | $(CC) -g $(CFLAGS) -Wl,-rpath=/usr/local/cuda/lib64 $(INCLUDES) $(LIBS) src/test/native/main.cpp -o $(BIN_DIR)/main.o $(LOAD_LIBS) 74 | 75 | $(LIBRARY): $(OBJECTS) $(LIB_DSSTNE) 76 | $(info ========== Building JNI Libs ==========) 77 | mkdir -p $(LIBRARY_DIR) 78 | $(CC) -shared $(LDFLAGS) $(LIBS) $(OBJECTS) -o $@ $(LOAD_LIBS) 79 | 80 | $(KNN_LIBRARY): $(KNN_OBJECTS) $(LIB_DSSTNE) 81 | $(info ========== Building KNN JNI Libs ==========) 82 | mkdir -p $(LIBRARY_DIR) 83 | $(CC) -shared $(LDFLAGS) $(LIBS) $(KNN_OBJECTS) -o $@ $(LOAD_LIBS) 84 | 85 | $(OBJECTS_DIR)/%.o: $(SOURCES_DIR)/%.cpp 86 | mkdir -p $(OBJECTS_DIR) 87 | $(CC) -c $(CFLAGS) $(INCLUDES) $< -o $@ 88 | 89 | $(KNN_OBJECTS_DIR)/%.o: $(KNN_SOURCES_DIR)/%.cpp 90 | mkdir -p $(KNN_OBJECTS_DIR) 91 | $(CC) -c $(CFLAGS) $(INCLUDES) $< -o $@ 92 | 93 | clean: 94 | rm -rf $(OBJECTS_DIR) $(LIBRARY) $(BIN_DIR) 95 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/Dim.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | /** 21 | * Describes a dimension (typically of either {@link NNDataSet} or {@link NNLayer}). 22 | */ 23 | public class Dim { 24 | /** 25 | * 1 for 1-d, 2 for 2-d, 3 for 3-d. 26 | */ 27 | public final int dimensions; 28 | 29 | /** 30 | * Length of the first dimension. 31 | */ 32 | public final int x; 33 | 34 | /** 35 | * Length of the second dimension. (1 for 1-d data). 36 | */ 37 | public final int y; 38 | 39 | /** 40 | * Length of the third dimension. (1 for 1-d and 2-d data). 41 | */ 42 | public final int z; 43 | 44 | /** 45 | * Number of examples. 46 | */ 47 | public final int examples; 48 | 49 | /** 50 | * Stride between examples. 51 | */ 52 | public final int stride; 53 | 54 | public Dim(final int dimensions, final int x, final int y, final int z, final int examples) { 55 | this.dimensions = dimensions; 56 | this.x = x; 57 | this.y = y; 58 | this.z = z; 59 | this.examples = examples; 60 | this.stride = x * y * z; 61 | } 62 | 63 | /** 64 | * Creates a new Dim object with the (x,y,z) dimensions equal to the 65 | * one provided and a new number of examples. 66 | */ 67 | public Dim(final Dim dim, final int examples) { 68 | this(dim.dimensions, dim.x, dim.y, dim.z, examples); 69 | } 70 | 71 | /** 72 | * Creates a 1-d dimension. 73 | */ 74 | public static Dim _1d(final int x, final int examples) { 75 | return new Dim(1, x, 1, 1, examples); 76 | } 77 | 78 | /** 79 | * Creates a 2-d dimension. 80 | */ 81 | public static Dim _2d(final int x, final int y, final int examples) { 82 | return new Dim(2, x, y, 1, examples); 83 | } 84 | 85 | /** 86 | * Creates a 2-d dimension. 87 | */ 88 | public static Dim _3d(final int x, final int y, final int z, final int examples) { 89 | return new Dim(3, x, y, z, examples); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/Dsstne.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import java.util.List; 21 | 22 | import com.amazon.dsstne.NNLayer.Kind; 23 | 24 | /** 25 | * Entry point to obtaining {@link NNNetwork}. Defines all the native methods. 26 | */ 27 | public class Dsstne { 28 | public static final long NULLPTR = 0x0; 29 | 30 | static { 31 | System.loadLibrary("dsstne_java"); 32 | } 33 | 34 | public static NNNetwork load(final NetworkConfig config) { 35 | long ptr = load(config.getNetworkFilePath(), config.getBatchSize(), config.getK()); 36 | if (ptr == NULLPTR) { 37 | throw new RuntimeException("Failed to load network from config: " + config); 38 | } 39 | 40 | List inputLayers = get_layers(ptr, Kind.Input.ordinal()); 41 | List outputLayers = get_layers(ptr, Kind.Output.ordinal()); 42 | 43 | if (inputLayers.isEmpty()) { 44 | throw new RuntimeException("No input layers found in: " + config); 45 | } 46 | if (outputLayers.isEmpty()) { 47 | throw new RuntimeException("No output layers found in: " + config); 48 | } 49 | 50 | return new NNNetwork(config, ptr, inputLayers, outputLayers); 51 | } 52 | 53 | /** 54 | * Loads the network from the netcdf file. Returns a pointer to a context data structure 55 | * that is used to access the network. 56 | */ 57 | private static native long load(final String networkFilePath, final int batchSize, final int maxK); 58 | 59 | public static native void load_datasets(final long ptr, NNDataSet[] datasets); 60 | 61 | /** 62 | * Shuts down this model and GPU context and releases all resources. Once shutdown, the init method 63 | * must be called again to start up the context. 64 | */ 65 | static native void shutdown(final long ptr); 66 | 67 | /** 68 | * Returns the metadata (e.g. dimensions, name, etc) of the layer (one entry per layer) 69 | * of the specified {@link NNLayer.Kind}. 70 | */ 71 | private static native List get_layers(final long ptr, final int kind); 72 | 73 | static native void predict(final long ptr, final int k, final NNDataSet[] inputs, final TopKOutput[] outputs); 74 | } 75 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/NNDataSetEnums.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | /** 21 | * Same as src/amazon/dsstne/engine/NNDataSetEnums.h. This class exists 22 | * so that we can pass attribute information of the layers in the network between 23 | * C++ and Java. The values here must be in sync with the ones in the NNDataSetEnums.h. 24 | */ 25 | public class NNDataSetEnums { 26 | 27 | public static final class Attribute { 28 | public static final int None = 0x0; 29 | public static final int Sparse = 0x1; 30 | public static final int Boolean = 0x2; 31 | public static final int Compressed = 0x4; 32 | public static final int Recurrent = 0x8; 33 | public static final int Mutable = 0xF; 34 | public static final int SparseIgnoreZero = 0x20; 35 | public static final int Indexed = 0x40; 36 | public static final int Weighted = 0x80; 37 | } 38 | 39 | /** 40 | * Corresponds to NNDataSetEnums::Kind. 41 | */ 42 | public enum Kind { 43 | /* 44 | * DO NOT CHANGE ORDER 45 | * THE ENUM POSITION NEEDS TO MATCH 46 | * THE ENUM VALUES OF C++ version of NNDataSetEnums::Kind 47 | */ 48 | Numeric, // 0 49 | Image, // 1 50 | Audio; // 2 51 | 52 | } 53 | 54 | /** 55 | * Corresponds to NNDataSetEnums::Sharding. 56 | */ 57 | public enum Sharding { 58 | /* 59 | * DO NOT CHANGE ORDER 60 | * THE ENUM POSITION NEEDS TO MATCH 61 | * THE ENUM VALUES OF C++ version of NNDataSetEnums::Sharding 62 | */ 63 | None, // 0 64 | Model, // 1 65 | Data; // 2 66 | } 67 | 68 | /** 69 | * Corresponds to NNDataSetEnums::DataType. 70 | */ 71 | public enum DataType { 72 | /* 73 | * DO NOT CHANGE ORDER 74 | * THE ENUM POSITION NEEDS TO MATCH 75 | * THE ENUM VALUES OF C++ version of NNDataSetEnums::Sharding 76 | */ 77 | UInt, // 0 78 | Int, // 1 79 | LLInt, // 2 80 | ULLInt, // 3 81 | Float, // 4 82 | Double, // 5 83 | RGB8, // 6 84 | RGB16, // 7 85 | UChar, // 8 86 | Char; // 9 87 | 88 | /** 89 | * The size of the data type in bytes. 90 | */ 91 | public static int sizeof(final DataType dataType) { 92 | switch (dataType) { 93 | case Int: 94 | return Integer.SIZE / 8; 95 | case LLInt: 96 | return Long.SIZE / 8; 97 | case Float: 98 | return java.lang.Float.SIZE / 8; 99 | case Double: 100 | return java.lang.Double.SIZE / 8; 101 | case Char: 102 | return Character.SIZE / 8; 103 | default: 104 | throw new IllegalArgumentException(dataType + " not supported in java binding"); 105 | } 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/NNLayer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import lombok.Value; 21 | 22 | /** 23 | * Represents a neural network layer. This is a data-only object around 24 | * DSSTNE's NNLayer with no functionality other than providing information 25 | * about a layer. 26 | * 27 | * @author kiuk 28 | */ 29 | @Value 30 | public class NNLayer { 31 | 32 | /** 33 | * Same as C++ NNLayer::Kind. Enums ordinals/values must be in sync. 34 | */ 35 | public enum Kind { 36 | /* 37 | * DO NOT CHANGE ORDER 38 | * THE ENUM POSITION NEEDS TO MATCH 39 | * THE ENUM VALUES OF C++ version of NNDataSetEnums::Kind 40 | */ 41 | Input, Hidden, Output, Target; 42 | 43 | } 44 | 45 | /** 46 | * Same as C++ NNLayer::Attribute. Since attribute is a mask we represent 47 | * them as int rather than enums. 48 | */ 49 | public static final class Attribute { 50 | /* 51 | * DO NOT CHANGE MASK VALUES 52 | * UNLESS THEY ALSO CHANGE IN NNLayer::Attribute 53 | */ 54 | public static final int None = 0x0; 55 | public static final int Sparse = 0x1; 56 | public static final int Denoising = 0x2; 57 | public static final int BatchNormalization = 0x4; 58 | } 59 | 60 | /** 61 | * Name of this layer. 62 | */ 63 | private final String name; 64 | 65 | /** 66 | * Name of the dataset for this layer. 67 | */ 68 | private final String datasetName; 69 | 70 | /** 71 | * {@link Kind} of this layer. 72 | */ 73 | private final Kind kind; 74 | 75 | /** 76 | * Attributes of this layer. 77 | */ 78 | private final int attributes; 79 | 80 | /** 81 | * Number of dimensions of this layer. (e.g. 3-d is represented in (x,y,z)) 82 | */ 83 | private final int dimensions; 84 | 85 | /** 86 | * Size of dimension X. Zero if dimension < 1. 87 | */ 88 | private final int dimX; 89 | 90 | /** 91 | * Size of dimension Y. Zero if dimension < 2. 92 | */ 93 | private final int dimY; 94 | 95 | /** 96 | * Size of dimension Z. Zero if dimension < 3. 97 | */ 98 | private final int dimZ; 99 | 100 | /** 101 | * Creates a layer. Use takes int as kind (instead of the enum {@link Kind}) to facilitate creating 102 | * this class from JNI. Note that this will break if the ordinal of the {@link Kind} enum in Java and C++ 103 | * do not match. 104 | */ 105 | public NNLayer(final String name, final String datasetName, final int kind, final int attributes, 106 | final int dimensions, final int dimX, 107 | final int dimY, final int dimZ) { 108 | this.name = name; 109 | this.datasetName = datasetName; 110 | this.kind = Kind.values()[kind]; 111 | this.attributes = attributes; 112 | this.dimensions = dimensions; 113 | this.dimX = dimX; 114 | this.dimY = dimY; 115 | this.dimZ = dimZ; 116 | } 117 | 118 | public Dim getDim() { 119 | return new Dim(dimensions, dimX, dimY, dimZ, 0); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/NetworkConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import java.nio.file.Paths; 21 | import java.util.Map; 22 | 23 | import lombok.Builder; 24 | import lombok.Data; 25 | import lombok.Singular; 26 | 27 | /** 28 | * Configurations to {@link NNNetwork}. 29 | * 30 | * @author kiuk 31 | */ 32 | @Data 33 | @Builder(builderMethodName = "with") 34 | public class NetworkConfig { 35 | private static final char EXTENSION_SEPARATOR = '.'; 36 | public static final int ALL = -1; 37 | 38 | /** 39 | * Location of the network netcdf file. 40 | */ 41 | private String networkFilePath; 42 | 43 | /** 44 | * Name of the dataset (model) in the model netcdf file. 45 | * Default: name of the network netcdf file (not including the .nc suffix) 46 | */ 47 | private String networkName; 48 | 49 | /** 50 | * Input batch size for the prediction (feed-forward) 51 | * Default: 32 52 | */ 53 | @Builder.Default 54 | private int batchSize = 32; 55 | 56 | /** 57 | * Number of predictions to generate per input. 58 | * If negative, returns the entire output layer. 59 | * Default: {@link NetworkConfig#ALL} 60 | */ 61 | @Builder.Default 62 | private int k = ALL; 63 | 64 | /** 65 | * Specifications of input data mapped to each input layer by layer name. 66 | */ 67 | @Singular 68 | private Map inputDataSets; 69 | 70 | @Singular 71 | private Map outputDataSets; 72 | 73 | public String getNetworkName() { 74 | if (this.networkName == null || this.networkName.isEmpty()) { 75 | // default to the name of the network file 76 | String fileName = Paths.get(networkFilePath).getFileName().toString(); 77 | int index = fileName.lastIndexOf(EXTENSION_SEPARATOR); 78 | return fileName.substring(0, index); 79 | } else { 80 | return this.networkName; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/TopKOutput.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import lombok.Getter; 21 | import lombok.Setter; 22 | 23 | /** 24 | * Data set to hold outputs of the predictions from a network. 25 | * Output data is comprised of two datasets: indexes and scores. 26 | * The indexes hold the indexes of the top-k results. 27 | * The scores hold the output values of the top-k results. 28 | */ 29 | @Getter 30 | public class TopKOutput { 31 | 32 | private final Dim dim; 33 | 34 | @Setter 35 | private String name = ""; 36 | 37 | /** 38 | * Name of the output layer this dataset is for. 39 | */ 40 | @Setter 41 | private String layerName = ""; 42 | 43 | private final float[] scores; 44 | private final long[] indexes; 45 | 46 | /* package private */ TopKOutput(final Dim dim) { 47 | this.dim = dim; 48 | this.scores = new float[dim.x * dim.y * dim.z * dim.examples]; 49 | this.indexes = new long[dim.x * dim.y * dim.z * dim.examples]; 50 | } 51 | 52 | public static TopKOutput create(final NetworkConfig config, final NNLayer outputLayer) { 53 | int k = config.getK(); 54 | int batchSize = config.getBatchSize(); 55 | Dim outputLayerDim = outputLayer.getDim(); 56 | 57 | TopKOutput outputDataset; 58 | if (config.getK() == NetworkConfig.ALL) { 59 | outputDataset = new TopKOutput(new Dim(outputLayerDim, batchSize)); 60 | } else { 61 | if(outputLayerDim.dimensions > 1) { 62 | throw new IllegalArgumentException("Top k outputs only supported on 1-D outputs"); 63 | } 64 | outputDataset = new TopKOutput(Dim._1d(k, batchSize)); 65 | } 66 | outputDataset.setName(outputLayer.getDatasetName()); 67 | outputDataset.setLayerName(outputLayer.getName()); 68 | return outputDataset; 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/data/DenseWeightedNNDataSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import java.util.Arrays; 21 | 22 | import com.amazon.dsstne.Dim; 23 | import com.amazon.dsstne.NNDataSetEnums.DataType; 24 | 25 | /** 26 | * Dense dataset with additional weight information 27 | * associated with each element of the data. 28 | */ 29 | public class DenseWeightedNNDataSet extends DenseNNDataSet { 30 | 31 | private final float[] weights; 32 | 33 | public DenseWeightedNNDataSet(final Dim dim, final DataType dataType) { 34 | super(dim, dataType); 35 | this.weights = new float[dim.stride * dim.examples]; 36 | } 37 | 38 | private void putWeightOne(final int index) { 39 | Arrays.fill(this.weights, index * getStride(), index + getStride(), 1f); 40 | } 41 | 42 | private void putWeight(final int index, final float[] weights) { 43 | System.arraycopy(weights, 0, this.weights, index * getStride(), getStride()); 44 | } 45 | 46 | @Override 47 | public float[] getWeights() { 48 | return weights; 49 | } 50 | 51 | @Override 52 | public void add(final int index, final char[] data) { 53 | super.add(index, data); 54 | putWeightOne(index); 55 | } 56 | 57 | @Override 58 | public void add(final int index, final int[] data) { 59 | super.add(index, data); 60 | putWeightOne(index); 61 | } 62 | 63 | @Override 64 | public void add(final int index, final float[] data) { 65 | super.add(index, data); 66 | putWeightOne(index); 67 | } 68 | 69 | @Override 70 | public void add(final int index, final double[] data) { 71 | super.add(index, data); 72 | putWeightOne(index); 73 | } 74 | 75 | @Override 76 | public void addWeighted(final int index, final char[] data, final float[] weights) { 77 | super.add(index, data); 78 | putWeight(index, weights); 79 | } 80 | 81 | @Override 82 | public void addWeighted(final int index, final int[] data, final float[] weights) { 83 | super.add(index, data); 84 | putWeight(index, weights); 85 | } 86 | 87 | @Override 88 | public void addWeighted(final int index, final float[] data, final float[] weights) { 89 | super.add(index, data); 90 | putWeight(index, weights); 91 | } 92 | 93 | @Override 94 | public void addWeighted(final int index, final double[] data, final float[] weights) { 95 | super.add(index, data); 96 | putWeight(index, weights); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/data/SparseWeightedNNDataSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import java.util.Arrays; 21 | 22 | import com.amazon.dsstne.Dim; 23 | import com.amazon.dsstne.NNDataSetEnums.DataType; 24 | 25 | /** 26 | * Sparse dataset that additionally has weights that are associated with 27 | * each element of the data. 28 | */ 29 | public class SparseWeightedNNDataSet extends SparseNNDataSet { 30 | 31 | private final float[] weights; 32 | 33 | public SparseWeightedNNDataSet(final Dim dim, final DataType dataType, final double sparseDensity) { 34 | super(dim, dataType, sparseDensity); 35 | this.weights = new float[getStride() * dim.examples]; 36 | } 37 | 38 | private void putWeightOne(final int index) { 39 | Arrays.fill(this.weights, index * getStride(), index + getStride(), 1f); 40 | } 41 | 42 | private void putWeight(final int index, final float[] weights) { 43 | System.arraycopy(weights, 0, this.weights, index * getStride(), getStride()); 44 | } 45 | 46 | @Override 47 | public float[] getWeights() { 48 | return weights; 49 | } 50 | 51 | @Override 52 | public void addSparse(final int index, final long[] sparseIndex, final char[] data) { 53 | super.addSparse(index, sparseIndex, data); 54 | putWeightOne(index); 55 | } 56 | 57 | @Override 58 | public void addSparse(final int index, final long[] sparseIndex, final int[] data) { 59 | super.addSparse(index, sparseIndex, data); 60 | putWeightOne(index); 61 | } 62 | 63 | @Override 64 | public void addSparse(final int index, final long[] sparseIndex, final float[] data) { 65 | super.addSparse(index, sparseIndex, data); 66 | putWeightOne(index); 67 | } 68 | 69 | @Override 70 | public void addSparse(final int index, final long[] sparseIndex, final double[] data) { 71 | super.addSparse(index, sparseIndex, data); 72 | putWeightOne(index); 73 | } 74 | 75 | @Override 76 | public void addSparseWeighted(final int index, final long[] sparseIndex, final float[] weights, final char[] data) { 77 | super.addSparse(index, sparseIndex, data); 78 | putWeight(index, weights); 79 | } 80 | 81 | @Override 82 | public void addSparseWeighted(final int index, final long[] sparseIndex, final float[] weights, final int[] data) { 83 | super.addSparse(index, sparseIndex, data); 84 | putWeight(index, weights); 85 | } 86 | 87 | @Override 88 | public void addSparseWeighted(final int index, final long[] sparseIndex, final float[] weights, 89 | final float[] data) { 90 | super.addSparse(index, sparseIndex, data); 91 | putWeight(index, weights); 92 | } 93 | 94 | @Override 95 | public void addSparseWeighted(final int index, final long[] sparseIndex, final float[] weights, 96 | final double[] data) { 97 | super.addSparse(index, sparseIndex, data); 98 | putWeight(index, weights); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/AbstractKNearestNeighbors.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import java.util.List; 21 | 22 | import lombok.AccessLevel; 23 | import lombok.Getter; 24 | import lombok.RequiredArgsConstructor; 25 | 26 | /** 27 | * Provides common functionality to the {@link KNearestNeighbors} implementation. 28 | * One should prefer to extend from this class rather than implementing {@link KNearestNeighbors} directly. 29 | */ 30 | @RequiredArgsConstructor 31 | @Getter(AccessLevel.PROTECTED) 32 | public abstract class AbstractKNearestNeighbors implements KNearestNeighbors { 33 | 34 | private final String label; 35 | private final int maxK; 36 | private final int featureSize; 37 | private final int batchSize; 38 | 39 | protected void validateFeatureSize(final Vector inputVector) { 40 | int size = inputVector.getCoordinates().size(); 41 | if (size != featureSize) { 42 | throw new IllegalArgumentException("feature size: " + size + " should equal: " + featureSize); 43 | } 44 | } 45 | 46 | protected void validateFeatureSize(final List inputBatch) { 47 | inputBatch.forEach(this::validateFeatureSize); 48 | } 49 | 50 | protected void validateK(final int k) { 51 | if (k <= 0 || k > maxK) { 52 | throw new IllegalArgumentException("k should be > 0 and < " + maxK + ". Given: " + k); 53 | } 54 | } 55 | 56 | @Override 57 | public int getMaxK() { 58 | return maxK; 59 | } 60 | 61 | @Override 62 | public int getFeatureSize() { 63 | return featureSize; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/DataType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | /** 21 | * Data type on the GPU. Loads and operates in either fp32 (single precision) or fp16 (half precision). 22 | */ 23 | public enum DataType { 24 | /* DO NOT CHANGE THE ORDER, IT IS PASSED TO C++ VIA JNI */ 25 | FP32, FP16; 26 | 27 | /** 28 | * Case insensitive {@link #valueOf(String)}. 29 | */ 30 | public static DataType fromString(final String dt) { 31 | return DataType.valueOf(dt.toUpperCase()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/KNearestNeighbors.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * @author kiuk 24 | */ 25 | public interface KNearestNeighbors { 26 | 27 | /** 28 | * Finds and returns the k-nearest-neighbors for the input vector. 29 | */ 30 | NearestNeighbors findKnn(final int k, final Vector inputVector); 31 | 32 | /** 33 | * Batch call to {@link #findKnn(int, Vector)}. 34 | */ 35 | List findKnnBatch(final int k, final List inputVectors); 36 | 37 | /** 38 | * Maximum k supported by this knn. 39 | */ 40 | int getMaxK(); 41 | 42 | /** 43 | * The length of the features in the matrix. The input vector should have the same length. 44 | */ 45 | int getFeatureSize(); 46 | 47 | } 48 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/KnnResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import lombok.Value; 21 | 22 | /** 23 | * Result of {@link KNearestNeighborsCuda#findKnn(int, float[], int)} . 24 | * The ids and scores are aligned by index. That is, (ids[i], scores[i]) defines the 25 | * neighbor's id and score. The scores are sorted in ascending order (closest neighbors first). 26 | * 27 | * @author kiuk 28 | */ 29 | @Value 30 | public class KnnResult { 31 | private final String[] keys; 32 | private final float[] scores; 33 | private final int k; 34 | 35 | public KnnResult(final String[] keys, final float[] scores, final int k) { 36 | int slen = scores.length; 37 | int ilen = keys.length; 38 | if (slen != ilen) { 39 | throw new IllegalArgumentException( 40 | "scores and indexes have different lengths (scores: " + slen + ", indexes: " + ilen + ")"); 41 | } 42 | if (scores.length % k != 0) { 43 | throw new IllegalArgumentException("k: " + k + " must divide the length of data: " + scores.length); 44 | } 45 | 46 | this.keys = keys; 47 | this.scores = scores; 48 | this.k = k; 49 | } 50 | 51 | /** 52 | * Returns the batch size (the number of input vectors in the findKnn batch). 53 | */ 54 | public int getBatchSize() { 55 | return scores.length / k; 56 | } 57 | 58 | /** 59 | * Id of the i^th neighbor in the row^th row in the batch. 60 | */ 61 | public String getKeyAt(final int row, final int i) { 62 | return keys[row * k + i]; 63 | } 64 | 65 | /** 66 | * Score of the i^th neighbor in the row^th row in the batch. 67 | */ 68 | public float getScoreAt(final int row, final int i) { 69 | return scores[row * k + i]; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/NearestNeighbors.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import java.util.Arrays; 21 | import java.util.List; 22 | import java.util.Objects; 23 | 24 | public class NearestNeighbors { 25 | 26 | /** 27 | * Statically creates a builder instance for NearestNeighbors. 28 | */ 29 | public static Builder builder() { 30 | return new Builder(); 31 | } 32 | 33 | /** 34 | * Fluent builder for instances of NearestNeighbors. 35 | */ 36 | public static class Builder { 37 | 38 | private int index; 39 | /** 40 | * Sets the value of the field "index" to be used for the constructed object. 41 | * @param index 42 | * The value of the "index" field. 43 | * @return 44 | * This builder. 45 | */ 46 | public Builder withIndex(int index) { 47 | this.index = index; 48 | return this; 49 | } 50 | 51 | private List neighbors; 52 | /** 53 | * Sets the value of the field "neighbors" to be used for the constructed object. 54 | * @param neighbors 55 | * The value of the "neighbors" field. 56 | * @return 57 | * This builder. 58 | */ 59 | public Builder withNeighbors(List neighbors) { 60 | this.neighbors = neighbors; 61 | return this; 62 | } 63 | 64 | /** 65 | * Sets the fields of the given instances to the corresponding values recorded when calling the "with*" methods. 66 | * @param instance 67 | * The instance to be populated. 68 | */ 69 | protected void populate(NearestNeighbors instance) { 70 | instance.setIndex(this.index); 71 | instance.setNeighbors(this.neighbors); 72 | } 73 | 74 | /** 75 | * Builds an instance of NearestNeighbors. 76 | *

77 | * The built object has its fields set to the values given when calling the "with*" methods of this builder. 78 | *

79 | */ 80 | public NearestNeighbors build() { 81 | NearestNeighbors instance = new NearestNeighbors(); 82 | 83 | populate(instance); 84 | 85 | return instance; 86 | } 87 | }; 88 | 89 | private int index; 90 | private List neighbors; 91 | 92 | public int getIndex() { 93 | return this.index; 94 | } 95 | 96 | public void setIndex(int index) { 97 | this.index = index; 98 | } 99 | 100 | public List getNeighbors() { 101 | return this.neighbors; 102 | } 103 | 104 | public void setNeighbors(List neighbors) { 105 | this.neighbors = neighbors; 106 | } 107 | 108 | private static final int classNameHashCode = 109 | internalHashCodeCompute("com.amazon.dsstne.knn.NearestNeighbors"); 110 | 111 | /** 112 | * HashCode implementation for NearestNeighbors 113 | * based on java.util.Arrays.hashCode 114 | */ 115 | @Override 116 | public int hashCode() { 117 | return internalHashCodeCompute( 118 | classNameHashCode, 119 | getIndex(), 120 | getNeighbors()); 121 | } 122 | 123 | private static int internalHashCodeCompute(Object... objects) { 124 | return Arrays.hashCode(objects); 125 | } 126 | 127 | /** 128 | * Equals implementation for NearestNeighbors 129 | * based on instanceof and Object.equals(). 130 | */ 131 | @Override 132 | public boolean equals(final Object other) { 133 | if (!(other instanceof NearestNeighbors)) { 134 | return false; 135 | } 136 | 137 | NearestNeighbors that = (NearestNeighbors) other; 138 | 139 | return 140 | Objects.equals(getIndex(), that.getIndex()) 141 | && Objects.equals(getNeighbors(), that.getNeighbors()); 142 | } 143 | @Override 144 | public String toString() { 145 | StringBuilder ret = new StringBuilder(); 146 | ret.append("NearestNeighbors("); 147 | 148 | ret.append("index="); 149 | ret.append(String.valueOf(index)); 150 | ret.append(", "); 151 | 152 | ret.append("neighbors="); 153 | ret.append(String.valueOf(neighbors)); 154 | ret.append(")"); 155 | 156 | return ret.toString(); 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/Neighbor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import java.util.Arrays; 21 | import java.util.Objects; 22 | 23 | public class Neighbor { 24 | 25 | /** 26 | * Statically creates a builder instance for Neighbor. 27 | */ 28 | public static Builder builder() { 29 | return new Builder(); 30 | } 31 | 32 | /** 33 | * Fluent builder for instances of Neighbor. 34 | */ 35 | public static class Builder { 36 | 37 | private String id; 38 | /** 39 | * Sets the value of the field "id" to be used for the constructed object. 40 | * @param id 41 | * The value of the "id" field. 42 | * @return 43 | * This builder. 44 | */ 45 | public Builder withId(String id) { 46 | this.id = id; 47 | return this; 48 | } 49 | 50 | private float score; 51 | /** 52 | * Sets the value of the field "score" to be used for the constructed object. 53 | * @param score 54 | * The value of the "score" field. 55 | * @return 56 | * This builder. 57 | */ 58 | public Builder withScore(float score) { 59 | this.score = score; 60 | return this; 61 | } 62 | 63 | /** 64 | * Sets the fields of the given instances to the corresponding values recorded when calling the "with*" methods. 65 | * @param instance 66 | * The instance to be populated. 67 | */ 68 | protected void populate(Neighbor instance) { 69 | instance.setId(this.id); 70 | instance.setScore(this.score); 71 | } 72 | 73 | /** 74 | * Builds an instance of Neighbor. 75 | *

76 | * The built object has its fields set to the values given when calling the "with*" methods of this builder. 77 | *

78 | */ 79 | public Neighbor build() { 80 | Neighbor instance = new Neighbor(); 81 | 82 | populate(instance); 83 | 84 | return instance; 85 | } 86 | }; 87 | 88 | private String id; 89 | private float score; 90 | 91 | public String getId() { 92 | return this.id; 93 | } 94 | 95 | public void setId(String id) { 96 | this.id = id; 97 | } 98 | 99 | public float getScore() { 100 | return this.score; 101 | } 102 | 103 | public void setScore(float score) { 104 | this.score = score; 105 | } 106 | 107 | private static final int classNameHashCode = 108 | internalHashCodeCompute("com.amazon.dsstne.knn.Neighbor"); 109 | 110 | /** 111 | * HashCode implementation for Neighbor 112 | * based on java.util.Arrays.hashCode 113 | */ 114 | @Override 115 | public int hashCode() { 116 | return internalHashCodeCompute( 117 | classNameHashCode, 118 | getId(), 119 | getScore()); 120 | } 121 | 122 | private static int internalHashCodeCompute(Object... objects) { 123 | return Arrays.hashCode(objects); 124 | } 125 | 126 | /** 127 | * Equals implementation for Neighbor 128 | * based on instanceof and Object.equals(). 129 | */ 130 | @Override 131 | public boolean equals(final Object other) { 132 | if (!(other instanceof Neighbor)) { 133 | return false; 134 | } 135 | 136 | Neighbor that = (Neighbor) other; 137 | 138 | return 139 | Objects.equals(getId(), that.getId()) 140 | && Objects.equals(getScore(), that.getScore()); 141 | } 142 | @Override 143 | public String toString() { 144 | StringBuilder ret = new StringBuilder(); 145 | ret.append("Neighbor("); 146 | 147 | ret.append("id="); 148 | ret.append(String.valueOf(id)); 149 | ret.append(", "); 150 | 151 | ret.append("score="); 152 | ret.append(String.valueOf(score)); 153 | ret.append(")"); 154 | 155 | return ret.toString(); 156 | } 157 | 158 | } 159 | -------------------------------------------------------------------------------- /java/src/main/java/com/amazon/dsstne/knn/UnbatchedKNearestNeighbors.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * Simply delegates the findKnn call to the underlying KnnCuda implementation. 25 | * NOT thread-safe. Intended to use for testing since this method does not batch properly. 26 | * 27 | * @author kiuk 28 | */ 29 | public class UnbatchedKNearestNeighbors extends AbstractKNearestNeighbors { 30 | 31 | private final KNearestNeighborsCuda knnCuda; 32 | private final float[] inputs; // input vectors 33 | private final float[] scores; // output scores 34 | private final String[] ids; // output ids (keys) 35 | 36 | UnbatchedKNearestNeighbors(final KNearestNeighborsCuda knnCuda) { 37 | this(null, knnCuda); 38 | } 39 | 40 | public UnbatchedKNearestNeighbors(final String label, final KNearestNeighborsCuda knnCuda) { 41 | super(label, knnCuda.getMaxK(), knnCuda.getFeatureSize(), knnCuda.getBatchSize()); 42 | int batchSize = getBatchSize(); 43 | int featureSize = getFeatureSize(); 44 | int maxK = getMaxK(); 45 | 46 | this.knnCuda = knnCuda; 47 | this.inputs = new float[batchSize * featureSize]; 48 | this.scores = new float[batchSize * maxK]; 49 | this.ids = new String[batchSize * maxK]; 50 | } 51 | 52 | @Override 53 | public NearestNeighbors findKnn(final int k, final Vector inputVector) { 54 | validateK(k); 55 | validateFeatureSize(inputVector); 56 | 57 | List coordinates = inputVector.getCoordinates(); 58 | 59 | for (int i = 0; i < coordinates.size(); i++) { 60 | inputs[i] = coordinates.get(i); 61 | } 62 | 63 | knnCuda.findKnn(inputs, scores, ids); 64 | 65 | List nearestNeighbors = new ArrayList<>(k); 66 | for (int i = 0; i < k; i++) { 67 | float score = scores[i]; 68 | String id = ids[i]; 69 | Neighbor neighbor = Neighbor.builder().withId(id).withScore(score).build(); 70 | nearestNeighbors.add(neighbor); 71 | } 72 | 73 | return NearestNeighbors.builder().withIndex(inputVector.getIndex()).withNeighbors(nearestNeighbors).build(); 74 | } 75 | 76 | @Override 77 | public List findKnnBatch(final int k, final List inputBatch) { 78 | List results = new ArrayList<>(inputBatch.size()); 79 | for (Vector vector : inputBatch) { 80 | results.add(findKnn(k, vector)); 81 | } 82 | return results; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /java/src/main/native/com_amazon_dsstne_Dsstne.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_amazon_dsstne_Dsstne */ 4 | 5 | #ifndef _Included_com_amazon_dsstne_Dsstne 6 | #define _Included_com_amazon_dsstne_Dsstne 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #undef com_amazon_dsstne_Dsstne_NULLPTR 11 | #define com_amazon_dsstne_Dsstne_NULLPTR 0LL 12 | /* 13 | * Class: com_amazon_dsstne_Dsstne 14 | * Method: load 15 | * Signature: (Ljava/lang/String;II)J 16 | */ 17 | JNIEXPORT jlong JNICALL Java_com_amazon_dsstne_Dsstne_load 18 | (JNIEnv *, jclass, jstring, jint, jint); 19 | 20 | /* 21 | * Class: com_amazon_dsstne_Dsstne 22 | * Method: load_datasets 23 | * Signature: (J[Lcom/amazon/dsstne/NNDataSet;)V 24 | */ 25 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_Dsstne_load_1datasets 26 | (JNIEnv *, jclass, jlong, jobjectArray); 27 | 28 | /* 29 | * Class: com_amazon_dsstne_Dsstne 30 | * Method: shutdown 31 | * Signature: (J)V 32 | */ 33 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_Dsstne_shutdown 34 | (JNIEnv *, jclass, jlong); 35 | 36 | /* 37 | * Class: com_amazon_dsstne_Dsstne 38 | * Method: get_layers 39 | * Signature: (JI)Ljava/util/List; 40 | */ 41 | JNIEXPORT jobject JNICALL Java_com_amazon_dsstne_Dsstne_get_1layers 42 | (JNIEnv *, jclass, jlong, jint); 43 | 44 | /* 45 | * Class: com_amazon_dsstne_Dsstne 46 | * Method: predict 47 | * Signature: (JI[Lcom/amazon/dsstne/NNDataSet;[Lcom/amazon/dsstne/TopKOutput;)V 48 | */ 49 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_Dsstne_predict 50 | (JNIEnv *, jclass, jlong, jint, jobjectArray, jobjectArray); 51 | 52 | #ifdef __cplusplus 53 | } 54 | #endif 55 | #endif 56 | -------------------------------------------------------------------------------- /java/src/main/native/jni_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef JNI_UTIL_H_ 19 | #define JNI_UTIL_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace dsstne { 27 | namespace jni { 28 | 29 | /* exceptions */ 30 | extern const std::string RuntimeException; 31 | extern const std::string NullPointerException; 32 | extern const std::string IllegalStateException; 33 | extern const std::string IllegalArgumentException; 34 | extern const std::string ClassNotFoundException; 35 | extern const std::string NoSuchMethodException; 36 | extern const std::string FileNotFoundException; 37 | extern const std::string UnsupportedOperationException; 38 | 39 | /* collections */ 40 | extern const std::string ArrayList; 41 | 42 | /* java types */ 43 | extern const std::string String; 44 | 45 | /* methods */ 46 | extern const std::string NO_ARGS_CONSTRUCTOR; 47 | 48 | struct References; 49 | 50 | void deleteReferences(JNIEnv *env, References &refs); 51 | 52 | /** 53 | * Container for jclass and jmethodID references 54 | */ 55 | struct References { 56 | private: 57 | std::map classGlobalRefs; 58 | public: 59 | friend void deleteReferences(JNIEnv *env, References &refs); 60 | 61 | jclass getClassGlobalRef(const std::string &className) const; 62 | 63 | bool containsClassGlobalRef(const std::string &className) const; 64 | 65 | void putClassGlobalRef(const std::string &className, jclass classRef); 66 | }; 67 | 68 | void throwJavaException(JNIEnv* env, const std::string &exceptionType, const char *fmt, ...); 69 | 70 | jclass findClassGlobalRef(JNIEnv *env, References &refs, const std::string &className); 71 | 72 | jmethodID findMethodId(JNIEnv *env, References &refs, const std::string &className, const std::string &methodName, 73 | const std::string &methodDescriptor); 74 | 75 | /** 76 | * Finds constructors. methodDescriptor for constructors should return void (V) (e.g. ([Ljava/lang/String;[FI)V ). 77 | */ 78 | jmethodID findConstructorId(JNIEnv *env, References &refs, const std::string &className, 79 | const std::string &methodDescriptor); 80 | 81 | jobject newObject(JNIEnv *env, const References &refs, const std::string &className, jmethodID jConstructor, ...); 82 | 83 | } // namespace jni 84 | } // namespace dsstne 85 | #endif /* JNI_UTIL_H_ */ 86 | -------------------------------------------------------------------------------- /java/src/main/native_knn/com_amazon_dsstne_knn_KNearestNeighborsCuda.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_amazon_dsstne_knn_KNearestNeighborsCuda */ 4 | 5 | #ifndef _Included_com_amazon_dsstne_knn_KNearestNeighborsCuda 6 | #define _Included_com_amazon_dsstne_knn_KNearestNeighborsCuda 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #undef com_amazon_dsstne_knn_KNearestNeighborsCuda_NULLPTR 11 | #define com_amazon_dsstne_knn_KNearestNeighborsCuda_NULLPTR 0LL 12 | #undef com_amazon_dsstne_knn_KNearestNeighborsCuda_DEFAULT_KEYVAL_DELIM 13 | #define com_amazon_dsstne_knn_KNearestNeighborsCuda_DEFAULT_KEYVAL_DELIM 9L 14 | #undef com_amazon_dsstne_knn_KNearestNeighborsCuda_DEFAULT_VEC_DELIM 15 | #define com_amazon_dsstne_knn_KNearestNeighborsCuda_DEFAULT_VEC_DELIM 32L 16 | /* 17 | * Class: com_amazon_dsstne_knn_KNearestNeighborsCuda 18 | * Method: initialize 19 | * Signature: (IIII)J 20 | */ 21 | JNIEXPORT jlong JNICALL Java_com_amazon_dsstne_knn_KNearestNeighborsCuda_initialize 22 | (JNIEnv *, jclass, jint, jint, jint, jint); 23 | 24 | /* 25 | * Class: com_amazon_dsstne_knn_KNearestNeighborsCuda 26 | * Method: load 27 | * Signature: ([Ljava/lang/String;[ICCJ)V 28 | */ 29 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_knn_KNearestNeighborsCuda_load 30 | (JNIEnv *, jclass, jobjectArray, jintArray, jchar, jchar, jlong); 31 | 32 | /* 33 | * Class: com_amazon_dsstne_knn_KNearestNeighborsCuda 34 | * Method: shutdown 35 | * Signature: (J)V 36 | */ 37 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_knn_KNearestNeighborsCuda_shutdown 38 | (JNIEnv *, jclass, jlong); 39 | 40 | /* 41 | * Class: com_amazon_dsstne_knn_KNearestNeighborsCuda 42 | * Method: findKnn 43 | * Signature: (I[FII[F[Ljava/lang/String;J)V 44 | */ 45 | JNIEXPORT void JNICALL Java_com_amazon_dsstne_knn_KNearestNeighborsCuda_findKnn__I_3FII_3F_3Ljava_lang_String_2J 46 | (JNIEnv *, jclass, jint, jfloatArray, jint, jint, jfloatArray, jobjectArray, jlong); 47 | 48 | /* 49 | * Class: com_amazon_dsstne_knn_KNearestNeighborsCuda 50 | * Method: findKnn 51 | * Signature: (I[FIIJ)Lcom/amazon/dsstne/knn/KnnResult; 52 | */ 53 | JNIEXPORT jobject JNICALL Java_com_amazon_dsstne_knn_KNearestNeighborsCuda_findKnn__I_3FIIJ 54 | (JNIEnv *, jclass, jint, jfloatArray, jint, jint, jlong); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | #endif 60 | -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/NNLayerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | import com.amazon.dsstne.NNLayer.Kind; 25 | 26 | /** 27 | * @author kiuk 28 | */ 29 | public class NNLayerTest { 30 | 31 | @Test 32 | public void testKindOrdinal() { 33 | assertEquals(0, Kind.Input.ordinal()); 34 | assertEquals(1, Kind.Hidden.ordinal()); 35 | assertEquals(2, Kind.Output.ordinal()); 36 | assertEquals(3, Kind.Target.ordinal()); 37 | } 38 | 39 | @Test 40 | public void testAttributeMaskValues() { 41 | assertEquals(0x0, NNLayer.Attribute.None); 42 | assertEquals(0x1, NNLayer.Attribute.Sparse); 43 | assertEquals(0x2, NNLayer.Attribute.Denoising); 44 | assertEquals(0x4, NNLayer.Attribute.BatchNormalization); 45 | } 46 | 47 | @Test 48 | public void testCreateNNLayer() { 49 | NNLayer inputLayer = new NNLayer("input-layer", "input-layer-data", 0, 1, 1, 128, 1, 1); 50 | NNLayer hiddenLayer = new NNLayer("hidden-layer", "", 1, 1, 1, 128, 1, 1); 51 | NNLayer outputLayer = new NNLayer("output-layer", "output-layer-data", 2, 1, 1, 128, 1, 1); 52 | NNLayer targetLayer = new NNLayer("target-layer", "", 3, 1, 1, 128, 1, 1); 53 | 54 | assertEquals(Kind.Input, inputLayer.getKind()); 55 | assertEquals(Kind.Hidden, hiddenLayer.getKind()); 56 | assertEquals(Kind.Output, outputLayer.getKind()); 57 | assertEquals(Kind.Target, targetLayer.getKind()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/NetworkConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | /** 25 | * @author kiuk 26 | */ 27 | public class NetworkConfigTest { 28 | 29 | @Test 30 | public void testGetNetworkName() { 31 | NetworkConfig config = 32 | NetworkConfig.with().networkFilePath("/tmp/test-model.nc").networkName("my-model").build(); 33 | assertEquals("my-model", config.getNetworkName()); 34 | } 35 | 36 | @Test 37 | public void testDefaultGetNetworkName() { 38 | NetworkConfig config = NetworkConfig.with().networkFilePath("/tmp/test-model.nc").build(); 39 | assertEquals("test-model", config.getNetworkName()); 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/TopKOutputTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne; 19 | 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import com.amazon.dsstne.NNLayer.Attribute; 24 | import com.amazon.dsstne.NNLayer.Kind; 25 | 26 | public class TopKOutputTest { 27 | 28 | private String layerName = "test-layer"; 29 | private String datasetName = "test-dataset"; 30 | private int x = 128; 31 | private int y = 1; 32 | private int z = 1; 33 | private int batchSize = 128; 34 | private NNLayer layer = new NNLayer(layerName, datasetName, Kind.Input.ordinal(), Attribute.None, 1, x, y, z); 35 | 36 | @Test 37 | public void testNamesAreSet() { 38 | NetworkConfig config = NetworkConfig.with().batchSize(batchSize).build(); 39 | TopKOutput output = TopKOutput.create(config, layer); 40 | Assert.assertEquals(datasetName, output.getName()); 41 | Assert.assertEquals(layerName, output.getLayerName()); 42 | } 43 | 44 | @Test 45 | public void testOutputAllUnitBuffer() { 46 | NetworkConfig config = NetworkConfig.with().batchSize(batchSize).build(); 47 | TopKOutput output = TopKOutput.create(config, layer); 48 | Assert.assertEquals(x, output.getDim().x); 49 | Assert.assertEquals(y, output.getDim().y); 50 | Assert.assertEquals(z, output.getDim().z); 51 | Assert.assertEquals(1, output.getDim().dimensions); 52 | Assert.assertEquals(batchSize, output.getDim().examples); 53 | } 54 | 55 | @Test 56 | public void testOutputTopK() { 57 | int k = 100; 58 | NetworkConfig config = NetworkConfig.with().batchSize(batchSize).k(100).build(); 59 | TopKOutput output = TopKOutput.create(config, layer); 60 | Assert.assertEquals(k, output.getDim().x); 61 | Assert.assertEquals(y, output.getDim().y); 62 | Assert.assertEquals(z, output.getDim().z); 63 | Assert.assertEquals(1, output.getDim().dimensions); 64 | Assert.assertEquals(batchSize, output.getDim().examples); 65 | } 66 | 67 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/data/DenseNNDataSetTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import java.nio.IntBuffer; 23 | 24 | import org.junit.Test; 25 | 26 | import com.amazon.dsstne.Dim; 27 | import com.amazon.dsstne.NNDataSetEnums.DataType; 28 | 29 | public class DenseNNDataSetTest { 30 | 31 | @Test 32 | public void testAddInt() { 33 | DenseNNDataSet ds = new DenseNNDataSet(Dim._1d(4, 3), DataType.Int); 34 | ds.add(2, new int[] {2, 2, 2, 2}); 35 | ds.add(1, new int[] {1, 1, 1, 1}); 36 | 37 | IntBuffer buf = ds.getData().asIntBuffer(); 38 | for (int i = 0; i < 4; i++) { 39 | assertEquals(0, buf.get(i)); 40 | } 41 | 42 | for (int i = 0; i < 4; i++) { 43 | assertEquals(0, buf.get(1)); 44 | } 45 | 46 | for (int i = 0; i < 4; i++) { 47 | assertEquals(0, buf.get(2)); 48 | } 49 | } 50 | 51 | @Test(expected = UnsupportedOperationException.class) 52 | public void testAddWeighted() { 53 | DenseNNDataSet ds = new DenseNNDataSet(Dim._1d(4, 3), DataType.Int); 54 | ds.addWeighted(0, new int[0], new float[0]); 55 | } 56 | 57 | @Test(expected = UnsupportedOperationException.class) 58 | public void testAddSparse() { 59 | DenseNNDataSet ds = new DenseNNDataSet(Dim._1d(4, 3), DataType.Int); 60 | ds.addSparse(0, new long[0], new int[0]); 61 | } 62 | 63 | @Test(expected = UnsupportedOperationException.class) 64 | public void testAddSparseWeighted() { 65 | DenseNNDataSet ds = new DenseNNDataSet(Dim._1d(4, 3), DataType.Int); 66 | ds.addSparseWeighted(0, new long[0], new float[0], new int[0]); 67 | } 68 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/data/DenseWeightedNNDataSetTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import static org.junit.Assert.assertArrayEquals; 21 | import static org.junit.Assert.assertEquals; 22 | 23 | import java.nio.FloatBuffer; 24 | 25 | import org.junit.Test; 26 | 27 | import com.amazon.dsstne.Dim; 28 | import com.amazon.dsstne.NNDataSetEnums.DataType; 29 | 30 | /** 31 | * @author kiuk 32 | */ 33 | public class DenseWeightedNNDataSetTest { 34 | 35 | @Test 36 | public void testAddFloat() { 37 | DenseWeightedNNDataSet ds = new DenseWeightedNNDataSet(Dim._1d(4, 2), DataType.Float); 38 | ds.add(0, new float[] {1.2f, 2.3f, 3.4f, 4.5f}); 39 | ds.addWeighted(1, new float[] {1.1f, 2.2f, 3.3f, 4.4f}, new float[] {4.4f, 3.3f, 2.2f, 1.1f}); 40 | 41 | FloatBuffer buf = ds.getData().asFloatBuffer(); 42 | float[] temp = new float[ds.getStride()]; 43 | buf.get(temp); 44 | assertArrayEquals(new float[] {1.2f, 2.3f, 3.4f, 4.5f}, temp, 0f); 45 | 46 | buf.get(temp); 47 | assertArrayEquals(new float[] {1.1f, 2.2f, 3.3f, 4.4f}, temp, 0f); 48 | 49 | for (int i = 0; i < ds.getStride(); i++) { 50 | assertEquals(1f, ds.getWeights()[i], 0f); 51 | } 52 | 53 | assertEquals(4.4f, ds.getWeights()[ds.getStride() + 0], 0f); 54 | assertEquals(3.3f, ds.getWeights()[ds.getStride() + 1], 0f); 55 | assertEquals(2.2f, ds.getWeights()[ds.getStride() + 2], 0f); 56 | assertEquals(1.1f, ds.getWeights()[ds.getStride() + 3], 0f); 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/data/DimTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | import com.amazon.dsstne.Dim; 25 | 26 | /** 27 | * @author kiuk 28 | */ 29 | public class DimTest { 30 | 31 | @Test 32 | public void testDim() { 33 | Dim oneD = Dim._1d(128, 32); 34 | Dim twoD = Dim._2d(128, 64, 32); 35 | Dim threeD = Dim._3d(128, 64, 32, 16); 36 | 37 | assertEquals(1, oneD.dimensions); 38 | assertEquals(2, twoD.dimensions); 39 | assertEquals(3, threeD.dimensions); 40 | 41 | assertEquals(128, oneD.stride); 42 | assertEquals(128 * 64, twoD.stride); 43 | assertEquals(128 * 64 * 32, threeD.stride); 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/data/NNDataSetEnumsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.data; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | import com.amazon.dsstne.NNDataSetEnums.DataType; 25 | import com.amazon.dsstne.NNDataSetEnums.Kind; 26 | import com.amazon.dsstne.NNDataSetEnums.Sharding; 27 | 28 | /** 29 | * Test that the ordinals of the enums match 30 | * the ones in C++ NNDataSetEnums. If the order 31 | * of the enums are changed then these tests should fail. 32 | */ 33 | public class NNDataSetEnumsTest { 34 | 35 | @Test 36 | public void testKind() { 37 | assertEquals(0, Kind.Numeric.ordinal()); 38 | assertEquals(1, Kind.Image.ordinal()); 39 | assertEquals(2, Kind.Audio.ordinal()); 40 | } 41 | 42 | @Test 43 | public void testSharding() { 44 | assertEquals(0, Sharding.None.ordinal()); 45 | assertEquals(1, Sharding.Model.ordinal()); 46 | assertEquals(2, Sharding.Data.ordinal()); 47 | } 48 | 49 | @Test 50 | public void testDataType() { 51 | assertEquals(0, DataType.UInt.ordinal()); 52 | assertEquals(1, DataType.Int.ordinal()); 53 | assertEquals(2, DataType.LLInt.ordinal()); 54 | assertEquals(3, DataType.ULLInt.ordinal()); 55 | assertEquals(4, DataType.Float.ordinal()); 56 | assertEquals(5, DataType.Double.ordinal()); 57 | assertEquals(6, DataType.RGB8.ordinal()); 58 | assertEquals(7, DataType.RGB16.ordinal()); 59 | assertEquals(8, DataType.UChar.ordinal()); 60 | assertEquals(9, DataType.Char.ordinal()); 61 | 62 | } 63 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/knn/DataTypeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | public class DataTypeTest { 25 | 26 | @Test 27 | public void testDataTypeFromString() { 28 | assertEquals(DataType.FP16, DataType.fromString("fp16")); 29 | assertEquals(DataType.FP32, DataType.fromString("fp32")); 30 | } 31 | 32 | /** 33 | * Ordinal of {@link DataType} should not change since we pass it to C++ via JNI. 34 | * If it needs to change then also need to change in the brazil package AstdlKnnCoreJNI. 35 | */ 36 | @Test 37 | public void testDataTypeFromString_ordinal() { 38 | assertEquals(0, DataType.FP32.ordinal()); 39 | assertEquals(1, DataType.FP16.ordinal()); 40 | } 41 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/knn/KnnResultTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | package com.amazon.dsstne.knn; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | 22 | import org.junit.Test; 23 | 24 | public class KnnResultTest { 25 | 26 | @Test(expected = IllegalArgumentException.class) 27 | public void testInvalidResults() { 28 | new KnnResult(new String[1], new float[2], 1); 29 | } 30 | 31 | @Test(expected = IllegalArgumentException.class) 32 | public void testInvalidK() { 33 | new KnnResult(new String[3], new float[3], 2); 34 | } 35 | 36 | @Test 37 | public void testGetK() { 38 | KnnResult result = new KnnResult(new String[6], new float[6], 3); 39 | assertEquals(3, result.getK()); 40 | } 41 | 42 | @Test 43 | public void testGetBatchSize() { 44 | KnnResult result = new KnnResult(new String[32], new float[32], 8); 45 | assertEquals(32 / 8, result.getBatchSize()); 46 | } 47 | } -------------------------------------------------------------------------------- /java/src/test/java/com/amazon/dsstne/test-data/movielens_20m_autoencoder.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/java/src/test/java/com/amazon/dsstne/test-data/movielens_20m_autoencoder.nc -------------------------------------------------------------------------------- /python/Makefile: -------------------------------------------------------------------------------- 1 | libdsstnecuda.a : dsstnecuda.c 2 | mpiCC -DOMPI_SKIP_MPICXX -std=c++0x -O3 -fPIC -I/usr/local/cuda/include -I/usr/include/jsoncpp -I../src/amazon/dsstne/engine -c dsstnecuda.c 3 | ar -cvr libdsstnecuda.a dsstnecuda.o 4 | -------------------------------------------------------------------------------- /python/encoder/README.md: -------------------------------------------------------------------------------- 1 | The encoder.py program requires "eval.nc" and "search_network.nc" files for which the data are Amazon proprietary, so this program 2 | cannot be executed and serves only as an additional example of the use of the dsstne Python-C++ extension API. -------------------------------------------------------------------------------- /python/encoder/embedding.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "Search Embeddings", 4 | "Kind" : "FeedForward", 5 | "ShuffleIndices" : false, 6 | "Layers" : [ 7 | 8 | { "Name" : "Input_asin", "Kind" : "Input", "N" : "auto", "DataSet" : "asin", "Sparse" : true }, 9 | { "Name" : "Hidden_asin", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input_asin", "N" : 512, "Activation" : "tanh" } 10 | ], 11 | 12 | "ErrorFunction" : "L2Hinge" 13 | } 14 | -------------------------------------------------------------------------------- /python/encoder/inference.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "Prediction", 4 | "Kind" : "FeedForward", 5 | "ShuffleIndices" : false, 6 | "Layers" : [ 7 | 8 | { "Name" : "Input_query", "Kind" : "Input", "N" : "auto", "DataSet" : "query", "Sparse" : true }, 9 | { "Name" : "Hidden_query", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input_query", "N" : 512, "Activation" : "tanh" }, 10 | { "Name" : "Output_asin", "Kind" : "Output", "N" : "auto", "DataSet" : "target", "Sparse" : true, "Activation" : "linear" } 11 | ], 12 | 13 | "ErrorFunction" : "L2Hinge" 14 | } 15 | -------------------------------------------------------------------------------- /python/images/README.md: -------------------------------------------------------------------------------- 1 | Execute the images.py program via 'python images.py train.cdl' to create the results/network.nc file followed by 'python images.py predict.cdl' -------------------------------------------------------------------------------- /python/images/checkpoint/README.md: -------------------------------------------------------------------------------- 1 | Checkpoint files (check*.nc) for image recognition training are written to this directory. -------------------------------------------------------------------------------- /python/images/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.81, 3 | "Name" : "CIFAR-100", 4 | "Kind" : "FeedForward", 5 | "LocalResponseNormalization" : 6 | { 7 | "k" : 2, 8 | "n" : 5, 9 | "alpha" : 0.0001, 10 | "beta" : 0.75 11 | }, 12 | "Layers" : [ 13 | { "Name" : "Input", "Kind" : "Input", "Type" : "Convolutional", "N" : "auto", "DataSet" : "input"}, 14 | { "Name" : "C1", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 15 | { "Name" : "C1a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 16 | { "Name" : "P1", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [4, 4], "KernelStride" : [2, 2]}, 17 | 18 | { "Name" : "C2", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 19 | { "Name" : "C2a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 20 | { "Name" : "P2", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [4, 4], "KernelStride" : [2, 2]}, 21 | 22 | { "Name" : "C3", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 96, "Kernel" : [5, 5], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 23 | { "Name" : "C3a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 96, "Kernel" : [5, 5], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 24 | { "Name" : "P3", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [2, 2], "KernelStride" : [2, 2]}, 25 | 26 | { "Kind" : "Hidden", "Type" : "FullyConnected", "BatchNormalization" : true, "N" : 1024, "Activation" : "Relu", "pDropout" : 0.5 }, 27 | 28 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "N" : "auto", "DataSet" : "output", "Activation" : "SoftMax", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 , "Bias" : 0.0 } } 29 | ], 30 | "ErrorFunction" : "CrossEntropy" 31 | } 32 | 33 | -------------------------------------------------------------------------------- /python/images/images.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 5 | 6 | http://aws.amazon.com/apache2.0/ 7 | 8 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 9 | ''' 10 | 11 | ''' 12 | This Python program is intended for use with the CIFAR-10 image recognition test data that may be installed by 13 | following the instructions explained in the comments in the "amazon-dsstne/samples/cifar-10/dparse.cpp" file. 14 | ''' 15 | 16 | # Import MPI before calling dsstnecuda.Startup to avoid opal_init(), orte_init(), and ompi_rte_init() failures. 17 | from mpi4py import MPI 18 | import dsstne as dn 19 | import numpy as np 20 | # import ctypes 21 | # import math 22 | import time 23 | import sys 24 | 25 | startTime = time.time() 26 | 27 | # Initialize GPU network 28 | dn.Startup(sys.argv) # The command-line arguments don't get used because 'import MPI' previously called MPI_Init(). 29 | if (len(sys.argv) == 2): 30 | try: 31 | CDL = dn.CreateCDLFromJSON(sys.argv[1]) 32 | except: 33 | print "**** error,", sys.argv[0], "could not parse CDL file", sys.argv[1] 34 | sys.exit() 35 | else: 36 | print "**** error: you must provide a CDL file to", sys.argv[0], "(typically either train.cdl or predict.cdl)" 37 | sys.exit() 38 | 39 | print "**** random seed =", dn.GetCDLRandomSeed(CDL) 40 | dn.SetRandomSeed(dn.GetCDLRandomSeed(CDL)) 41 | 42 | # Load training data 43 | print "*** data filename =", dn.GetCDLDataFileName(CDL) 44 | DataSetList = dn.LoadNetCDF(dn.GetCDLDataFileName(CDL)) 45 | 46 | # Create neural network 47 | print "**** network filename =", dn.GetCDLNetworkFileName(CDL) 48 | if (dn.GetCDLMode(CDL) == "Prediction"): 49 | Network = dn.LoadNeuralNetworkNetCDF(dn.GetCDLNetworkFileName(CDL), dn.GetCDLBatch(CDL)) 50 | else: 51 | Network = dn.LoadNeuralNetworkJSON(dn.GetCDLNetworkFileName(CDL), dn.GetCDLBatch(CDL), DataSetList) 52 | 53 | # Dump memory usage 54 | gpuMemoryUsage, cpuMemoryUsage = dn.GetMemoryUsage() 55 | print "**** GPU memory usage:", gpuMemoryUsage, " KB" 56 | print "**** CPU memory usage:", cpuMemoryUsage, " KB" 57 | 58 | # Load the data sets 59 | dn.LoadDataSets(Network, DataSetList) 60 | 61 | if (dn.GetCDLMode(CDL) == "Training"): 62 | dn.SetCheckpoint(Network, dn.GetCDLCheckpointFileName(CDL), dn.GetCDLCheckpointInterval(CDL)); 63 | CheckpointFileName, CheckpointInterval = dn.GetCheckpoint(Network) 64 | print "**** checkpoint filename =", CheckpointFileName 65 | print "**** checkpoint interval =", CheckpointInterval 66 | 67 | # Train or predict based on operating mode 68 | if (dn.GetCDLMode(CDL) == "Training"): 69 | dn.SetTrainingMode(Network, dn.GetCDLOptimizer(CDL)) 70 | alpha = dn.GetCDLAlpha(CDL) 71 | lambda1 = 0.0 72 | mu1 = 0.0 73 | epochs = 0 74 | while (epochs < dn.GetCDLEpochs(CDL)): 75 | dn.Train(Network, dn.GetCDLAlphaInterval(CDL), alpha, dn.GetCDLLambda(CDL), lambda1, dn.GetCDLMu(CDL), mu1) 76 | alpha = alpha * dn.GetCDLAlphaMultiplier(CDL) 77 | epochs = epochs + dn.GetCDLAlphaInterval(CDL) 78 | print "**** saving training results to file: ", dn.GetCDLResultsFileName(CDL) 79 | dn.SaveNetCDF(Network, dn.GetCDLResultsFileName(CDL)) 80 | else: 81 | K = 10 82 | topK = dn.PredictTopK(Network, DataSetList, CDL, K) 83 | print "**** top", K, "results follow:" 84 | # Formatted printing is cumbersome 85 | for index, value in enumerate(topK, 1): 86 | print format(index, '3d'), 87 | for i, x in enumerate(value): print format(x, '.3f'), 88 | print 89 | 90 | # Dump memory usage 91 | gpuMemoryUsage, cpuMemoryUsage = dn.GetMemoryUsage() 92 | print "**** GPU memory usage:", gpuMemoryUsage, " KB" 93 | print "**** CPU memory usage:", cpuMemoryUsage, " KB" 94 | 95 | # Report the network name and some other details 96 | print "**** network name =", dn.GetName(Network) 97 | Layers = dn.GetLayers(Network) 98 | print "**** network layers =", Layers 99 | 100 | # Delete the CDL instance, the training data, and the neural network and then shut down the GPU network 101 | dn.DeleteCDL(CDL) 102 | for i in range(0, len(DataSetList)): 103 | dn.DeleteDataSet(DataSetList[i]); 104 | dn.DeleteNNNetwork(Network) 105 | dn.Shutdown() 106 | 107 | endTime = time.time() 108 | print "**** time for cuda: ", endTime-startTime 109 | -------------------------------------------------------------------------------- /python/images/predict.cdl: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.2, 3 | "Network" : "results/network.nc", 4 | "Command" : "Predict", 5 | "RandomSeed" : 12345, 6 | 7 | "Data" : "../../samples/cifar-10/cifar-10-batches-bin/cifar10_test.nc" 8 | } 9 | -------------------------------------------------------------------------------- /python/images/results/README.md: -------------------------------------------------------------------------------- 1 | The results file (network.nc) for image recognition training is written to this directory. -------------------------------------------------------------------------------- /python/images/train.cdl: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.2, 3 | "Network" : "config.json", 4 | "Command" : "Train", 5 | "RandomSeed" : 12345, 6 | 7 | "TrainingParameters" : { 8 | "Optimizer" : "Nesterov", 9 | "Epochs" : 60, 10 | "Alpha" : 0.025, 11 | "AlphaInterval" : 20, 12 | "AlphaMultiplier" : 0.8, 13 | "mu": 0.5, 14 | "lambda" : 0.0001, 15 | "Checkpointname" : "checkpoint/check", 16 | "Results" : "results/network.nc" 17 | }, 18 | "Data" : "../../samples/cifar-10/cifar-10-batches-bin/cifar10_training.nc" 19 | } 20 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup, Extension 2 | import os 3 | import numpy as np 4 | 5 | os.environ["CC"] = "mpiCC" 6 | os.environ["LDSHARED"] = "mpiCC -shared -Wl,--no-undefined" 7 | os.environ["LDSHARED"] = "mpiCC -shared" # comment this line out to see what isn't linking correctly 8 | 9 | module1 = Extension('dsstne', 10 | include_dirs = [np.get_include(), 11 | '/usr/local/cuda/include', 12 | 'B40C', 13 | 'B40C/KernelCommon', 14 | '/usr/local/include', 15 | '/usr/lib/openmpi/include', 16 | '/usr/local/include', 17 | '/usr/include/jsoncpp', 18 | '../src/amazon/dsstne/utils', 19 | '../src/amazon/dsstne/engine', 20 | '/usr/include/cppunit'], 21 | libraries = ['dsstne', 22 | 'cudnn', 23 | 'curand', 24 | 'cublas', 25 | 'cudart', 26 | 'jsoncpp', 27 | 'netcdf', 28 | 'blas', 29 | 'dl', 30 | 'stdc++', 31 | 'netcdf_c++4'], 32 | runtime_library_dirs = ['/usr/lib/x86_64-linux-gnu'], 33 | library_dirs = [os.path.dirname(os.path.realpath(__file__)), 34 | '/usr/lib/atlas-base', 35 | '/usr/local/cuda/lib64', 36 | '/usr/local/lib', 37 | '../amazon-dsstne/lib'], 38 | language='c++11', 39 | extra_compile_args=['-std=c++11', 40 | '-DOMPI_SKIP_MPICXX'], 41 | sources = ['dsstnemodule.cc', 42 | '../src/amazon/dsstne/utils/cdl.cpp']) 43 | 44 | setup (name = 'DSSTNE', 45 | version = '1.0', 46 | description = 'This is a package that links dsstne functions to Python.', 47 | ext_modules = [module1]) 48 | -------------------------------------------------------------------------------- /samples/cifar-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.81, 3 | "Name" : "CIFAR-100", 4 | "Kind" : "FeedForward", 5 | "LocalResponseNormalization" : 6 | { 7 | "k" : 2, 8 | "n" : 5, 9 | "alpha" : 0.0001, 10 | "beta" : 0.75 11 | }, 12 | "Layers" : [ 13 | { "Name" : "Input", "Kind" : "Input", "Type" : "Convolutional", "N" : "auto", "DataSet" : "input"}, 14 | { "Name" : "C1", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 15 | { "Name" : "C1a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 16 | { "Name" : "P1", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [4, 4], "KernelStride" : [2, 2]}, 17 | 18 | { "Name" : "C2", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 19 | { "Name" : "C2a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 128, "Kernel" : [7, 7], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 20 | { "Name" : "P2", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [4, 4], "KernelStride" : [2, 2]}, 21 | 22 | { "Name" : "C3", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 96, "Kernel" : [5, 5], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 23 | { "Name" : "C3a", "Kind" : "Hidden", "Type" : "Convolutional", "N" : 96, "Kernel" : [5, 5], "KernelStride" : [1, 1], "Activation" : "Linear", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.02 , "Bias" : 0.0 } }, 24 | { "Name" : "P3", "Kind" : "Hidden", "Type" : "Pooling", "Function" : "Max", "Kernel" : [2, 2], "KernelStride" : [2, 2]}, 25 | 26 | { "Kind" : "Hidden", "Type" : "FullyConnected", "BatchNormalization" : true, "N" : 1024, "Activation" : "Relu", "pDropout" : 0.5 }, 27 | 28 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "N" : "auto", "DataSet" : "output", "Activation" : "SoftMax", "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 , "Bias" : 0.0 } } 29 | ], 30 | "ErrorFunction" : "CrossEntropy" 31 | } 32 | 33 | -------------------------------------------------------------------------------- /samples/cifar-10/predict.cdl: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.2, 3 | "Network" : "network.nc", 4 | "Command" : "Predict", 5 | "RandomSeed" : 12345, 6 | 7 | "Data" : "cifar10_test.nc" 8 | } 9 | -------------------------------------------------------------------------------- /samples/cifar-10/train.cdl: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.2, 3 | "Network" : "config.json", 4 | "Command" : "Train", 5 | "RandomSeed" : 12345, 6 | 7 | "TrainingParameters" : { 8 | "Optimizer" : "Nesterov", 9 | "Epochs" : 60, 10 | "Alpha" : 0.025, 11 | "AlphaInterval" : 20, 12 | "AlphaMultiplier" : 0.8, 13 | "mu": 0.5, 14 | "lambda" : 0.0001, 15 | "Results" : "network.nc" 16 | }, 17 | "Data" : "cifar10_training.nc" 18 | } 19 | -------------------------------------------------------------------------------- /samples/movielens/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.7, 3 | "Name" : "AE", 4 | "Kind" : "FeedForward", 5 | "SparsenessPenalty" : { 6 | "p" : 0.5, 7 | "beta" : 2.0 8 | }, 9 | 10 | "ShuffleIndices" : false, 11 | 12 | "Denoising" : { 13 | "p" : 0.2 14 | }, 15 | 16 | "ScaledMarginalCrossEntropy" : { 17 | "oneTarget" : 1.0, 18 | "zeroTarget" : 0.0, 19 | "oneScale" : 1.0, 20 | "zeroScale" : 1.0 21 | }, 22 | "Layers" : [ 23 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "gl_input", "Sparse" : true }, 24 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "N" : 128, "Activation" : "Sigmoid", "Sparse" : true }, 25 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "gl_output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true } 26 | ], 27 | 28 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 29 | } 30 | -------------------------------------------------------------------------------- /samples/movielens/convert_ratings.awk: -------------------------------------------------------------------------------- 1 | BEGIN { 2 | FS = "," 3 | RS = "\r|\n|\r\n|\n\r" 4 | ORS = "\n" 5 | } 6 | 7 | NR > 1 { 8 | if (!prev) { 9 | printf $1 "\t" 10 | } else if (prev != $1) { 11 | printf "\n" $1 "\t" 12 | } else { 13 | printf ":" 14 | } 15 | printf $2 "," $4 16 | prev = $1 17 | } 18 | 19 | END { 20 | printf "\n" 21 | } 22 | -------------------------------------------------------------------------------- /samples/movielens/run_movielens_sample.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Downloads the MovieLens dataset and trains a small model for 10 epochs before 4 | # writing predictions to a file called 'recs'. 5 | # 6 | 7 | # Fetch Movielens dataset if necessary 8 | if [ -f ml-20m.zip ]; then 9 | echo "Already downloaded MovieLens dataset." 10 | else 11 | echo "Downloading MovieLens dataset..." 12 | wget http://files.grouplens.org/datasets/movielens/ml-20m.zip 13 | fi 14 | 15 | # Extract ratings from dataset 16 | echo "Extracting ml-20/ratings.csv from ml-20m.zip to ml-20m_ratings.csv" 17 | unzip -p ml-20m.zip ml-20m/ratings.csv > ml-20m_ratings.csv 18 | 19 | # Convert ml-20m_ratings.csv to format supported by generateNetCDF 20 | echo "Converting ml-20m_ratings.csv to DSSTNE format" 21 | awk -f convert_ratings.awk ml-20m_ratings.csv > ml-20m_ratings 22 | 23 | # Generate NetCDF files for input and output layers 24 | generateNetCDF -d gl_input -i ml-20m_ratings -o gl_input.nc -f features_input -s samples_input -c 25 | generateNetCDF -d gl_output -i ml-20m_ratings -o gl_output.nc -f features_output -s samples_input -c 26 | 27 | # Train the network 28 | train -c config.json -i gl_input.nc -o gl_output.nc -n gl.nc -b 256 -e 10 29 | 30 | # Generate predictions 31 | predict -b 256 -d gl -i features_input -o features_output -k 10 -n gl.nc -f ml-20m_ratings -s recs -r ml-20m_ratings 32 | -------------------------------------------------------------------------------- /samples/network/config_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.7, 3 | "Name" : "AE", 4 | "Kind" : "FeedForward", 5 | "SparsenessPenalty" : { 6 | "p" : 0.5, 7 | "beta" : 2.0 8 | }, 9 | 10 | "ShuffleIndices" : false, 11 | 12 | "Denoising" : { 13 | "p" : 0.2 14 | }, 15 | 16 | "ScaledMarginalCrossEntropy" : { 17 | "oneTarget" : 1.0, 18 | "zeroTarget" : 0.0, 19 | "oneScale" : 1.0, 20 | "zeroScale" : 1.0 21 | }, 22 | "Layers" : [ 23 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "input", "Sparse" : true }, 24 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "N" : 128, "Activation" : "Sigmoid", "Sparse" : true }, 25 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "input", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true } 26 | ], 27 | 28 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 29 | } 30 | -------------------------------------------------------------------------------- /samples/network/config_2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "2 Hidden Layer", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | 9 | "ScaledMarginalCrossEntropy" : { 10 | "oneTarget" : 1.0, 11 | "zeroTarget" : 0.0, 12 | "oneScale" : 1.0, 13 | "zeroScale" : 1.0 14 | }, 15 | "Layers" : [ 16 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "input", "Sparse" : true }, 17 | { "Name" : "Hidden1", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 1024, "Activation" : "Relu", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 18 | { "Name" : "Hidden2", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : ["Hidden1"], "N" : 1024, "Activation" : "Relu", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 19 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true , "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01, "Bias" : -10.2 } } 20 | ], 21 | 22 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 23 | } 24 | -------------------------------------------------------------------------------- /samples/network/config_3.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.7, 3 | "Name" : "AutoEncoder Test", 4 | "Kind" : "FeedForward", 5 | "SparsenessPenalty" : { 6 | "p" : 0.5, 7 | "beta" : 2.0 8 | }, 9 | 10 | "DeltaBoost" : { 11 | "one" : 30.0, 12 | "zero" : 1.0 13 | }, 14 | "Layers" : [ 15 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "input", "Sparse" : true }, 16 | { "Name" : "Hidden1", "Kind" : "Hidden", "Type" : "FullyConnected", "N" : 128, "Activation" : "Sigmoid", "Sparse" : true }, 17 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "input", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true } 18 | ], 19 | 20 | "ErrorFunction" : "CrossEntropy" 21 | } 22 | -------------------------------------------------------------------------------- /samples/network/config_4.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "2 layer ReLu", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | 9 | "ScaledMarginalCrossEntropy" : { 10 | "oneTarget" : 1.0, 11 | "zeroTarget" : 0.0, 12 | "oneScale" : 1.0, 13 | "zeroScale" : 1.0 14 | }, 15 | "Layers" : [ 16 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "input", "Sparse" : true }, 17 | { "Name" : "Hidden1", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 1536, "Activation" : "Relu", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 18 | { "Name" : "Hidden2", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : ["Hidden1"], "N" : 1536, "Activation" : "Relu", "Sparse" : false, "pDropout" : 0.5, "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01 } }, 19 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true , "WeightInit" : { "Scheme" : "Gaussian", "Scale" : 0.01, "Bias" : -10.2 } } 20 | ], 21 | 22 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 23 | } 24 | -------------------------------------------------------------------------------- /samples/network/config_5.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "One layer Sigmoid", 4 | "Kind" : "FeedForward", 5 | "SparsenessPenalty" : { 6 | "p" : 0.5, 7 | "beta" : 2.0 8 | }, 9 | 10 | "ShuffleIndices" : false, 11 | 12 | "Denoising" : { 13 | "p" : 0.4 14 | }, 15 | 16 | "ScaledMarginalCrossEntropy" : { 17 | "oneTarget" : 1.0, 18 | "zeroTarget" : 0.0, 19 | "oneScale" : 30.0, 20 | "zeroScale" : 1.0 21 | }, 22 | "Layers" : [ 23 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "input", "Sparse" : true }, 24 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : [ "Input" ], "N" : 256, "Activation" : "Sigmoid", "Sparse" : true }, 25 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "Source" : [ "Hidden" ], "DataSet" : "output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true } 26 | ], 27 | 28 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 29 | } 30 | -------------------------------------------------------------------------------- /samples/network/config_6.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.7, 3 | "Name" : "Denoising ", 4 | "Kind" : "FeedForward", 5 | "SparsenessPenalty" : { 6 | "p" : 0.5, 7 | "beta" : 2.0 8 | }, 9 | 10 | "ShuffleIndices" : false, 11 | 12 | "Denoising" : { 13 | "p" : 0.28 14 | }, 15 | 16 | "ScaledMarginalCrossEntropy" : { 17 | "oneTarget" : 1.0, 18 | "zeroTarget" : 0.0, 19 | "oneScale" : 30.0, 20 | "zeroScale" : 1.0 21 | }, 22 | "Layers" : [ 23 | { "Name" : "Input", "Kind" : "Input", "N" : "auto", "DataSet" : "glinput", "Sparse" : true }, 24 | { "Name" : "Hidden1", "Kind" : "Hidden", "Type" : "FullyConnected", "N" : 256, "Activation" : "Sigmoid", "Sparse" : true }, 25 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "gl_output", "N" : "auto", "Activation" : "Sigmoid", "Sparse" : true } 26 | ], 27 | 28 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 29 | } 30 | -------------------------------------------------------------------------------- /src/amazon/dsstne/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # THIS MAKEFILE IS DEPRECATED PLEASE USE THE ONE AT THE ROOT OF THE DSSTNE REPOSITORY! 3 | # Keeping this here for backwards compatibility 4 | # 5 | 6 | $(warning **************************************************************************************) 7 | $(warning ****************** USE OF DEPRECATED MAKEFILE ******************) 8 | $(warning ****************** PLEASE USE THE ONE AT THE ROOT OF THE REPOSITORY ******************) 9 | $(warning **************************************************************************************) 10 | 11 | SHELL=/bin/sh 12 | VPATH= 13 | 14 | # Prefix of the directory to install dsstne. Override-able by setting PREFIX before calling make 15 | PREFIX ?= $(dir $(shell pwd))../../amazon-dsstne 16 | BUILD_DIR ?= $(dir $(shell pwd))../../../build 17 | 18 | all: $(BUILD_DIR)/lib/libdsstne.a $(BUILD_DIR)/bin/train $(BUILD_DIR)/bin/predict $(BUILD_DIR)/bin/generateNetCDF 19 | cd engine && make 20 | cd utils && make 21 | 22 | $(BUILD_DIR)/lib/libdsstne.a: 23 | cd engine && make 24 | 25 | install: all 26 | mkdir -p $(PREFIX) 27 | cp -rfp $(BUILD_DIR)/lib $(PREFIX)/lib 28 | cp -rfp $(BUILD_DIR)/bin $(PREFIX)/bin 29 | cp -rfp $(BUILD_DIR)/include $(PREFIX)/include 30 | 31 | $(BUILD_DIR)/bin/train: 32 | cd utils && make 33 | 34 | $(BUILD_DIR)/bin/predict: 35 | cd utils && make 36 | 37 | $(BUILD_DIR)/bin/generateNetCDF: 38 | cd utils && make 39 | 40 | clean: 41 | cd engine && make clean 42 | cd utils && make clean 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/amazon/dsstne/Makefile.inc: -------------------------------------------------------------------------------- 1 | # 2 | # For debugging add '-DMEMTRACKING -gdwarf-3' to CPPFLAGS and CFLAGS 3 | # 4 | CPP ?= /lib/cpp 5 | #CPPFLAGS = -traditional -P -std=c++0x 6 | CC = mpiCC 7 | NVCC = nvcc 8 | LOAD = mpiCC 9 | 10 | # Build directory 11 | BUILD_DIR ?= $(dir $(shell pwd))../../../build 12 | 13 | # 14 | # Used to exclude cuda headers from auto dependency generation 15 | # For some odd reason nvcc includes the directory relative to the bin directory 16 | # rather than the usual /usr/local/cuda/include 17 | # 18 | CUDA_SYSTEM_INCLUDE_DIR := $(dir $(shell which $(NVCC)))../target/x86_64/include 19 | 20 | # switch to 1 for debugging 21 | DEBUG ?= 0 22 | ifeq ($(DEBUG), 1) 23 | $(info ************ DEBUG mode ************) 24 | CFLAGS = \ 25 | -g \ 26 | -O0 \ 27 | -Wall \ 28 | -std=c++11 \ 29 | -fPIC \ 30 | -DOMPI_SKIP_MPICXX \ 31 | -MMD \ 32 | -MP 33 | 34 | CU_FLAGS = \ 35 | -g \ 36 | -O0 \ 37 | --device-debug \ 38 | --generate-line-info \ 39 | -std=c++11 \ 40 | --compiler-options=-fPIC \ 41 | --compiler-options=-Wall \ 42 | -use_fast_math \ 43 | --ptxas-options="-v" \ 44 | -gencode arch=compute_70,code=sm_70 \ 45 | -gencode arch=compute_61,code=sm_61 \ 46 | -gencode arch=compute_52,code=sm_52 \ 47 | -gencode arch=compute_30,code=sm_30 \ 48 | -DOMPI_SKIP_MPICXX 49 | 50 | else 51 | $(info ************ RELEASE mode ************) 52 | CFLAGS = \ 53 | -O3 \ 54 | -std=c++11 \ 55 | -fPIC \ 56 | -DOMPI_SKIP_MPICXX \ 57 | -MMD \ 58 | -MP 59 | 60 | CU_FLAGS = \ 61 | -O3 \ 62 | -std=c++11 \ 63 | --compiler-options=-fPIC \ 64 | -use_fast_math \ 65 | --ptxas-options="-v" \ 66 | -gencode arch=compute_70,code=sm_70 \ 67 | -gencode arch=compute_61,code=sm_61 \ 68 | -gencode arch=compute_52,code=sm_52 \ 69 | -gencode arch=compute_30,code=sm_30 \ 70 | -DOMPI_SKIP_MPICXX 71 | 72 | endif 73 | 74 | # add dependency include dirs as -isystem so that they are 75 | # not included during auto dependency generation (see -MMD option for gcc) 76 | CU_INCLUDES ?= \ 77 | -I/usr/local/include \ 78 | -isystem /usr/local/cuda/include \ 79 | -isystem /usr/lib/openmpi/include \ 80 | -isystem /usr/include/jsoncpp \ 81 | -IB40C \ 82 | -IB40C/KernelCommon \ 83 | -I$(BUILD_DIR)/include 84 | 85 | CU_LIBS ?= \ 86 | -L/usr/lib/atlas-base \ 87 | -L/usr/local/cuda/lib64 \ 88 | -L/usr/local/lib/ 89 | 90 | CU_LOADLIBS = \ 91 | -lcudnn \ 92 | -lcurand \ 93 | -lcublas \ 94 | -lcudart \ 95 | -ljsoncpp \ 96 | -lnetcdf_c++4 \ 97 | -lnetcdf \ 98 | -lblas \ 99 | -ldl \ 100 | -lstdc++ 101 | -------------------------------------------------------------------------------- /src/amazon/dsstne/engine/GpuSort.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 7 | 8 | http://aws.amazon.com/apache2.0/ 9 | 10 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | */ 12 | 13 | #ifndef __GPUSORT_H__ 14 | #define __GPUSORT_H__ 15 | 16 | #include 17 | 18 | template class GpuSort 19 | { 20 | private: 21 | unsigned int _items; 22 | unsigned int _itemStride; 23 | unique_ptr> _pbKey; 24 | KeyType* _pKey0; 25 | KeyType* _pKey1; 26 | unique_ptr> _pbValue; 27 | ValueType* _pValue0; 28 | ValueType* _pValue1; 29 | size_t _tempBytes; 30 | unique_ptr> _pbTemp; 31 | KeyType* _pKey; 32 | ValueType* _pValue; 33 | 34 | 35 | 36 | public: 37 | GpuSort(unsigned int items) : 38 | _items(items), 39 | _itemStride(((items + 511) >> 9) << 9), 40 | _pbKey(new GpuBuffer(_itemStride * 2)), 41 | _pKey0(_pbKey->_pDevData), 42 | _pKey1(_pbKey->_pDevData + _itemStride), 43 | _pbValue(new GpuBuffer(_itemStride * 2)), 44 | _pValue0(_pbValue->_pDevData), 45 | _pValue1(_pbValue->_pDevData + _itemStride), 46 | _tempBytes(kInitSort(_items, _pbValue.get(), _pbKey.get())), 47 | _pbTemp(new GpuBuffer(_tempBytes)) 48 | { 49 | _pKey = _pKey0; 50 | _pValue = _pValue0; 51 | } 52 | 53 | ~GpuSort() 54 | { 55 | } 56 | bool Sort() { return kSort(_items, _pKey0, _pKey1, _pValue0, _pValue1, _pbTemp->_pDevData, _tempBytes); } 57 | GpuBuffer* GetKeyBuffer() { return _pbKey.get(); } 58 | GpuBuffer* GetValueBuffer() { return _pbValue.get(); } 59 | KeyType* GetKeyPointer() { return _pKey;} 60 | ValueType* GetValuePointer() { return _pValue; } 61 | }; 62 | #endif 63 | -------------------------------------------------------------------------------- /src/amazon/dsstne/engine/Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL=/bin/sh 3 | VPATH= 4 | 5 | include ../Makefile.inc 6 | 7 | SOURCES_DIR := . 8 | 9 | HEADERS := $(wildcard *.h *.hpp) 10 | HEADERS_BUILD_DIR := $(BUILD_DIR)/include/amazon/dsstne/engine 11 | 12 | OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/engine/cpp 13 | CU_OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/engine/cuda 14 | $(shell mkdir -p $(OBJS_BUILD_DIR)) 15 | $(shell mkdir -p $(CU_OBJS_BUILD_DIR)) 16 | 17 | SOURCES=$(wildcard *.c *.cc *.cpp) 18 | CU_SOURCES=$(wildcard *.cu) 19 | 20 | OBJECTS := $(SOURCES:%.cpp=$(OBJS_BUILD_DIR)/%.o) 21 | CU_OBJECTS := $(CU_SOURCES:%.cu=$(CU_OBJS_BUILD_DIR)/%.o) 22 | 23 | DEP := $(OBJECTS:.o=.d) 24 | CU_DEP := $(CU_OBJECTS:.o=.d) 25 | 26 | CU_INCLUDES += -I../utils 27 | 28 | LIB_BUILD_DIR := $(BUILD_DIR)/lib 29 | 30 | all: $(LIB_BUILD_DIR)/libdsstne.a $(LIB_BUILD_DIR)/libdsstne_engine.so 31 | 32 | $(LIB_BUILD_DIR)/libdsstne.a: $(OBJECTS) $(CU_OBJECTS) 33 | $(info ========== Creating libdsstne.a ==========) 34 | mkdir -p $(LIB_BUILD_DIR) 35 | ar -cvr $@ $^ 36 | $(info ========== Copying amazon/dsstne/engine headers ==========) 37 | mkdir -p $(HEADERS_BUILD_DIR) 38 | cp $(HEADERS) $(HEADERS_BUILD_DIR) 39 | 40 | $(LIB_BUILD_DIR)/libdsstne_engine.so: $(OBJECTS) $(CU_OBJECTS) 41 | $(info ========== Creating libdsstne_engine.so ==========) 42 | mkdir -p $(BUILD_DIR)/lib 43 | $(CC) -shared $(LDFLAGS) $(CU_LIBS) $(OBJECTS) $(CU_OBJECTS) -o $@ $(CU_LOADLIBS) 44 | 45 | clean: 46 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 47 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) $(HEADERS_BUILD_DIR) $(LIB_BUILD_DIR)/libdsstne.a 48 | 49 | distclean: 50 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 51 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) 52 | 53 | # The following statement keeps the .f90 files from getting clobbered; this 54 | # would happen because they are intermediates. 55 | 56 | # Pattern rule for compiling C++ 57 | $(OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cpp 58 | $(CC) $(CFLAGS) $(CU_INCLUDES) -c $< -o $@ 59 | 60 | # Pattern rule for compiling CUDA 61 | $(CU_OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cu GpuTypes.h $(CU_OBJS_BUILD_DIR)/%.d 62 | $(NVCC) $(CU_FLAGS) --keep-dir $(CU_OBJS_BUILD_DIR) $(CU_INCLUDES) $($*.cu.CU_FLAGS) -c $< -o $@ 63 | 64 | # Pattern rule for auto dependency generation of *.cu files 65 | $(CU_OBJS_BUILD_DIR)/%.d: $(SOURCES_DIR)/%.cu 66 | $(NVCC) -E -std=c++11 -Xcompiler "-isystem $(CUDA_SYSTEM_INCLUDE_DIR) -MP,-MM" $(CU_INCLUDES) $< -o $@ 67 | 68 | -include $(DEP) 69 | -include $(CU_DEP) 70 | -------------------------------------------------------------------------------- /src/amazon/dsstne/engine/NNEnum.h: -------------------------------------------------------------------------------- 1 | #ifndef NNENUM_H 2 | #define NNENUM_H 3 | 4 | #include 5 | 6 | namespace NNDataSetEnums 7 | { 8 | 9 | enum Attributes 10 | { 11 | Sparse = 1, // Sparse dataset 12 | Boolean = 2, // All datapoints are 0/1 13 | Compressed = 4, // Data uses type-specific compression 14 | Recurrent = 8, // Data has a time dimension 15 | Mutable = 16, // Data can be modified by running network backwards 16 | SparseIgnoreZero = 32, // Only calculate errors and deltas on non-zero values 17 | Indexed = 64, // Data set is indexed 18 | Weighted = 128 // Data points have weight associated with them 19 | }; 20 | 21 | enum Kind 22 | { 23 | Numeric = 0, 24 | Image = 1, 25 | Audio = 2 26 | }; 27 | 28 | enum Sharding 29 | { 30 | None = 0, 31 | Model = 1, 32 | Data = 2, 33 | }; 34 | 35 | enum DataType 36 | { 37 | UInt = 0, 38 | Int = 1, 39 | LLInt = 2, 40 | ULLInt = 3, 41 | Float = 4, 42 | Double = 5, 43 | RGB8 = 6, 44 | RGB16 = 7, 45 | UChar = 8, 46 | Char = 9 47 | }; 48 | 49 | /** 50 | * Template specialization to get DataType enum. 51 | * Support types listed in NNTypes.cpp#LoadNetCDF(). 52 | * Template types are the same as listed in kernels.cu#EXPLICITLY_INSTANTIATE_KERNELS 53 | */ 54 | template inline DataType getDataType() 55 | { 56 | throw std::runtime_error("Default data type not defined"); 57 | } 58 | 59 | template<> inline DataType getDataType() 60 | { 61 | return DataType::UInt; 62 | } 63 | 64 | template<> inline DataType getDataType() 65 | { 66 | return NNDataSetEnums::DataType::Int; 67 | } 68 | 69 | template<> inline DataType getDataType() 70 | { 71 | return NNDataSetEnums::DataType::LLInt; 72 | } 73 | 74 | template<> inline DataType getDataType() 75 | { 76 | return NNDataSetEnums::DataType::ULLInt; 77 | } 78 | 79 | template<> inline DataType getDataType() 80 | { 81 | return DataType::Float; 82 | } 83 | 84 | template<> inline DataType getDataType() 85 | { 86 | return DataType::Double; 87 | } 88 | 89 | template<> inline DataType getDataType() 90 | { 91 | return NNDataSetEnums::DataType::Char; 92 | } 93 | 94 | template<> inline DataType getDataType() 95 | { 96 | return NNDataSetEnums::DataType::UChar; 97 | } 98 | 99 | } // namespace NNDataSetEnums 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /src/amazon/dsstne/engine/NcExcptionWrap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #define NC_EXCEPTION(errorStr, msg, filename, line) NcException(std::string(msg).c_str(), filename, line) 5 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/DataReader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "DataReader.h" 25 | 26 | namespace 27 | { 28 | static const int success = 0; 29 | static const int failure = 1; 30 | 31 | int splitKeyVector(const std::string &line, std::string &key, std::string &vector, char keyValueDelimiter) 32 | { 33 | int keyValDelimIndex = line.find_first_of(keyValueDelimiter); 34 | 35 | if (keyValDelimIndex == std::string::npos) 36 | { 37 | return failure; 38 | } 39 | 40 | key = line.substr(0, keyValDelimIndex); 41 | vector = line.substr(keyValDelimIndex + 1, line.size()); 42 | return success; 43 | } 44 | } 45 | 46 | uint32_t DataReader::getRows() const 47 | { 48 | return rows; 49 | } 50 | 51 | int DataReader::getColumns() const 52 | { 53 | return columns; 54 | } 55 | 56 | TextFileDataReader::TextFileDataReader(const std::string &fileName, char keyValueDelimiter, char vectorDelimiter) : 57 | fileName(fileName), 58 | fileStream(fileName, std::ios_base::in), 59 | keyValueDelimiter(keyValueDelimiter), 60 | vectorDelimiter(vectorDelimiter) 61 | { 62 | 63 | findDataDimensions(fileName, rows, columns, keyValueDelimiter, vectorDelimiter); 64 | } 65 | 66 | void TextFileDataReader::findDataDimensions(const std::string &fileName, uint32_t &rows, int &columns, 67 | char keyValueDelimiter, char vectorDelimiter) 68 | { 69 | std::fstream fs(fileName, std::ios_base::in); 70 | 71 | rows = 0; 72 | columns = 0; 73 | 74 | std::string line; 75 | while (std::getline(fs, line)) 76 | { 77 | if (line.length() == 0) 78 | { 79 | // skip empty lines 80 | continue; 81 | } 82 | 83 | ++rows; 84 | 85 | std::string key; 86 | std::string vectorStr; 87 | 88 | if (splitKeyVector(line, key, vectorStr, keyValueDelimiter)) 89 | { 90 | std::stringstream msg; 91 | msg << "In file: " << fileName << "#" << rows << ". Malformed line. key-value delimiter [" << keyValueDelimiter 92 | << "] not found in: " << line; 93 | throw std::invalid_argument(msg.str()); 94 | } 95 | 96 | std::stringstream vectorStrStream(vectorStr); 97 | std::string elementStr; 98 | int columnsInRow = 0; 99 | while (std::getline(vectorStrStream, elementStr, vectorDelimiter)) 100 | { 101 | ++columnsInRow; 102 | } 103 | 104 | // check all rows have same nColumns 105 | if (columns == 0) 106 | { 107 | columns = columnsInRow; 108 | } else 109 | { 110 | if (columns != columnsInRow) 111 | { 112 | std::stringstream msg; 113 | msg << "In file: " << fileName << "#" << rows << ". Inconsistent num columns detected. Expected : " << columns 114 | << " Actual: " << columnsInRow; 115 | throw std::invalid_argument(msg.str()); 116 | } 117 | } 118 | } 119 | 120 | fs.close(); 121 | } 122 | 123 | bool TextFileDataReader::readRow(std::string *key, float *vector) 124 | { 125 | std::string line; 126 | if (std::getline(fileStream, line)) 127 | { 128 | std::string vectorStr; 129 | splitKeyVector(line, *key, vectorStr, keyValueDelimiter); 130 | 131 | std::stringstream vectorStrStream(vectorStr); 132 | std::string elementStr; 133 | size_t idx; 134 | 135 | for (int i = 0; std::getline(vectorStrStream, elementStr, vectorDelimiter); ++i) 136 | { 137 | try 138 | { 139 | vector[i] = std::stof(elementStr, &idx); 140 | if(idx != elementStr.size()) { 141 | std::stringstream msg; 142 | msg << "Malformed vector element: " << elementStr; 143 | throw std::invalid_argument(msg.str()); 144 | } 145 | } catch (const std::exception& e) 146 | { 147 | std::stringstream msg; 148 | msg << "ERROR: " << elementStr << " cannot be parsed as float. Column " << i << " of: " << line; 149 | fprintf(stderr, "ERROR: %s cannot be parsed as float. Column %d, line: %s\n", elementStr.c_str(), i, 150 | line.c_str()); 151 | throw; 152 | } 153 | } 154 | 155 | return true; 156 | } else 157 | { 158 | return false; 159 | } 160 | } 161 | 162 | TextFileDataReader::~TextFileDataReader() 163 | { 164 | fileStream.close(); 165 | } 166 | 167 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/DataReader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_DATAREADER_H_ 19 | #define LIBKNN_DATAREADER_H_ 20 | 21 | #include 22 | 23 | class DataReader 24 | { 25 | 26 | public: 27 | 28 | /** 29 | * Reads and parses the next row into the key and vector variables. 30 | */ 31 | virtual bool readRow(std::string *key, float *vector) = 0; 32 | 33 | uint32_t getRows() const; 34 | 35 | int getColumns() const; 36 | 37 | virtual ~DataReader() 38 | { 39 | } 40 | 41 | protected: 42 | uint32_t rows; 43 | int columns; 44 | }; 45 | 46 | class TextFileDataReader: public DataReader 47 | { 48 | 49 | public: 50 | TextFileDataReader(const std::string &fileName, char keyValueDelimiter = '\t', char vectorDelimiter = ' '); 51 | 52 | bool readRow(std::string *key, float *vector); 53 | 54 | static void findDataDimensions(const std::string &fileName, uint32_t &rows, int &columns, char keyValueDelimiter = 55 | '\t', char vectorDelimiter = ' '); 56 | 57 | ~TextFileDataReader(); 58 | 59 | private: 60 | std::string fileName; 61 | std::fstream fileStream; 62 | char keyValueDelimiter; 63 | char vectorDelimiter; 64 | }; 65 | 66 | #endif /* LIBKNN_DATAREADER_H_ */ 67 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/KnnData.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_KNN_HANDLE_H_ 19 | #define LIBKNN_KNN_HANDLE_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | #include "DataReader.h" 29 | 30 | namespace astdl 31 | { 32 | namespace knn 33 | { 34 | 35 | enum class DataType 36 | { 37 | FP32 = 0, FP16 = 1 38 | }; 39 | 40 | std::string getDataTypeString(DataType dataType); 41 | 42 | DataType getDataTypeFromString(const std::string &dataTypeLiteral); 43 | 44 | /** 45 | * Holds the data pointer and row, column, and element size 46 | * information for a 2-d array in either host or device memory. 47 | */ 48 | struct Matrix 49 | { 50 | void *data; 51 | uint32_t numRows; 52 | int numColumns; 53 | size_t elementSize; 54 | cudaMemoryType memoryType; 55 | 56 | Matrix(); 57 | 58 | Matrix(void *data, uint32_t numRows, int numColumns, size_t elementSize, cudaMemoryType memoryType); 59 | 60 | size_t getSizeInBytes(); 61 | 62 | size_t getLength(); 63 | }; 64 | 65 | Matrix loadDataOnHost(DataReader *dataReader); 66 | 67 | struct KnnData 68 | { 69 | const int numGpus; 70 | const int batchSize; 71 | const int maxK; 72 | 73 | /* 74 | * cublas handles 75 | */ 76 | std::vector cublasHandles; 77 | 78 | /* 79 | * data 80 | */ 81 | std::vector dCollectionPartitions; // column major 82 | std::vector dInputBatches; 83 | std::vector dProducts; 84 | 85 | std::vector dResultScores; 86 | std::vector dResultIndexes; 87 | 88 | std::vector hResultScores; 89 | std::vector hResultIndexes; 90 | std::vector> hKeys; 91 | 92 | /* 93 | * tmp buffers 94 | */ 95 | // used for converting float -> half when data type is FP16 96 | std::vector dInputBatchTmpBuffers; 97 | 98 | /* 99 | * number of rows padded per device. 100 | * we pad rows to multiples of 4 for cublasSgemm performance 101 | * actualNumRows = dCollectionPartitions[device].rows - collectionRowsPadded[device] 102 | */ 103 | std::vector collectionRowsPadded; 104 | 105 | /* 106 | * metric data 107 | */ 108 | std::vector elapsedSgemm; 109 | std::vector elapsedTopK; 110 | 111 | /** 112 | * Stores elements in fp16 instead of fp32. Also uses fp16 math on V100. 113 | */ 114 | const DataType dataType; 115 | 116 | KnnData(int numGpus, int batchSize, int maxK, DataType dataType); 117 | 118 | void load(int device, DataReader *dataReader); 119 | 120 | void load(const std::map &deviceToData); 121 | 122 | void load(const std::map &deviceToFile, char keyValDelim, char vecDelim); 123 | 124 | int getFeatureSize() const; 125 | 126 | ~KnnData(); 127 | }; 128 | 129 | Matrix allocateMatrixOnHost(uint32_t numRows, int numColumns, size_t elementSize); 130 | 131 | Matrix allocateMatrixOnDevice(uint32_t numRows, int numColumns, size_t elementSize); 132 | 133 | void freeMatrix(const Matrix &matrix); 134 | 135 | } // namespace knn 136 | } // namepace astdl 137 | 138 | #endif /* KNN_HANDLE_H_ */ 139 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/KnnExactGpu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_KNN_EXACT_GPU_H_ 19 | #define LIBKNN_KNN_EXACT_GPU_H_ 20 | 21 | #include "KnnData.h" 22 | 23 | namespace astdl 24 | { 25 | namespace knn 26 | { 27 | class Knn 28 | { 29 | public: 30 | 31 | virtual void search(int k, const float *inputs, std::string *keys, float *scores) = 0; 32 | 33 | virtual ~Knn() 34 | { 35 | 36 | } 37 | 38 | protected: 39 | KnnData *data; 40 | 41 | Knn(KnnData *data); 42 | }; 43 | 44 | class KnnExactGpu: public Knn 45 | { 46 | public: 47 | KnnExactGpu(KnnData *data); 48 | 49 | void search(int k, const float *inputs, int size, std::string *keys, float *scores); 50 | 51 | void search(int k, const float *inputs, std::string *keys, float *scores) 52 | { 53 | search(k, inputs, data->batchSize, keys, scores); 54 | } 55 | 56 | }; 57 | 58 | /** 59 | * merges the top k results from each of the GPUs. 60 | * batchSize - number of rows in g_scores[i], and g_indexes[i] 61 | * k - number of columns in g_scores[i] and g_indexes[i] 62 | * g_scores - top k scores (k x batchSize) from each GPU 63 | * g_indexes - top k indexes (k x batchSize) from each GPU 64 | * scores - where to store the merged top k scores (must malloc k x batchSize x sizeof(float)) 65 | * indexes - where to store the merged top k indexes (must malloc k x batchSize x sizeof(float)) 66 | */ 67 | void mergeKnn(int k, int batchSize, int width, int numGpus, const std::vector &allScores, 68 | const std::vector &allIndexes, const std::vector> &allKeys, float *scores, 69 | std::string *keys); 70 | } // namespace knn 71 | } // namespace astdl 72 | 73 | #endif /* LIBKNN_KNN_EXACT_GPU_H_ */ 74 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL=/bin/sh 3 | VPATH= 4 | 5 | include ../Makefile.inc 6 | 7 | CC = g++ 8 | 9 | SOURCES_DIR := . 10 | 11 | HEADERS := $(wildcard *.h *.hpp) 12 | HEADERS_BUILD_DIR := $(BUILD_DIR)/include/amazon/dsstne/knn 13 | 14 | OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/knn/cpp 15 | CU_OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/knn/cuda 16 | $(shell mkdir -p $(OBJS_BUILD_DIR)) 17 | $(shell mkdir -p $(CU_OBJS_BUILD_DIR)) 18 | 19 | SOURCES=$(wildcard *.c *.cc *.cpp) 20 | CU_SOURCES=$(wildcard *.cu) 21 | 22 | OBJECTS := $(SOURCES:%.cpp=$(OBJS_BUILD_DIR)/%.o) 23 | CU_OBJECTS := $(CU_SOURCES:%.cu=$(CU_OBJS_BUILD_DIR)/%.o) 24 | 25 | DEP := $(OBJECTS:.o=.d) 26 | CU_DEP := $(CU_OBJECTS:.o=.d) 27 | 28 | CU_INCLUDES += -I../utils 29 | 30 | LIB_BUILD_DIR := $(BUILD_DIR)/lib 31 | 32 | all: $(LIB_BUILD_DIR)/libdsstne_knn.so 33 | 34 | $(LIB_BUILD_DIR)/libdsstne_knn.so: $(OBJECTS) $(CU_OBJECTS) 35 | $(info ========== Creating libdsstne_knn.so ==========) 36 | mkdir -p $(BUILD_DIR)/lib 37 | $(CC) -shared $(LDFLAGS) $(CU_LIBS) $(OBJECTS) $(CU_OBJECTS) -o $@ $(CU_LOADLIBS) 38 | $(info ========== Copying amazon/dsstne/knn headers ==========) 39 | mkdir -p $(HEADERS_BUILD_DIR) 40 | cp $(HEADERS) $(HEADERS_BUILD_DIR) 41 | 42 | clean: 43 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 44 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) $(HEADERS_BUILD_DIR) $(LIB_BUILD_DIR)/libdsstne_knn.a 45 | 46 | distclean: 47 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 48 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) 49 | 50 | # The following statement keeps the .f90 files from getting clobbered; this 51 | # would happen because they are intermediates. 52 | 53 | # Pattern rule for compiling C++ 54 | $(OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cpp 55 | $(CC) $(CFLAGS) $(CU_INCLUDES) -c $< -o $@ 56 | 57 | # Pattern rule for compiling CUDA 58 | $(CU_OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cu $(CU_OBJS_BUILD_DIR)/%.d 59 | $(NVCC) $(CU_FLAGS) --keep-dir $(CU_OBJS_BUILD_DIR) $(CU_INCLUDES) $($*.cu.CU_FLAGS) -c $< -o $@ 60 | 61 | # Pattern rule for auto dependency generation of *.cu files 62 | $(CU_OBJS_BUILD_DIR)/%.d: $(SOURCES_DIR)/%.cu 63 | $(NVCC) -E -std=c++11 -Xcompiler "-isystem $(CUDA_SYSTEM_INCLUDE_DIR) -MP,-MM" $(CU_INCLUDES) $< -o $@ 64 | 65 | -include $(DEP) 66 | -include $(CU_DEP) 67 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/MathUtil.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | #include "cudautil.h" 24 | 25 | namespace astdl 26 | { 27 | namespace math 28 | { 29 | /** 30 | * Copies size elements from src[0...size] to dst[0...size], converting each float value 31 | * in src to half in dst. 32 | */ 33 | __global__ void kFloatToHalf_kernel(const float *src, size_t length, half *dst) 34 | { 35 | size_t idx = blockIdx.x * blockDim.x + threadIdx.x; 36 | 37 | if (idx < length) 38 | { 39 | dst[idx] = __float2half(src[idx]); 40 | } 41 | } 42 | 43 | void kFloatToHalf(const float *hSource, size_t sourceSizeInBytes, half *dDest, float *dBuffer, size_t bufferSizeInBytes) 44 | { 45 | if(sourceSizeInBytes % sizeof(float) != 0) 46 | { 47 | throw std::invalid_argument("sourceSizeInBytes must be divisible by sizeof(float)"); 48 | } 49 | 50 | if (bufferSizeInBytes % sizeof(float) != 0) 51 | { 52 | throw std::invalid_argument("bufferSizeInBytes must be divisible by sizeof(float)"); 53 | } 54 | 55 | dim3 threads(128); 56 | size_t bufferLen = bufferSizeInBytes / sizeof(float); 57 | dim3 blocks((bufferLen + (threads.x - 1)) / threads.x); 58 | 59 | size_t srcLeftBytes = sourceSizeInBytes; 60 | size_t offset = 0; 61 | 62 | while (srcLeftBytes > 0) 63 | { 64 | size_t cpyBytes = srcLeftBytes < bufferSizeInBytes ? srcLeftBytes : bufferSizeInBytes; 65 | size_t cpyLength = cpyBytes / sizeof(float); 66 | 67 | CHECK_ERR(cudaMemcpy(dBuffer, hSource + offset, cpyBytes, cudaMemcpyHostToDevice)); 68 | LAUNCH_ERR((kFloatToHalf_kernel<<>>(dBuffer, cpyLength, dDest + offset))); 69 | 70 | offset += cpyLength; 71 | srcLeftBytes -= cpyBytes; 72 | } 73 | } 74 | 75 | void kFloatToHalf(const float *hSource, size_t sourceSizeInBytes, half *dDest, size_t bufferSizeInBytes) 76 | { 77 | float *dBuffer; 78 | CHECK_ERR(cudaMalloc(&dBuffer, bufferSizeInBytes)); 79 | kFloatToHalf(hSource, sourceSizeInBytes, dDest, dBuffer, bufferSizeInBytes); 80 | CHECK_ERR(cudaFree(dBuffer)); 81 | } 82 | 83 | __global__ void kHalfToFloat_kernel(const half *src, size_t length, float *dst) 84 | { 85 | size_t idx = blockIdx.x * blockDim.x + threadIdx.x; 86 | if (idx < length) 87 | { 88 | dst[idx] = __half2float(src[idx]); 89 | } 90 | } 91 | 92 | void kHalfToFloat(const half *dSource, size_t sourceSizeInBytes, float *hDest, size_t bufferSizeInBytes) 93 | { 94 | if(sourceSizeInBytes % sizeof(half) != 0) { 95 | throw std::invalid_argument("sourceSizeInBytes must be divisible by sizeof(half)"); 96 | } 97 | 98 | if (bufferSizeInBytes % sizeof(float) != 0) 99 | { 100 | throw std::invalid_argument("bufferSizeInBytes must be divisible by sizeof(float)"); 101 | } 102 | 103 | 104 | dim3 threads(128); 105 | size_t bufferLen = bufferSizeInBytes / sizeof(float); 106 | dim3 blocks((bufferLen + (threads.x - 1)) / threads.x); 107 | 108 | float *dBuffer; 109 | CHECK_ERR(cudaMalloc(&dBuffer, bufferLen * sizeof(float))); 110 | 111 | size_t sourceLength = sourceSizeInBytes / sizeof(half); 112 | size_t srcLeftBytes = sourceLength * sizeof(float); 113 | size_t offset = 0; 114 | 115 | while (srcLeftBytes > 0) 116 | { 117 | size_t cpyBytes = srcLeftBytes < bufferSizeInBytes ? srcLeftBytes : bufferSizeInBytes; 118 | size_t cpyLength = cpyBytes / sizeof(float); 119 | 120 | LAUNCH_ERR((kHalfToFloat_kernel<<>>(dSource + offset, cpyLength, dBuffer))); 121 | CHECK_ERR(cudaMemcpy(hDest + offset, dBuffer, cpyBytes, cudaMemcpyDeviceToHost)); 122 | 123 | offset += cpyLength; 124 | srcLeftBytes -= cpyBytes; 125 | } 126 | 127 | CHECK_ERR(cudaFree(dBuffer)); 128 | } 129 | } // namespace math 130 | } // namespace astdl 131 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/MathUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_MATHUTIL_H_ 19 | #define LIBKNN_MATHUTIL_H_ 20 | 21 | #include 22 | 23 | namespace astdl 24 | { 25 | namespace math 26 | { 27 | /** 28 | * Converts and loads the fp32 array on the host to fp16 array on device. 29 | * Temporarily allocates and frees a copy buffer on the device of length bufferSizeInBytes (must be a multiple of 4). 30 | */ 31 | void kFloatToHalf(const float *hSource, size_t sourceLength, half *dDest, size_t bufferSizeInBytes = 4 * 1024 * 1024); 32 | 33 | /** 34 | * Converts and loads the fp32 array on the host to fp16 array on device. 35 | * Uses the provided dBuffer on the device to copy the source floats on the host to device. 36 | */ 37 | void kFloatToHalf(const float *hSource, size_t sourceSizeInBytes, half *dDest, float *dBuffer, size_t bufferSizeInBytes); 38 | 39 | /** 40 | * Converts and loads the fp16 array on device to fp32 array on host. 41 | * bufferSizeInBytes MUST be multiples of 4. 42 | */ 43 | void kHalfToFloat(const half *dSource, size_t sourceLength, float *hDest, size_t bufferSizeInBytes = 4 * 1024 * 1024); 44 | } // namespace math 45 | } // namespace astdl 46 | #endif /* LIBKNN_MATHUTIL_H_ */ 47 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/cudautil.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #include 19 | 20 | #include "cudautil.h" 21 | 22 | namespace astdl 23 | { 24 | namespace cuda_util 25 | { 26 | void printMemInfo(const char *header) 27 | { 28 | int bytesInMb = 1024 * 1024; 29 | int device; 30 | size_t free; 31 | size_t total; 32 | CHECK_ERR(cudaGetDevice(&device)); 33 | CHECK_ERR(cudaMemGetInfo(&free, &total)); 34 | 35 | long freeMb = free / bytesInMb; 36 | long usedMb = (total - free) / bytesInMb; 37 | long totalMb = total / bytesInMb; 38 | 39 | printf("--%-50s GPU [%d] Mem Used: %-6ld MB. Free: %-6ld MB. Total: %-6ld MB\n", header, device, usedMb, freeMb, 40 | totalMb); 41 | } 42 | 43 | void getDeviceMemoryInfoInMb(int device, size_t *total, size_t *free) { 44 | static const int bytesInMb = 1024 * 1024; 45 | size_t freeInBytes; 46 | size_t totalInBytes; 47 | CHECK_ERR(cudaGetDevice(&device)); 48 | CHECK_ERR(cudaMemGetInfo(&freeInBytes, &totalInBytes)); 49 | *total = totalInBytes / bytesInMb; 50 | *free = freeInBytes / bytesInMb; 51 | } 52 | 53 | int getDeviceCount() 54 | { 55 | int deviceCount; 56 | cudaError_t err = cudaGetDeviceCount(&deviceCount); 57 | if (err != cudaSuccess) 58 | { 59 | fprintf(stderr, "** ERROR (%d - %s) calling cudaGetDeviceCount()." 60 | " The host probably does not have any GPUs or the driver is not installed." 61 | " Returning -1\n", err, cudaGetErrorString(err)); 62 | return -1; 63 | } else 64 | { 65 | return deviceCount; 66 | } 67 | } 68 | 69 | bool hasGpus() 70 | { 71 | int deviceCount = getDeviceCount(); 72 | return deviceCount > 0; 73 | } 74 | 75 | } // namespace cudautil 76 | } // namespace astdl 77 | 78 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/cudautil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_CUDAUTIL_H_ 19 | #define LIBKNN_CUDAUTIL_H_ 20 | 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | static void CHECK_ERR2(cudaError_t e, const char *fname, int line) 28 | { 29 | if (e != cudaSuccess) 30 | { 31 | fprintf(stderr, "FATAL ERROR: cuda failure(%d): %s in %s#%d\n", e, cudaGetErrorString(e), fname, 32 | line); 33 | exit(-1); 34 | } 35 | } 36 | 37 | static void STATUS_ERR2(cublasStatus_t e, const char *fname, int line) 38 | { 39 | if (e != CUBLAS_STATUS_SUCCESS) 40 | { 41 | fprintf(stderr, "FATAL ERROR: cublas failure %d in %s#%d\n", e, fname, line); 42 | exit(-1); 43 | } 44 | } 45 | 46 | static void LAUNCH_ERR2(const char *kernelName, const char *fname, int line) 47 | { 48 | cudaError_t e = cudaGetLastError(); 49 | if (e != cudaSuccess) 50 | { 51 | fprintf(stderr, "FATAL ERROR: %s launching kernel: %s\n in %s#%d\n", cudaGetErrorString(e), kernelName, fname, line); 52 | exit(-1); 53 | } 54 | } 55 | 56 | #define CHECK_ERR(e) {CHECK_ERR2(e, __FILE__, __LINE__);} 57 | 58 | #define STATUS_ERR(e) {STATUS_ERR2(e, __FILE__, __LINE__);} 59 | 60 | #define LAUNCH_ERR(expression) { \ 61 | expression; \ 62 | LAUNCH_ERR2(#expression, __FILE__, __LINE__); \ 63 | } 64 | 65 | namespace astdl 66 | { 67 | namespace cuda_util 68 | { 69 | void printMemInfo(const char *header = ""); 70 | 71 | void getDeviceMemoryInfoInMb(int device, size_t *total, size_t *free); 72 | 73 | int getDeviceCount(); 74 | 75 | /** 76 | * Returns true if the host has GPUs, false otherwise. 77 | */ 78 | bool hasGpus(); 79 | 80 | } // namespace cuda 81 | } // namespace astdl 82 | 83 | /** 84 | * When used as the first statement of a function, returns immediately 85 | * if the host does not have any GPUs. 86 | */ 87 | #define REQUIRE_GPU if(!astdl::cuda_util::hasGpus()) return; 88 | 89 | /** 90 | * When used as the first statement of a function, returns immediately 91 | * if the host does not have at least numGpus. 92 | */ 93 | #define REQUIRE_GPUS(numGpus) if(astdl::cuda_util::getDeviceCount() < numGpus) return; 94 | 95 | #endif /* LIBKNN_CUDAUTIL_H_ */ 96 | -------------------------------------------------------------------------------- /src/amazon/dsstne/knn/topk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef LIBKNN_TOPK_H_ 19 | #define LIBKNN_TOPK_H_ 20 | 21 | #include 22 | #include 23 | 24 | #define NNFloat float 25 | 26 | static const float MAX_VALUE = 999999999999999.0f; 27 | 28 | 29 | static const int SM_3X_THREADS_PER_BLOCK = 128; 30 | static const int SM_5X_THREADS_PER_BLOCK = 128; 31 | static const int SM_6X_THREADS_PER_BLOCK = 128; 32 | 33 | #if (__CUDA_ARCH__ >= 600) 34 | #define LAUNCH_BOUNDS() __launch_bounds__(SM_6X_THREADS_PER_BLOCK, 8) 35 | #define LAUNCH_BOUNDS256() __launch_bounds__(256, 5) 36 | #elif (__CUDA_ARCH__ >= 500) 37 | #define LAUNCH_BOUNDS() __launch_bounds__(SM_5X_THREADS_PER_BLOCK, 8) 38 | #define LAUNCH_BOUNDS256() __launch_bounds__(256, 5) 39 | #else 40 | #define LAUNCH_BOUNDS() __launch_bounds__(SM_3X_THREADS_PER_BLOCK, 10) 41 | #define LAUNCH_BOUNDS256() __launch_bounds__(256, 4) 42 | #endif 43 | #define LAUNCH_BOUNDS512() __launch_bounds__(512, 2) 44 | #define LAUNCH_BOUNDS1024() __launch_bounds__(1024, 1) 45 | 46 | #ifdef SYNCHRONOUS 47 | #define LAUNCHERROR(s) \ 48 | { \ 49 | cudaError_t status = cudaGetLastError(); \ 50 | if (status != cudaSuccess) { \ 51 | printf("Error: %s launching kernel %s\n", cudaGetErrorString(status), s); \ 52 | exit(-1); \ 53 | } \ 54 | cudaDeviceSynchronize(); \ 55 | } 56 | #else 57 | #define LAUNCHERROR(s) \ 58 | { \ 59 | cudaError_t status = cudaGetLastError(); \ 60 | if (status != cudaSuccess) { \ 61 | printf("Error: %s launching kernel %s\n", cudaGetErrorString(status), s); \ 62 | exit(-1); \ 63 | } \ 64 | } 65 | #endif 66 | 67 | /* 68 | * TODO use kCalculateTopK from amazon/dsstne/engine/kernels.h 69 | * The reason why we have two versions is that this version accounts 70 | * for row padding which is required to make rows a multiple of 4 or 8 71 | * to enable cublas to run the fp16 sgemm kernels on tensorcores. 72 | */ 73 | void kCalculateTopK(NNFloat* pOutput, NNFloat *pKey, uint32_t* pValue, uint32_t batch, uint32_t width, uint32_t widthPadding, uint32_t k); 74 | 75 | #endif /* LIBKNN_TOPK_H_ */ 76 | -------------------------------------------------------------------------------- /src/amazon/dsstne/runtime/DsstneContext.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #include "DsstneContext.h" 19 | 20 | namespace 21 | { 22 | const int ARGC = 1; 23 | char *ARGV = "dsstne-faux-process"; 24 | const unsigned long SEED = 12134ULL; 25 | } 26 | 27 | using namespace std; 28 | 29 | DsstneContext::DsstneContext(const string &networkFilename, uint32_t batchSize, int maxK) : 30 | networkFilename(networkFilename), 31 | batchSize(batchSize), 32 | maxK(maxK) 33 | { 34 | getGpu().Startup(ARGC, &ARGV); 35 | getGpu().SetRandomSeed(SEED); 36 | NNNetwork *network = LoadNeuralNetworkNetCDF(networkFilename, batchSize); 37 | getGpu().SetNeuralNetwork(network); 38 | 39 | vector outputLayers; 40 | vector::iterator it = network->GetLayers(NNLayer::Kind::Output, outputLayers); 41 | 42 | for (; it != outputLayers.end(); ++it) 43 | { 44 | const string &layerName = (*it)->GetName(); 45 | 46 | if(maxK != ALL) 47 | { 48 | // FIXME this only works for 1-D outputs 49 | if ((*it)->GetNumDimensions() > 1) 50 | { 51 | std::runtime_error("topK only supported on 1-D output layers"); 52 | } 53 | size_t outputBufferLength = maxK * batchSize; 54 | printf( 55 | "DsstneContext::DsstneContext: Allocating output score and index buffers, each of size %zu for output layer %s\n", 56 | 57 | outputBufferLength, layerName.c_str()); 58 | GpuBuffer *outputScores = new GpuBuffer(outputBufferLength, false, false); 59 | GpuBuffer *outputIndexes = new GpuBuffer(outputBufferLength, false, false); 60 | 61 | dOutputScores[layerName] = outputScores; 62 | dOutputIndexes[layerName] = outputIndexes; 63 | } 64 | } 65 | } 66 | 67 | GpuBuffer* DsstneContext::getOutputScoresBuffer(const std::string &layerName) 68 | { 69 | return dOutputScores.at(layerName); 70 | } 71 | 72 | GpuBuffer* DsstneContext::getOutputIndexesBuffer(const std::string &layerName) 73 | { 74 | return dOutputIndexes.at(layerName); 75 | } 76 | 77 | DsstneContext::~DsstneContext() 78 | { 79 | const string networkName = getNetwork()->GetName(); 80 | for (const auto &kv : dOutputScores) 81 | { 82 | delete (kv.second); 83 | } 84 | for (const auto &kv : dOutputIndexes) 85 | { 86 | delete (kv.second); 87 | } 88 | 89 | dOutputScores.clear(); 90 | dOutputIndexes.clear(); 91 | 92 | delete getNetwork(); 93 | getGpu().Shutdown(); 94 | printf("DsstneContext::~DsstneContext: Destroyed context for network %s\n", networkName.c_str()); 95 | } 96 | 97 | NNNetwork* DsstneContext::getNetwork() const 98 | { 99 | return getGpu()._pNetwork; 100 | } 101 | 102 | void DsstneContext::initInputLayerDataSets(const vector datasetDescriptors) 103 | { 104 | vector datasets; 105 | for (const auto &descriptor : datasetDescriptors) 106 | { 107 | NNDataSetBase *dataset = createNNDataSet(descriptor); 108 | datasets.push_back(dataset); 109 | } 110 | 111 | /* 112 | * LoadDataSet marks the network as "dirty" meaning that the next time the Predict() 113 | * method is called on the network, it will refresh the state of the network 114 | * which is expensive as it tries to re-allocate the GpuBuffers by calling the Shard() 115 | * method on the dataset. 116 | * Run through a prediction once on the dataset to prime (allocate) the GpuBuffers 117 | * and RefreshState() of the network once. Going forward we will avoid marking the network 118 | * as dirty. 119 | */ 120 | NNNetwork *network = getNetwork(); 121 | network->LoadDataSets(datasets); 122 | network->PredictBatch(); 123 | network->SetPosition(0); 124 | } 125 | 126 | -------------------------------------------------------------------------------- /src/amazon/dsstne/runtime/DsstneContext.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef DSSTNECONTEXT_H_ 19 | #define DSSTNECONTEXT_H_ 20 | 21 | #include "amazon/dsstne/engine/GpuTypes.h" 22 | #include "amazon/dsstne/engine/NNTypes.h" 23 | #include "amazon/dsstne/engine/NNLayer.h" 24 | 25 | /** 26 | * Holds the context for an instance of NNNetwork, 27 | * which is the network itself along with data structures 28 | * needed to support operations (e.g. predict) on the network. 29 | * Currently can only support one context per process 30 | * since GpuContext is a static. 31 | */ 32 | class DsstneContext 33 | { 34 | private: 35 | static const int ALL = -1; 36 | 37 | const std::string networkFilename; 38 | const uint32_t batchSize; 39 | const uint32_t maxK; 40 | 41 | std::map*> dOutputScores; 42 | std::map*> dOutputIndexes; 43 | 44 | public: 45 | DsstneContext(const std::string &networkFilename, uint32_t batchSize, int maxK = ALL); 46 | 47 | ~DsstneContext(); 48 | 49 | NNNetwork* getNetwork() const; 50 | 51 | /** 52 | * Initializes empty datasets for the input layers given the NNDataSetDescriptors. 53 | */ 54 | void initInputLayerDataSets(const std::vector datasetDescriptors); 55 | 56 | GpuBuffer* getOutputScoresBuffer(const std::string &layerName); 57 | 58 | GpuBuffer* getOutputIndexesBuffer(const std::string &layerName); 59 | 60 | static DsstneContext* fromPtr(long ptr) 61 | { 62 | DsstneContext * dc = (DsstneContext *) ptr; 63 | if (dc == nullptr) 64 | { 65 | std::runtime_error("Cannot convert nullptr to DsstneContext"); 66 | } 67 | return dc; 68 | } 69 | }; 70 | 71 | #endif /* DSSTNECONTEXT_H_ */ 72 | -------------------------------------------------------------------------------- /src/amazon/dsstne/runtime/Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL=/bin/sh 3 | VPATH= 4 | 5 | include ../Makefile.inc 6 | 7 | SOURCES_DIR := . 8 | 9 | HEADERS := $(wildcard *.h *.hpp) 10 | HEADERS_BUILD_DIR := $(BUILD_DIR)/include/amazon/dsstne/runtime 11 | 12 | OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/runtime/cpp 13 | CU_OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/runtime/cuda 14 | $(shell mkdir -p $(OBJS_BUILD_DIR)) 15 | $(shell mkdir -p $(CU_OBJS_BUILD_DIR)) 16 | 17 | SOURCES=$(wildcard *.c *.cc *.cpp) 18 | CU_SOURCES=$(wildcard *.cu) 19 | 20 | OBJECTS := $(SOURCES:%.cpp=$(OBJS_BUILD_DIR)/%.o) 21 | CU_OBJECTS := $(CU_SOURCES:%.cu=$(CU_OBJS_BUILD_DIR)/%.o) 22 | 23 | LIB_DSSTNE :=$(BUILD_DIR)/lib/libdsstne.a 24 | 25 | LDFLAGS ?= -fPIC 26 | #LDFLAGS ?= -fPIC -Wl,--whole-archive $(LIB_DSSTNE) -Wl,--no-whole-archive 27 | 28 | LIBS = \ 29 | $(CU_LIBS) \ 30 | -L$(BUILD_DIR)/lib 31 | 32 | LOAD_LIBS = \ 33 | $(CU_LOADLIBS) \ 34 | -ldsstne_engine 35 | 36 | DEP := $(OBJECTS:.o=.d) 37 | CU_DEP := $(CU_OBJECTS:.o=.d) 38 | 39 | LIB_BUILD_DIR := $(BUILD_DIR)/lib 40 | 41 | all: $(LIB_BUILD_DIR)/libdsstne_runtime.so 42 | 43 | $(LIB_BUILD_DIR)/libdsstne_runtime.so: $(OBJECTS) $(CU_OBJECTS) 44 | $(info ========== Creating libdsstne_runtime.so ==========) 45 | mkdir -p $(BUILD_DIR)/lib 46 | $(CC) -shared $(LDFLAGS) $(LIBS) $(OBJECTS) $(CU_OBJECTS) -o $@ $(LOAD_LIBS) 47 | $(info ========== Copying amazon/dsstne/runtime headers ==========) 48 | mkdir -p $(HEADERS_BUILD_DIR) 49 | cp $(HEADERS) $(HEADERS_BUILD_DIR) 50 | 51 | clean: 52 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) $(HEADERS_BUILD_DIR) $(LIB_BUILD_DIR)/libdsstne_runtime.so 53 | 54 | distclean: 55 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) 56 | 57 | # Pattern rule for compiling C++ 58 | $(OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cpp 59 | $(CC) $(CFLAGS) $(CU_INCLUDES) -c $< -o $@ 60 | 61 | # Pattern rule for compiling CUDA 62 | $(CU_OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cu GpuTypes.h $(CU_OBJS_BUILD_DIR)/%.d 63 | $(NVCC) $(CU_FLAGS) --keep-dir $(CU_OBJS_BUILD_DIR) $(CU_INCLUDES) $($*.cu.CU_FLAGS) -c $< -o $@ 64 | 65 | # Pattern rule for auto dependency generation of *.cu files 66 | $(CU_OBJS_BUILD_DIR)/%.d: $(SOURCES_DIR)/%.cu 67 | $(NVCC) -E -std=c++11 -Xcompiler "-isystem $(CUDA_SYSTEM_INCLUDE_DIR) -MP,-MM" $(CU_INCLUDES) $< -o $@ 68 | 69 | -include $(DEP) 70 | -include $(CU_DEP) 71 | -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/FilterHelper.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 7 | 8 | http://aws.amazon.com/apache2.0/ 9 | 10 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | */ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "Utils.h" 23 | #include"Filters.h" 24 | 25 | using namespace Json; 26 | using namespace std; 27 | 28 | /** 29 | TODO Add the helper with Test Cases 30 | */ 31 | int main(int argc, char** argv) { 32 | } 33 | -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/Filters.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 7 | 8 | http://aws.amazon.com/apache2.0/ 9 | 10 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | */ 12 | #ifndef FILTERS_H 13 | #define FILTERS_H 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | class AbstractFilter 21 | { 22 | public: 23 | virtual ~AbstractFilter() = default; 24 | 25 | virtual void loadFilter(std::unordered_map &xMInput, 26 | std::unordered_map &xMSamples, 27 | const std::string &filePath) = 0; 28 | 29 | virtual void applyFilter(float *xArray, int xSamplesIndex) const = 0; 30 | virtual void applyFilter(float *xArray, int xSamplesIndex, int offset, int width) const = 0; 31 | 32 | virtual std::string getFilterType() const = 0; 33 | 34 | protected: 35 | void updateRecords(float *xArray, const std::unordered_map *xFilter) const; 36 | void updateRecords(float *xArray, const std::unordered_map *xFilter, int offset, int width) const; 37 | }; 38 | 39 | class SamplesFilter : public AbstractFilter 40 | { 41 | std::unique_ptr>>> samplefilters; 42 | 43 | void loadSingleFilter(std::unordered_map &xMInput, 44 | std::unordered_map &xMSamples, 45 | std::vector>> &sampleFilters, 46 | const std::string &filePath); 47 | 48 | public: 49 | void loadFilter(std::unordered_map &xMInput, 50 | std::unordered_map &xMSamples, 51 | const std::string &filePath); 52 | 53 | void applyFilter(float *xArray, int xSamplesIndex) const; 54 | void applyFilter(float *xArray, int xSamplesIndex, int offset, int width) const; 55 | 56 | std::string getFilterType() const 57 | { 58 | return "samplesFilterType"; 59 | } 60 | }; 61 | 62 | class FilterConfig 63 | { 64 | std::unique_ptr sampleFilter; 65 | std::string outputFileName; 66 | 67 | public : 68 | void setOutputFileName(const std::string &xOutputFileName) 69 | { 70 | outputFileName = xOutputFileName; 71 | } 72 | 73 | std::string getOutputFileName() const 74 | { 75 | return outputFileName; 76 | } 77 | 78 | void setSamplesFilter(SamplesFilter *xSampleFilter) 79 | { 80 | sampleFilter.reset(xSampleFilter); 81 | } 82 | 83 | void applySamplesFilter(float *xInput, int xSampleIndex, int offset, int width) const 84 | { 85 | if (sampleFilter) 86 | { 87 | sampleFilter->applyFilter(xInput, xSampleIndex, offset, width); 88 | } 89 | } 90 | }; 91 | 92 | /** 93 | * Parses a filterConfig file, which should be in JSON format 94 | * 95 | * This filter will be created based on the indexes given for the input layer, 96 | * mInput, and samples, mSamples. 97 | * 98 | * Sample Filters.json: 99 | * 100 | * "filters": [ 101 | * {"sampleFilters": "watches", "nodeFilters": "primeFilters", "outputFile":"primerecs" } 102 | * ] 103 | */ 104 | FilterConfig* loadFilters(const std::string &samplesFilterFileName, 105 | const std::string &outputFileName, 106 | std::unordered_map &xMInput, 107 | std::unordered_map &xMSamples); 108 | 109 | #endif 110 | -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL=/bin/sh 3 | VPATH= 4 | 5 | # Platform-specific info should be found in ../config.h 6 | 7 | include ../Makefile.inc 8 | 9 | SOURCES_DIR := . 10 | 11 | HEADERS := $(wildcard *.h *.hpp) 12 | HEADERS_BUILD_DIR := $(BUILD_DIR)/include/amazon/dsstne/utils 13 | 14 | OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/utils/cpp 15 | CU_OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/utils/cuda 16 | $(shell mkdir -p $(OBJS_BUILD_DIR)) 17 | $(shell mkdir -p $(CU_OBJS_BUILD_DIR)) 18 | 19 | SOURCES=$(wildcard *.c *.cc *.cpp) 20 | CU_SOURCES=$(wildcard *.cu) 21 | 22 | OBJECTS := $(SOURCES:%.cpp=$(OBJS_BUILD_DIR)/%.o) 23 | CU_OBJECTS := $(CU_SOURCES:%.cu=$(CU_OBJS_BUILD_DIR)/%.o) 24 | # objects for the shared lib (e.g. the cpp files that have a header) 25 | OBJS := $(addprefix $(OBJS_BUILD_DIR)/, $(addsuffix .o, $(basename $(HEADERS)))) 26 | 27 | LIB_BUILD_DIR := $(BUILD_DIR)/lib 28 | 29 | LIB_DSSTNE :=$(BUILD_DIR)/lib/libdsstne.a 30 | 31 | BIN_BUILD_DIR := $(BUILD_DIR)/bin 32 | $(shell mkdir -p $(BIN_BUILD_DIR)) 33 | 34 | INCLUDES = \ 35 | $(CU_INCLUDES) \ 36 | -I../engine 37 | 38 | LIBS = \ 39 | $(CU_LIBS) \ 40 | 41 | LOAD_LIBS = $(CU_LOADLIBS) 42 | 43 | LDFLAGS ?= -fPIC -Wl,--whole-archive $(LIB_DSSTNE) -Wl,--no-whole-archive 44 | 45 | EXECUTABLES := \ 46 | $(BIN_BUILD_DIR)/generateNetCDF \ 47 | $(BIN_BUILD_DIR)/train \ 48 | $(BIN_BUILD_DIR)/predict \ 49 | $(BIN_BUILD_DIR)/encoder 50 | 51 | all: $(EXECUTABLES) $(LIB_BUILD_DIR)/libdsstne_utils.so 52 | 53 | $(LIB_DSSTNE): 54 | cd ../engine && make 55 | 56 | $(LIB_BUILD_DIR)/libdsstne_utils.so: $(OBJS) 57 | $(info ========== Creating libdsstne_utils.so ==========) 58 | mkdir -p $(BUILD_DIR)/lib 59 | $(CC) -shared $(LDFLAGS) $(LIBS) $^ -o $@ $(LOAD_LIBS) 60 | $(info ========== Copying amazon/dsstne/utils headers ==========) 61 | mkdir -p $(HEADERS_BUILD_DIR) 62 | cp $(HEADERS) $(HEADERS_BUILD_DIR) 63 | 64 | $(BIN_BUILD_DIR)/generateNetCDF: $(OBJS) $(LIB_DSSTNE) $(OBJS_BUILD_DIR)/NetCDFGenerator.o 65 | $(LOAD) $(LOADFLAGS) $(LIBS) $^ -o $@ $(LOAD_LIBS) 66 | 67 | $(BIN_BUILD_DIR)/train: $(OBJS) $(LIB_DSSTNE) $(OBJS_BUILD_DIR)/Train.o 68 | $(LOAD) $(LOADFLAGS) $(LIBS) $^ -o $@ $(LOAD_LIBS) 69 | 70 | $(BIN_BUILD_DIR)/encoder: $(OBJS) $(LIB_DSSTNE) $(OBJS_BUILD_DIR)/main.o 71 | $(LOAD) $(LOADFLAGS) $(LIBS) $^ -o $@ $(LOAD_LIBS) 72 | 73 | $(BIN_BUILD_DIR)/predict: $(OBJS) $(LIB_DSSTNE) $(OBJS_BUILD_DIR)/Predict.o 74 | $(LOAD) $(LOADFLAGS) $(LIBS) $^ -o $@ $(LOAD_LIBS) 75 | 76 | clean: 77 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 78 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) $(BIN_BUILD_DIR) $(HEADERS_BUILD_DIR) $(LIB_BUILD_DIR)/libdsstne_utils.so 79 | 80 | distclean: 81 | rm -f *cudafe* *.fatbin.* *.fatbin *.ii *.cubin *cu.cpp *.ptx *.cpp?.* *.hash *.o *.d work.pc* 82 | rm -rf $(OBJS_BUILD_DIR) $(CU_OBJS_BUILD_DIR) 83 | 84 | # Pattern rule for compiling C++ 85 | $(OBJS_BUILD_DIR)/%.o: $(SOURCES_DIR)/%.cpp 86 | $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ 87 | 88 | # FIXME TestGPU belongs with tst, move this into the Makefile of tst 89 | TST_OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/utils/tst 90 | 91 | $(BIN_BUILD_DIR)/TestGPU: $(OBJS) $(LIB_DSSTNE) $(TST_OBJS_BUILD_DIR)/TestDune.o 92 | $(LOAD) $(LOADFLAGS) $(LIBS) $^ -o $(BIN_BUILD_DIR)/$@ $(LOAD_LIBS) -l:libcppunit.a 93 | 94 | $(TST_OBJS_BUILD_DIR)/TestDune.o: ../../../../tst/gputests/TestDune.cpp 95 | mkdir -p $(TST_OBJS_BUILD_DIR) 96 | $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ 97 | # FIXME END -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/NNRecsGenerator.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 7 | 8 | http://aws.amazon.com/apache2.0/ 9 | 10 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | */ 12 | #ifndef NNRECSGENERATOR_H 13 | #define NNRECSGENERATOR_H 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "GpuTypes.h" 20 | #include "NNTypes.h" 21 | 22 | class FilterConfig; 23 | class NNNetwork; 24 | 25 | class NNRecsGenerator 26 | { 27 | std::unique_ptr> pbKey; 28 | std::unique_ptr> pbUIValue; 29 | std::unique_ptr> pFilteredOutput; 30 | std::vector*> *vNodeFilters; 31 | std::string recsGenLayerLabel; 32 | std::string scorePrecision; 33 | 34 | public: 35 | static const std::string DEFAULT_LAYER_RECS_GEN_LABEL; 36 | static const unsigned int TOPK_SCALAR; 37 | static const std::string DEFAULT_SCORE_PRECISION; 38 | 39 | NNRecsGenerator(unsigned int xBatchSize, 40 | unsigned int xK, 41 | unsigned int xOutputBufferSize, 42 | const std::string &layer = DEFAULT_LAYER_RECS_GEN_LABEL, 43 | const std::string &precision = DEFAULT_SCORE_PRECISION); 44 | 45 | void generateRecs(NNNetwork *network, 46 | unsigned int topK, 47 | const FilterConfig *filters, 48 | const std::vector &customerIndex, 49 | const std::vector &featureIndex); 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/Utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 7 | 8 | http://aws.amazon.com/apache2.0/ 9 | 10 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | */ 12 | #pragma once 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | using std::string; 22 | using std::vector; 23 | 24 | const string INPUT_DATASET_SUFFIX = "_input"; 25 | const string OUTPUT_DATASET_SUFFIX = "_output"; 26 | const string NETCDF_FILE_EXTENTION = ".nc"; 27 | const unsigned long FIXED_SEED = 12134ull; 28 | 29 | class CWMetric 30 | { 31 | public: 32 | static void updateMetrics(string metric, string value); 33 | 34 | // This function accepts a value of any type that can be used with std::to_string. 35 | template ()))> 36 | static void updateMetrics(string metric, Value value) { 37 | updateMetrics(std::move(metric), std::to_string(value)); 38 | } 39 | }; 40 | 41 | char* getCmdOption(char ** , char **, const std::string & ); 42 | 43 | bool cmdOptionExists(char** , char**, const std::string& ); 44 | 45 | /** 46 | * Helper function to return the value of a given argument. If it isn't supplied, errors out. 47 | * @param argc has count of arguments 48 | * @param argv the command line arguments. 49 | * @param flag for the argument of interest. 50 | * @param message message in event the argument is not defined. 51 | * @param usage callback error function to call and exit with error code 1. 52 | */ 53 | string getRequiredArgValue(int argc, char** argv, string flag, string message, void (*usage)()); 54 | 55 | /** 56 | * Helper function to return the value of a given argument. If it isn't supplied, return the defaultValue. 57 | * @param argc has count of arguments 58 | * @param argv the command line arguments. 59 | * @param flag for the argument of interest. 60 | * @param defaultValue to return in the event it is not overridden in the arguments. 61 | */ 62 | string getOptionalArgValue(int argc, char** argv, string flag, string defaultValue); 63 | 64 | /** 65 | * Returns true if the argument flag is defined/set. 66 | * @param argc has count of arguments 67 | * @param argv the command line arguments. 68 | * @param flag for the argument of interest. 69 | */ 70 | bool isArgSet(int argc, char** argv, string flag); 71 | 72 | bool fileExists(const std::string &); 73 | 74 | /** 75 | * Return true if the file is a NetCDF file. 76 | */ 77 | bool isNetCDFfile(const string &filename); 78 | 79 | std::vector &split(const std::string &s, char delim, std::vector &elems); 80 | 81 | std::vector split(const std::string &s, char delim); 82 | 83 | /** 84 | * Return the number of seconds elapsed between two time points. 85 | * 86 | * The two time points must be issued from the same clock -- otherwise this 87 | * does not make sense. 88 | */ 89 | template 90 | double elapsed_seconds(std::chrono::time_point start, 91 | std::chrono::time_point end) 92 | { 93 | using FloatingPointSeconds = std::chrono::duration>; 94 | return std::chrono::duration_cast(end - start).count(); 95 | } 96 | 97 | /** 98 | * Returns true iff dirname is a directory 99 | */ 100 | bool isDirectory(const string &dirname); 101 | 102 | /** 103 | * Returns true iff filename is a file 104 | */ 105 | bool isFile(const string &filename); 106 | 107 | /** 108 | * Adds all files (not directories) under the specified dirname into files 109 | * Returns 0 if success and non-zero otherwise. 110 | * If the recursive flag is not set, only lists the first level files. Otherwise, it recurses into 111 | * all sub-directories. 112 | */ 113 | int listFiles(const string &dirname, const bool recursive, vector &files); 114 | 115 | // sort top K by keys and return top keys with top values 116 | template 117 | void topKsort(Tkey* keys, Tval* vals, const int size, Tkey* topKkeys, Tval* topKvals, const int topK, const bool sortByKey = true); 118 | 119 | 120 | // min max - inclusive 121 | inline int rand(int min, int max) { 122 | return rand() % (max - min + 1) + min; 123 | } 124 | 125 | inline float rand(float min, float max) { 126 | float r = (float)rand() / (float)RAND_MAX; 127 | return min + r * (max - min); 128 | } 129 | -------------------------------------------------------------------------------- /src/amazon/dsstne/utils/cdl.h: -------------------------------------------------------------------------------- 1 | #ifndef __CDL_H__ 2 | #define __CDL_H__ 3 | 4 | #include "GpuTypes.h" 5 | #include "NNTypes.h" 6 | 7 | struct CDL 8 | { 9 | CDL(); 10 | int Load_JSON(const string& fname); 11 | 12 | 13 | std::string _networkFileName; // NetCDF or JSON Object file name (required) 14 | int _randomSeed; // Initializes RNG for reproducible runs (default: sets from time of day) 15 | Mode _mode; 16 | std::string _dataFileName; 17 | 18 | // training params 19 | int _epochs; // total epochs 20 | int _batch; // used by inference as well: Mini-batch size (default 500, use 0 for entire dataset) 21 | float _alpha; 22 | float _lambda; 23 | float _mu; 24 | int _alphaInterval; // number of epochs per update to alpha - so this is the number of epochs per DSSTNE call 25 | float _alphaMultiplier; // amount to scale alpha every alphaInterval number of epochs 26 | TrainingMode _optimizer; 27 | std::string _checkpointFileName; 28 | int _checkpointInterval; 29 | bool _shuffleIndexes; 30 | std::string _resultsFileName; 31 | }; 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /talks/Data Science Summit.ppt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/Data Science Summit.ppt -------------------------------------------------------------------------------- /talks/GTC2016.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/GTC2016.pdf -------------------------------------------------------------------------------- /talks/ISCA2017.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/ISCA2017.pptx -------------------------------------------------------------------------------- /talks/OpenDataScienceConference2016.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/OpenDataScienceConference2016.odp -------------------------------------------------------------------------------- /talks/QCon2017.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/QCon2017.odp -------------------------------------------------------------------------------- /talks/TradeShowWest2017.odp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/amazon-dsstne/e429ea811135a2ba3d69b2f7af496b791a61e962/talks/TradeShowWest2017.odp -------------------------------------------------------------------------------- /tst/Makefile: -------------------------------------------------------------------------------- 1 | MODULE_ROOT = $(shell pwd) 2 | 3 | BUILD_DIR ?= $(shell pwd)/../build 4 | BIN_DIR = $(BUILD_DIR)/tst/bin 5 | TEST_SOURCES_DIR = amazon 6 | OBJS_BUILD_DIR := $(BUILD_DIR)/tmp/tst 7 | 8 | SOURCES=$(shell find '$(TEST_SOURCES_DIR)' -type f -name '*.cpp') 9 | OBJECTS := $(SOURCES:$(TEST_SOURCES_DIR)/%.cpp=$(OBJS_BUILD_DIR)/%.o) 10 | 11 | DEP := $(OBJECTS:.o=.d) 12 | 13 | DSSTNE_SRC_DIR = ../src/amazon/dsstne 14 | LIB_DSSTNE = $(BUILD_DIR)/lib/libdsstne.a 15 | LIB_DSSTNE_UTILS = $(BUILD_DIR)/lib/libdsstne_utils.so 16 | 17 | INCLUDES = \ 18 | -isystem /usr/local/cuda/include \ 19 | -isystem /usr/include/jsoncpp \ 20 | -isystem /usr/lib/openmpi/include \ 21 | -isystem /usr/lib/openmpi/include/openmpi \ 22 | -I../src # dsstne headers 23 | 24 | LIB = \ 25 | $(BUILD_DIR)/lib \ 26 | /usr/local/cuda/lib64 \ 27 | /usr/lib/openmpi/lib 28 | 29 | LIBS ?= $(LIB:%=-L%) 30 | 31 | LLIB = \ 32 | cudnn \ 33 | curand \ 34 | cublas \ 35 | cudart \ 36 | jsoncpp \ 37 | netcdf \ 38 | netcdf_c++4 \ 39 | blas \ 40 | dl \ 41 | stdc++ \ 42 | mpi_cxx \ 43 | mpi \ 44 | cppunit \ 45 | dsstne_utils 46 | 47 | LOAD_LIBS = $(LLIB:%=-l%) 48 | 49 | CC = g++ 50 | CFLAGS ?=-Wall -std=c++11 -pthread 51 | LDFLAGS ?= -Wl,-rpath=/usr/local/cuda/lib64 52 | 53 | all: $(BIN_DIR)/unittests 54 | 55 | $(LIB_DSSTNE): 56 | cd $(DSSTNE_SRC_DIR)/engine && make 57 | 58 | $(LIB_DSSTNE_UTILS): 59 | cd $(DSSTNE_SRC_DIR)/utils && make 60 | 61 | $(BIN_DIR)/unittests: $(LIB_DSSTNE) $(LIB_DSSTNE_UTILS) $(OBJECTS) 62 | $(info ========== Building unittests =============) 63 | mkdir -p $(BIN_DIR) 64 | $(CC) $(CFLAGS) $(LDFLAGS) $(LIBS) unittests.cpp $^ $(LIB_DSSTNE) -o $@ $(LOAD_LIBS) 65 | 66 | run-tests: $(BIN_DIR)/unittests 67 | $(info ========== Running unittests =============) 68 | LD_LIBRARY_PATH=$(BUILD_DIR)/lib $(BIN_DIR)/unittests 69 | 70 | clean: 71 | rm -rf $(BIN_DIR) 72 | 73 | # Pattern rule for compiling C++ 74 | $(OBJS_BUILD_DIR)/%.o: $(TEST_SOURCES_DIR)/%.cpp 75 | mkdir -p $(dir $@) 76 | $(CC) $(CFLAGS) -MMD -MP $(INCLUDES) -c $< -o $@ 77 | 78 | #-include $(DEP) 79 | 80 | -------------------------------------------------------------------------------- /tst/amazon/dsstne/engine/TestGpuBuffer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://aws.amazon.com/apache2.0/ 9 | * 10 | * or in the "license" file accompanying this file. 11 | * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. 13 | * 14 | * See the License for the specific language governing permissions and limitations under the License. 15 | * 16 | */ 17 | #include 18 | 19 | #include "amazon/dsstne/engine/GpuTypes.h" 20 | 21 | class TestGpuBuffer: public CppUnit::TestFixture 22 | { 23 | CPPUNIT_TEST_SUITE(TestGpuBuffer); 24 | 25 | CPPUNIT_TEST(testResize); 26 | 27 | CPPUNIT_TEST_SUITE_END(); 28 | 29 | public: 30 | 31 | void testResize() 32 | { 33 | size_t length = 1024; 34 | // create managed memory to make testing easier 35 | GpuBuffer buff(length, false, true); 36 | 37 | for(size_t i = 0; i < length ; ++i) 38 | { 39 | buff._pDevData[i] = i; 40 | } 41 | 42 | // check that the buffer contains what we put 43 | for(uint32_t i =0; i < length; ++i){ 44 | CPPUNIT_ASSERT_EQUAL(i, buff._pDevData[i]); 45 | } 46 | 47 | // shouldn't have resized (same length) check that buffer still contains the same data 48 | buff.Resize(length); 49 | for (uint32_t i = 0; i < length; ++i) 50 | { 51 | CPPUNIT_ASSERT_EQUAL(i, buff._pDevData[i]); 52 | } 53 | 54 | // shouldn't have resized (smaller length) check that buffer still contains the same data 55 | buff.Resize(length - 1); 56 | for (uint32_t i = 0; i < length; ++i) 57 | { 58 | CPPUNIT_ASSERT_EQUAL(i, buff._pDevData[i]); 59 | } 60 | 61 | // should resize (length larger) check that not all the data is the same 62 | buff.Resize(length + 1); 63 | bool isSame = true; 64 | for(uint32_t i=0; i< length; ++i){ 65 | isSame &= (buff._pDevData[i] == i); 66 | } 67 | CPPUNIT_ASSERT(!isSame); 68 | } 69 | }; 70 | 71 | CPPUNIT_TEST_SUITE_REGISTRATION(TestGpuBuffer); 72 | -------------------------------------------------------------------------------- /tst/amazon/dsstne/engine/TestNNDataSetDimensions.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "amazon/dsstne/engine/GpuTypes.h" 4 | #include "amazon/dsstne/engine/NNTypes.h" 5 | #include "amazon/dsstne/engine/NNLayer.h" 6 | 7 | class TestNNDataSetDimensions : public CppUnit::TestFixture 8 | { 9 | 10 | CPPUNIT_TEST_SUITE(TestNNDataSetDimensions); 11 | 12 | CPPUNIT_TEST(testNumDimensions); 13 | 14 | CPPUNIT_TEST_SUITE_END(); 15 | 16 | public: 17 | 18 | void testNumDimensions() 19 | { 20 | NNDataSetDimensions zero_d(1); 21 | NNDataSetDimensions one_d(2); 22 | NNDataSetDimensions two_d(2, 2); 23 | NNDataSetDimensions three_d(2, 2, 2); 24 | 25 | CPPUNIT_ASSERT_EQUAL(0U, zero_d._dimensions); 26 | CPPUNIT_ASSERT_EQUAL(1U, one_d._dimensions); 27 | CPPUNIT_ASSERT_EQUAL(2U, two_d._dimensions); 28 | } 29 | }; 30 | 31 | CPPUNIT_TEST_SUITE_REGISTRATION(TestNNDataSetDimensions); 32 | -------------------------------------------------------------------------------- /tst/gputests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.2) 2 | 3 | project (amazon-dsstne) 4 | 5 | ################################################################################ 6 | # 7 | # Check for C++11 support 8 | # 9 | ################################################################################ 10 | 11 | include(CheckCXXCompilerFlag) 12 | 13 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 14 | 15 | if(!COMPILER_SUPPORTS_CXX11) 16 | message(FATAL_ERROR "Your compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 17 | endif() 18 | 19 | ################################################################################ 20 | # 21 | # Dependencies 22 | # 23 | ################################################################################ 24 | 25 | find_package(CUDA) 26 | find_package(MPI) 27 | find_package(PkgConfig) 28 | 29 | PKG_CHECK_MODULES(CPPUNIT REQUIRED cppunit) 30 | PKG_CHECK_MODULES(NETCDF REQUIRED netcdf) 31 | PKG_CHECK_MODULES(NETCDF_CXX4 REQUIRED netcdf-cxx4) 32 | 33 | ################################################################################ 34 | # 35 | # Compiler flags 36 | # 37 | ################################################################################ 38 | 39 | SET(CUDA_PROPAGATE_HOST_FLAGS OFF) 40 | set(CUDA_NVCC_FLAGS "${CMAKE_CXX_FLAGS} ${CUDA_NVCC_FLAGS} -use_fast_math -gencode arch=compute_50,code=sm_50 -gencode arch=compute_30,code=sm_30 -DOMPI_SKIP_MPICXX -std=c++11") 41 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 42 | 43 | ################################################################################ 44 | # 45 | # Test suite 46 | # 47 | ################################################################################ 48 | 49 | set(ENGINE_DIR ../../src/amazon/dsstne/engine) 50 | set(UTILS_DIR ../../src/amazon/dsstne/utils) 51 | 52 | include_directories( 53 | ${ENGINE_DIR} 54 | ${UTILS_DIR} 55 | ${CPPUNIT_INCLUDE_DIR} 56 | ${CUDA_INCLUDE_DIRS} 57 | ${MPI_CXX_INCLUDE_PATH} 58 | ${NETCDF_INCLUDE_DIR} 59 | ${NETCDF_CXX4_INCLUDE_DIR} 60 | ) 61 | 62 | set(ENGINE_SOURCES 63 | ${ENGINE_DIR}/GpuTypes.cpp 64 | ${ENGINE_DIR}/kernels.cu 65 | ${ENGINE_DIR}/kActivation.cu 66 | ${ENGINE_DIR}/kDelta.cu 67 | ${ENGINE_DIR}/kLoss.cu 68 | ) 69 | 70 | set(UTILS_SOURCES 71 | ${UTILS_DIR}/Utils.cpp 72 | ) 73 | 74 | set(TEST_SOURCES 75 | TestDune.cpp 76 | ) 77 | 78 | cuda_add_executable(gputests 79 | ${ENGINE_SOURCES} 80 | ${TEST_SOURCES} 81 | ${UTILS_SOURCES} 82 | ) 83 | 84 | target_link_libraries(gputests 85 | ${CPPUNIT_LIBRARIES} 86 | ${CUDA_CUBLAS_LIBRARIES} 87 | ${CUDA_curand_LIBRARY} 88 | ${CUDA_LIBRARIES} 89 | ${MPI_CXX_LIBRARIES} 90 | ${NETCDF_LIBRARIES} 91 | ${NETCDF_CXX4_LIBRARIES} 92 | ) 93 | -------------------------------------------------------------------------------- /tst/gputests/TestActivationFunctions.cpp: -------------------------------------------------------------------------------- 1 | // CppUnit 2 | #include "cppunit/extensions/HelperMacros.h" 3 | #include "cppunit/ui/text/TestRunner.h" 4 | #include "cppunit/TestAssert.h" 5 | // STL 6 | #include 7 | 8 | #include "Utils.h" 9 | #include "GpuTypes.h" 10 | #include "NNTypes.h" 11 | #include "TestUtils.h" 12 | 13 | //---------------------------------------------------------------------------- 14 | class TestActivationFunctions: public CppUnit::TestFixture { 15 | public: 16 | // Interface 17 | void testActivationFunctions() { 18 | const size_t numberTests = 2; 19 | const std::string modelPaths[numberTests] = { 20 | std::string(TEST_DATA_PATH) + std::string("validate_L2_LRelu_01.json"), 21 | std::string(TEST_DATA_PATH) + std::string("validate_L2_LRelu_02.json")}; 22 | const size_t batches[numberTests] = {2, 4}; 23 | 24 | for (size_t i = 0; i < numberTests; i++) { 25 | DataParameters dataParameters; 26 | dataParameters.numberOfSamples = 1024; 27 | dataParameters.inpFeatureDimensionality = 2; 28 | dataParameters.outFeatureDimensionality = 2; 29 | bool result = validateNeuralNetwork(batches[i], modelPaths[i], Classification, dataParameters, std::cout); 30 | std::cout << "batches " << batches[i] << ", model " << modelPaths[i] << std::endl; 31 | CPPUNIT_ASSERT_MESSAGE("failed on testActivationFunctions", result); 32 | } 33 | } 34 | 35 | public: 36 | CPPUNIT_TEST_SUITE(TestActivationFunctions); 37 | CPPUNIT_TEST(testActivationFunctions); 38 | CPPUNIT_TEST_SUITE_END(); 39 | }; 40 | -------------------------------------------------------------------------------- /tst/gputests/TestCostFunctions.cpp: -------------------------------------------------------------------------------- 1 | // CppUnit 2 | #include "cppunit/extensions/HelperMacros.h" 3 | #include "cppunit/ui/text/TestRunner.h" 4 | #include "cppunit/TestAssert.h" 5 | // STL 6 | #include 7 | 8 | #include "Utils.h" 9 | #include "GpuTypes.h" 10 | #include "NNTypes.h" 11 | #include "TestUtils.h" 12 | 13 | class TestCostFunctions: public CppUnit::TestFixture { 14 | public: 15 | // Interface 16 | void testCostFunctions() { 17 | // Initialize GPU 18 | getGpu().SetRandomSeed(12345); 19 | getGpu().CopyConstants(); 20 | 21 | // test data scaled marginal cross entropy 22 | { 23 | const size_t batch = 2; 24 | const string modelPath = std::string(TEST_DATA_PATH) + "validate_DataScaledMarginalCrossEntropy_02.json"; 25 | DataParameters dataParameters; 26 | dataParameters.numberOfSamples = 1024; 27 | dataParameters.inpFeatureDimensionality = 2; 28 | dataParameters.outFeatureDimensionality = 2; 29 | bool result = validateNeuralNetwork(batch, modelPath, ClassificationAnalog, dataParameters, std::cout); 30 | CPPUNIT_ASSERT_MESSAGE("failed on DataScaledMarginalCrossEntropy", result); 31 | } 32 | 33 | // test marginal cross entropy 34 | { 35 | const size_t batch = 4; 36 | const string modelPath = std::string(TEST_DATA_PATH) + "validate_ScaledMarginalCrossEntropy_02.json"; 37 | DataParameters dataParameters; 38 | dataParameters.numberOfSamples = 1024; 39 | dataParameters.inpFeatureDimensionality = 2; 40 | dataParameters.outFeatureDimensionality = 2; 41 | bool result = validateNeuralNetwork(batch, modelPath, Classification, dataParameters, std::cout); 42 | CPPUNIT_ASSERT_MESSAGE("failed on DataScaledMarginalCrossEntropy", result); 43 | } 44 | 45 | // test L2 46 | { 47 | const size_t batch = 4; 48 | const string modelPath = std::string(TEST_DATA_PATH) + "validate_L2_02.json"; 49 | DataParameters dataParameters; 50 | dataParameters.numberOfSamples = 1024; 51 | dataParameters.inpFeatureDimensionality = 1; 52 | dataParameters.outFeatureDimensionality = 1; 53 | dataParameters.W0 = -2.f; 54 | dataParameters.B0 = 3.f; 55 | bool result = validateNeuralNetwork(batch, modelPath, Regression, dataParameters, std::cout); 56 | CPPUNIT_ASSERT_MESSAGE("failed on DataScaledMarginalCrossEntropy", result); 57 | } 58 | } 59 | 60 | public: 61 | CPPUNIT_TEST_SUITE(TestCostFunctions); 62 | CPPUNIT_TEST(testCostFunctions); 63 | CPPUNIT_TEST_SUITE_END(); 64 | }; 65 | -------------------------------------------------------------------------------- /tst/gputests/TestDune.cpp: -------------------------------------------------------------------------------- 1 | // CppUnit 2 | #include "cppunit/extensions/HelperMacros.h" 3 | #include "cppunit/ui/text/TestRunner.h" 4 | // STL 5 | #include 6 | 7 | #include "TestSort.cpp" 8 | #include "TestActivationFunctions.cpp" 9 | #include "TestCostFunctions.cpp" 10 | 11 | /** 12 | * In order to write a new test case, create a Test.cpp and write the test 13 | * methods in that file. Include the cpp file in this file and also 14 | * 15 | * add runner.addTest(Test::suite()); 16 | * Unit test file name has to start with Test 17 | * 18 | */ 19 | 20 | int main() { 21 | getGpu().Startup(0, NULL); 22 | getGpu().SetRandomSeed(12345); 23 | getGpu().CopyConstants(); 24 | CppUnit::TextUi::TestRunner runner; 25 | runner.addTest(TestSort::suite()); 26 | runner.addTest(TestActivationFunctions::suite()); 27 | runner.addTest(TestCostFunctions::suite()); 28 | const bool result = runner.run(); 29 | getGpu().Shutdown(); 30 | return result ? EXIT_SUCCESS : EXIT_FAILURE; 31 | } 32 | -------------------------------------------------------------------------------- /tst/gputests/TestGpu.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // CppUnit 4 | #include "cppunit/extensions/HelperMacros.h" 5 | #include "cppunit/ui/text/TestRunner.h" 6 | #include "cppunit/TestAssert.h" 7 | 8 | #include "filterKernels.h" 9 | 10 | class TestGpu : public CppUnit::TestFixture { 11 | 12 | public: 13 | TestGpu() { 14 | getGpu().Startup(0, NULL); 15 | } 16 | 17 | ~TestGpu() { 18 | getGpu().Shutdown(); 19 | } 20 | 21 | void TestApplyNodeFilter() { 22 | int outputKeySize = 6, filterSize = 3; 23 | NNFloat localOutputKey[outputKeySize] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; 24 | NNFloat expectedOutputKey[outputKeySize] = {7.0, 16.0, 27.0, 28.0, 40.0, 54.0}; 25 | NNFloat localFilter[filterSize] = {7.0, 8.0, 9.0}; 26 | NNFloat expectedFilter[filterSize] = {7.0, 8.0, 9.0}; 27 | GpuBuffer *deviceOutputKey = new GpuBuffer(outputKeySize); 28 | GpuBuffer *deviceFilter = new GpuBuffer(filterSize); 29 | 30 | deviceOutputKey->Upload(localOutputKey); 31 | deviceFilter->Upload(localFilter); 32 | 33 | kApplyNodeFilter(deviceOutputKey->_pDevData, deviceFilter->_pDevData, filterSize, 2); 34 | 35 | deviceOutputKey->Download(localOutputKey); 36 | deviceFilter->Download(localFilter); 37 | 38 | delete deviceOutputKey; 39 | delete deviceFilter; 40 | 41 | for (int i = 0; i < outputKeySize; ++i) { 42 | CPPUNIT_ASSERT_EQUAL_MESSAGE("OutputKey is different", expectedOutputKey[i], localOutputKey[i]); 43 | } 44 | 45 | for (int i = 0; i < filterSize; ++i) { 46 | CPPUNIT_ASSERT_EQUAL_MESSAGE("Filter is different", expectedFilter[i], localFilter[i]); 47 | } 48 | } 49 | 50 | CPPUNIT_TEST_SUITE(TestGpu); 51 | CPPUNIT_TEST(TestApplyNodeFilter); 52 | CPPUNIT_TEST_SUITE_END(); 53 | }; 54 | -------------------------------------------------------------------------------- /tst/test_data/validate_DataScaledMarginalCrossEntropy_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "DataScaledMarginalCrossEntropy", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "DataScaledMarginalCrossEntropy" : { 9 | "oneTarget" : 1.0, 10 | "zeroTarget" : 0.0, 11 | "oneScale" : 30.0, 12 | "zeroScale" : 1.0 13 | }, 14 | "Layers" : [ 15 | { "Name" : "Input", "Kind" : "Input", "N" : 2, "DataSet" : "input", "Sparse" : true }, 16 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 2, "Activation" : "Sigmoid", "Sparse" : true }, 17 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : 2, "Source" : ["Hidden"], "Activation" : "Sigmoid", "Sparse" : true } 18 | ], 19 | 20 | "ErrorFunction" : "DataScaledMarginalCrossEntropy" 21 | } 22 | -------------------------------------------------------------------------------- /tst/test_data/validate_L2_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "L2 1d regression", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "Layers" : [ 9 | { "Name" : "Input", "Kind" : "Input", "N" : 1, "DataSet" : "input", "Sparse" : true }, 10 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : 1, "Source" : ["Input"], "Activation" : "Linear", "Sparse" : true } 11 | ], 12 | 13 | "ErrorFunction" : "L2" 14 | } 15 | -------------------------------------------------------------------------------- /tst/test_data/validate_L2_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "L2 1d regression", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "Layers" : [ 9 | { "Name" : "Input", "Kind" : "Input", "N" : 1, "DataSet" : "input", "Sparse" : true }, 10 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 4, "Activation" : "Sigmoid", "Sparse" : true }, 11 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : 1, "Source" : ["Input"], "Activation" : "Linear", "Sparse" : true } 12 | ], 13 | 14 | "ErrorFunction" : "L2" 15 | } 16 | -------------------------------------------------------------------------------- /tst/test_data/validate_L2_LRelu_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "L2 1d regression", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "Layers" : [ 9 | { "Name" : "Input", 10 | "Kind" : "Input", 11 | "N" : 2, 12 | "DataSet" : "input", 13 | "Sparse" : true }, 14 | { "Name" : "Hidden", 15 | "Kind" : "Hidden", 16 | "Type" : "FullyConnected", 17 | "Source" : "Input", 18 | "N" : 2, 19 | "Activation" : "LRelu", 20 | "Slope" : 0.0, 21 | "Sparse" : true }, 22 | { "Name" : "Output", 23 | "Kind" : "Output", 24 | "Type" : "FullyConnected", 25 | "DataSet" : "output", 26 | "N" : 2, 27 | "Source" : ["Input"], 28 | "Activation" : "Linear", 29 | "Sparse" : true } 30 | ], 31 | 32 | "ErrorFunction" : "L2" 33 | } 34 | -------------------------------------------------------------------------------- /tst/test_data/validate_L2_LRelu_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "L2 1d regression", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "Layers" : [ 9 | { "Name" : "Input", 10 | "Kind" : "Input", 11 | "N" : 2, 12 | "DataSet" : "input", 13 | "Sparse" : true }, 14 | { "Name" : "Hidden", 15 | "Kind" : "Hidden", 16 | "Type" : "FullyConnected", 17 | "Source" : "Input", 18 | "N" : 2, 19 | "Activation" : "LRelu", 20 | "Slope" : 0.5, 21 | "Sparse" : true }, 22 | { "Name" : "Output", 23 | "Kind" : "Output", 24 | "Type" : "FullyConnected", 25 | "DataSet" : "output", 26 | "N" : 2, 27 | "Source" : ["Input"], 28 | "Activation" : "Linear", 29 | "Sparse" : true } 30 | ], 31 | 32 | "ErrorFunction" : "L2" 33 | } 34 | -------------------------------------------------------------------------------- /tst/test_data/validate_ScaledMarginalCrossEntropy_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "ScaledMarginalCrossEntropy", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "ScaledMarginalCrossEntropy" : { 9 | "oneTarget" : 1.0, 10 | "zeroTarget" : 0.0, 11 | "oneScale" : 30.0, 12 | "zeroScale" : 1.0 13 | }, 14 | "Layers" : [ 15 | { "Name" : "Input", "Kind" : "Input", "N" : 2, "DataSet" : "input", "Sparse" : true }, 16 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : 2, "Source" : ["Input"], "Activation" : "Sigmoid", "Sparse" : true } 17 | ], 18 | 19 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 20 | } 21 | -------------------------------------------------------------------------------- /tst/test_data/validate_ScaledMarginalCrossEntropy_02.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version" : 0.8, 3 | "Name" : "ScaledMarginalCrossEntropy", 4 | "Kind" : "FeedForward", 5 | 6 | "ShuffleIndices" : false, 7 | 8 | "ScaledMarginalCrossEntropy" : { 9 | "oneTarget" : 1.0, 10 | "zeroTarget" : 0.0, 11 | "oneScale" : 30.0, 12 | "zeroScale" : 1.0 13 | }, 14 | "Layers" : [ 15 | { "Name" : "Input", "Kind" : "Input", "N" : 2, "DataSet" : "input", "Sparse" : true }, 16 | { "Name" : "Hidden", "Kind" : "Hidden", "Type" : "FullyConnected", "Source" : "Input", "N" : 2, "Activation" : "Sigmoid", "Sparse" : true }, 17 | { "Name" : "Output", "Kind" : "Output", "Type" : "FullyConnected", "DataSet" : "output", "N" : 2, "Source" : ["Hidden"], "Activation" : "Sigmoid", "Sparse" : true } 18 | ], 19 | 20 | "ErrorFunction" : "ScaledMarginalCrossEntropy" 21 | } 22 | -------------------------------------------------------------------------------- /tst/unittests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | class PrintProgressListener: public CppUnit::TestListener 8 | { 9 | public: 10 | void startTest(CppUnit::Test *test) 11 | { 12 | std::cout << "Running [" << test->getName() << "]" << std::endl; 13 | } 14 | 15 | void endTest(CppUnit::Test *test) 16 | { 17 | 18 | std::cout << "Finished [" << test->getName() << "]" << std::endl; 19 | } 20 | }; 21 | 22 | int main(int argc, char **argv) 23 | { 24 | PrintProgressListener progressListener; 25 | CppUnit::TextUi::TestRunner runner; 26 | CppUnit::TestFactoryRegistry ®istry = CppUnit::TestFactoryRegistry::getRegistry(); 27 | runner.addTest(registry.makeTest()); 28 | runner.eventManager().addListener(&progressListener); 29 | 30 | bool wasSuccessful = runner.run("", false); 31 | return !wasSuccessful; 32 | } 33 | -------------------------------------------------------------------------------- /tst/unittests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.2) 2 | 3 | project (amazon-dsstne) 4 | 5 | ################################################################################ 6 | # 7 | # Compiler configuration 8 | # 9 | ################################################################################ 10 | 11 | include(CheckCXXCompilerFlag) 12 | 13 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 14 | 15 | if(COMPILER_SUPPORTS_CXX11) 16 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 17 | else() 18 | message(FATAL_ERROR "Your compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 19 | endif() 20 | 21 | ################################################################################ 22 | # 23 | # Dependencies 24 | # 25 | ################################################################################ 26 | 27 | find_package(PkgConfig) 28 | 29 | PKG_CHECK_MODULES(CPPUNIT REQUIRED cppunit) 30 | PKG_CHECK_MODULES(NETCDF REQUIRED netcdf) 31 | PKG_CHECK_MODULES(NETCDF_CXX4 REQUIRED netcdf-cxx4) 32 | 33 | ################################################################################ 34 | # 35 | # Test suite 36 | # 37 | ################################################################################ 38 | 39 | set(ENGINE_DIR ../../src/amazon/dsstne/engine) 40 | set(UTILS_DIR ../../src/amazon/dsstne/utils) 41 | 42 | include_directories( 43 | ${ENGINE_DIR} 44 | ${UTILS_DIR} 45 | ${CPPUNIT_INCLUDE_DIR} 46 | ${NETCDF_INCLUDE_DIR} 47 | ${NETCDF_CXX4_INCLUDE_DIR} 48 | ) 49 | 50 | set(UTILS_SOURCES 51 | ${UTILS_DIR}/NetCDFhelper.cpp 52 | ${UTILS_DIR}/Utils.cpp 53 | ) 54 | 55 | set(TEST_SOURCES 56 | main.cpp 57 | ) 58 | 59 | add_executable(unittests 60 | ${TEST_SOURCES} 61 | ${UTILS_SOURCES} 62 | ) 63 | 64 | target_link_libraries(unittests 65 | ${CPPUNIT_LIBRARIES} 66 | ${NETCDF_LIBRARIES} 67 | ${NETCDF_CXX4_LIBRARIES} 68 | ) 69 | -------------------------------------------------------------------------------- /tst/unittests/TestUtils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "Utils.h" 6 | 7 | class TestUtils : public CppUnit::TestFixture 8 | { 9 | public: 10 | void TestIsNetCDFfile() 11 | { 12 | bool result = isNetCDFfile("network.nc"); 13 | CPPUNIT_ASSERT(result); 14 | 15 | result = isNetCDFfile("network.nic"); 16 | CPPUNIT_ASSERT(!result); 17 | } 18 | 19 | CPPUNIT_TEST_SUITE(TestUtils); 20 | CPPUNIT_TEST(TestIsNetCDFfile); 21 | CPPUNIT_TEST_SUITE_END(); 22 | }; 23 | -------------------------------------------------------------------------------- /tst/unittests/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | // Test files 7 | #include "TestNetCDFhelper.cpp" 8 | #include "TestUtils.cpp" 9 | 10 | // 11 | // In order to write a new test case, create a Test.cpp and write the 12 | // test methods in that file. Include the cpp file in this file and add: 13 | // 14 | // runner.addTest(Test::suite()); 15 | // 16 | // Unit test file names have to start with 'Test' 17 | // 18 | int main() 19 | { 20 | CppUnit::TextUi::TestRunner runner; 21 | runner.addTest(TestNetCDFhelper::suite()); 22 | runner.addTest(TestUtils::suite()); 23 | return runner.run() ? EXIT_SUCCESS : EXIT_FAILURE; 24 | } 25 | --------------------------------------------------------------------------------