├── .appveyor.yml ├── .gitignore ├── .travis.yml ├── .travis ├── build_dynet.sh ├── build_linux_wheel.sh ├── build_macos_wheel.sh ├── deploy.sh ├── fix_version.sh ├── install_dependencies.sh └── test_dynet.sh ├── CMakeLists.txt ├── LICENSE.txt ├── MANIFEST.in ├── MEM.notes ├── README.md ├── bench └── sequence_transduction.py ├── cmake ├── FindDyNet.cmake └── FindEigen3.cmake ├── config.h.cmake ├── contrib ├── c │ ├── CMakeLists.txt │ ├── README.md │ ├── dynet_c │ │ ├── CMakeLists.txt │ │ ├── api.h │ │ ├── config.h.in │ │ ├── define.h │ │ ├── devices.cc │ │ ├── devices.h │ │ ├── dim.cc │ │ ├── dim.h │ │ ├── expr.cc │ │ ├── expr.h │ │ ├── graph.cc │ │ ├── graph.h │ │ ├── init.cc │ │ ├── init.h │ │ ├── internal.cc │ │ ├── internal.h │ │ ├── io.cc │ │ ├── io.h │ │ ├── model.cc │ │ ├── model.h │ │ ├── param-init.cc │ │ ├── param-init.h │ │ ├── rnn-builder.cc │ │ ├── rnn-builder.h │ │ ├── softmax-builder.cc │ │ ├── softmax-builder.h │ │ ├── status.cc │ │ ├── status.h │ │ ├── tensor.cc │ │ ├── tensor.h │ │ ├── training.cc │ │ └── training.h │ └── test │ │ ├── CMakeLists.txt │ │ ├── dim_test.cc │ │ ├── exec_test.cc │ │ ├── params_test.cc │ │ └── test_utils.h ├── csharp │ ├── README.md │ ├── dynetsharp │ │ ├── dynetsharp.sln │ │ └── dynetsharp │ │ │ ├── dynet.cpp │ │ │ ├── dynet.h │ │ │ ├── dynetsharp.vcxproj │ │ │ ├── dynetsharp.vcxproj.filters │ │ │ ├── dynetsharp.vcxproj.user │ │ │ └── dynetsharp_GPU.vcxproj │ └── examples │ │ ├── attention.cs │ │ ├── train_rnnlm.cs │ │ └── train_xor.cs ├── rust │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── dynet-sys │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ └── lib.rs │ ├── examples │ │ ├── mnist │ │ │ ├── download_data.sh │ │ │ └── mnist.rs │ │ ├── rnnlm │ │ │ ├── install-examples.sh │ │ │ └── rnnlm.rs │ │ └── xor │ │ │ └── xor.rs │ └── src │ │ ├── device.rs │ │ ├── dim.rs │ │ ├── expr.rs │ │ ├── graph.rs │ │ ├── init.rs │ │ ├── io.rs │ │ ├── lib.rs │ │ ├── model.rs │ │ ├── param_init.rs │ │ ├── rnn_builder.rs │ │ ├── softmax_builder.rs │ │ ├── status.rs │ │ ├── tensor.rs │ │ ├── training.rs │ │ └── util.rs └── swig │ ├── CMakeLists.txt │ ├── README.md │ ├── build.sbt │ ├── dynet_swig.i │ ├── project │ └── assembly.sbt │ └── src │ ├── main │ ├── java │ │ └── edu │ │ │ └── cmu │ │ │ └── dynet │ │ │ └── examples │ │ │ └── XorExample.java │ └── scala │ │ └── edu │ │ └── cmu │ │ └── dynet │ │ ├── ComputationGraph.scala │ │ ├── Dim.scala │ │ ├── Expression.scala │ │ ├── FastLstmBuilder.scala │ │ ├── GRUBuilder.scala │ │ ├── Initialize.scala │ │ ├── LSTMBuilder.scala │ │ ├── Parameter.scala │ │ ├── ParameterCollection.scala │ │ ├── Pointer.scala │ │ ├── RNNBuilder.scala │ │ ├── Serialization.scala │ │ ├── Tensor.scala │ │ ├── Trainer.scala │ │ ├── TreeLSTMBuilder.scala │ │ ├── Utilities.scala │ │ ├── VariableIndex.scala │ │ ├── Vector.scala │ │ └── examples │ │ ├── EncoderDecoder.scala │ │ ├── LinearRegression.scala │ │ ├── Mnist.scala │ │ ├── MultiLayerPerceptron.scala │ │ ├── PoissonRegression.scala │ │ ├── RnnLanguageModelBatch.scala │ │ ├── WordDict.scala │ │ └── XorScala.scala │ └── test │ └── scala │ └── edu │ └── cmu │ └── dynet │ ├── ComputationGraphSpec.scala │ ├── DimSpec.scala │ ├── ExpressionSpec.scala │ ├── LinearRegressionSpec.scala │ ├── ParameterInitSpec.scala │ ├── SampleSpec.scala │ ├── SerializationSpec.scala │ └── VectorSpec.scala ├── doc ├── Makefile ├── build_doc.sh ├── doxygen │ ├── Doxyfile │ └── references.bib ├── make.bat └── source │ ├── builders.rst │ ├── citing.rst │ ├── code_style.rst │ ├── commandline.rst │ ├── conf.py │ ├── contributing.rst │ ├── contributors.rst │ ├── core.rst │ ├── cpp_basic_tutorial.rst │ ├── cpp_ref.rst │ ├── cpp_saving_tutorial.rst │ ├── debugging.rst │ ├── doc_style.rst │ ├── doc_util.py │ ├── examples.rst │ ├── images │ ├── ai2.png │ ├── autobatch.gif │ ├── biu.jpg │ ├── cmu.png │ ├── dynet_logo.png │ ├── dynet_logo_bg.png │ ├── dynet_logo_white.png │ ├── dynet_logo_white_and_color.png │ ├── dynet_logomark.png │ ├── minibatch.png │ ├── naist.png │ ├── petuum.png │ └── uw.jpg │ ├── index.rst │ ├── install.rst │ ├── minibatch.rst │ ├── multiprocessing.rst │ ├── operations.rst │ ├── optimizers.rst │ ├── other_languages.rst │ ├── projects.rst │ ├── python.rst │ ├── python_ref.rst │ ├── python_saving_tutorial.rst │ ├── requirements.txt │ ├── tutorial.rst │ └── unorthodox.rst ├── docker ├── Dockerfile-i686 └── Dockerfile-x86_64 ├── dynet ├── CMakeLists.txt ├── aligned-mem-pool.cc ├── aligned-mem-pool.h ├── c2w.h ├── cfsm-builder.cc ├── cfsm-builder.h ├── cuda.cc ├── cuda.h ├── cudnn-ops.cu ├── cudnn-ops.h ├── deep-lstm.cc ├── deep-lstm.h ├── device-structs.h ├── devices.cc ├── devices.h ├── dict.cc ├── dict.h ├── dim.cc ├── dim.h ├── dynet-helper.h ├── dynet.cc ├── dynet.h ├── except.h ├── exec.cc ├── exec.h ├── expr.cc ├── expr.h ├── fast-lstm.cc ├── fast-lstm.h ├── functors.h ├── globals.cc ├── globals.h ├── gpu-kernels.h ├── gpu-ops.cu ├── gpu-ops.h ├── grad-check.cc ├── grad-check.h ├── graph.cc ├── graph.h ├── gru.cc ├── gru.h ├── hsm-builder.cc ├── hsm-builder.h ├── index-tensor.h ├── init.cc ├── init.h ├── io.cc ├── io.h ├── lstm.cc ├── lstm.h ├── matrix-multiply.h ├── mem.cc ├── mem.h ├── model.cc ├── model.h ├── mp.cc ├── mp.h ├── nodes-activations.cc ├── nodes-activations.h ├── nodes-affinetransform.cc ├── nodes-affinetransform.h ├── nodes-argmax.cc ├── nodes-argmax.h ├── nodes-arith-const.cc ├── nodes-arith-const.h ├── nodes-arith-cwise.cc ├── nodes-arith-cwise.h ├── nodes-arith-sum.cc ├── nodes-arith-sum.h ├── nodes-arith-unary.cc ├── nodes-arith-unary.h ├── nodes-concat.cc ├── nodes-concat.h ├── nodes-const.cc ├── nodes-const.h ├── nodes-contract.cc ├── nodes-contract.h ├── nodes-conv.cc ├── nodes-conv.h ├── nodes-conv2d.cc ├── nodes-conv2d.h ├── nodes-cumulative.cc ├── nodes-cumulative.h ├── nodes-def-macros.h ├── nodes-dropout.cc ├── nodes-dropout.h ├── nodes-flow.cc ├── nodes-flow.h ├── nodes-hinge.cc ├── nodes-hinge.h ├── nodes-impl-macros.h ├── nodes-linalg.cc ├── nodes-linalg.h ├── nodes-logsumexp.cc ├── nodes-logsumexp.h ├── nodes-losses.cc ├── nodes-losses.h ├── nodes-lstm.cc ├── nodes-lstm.h ├── nodes-matrixmultiply.cc ├── nodes-matrixmultiply.h ├── nodes-maxpooling2d.cc ├── nodes-maxpooling2d.h ├── nodes-minmax.cc ├── nodes-minmax.h ├── nodes-moments.cc ├── nodes-moments.h ├── nodes-normalization.cc ├── nodes-normalization.h ├── nodes-norms.cc ├── nodes-norms.h ├── nodes-pickneglogsoftmax.cc ├── nodes-pickneglogsoftmax.h ├── nodes-random.cc ├── nodes-random.h ├── nodes-rounding.cc ├── nodes-rounding.h ├── nodes-select.cc ├── nodes-select.h ├── nodes-similarities.cc ├── nodes-similarities.h ├── nodes-softmaxes.cc ├── nodes-softmaxes.h ├── nodes-to-device.cc ├── nodes-to-device.h ├── nodes-trig.cc ├── nodes-trig.h ├── nodes.h ├── param-init.cc ├── param-init.h ├── param-nodes.cc ├── param-nodes.h ├── pretrain.cc ├── pretrain.h ├── rnn-state-machine.cc ├── rnn-state-machine.h ├── rnn.cc ├── rnn.h ├── saxe-init.cc ├── saxe-init.h ├── shadow-params.cc ├── shadow-params.h ├── sig.h ├── simd-functors.h ├── str-util.h ├── tensor-eigen.h ├── tensor.cc ├── tensor.h ├── timing.h ├── training.cc ├── training.h ├── treelstm.cc ├── treelstm.h ├── virtual-cudnn.h ├── weight-decay.cc └── weight-decay.h ├── examples ├── CMakeLists.txt ├── README.md ├── autobatch │ ├── train_rnn-autobatch.cc │ └── train_xor-autobatch.cc ├── batching │ ├── minibatch.py │ ├── rnnlm-batch.h │ ├── rnnlm-batch.py │ ├── train_rnnlm-batch.cc │ ├── train_xor-batch.cc │ └── xor-batch-lookup │ │ └── train_xor-batch-lookup.cc ├── cpp-utils │ ├── cl-args.h │ ├── data-io.h │ └── getpid.h ├── devices │ ├── cpu_vs_gpu.py │ ├── xor-multidevice.py │ └── xor-multidevice │ │ └── train_xor-multidevice.cc ├── document-classification │ └── train_imdb.cc ├── example-data │ ├── clusters-hsm.txt │ ├── dev-hsm.txt │ ├── dev-poi.txt │ ├── fin-dev.txt │ ├── fin-toy.txt │ ├── fin-words-dev.txt │ ├── fin-words.txt │ ├── fin.txt │ ├── seg-sup.dev.txt │ ├── textcat.txt │ ├── train-hsm.txt │ └── train-poi.txt ├── fflm │ └── train_fflm.cc ├── jupyter-tutorials │ ├── API.ipynb │ ├── Autobatching.ipynb │ ├── RNNs.ipynb │ ├── imgs │ │ ├── autobatch.gif │ │ ├── bist-autobatch-speed.png │ │ └── treelstm-autobatch-speed.png │ └── tutorial-1-xor.ipynb ├── mnist │ ├── README.md │ ├── basic-mnist-benchmarks │ │ ├── README.md │ │ ├── mnist_dynet_autobatch.py │ │ ├── mnist_dynet_minibatch.py │ │ └── mnist_pytorch.py │ ├── mlp.h │ ├── mnist-autobatch.py │ └── train_mnist.cc ├── multiprocessing │ ├── rnnlm.h │ ├── train_rnnlm-mp.cc │ ├── train_xor-mp.cc │ └── train_xor-simple-mp.cc ├── noise-contrastive-estimation │ ├── nce.h │ ├── sampler.h │ └── train_rnnlm-batch-nce.cc ├── poisson-regression │ └── train_poisson-regression.cc ├── python-utils │ └── util.py ├── read-write │ └── train_read-write.cc ├── reinforcement-learning │ ├── README.md │ ├── ddpg.py │ ├── dqn.py │ ├── main_ddpg.py │ ├── main_dqn.py │ ├── memory.py │ ├── network.py │ ├── reduce_tree.py │ ├── results │ │ ├── Acrobot.gif │ │ ├── Cartpole.gif │ │ ├── MountainCar.gif │ │ ├── hopforward.gif │ │ ├── move.gif │ │ └── stand_still.gif │ └── train_test_utils.py ├── rnnlm │ ├── README.md │ ├── install-examples.sh │ ├── lstmlm-auto.py │ ├── rnnlm.py │ ├── rnnlm_transduce.py │ └── train_rnnlm.cc ├── segmental-rnn │ └── train_segrnn-sup.cc ├── sentence-embedding │ └── train_embed-cl.cc ├── sequence-to-sequence │ ├── attention.h │ ├── attention.py │ ├── encdec.h │ ├── train_attention.cc │ └── train_encdec.cc ├── softmax-builders │ └── train_rnnlm-cfsm.cc ├── tagger │ ├── bilstmtagger.py │ └── train_tag-bilstm.cc ├── tensorboard │ ├── README.md │ ├── rnnlm-batch.py │ ├── runcrayonserver.perl │ ├── rundocker.perl │ ├── tensorboardexample.png │ ├── tensorboardexample2.png │ └── util.py ├── textcat │ └── train_textcat.cc ├── transformer │ ├── batch-utils.h │ ├── data-utils.h │ ├── def.h │ ├── dict-utils.h │ ├── ensemble-decoder.h │ ├── expr-xtra.h │ ├── mask-utils.h │ ├── math-utils.h │ ├── str-utils.h │ ├── timer.h │ ├── transformer-decode.cc │ ├── transformer-train.cc │ ├── transformer.h │ ├── utils.h │ └── wrap-data.py ├── treelstm │ ├── dataloader.py │ ├── filter_glove.py │ ├── main.py │ ├── model.py │ ├── readme.md │ ├── scheduler.py │ └── utils.py ├── variational-autoencoder │ ├── basic-image-recon │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── utils.py │ │ └── vae.py │ └── train_rnnlm-aevb.cc ├── word-embedding │ └── train_tok-embed.cc └── xor │ ├── train_xor.cc │ └── xor.py ├── pyproject.toml ├── python ├── CHANGES.md ├── CMakeLists.txt ├── _dynet.pxd ├── _dynet.pyx ├── dynet.py.in ├── dynet_config.py ├── dynet_viz.py └── model_test.py ├── setup.cfg ├── setup.py ├── tests ├── CMakeLists.txt ├── README.md ├── python │ └── test.py ├── test-dim.cc ├── test-dynet.cc ├── test-exec.cc ├── test-io.cc ├── test-mem.cc ├── test-nodes.cc ├── test-params.cc ├── test-rnn.cc ├── test-serialize.cc ├── test-softmax.cc ├── test-tensor.cc ├── test-trainers-io.cc ├── test-trainers.cc └── test.h ├── third_party ├── eigen_backward_spatial_convolutions.h ├── eigen_pooling.h ├── eigen_spatial_convolutions.h ├── eigen_volume_patch.h ├── topk.h ├── topk_tf.h └── topk_tr.h └── tutorial ├── 0_multiply.cc ├── 1_linear_regression.cc └── CMakeLists.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # cmake stuff 2 | build*/ 3 | Testing/ 4 | dynet/Testing/ 5 | dynet/tests.bin/ 6 | CTestTestfile.cmake 7 | config.h 8 | Makefile 9 | CMakeCache.txt 10 | CMakeFiles 11 | cmake_install.cmake 12 | python/dynet.cpp 13 | python/dist/ 14 | python/dyNET.egg-info/ 15 | 16 | # binaries 17 | 18 | #data 19 | rnnlm/ptb-mikolov/ 20 | 21 | # Python temporary files 22 | *.pyc 23 | 24 | # Compiled Object files 25 | *.slo 26 | *.lo 27 | *.o 28 | *.obj 29 | 30 | # Precompiled Headers 31 | *.gch 32 | *.pch 33 | 34 | # Compiled Dynamic libraries 35 | *.so 36 | *.dylib 37 | *.dll 38 | 39 | # Fortran module files 40 | *.mod 41 | 42 | # Compiled Static libraries 43 | *.lai 44 | *.la 45 | *.a 46 | *.lib 47 | 48 | # Executables 49 | *.exe 50 | *.out 51 | *.app 52 | 53 | # Editor stuff 54 | *.swp 55 | 56 | # Doc stuff 57 | doc/doxygen/xml 58 | doc/source/tutorials_notebooks 59 | 60 | .RData 61 | .RHistory 62 | -------------------------------------------------------------------------------- /.travis/build_dynet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | cd "$TRAVIS_BUILD_DIR" 5 | if [[ "$BACKEND" == cuda ]]; then 6 | sed -i.bak "s/\(APPEND CUDA_NVCC_FLAGS.*;\)/\1-w;/" dynet/CMakeLists.txt 7 | export DYNET_TEST_DEVICES=CPU 8 | BACKEND_OPTION="-DBACKEND=cuda" 9 | fi 10 | if [[ "$PYTHON_INSTALL" == manual ]]; then 11 | mkdir build 12 | cd build 13 | cmake .. $BACKEND_OPTION -DEIGEN3_INCLUDE_DIR="$EIGEN3_INCLUDE_DIR" -DENABLE_BOOST=ON -DENABLE_CPP_EXAMPLES=ON -DENABLE_C=ON -DPYTHON=$(which python) -DCMAKE_INSTALL_PREFIX=$(dirname $(which python))/.. 14 | else # pip 15 | .travis/fix_version.sh 16 | if [[ "$TRAVIS_OS_NAME" == linux ]]; then 17 | docker build --rm -t "dynet-manylinux1-${BUILD_ARCH}-builder" -f "docker/Dockerfile-$BUILD_ARCH" . 18 | docker run -e PYVER -e BUILD_ARCH -e TRAVIS_BUILD_DIR=/build -v "$TRAVIS_BUILD_DIR":/build --rm "dynet-manylinux1-${BUILD_ARCH}-builder" /root/.travis/build_linux_wheel.sh 19 | elif [[ "$TRAVIS_OS_NAME" == osx ]]; then 20 | .travis/build_macos_wheel.sh 21 | fi 22 | fi 23 | 24 | -------------------------------------------------------------------------------- /.travis/build_linux_wheel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | # To be run inside docker container 5 | export CMAKE=cmake28 EIGEN3_INCLUDE_DIR="$TRAVIS_BUILD_DIR/eigen" LD_LIBRARY_PATH="$TRAVIS_BUILD_DIR/build/dynet:$LD_LIBRARY_PATH" 6 | cd "$TRAVIS_BUILD_DIR" 7 | 8 | if [[ "$BUILD_ARCH" == i686 ]]; then 9 | yum install -y openssl-devel 10 | else 11 | yum install -y gmp-devel 12 | fi 13 | # Compile wheels 14 | for PYBIN in /opt/python/*${PYVER/./}*/bin; do 15 | "$PYBIN/pip" install -U pip 16 | "$PYBIN/pip" install --prefer-binary cryptography 17 | "$PYBIN/pip" install -U numpy twine cython 18 | if [[ -n "$DYNET_TEST" ]]; then 19 | "$TRAVIS_BUILD_DIR"/.travis/test_dynet.sh 20 | else # build 21 | "$PYBIN/python" setup.py bdist_wheel 22 | fi 23 | done 24 | 25 | # Bundle external shared libraries into the wheels 26 | for whl in build/py*/python/dist/*.whl; do 27 | auditwheel repair "$whl" 28 | done 29 | mv wheelhouse dist 30 | 31 | # Fix permissions for any new files because user is root in Docker 32 | cd "$TRAVIS_BUILD_DIR" 33 | chmod -R a+rw . 34 | find . -type d -exec chmod a+x {} \; 35 | -------------------------------------------------------------------------------- /.travis/build_macos_wheel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | cd "$TRAVIS_BUILD_DIR" 5 | source activate "$PYVER" 6 | python setup.py bdist_wheel 7 | cp -vr build/py*/python/dist ./ 8 | -------------------------------------------------------------------------------- /.travis/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | cd "$TRAVIS_BUILD_DIR" 5 | twine check dist/* 6 | twine upload --verbose --skip-existing dist/* # Upload to PyPI 7 | -------------------------------------------------------------------------------- /.travis/fix_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | cd "$TRAVIS_BUILD_DIR" 5 | if [[ -n "$TRAVIS_TAG" ]]; then 6 | sed -i.bak "s/# version=.*/version=\"$TRAVIS_TAG\",/" setup.py 7 | sed -i.bak "s/ -march=native//" CMakeLists.txt 8 | fi 9 | 10 | -------------------------------------------------------------------------------- /.travis/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | # Boost, Python packages 5 | if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then 6 | sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y 7 | sudo apt-get -qq update 8 | sudo apt-get install -y gcc-4.8 g++-4.8 libssl-dev 9 | PYTHON_PACKAGES="numpy twine auditwheel cython" 10 | if [[ "$PYTHON_INSTALL" == manual ]]; then 11 | sudo apt-get install -y --allow-unauthenticated libboost-filesystem1.55-dev libboost-program-options1.55-dev libboost-serialization1.55-dev libboost-test1.55-dev libboost-regex1.55-dev 12 | sudo -H pip install -U $PYTHON_PACKAGES 13 | else 14 | pip install -U pip 15 | pip install --prefer-binary cryptography 16 | pip install -U $PYTHON_PACKAGES 17 | fi 18 | else 19 | brew update 20 | # Install Miniconda 21 | export MINICONDA_OS_NAME=MacOSX MINICONDA_ARCH=x86_64 22 | wget "https://repo.continuum.io/miniconda/Miniconda3-latest-$MINICONDA_OS_NAME-$MINICONDA_ARCH.sh" -O miniconda.sh 23 | bash miniconda.sh -b -p miniconda 24 | export PATH="$PWD/miniconda/bin:$PATH" 25 | hash -r 26 | conda config --set always_yes yes --set changeps1 no 27 | conda update -q conda 28 | conda create -c conda-forge -q -n "$PYVER" python="$PYVER" numpy cython 29 | # Useful for debugging any issues with conda 30 | conda info -a 31 | source activate "$PYVER" 32 | pip install twine 33 | fi 34 | 35 | # CUDA 36 | if [[ "$BACKEND" == cuda ]]; then 37 | CUDA_VERSION_MAJOR="8" CUDA_VERSION_MINOR="0" 38 | CUDA_PKG_LONGVERSION="${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}.61-1" 39 | CUDA_PKG_VERSION="${CUDA_VERSION_MAJOR}-${CUDA_VERSION_MINOR}" 40 | CUDA_REPO_PKG=cuda-repo-ubuntu1404_${CUDA_PKG_LONGVERSION}_amd64.deb 41 | wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/$CUDA_REPO_PKG 42 | sudo dpkg -i $CUDA_REPO_PKG 43 | rm $CUDA_REPO_PKG 44 | sudo apt-get -y update 45 | sudo apt-get install -y --no-install-recommends cuda-drivers cuda-core-$CUDA_PKG_VERSION cuda-cudart-dev-$CUDA_PKG_VERSION cuda-cublas-dev-$CUDA_PKG_VERSION cuda-curand-dev-$CUDA_PKG_VERSION 46 | sudo ln -s /usr/local/cuda-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR} /usr/local/cuda 47 | fi 48 | 49 | # Eigen 50 | mkdir eigen 51 | cd eigen 52 | wget https://github.com/clab/dynet/releases/download/2.1/eigen-b2e267dc99d4.zip 53 | unzip eigen-b2e267dc99d4.zip 54 | mkdir build && cd build 55 | cmake .. 56 | sudo make install 57 | 58 | -------------------------------------------------------------------------------- /.travis/test_dynet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xe 3 | 4 | cd "$TRAVIS_BUILD_DIR" 5 | if [[ "$BUILD_ARCH" == i686 && `arch` == x86_64 ]]; then 6 | echo Skipping Python test on $BUILD_ARCH 7 | #echo Using Docker to test $BUILD_ARCH build on `arch` 8 | #docker run -e PYVER -e TRAVIS_OS_NAME -e CONDA_PACKAGES -e TRAVIS_BUILD_DIR=/build -e DYNET_TEST=1 -v "$TRAVIS_BUILD_DIR":/build --rm "dynet-manylinux1-${BUILD_ARCH}-builder" /root/.travis/build_manylinux_wheel.sh 9 | else 10 | if [[ "$PYTHON_INSTALL" == manual ]]; then 11 | cd build 12 | if [[ "$TRAVIS_OS_NAME" == linux ]]; then 13 | make -j$(nproc) 14 | elif [[ "$TRAVIS_OS_NAME" == osx ]]; then 15 | source activate "$PYVER" 16 | make -j$(sysctl -n hw.ncpu) 17 | export DYLD_LIBRARY_PATH="$TRAVIS_BUILD_DIR/build/dynet" 18 | fi 19 | if [[ "$BACKEND" != cuda ]]; then 20 | make install 21 | export CTEST_OUTPUT_ON_FAILURE=1 22 | make test 23 | cd python 24 | python ../../setup.py build --build-dir=.. --skip-build install --user 25 | fi 26 | else # PYTHON_INSTALL is pip 27 | if [[ "$TRAVIS_OS_NAME" == osx ]]; then 28 | source activate "$PYVER" 29 | fi 30 | pip install dynet --no-index -f dist 31 | fi 32 | 33 | if [[ "$BACKEND" != cuda ]]; then 34 | cd "$TRAVIS_BUILD_DIR"/tests/python 35 | python test.py 36 | fi 37 | fi 38 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft python 2 | global-include *.py *.cc *.h *.cu CMakeLists.txt *.cmake 3 | include README.md 4 | include LICENSE.txt 5 | -------------------------------------------------------------------------------- /MEM.notes: -------------------------------------------------------------------------------- 1 | The code that computes the l2 norm of the gradient is going to need 2 | scratch space on every device that DYNET is using that has a parameter. 3 | 1) devices should know whether they have parameters/gradients 4 | 5 | alignment code is hidden away. it's all hard coded, but it looks like 6 | Intel at least is getting more foregiving about alingment problems so 7 | we might not notice opportunities for speedups if something changes. 8 | GPU memory is aligned mostly by CUDA 9 | 10 | the MP stuff needs to be tested by Austin. 11 | -------------------------------------------------------------------------------- /bench/sequence_transduction.py: -------------------------------------------------------------------------------- 1 | import dynet as dy 2 | import random 3 | import time 4 | import sys 5 | random.seed(1) 6 | 7 | SEQ_LENGTH=2 8 | BATCH_SIZE=2 9 | HIDDEN=1 10 | NCLASSS=2 11 | EMBED_SIZE=1 12 | N_SEQS=1000 13 | autobatching=True 14 | 15 | dy.renew_cg() 16 | 17 | random_seq = lambda ln,t: [random.randint(0,t-1) for _ in xrange(ln)] 18 | seq_lengths = [SEQ_LENGTH for _ in range(N_SEQS)] 19 | #seq_lengths = [random.randint(10, SEQ_LENGTH) for _ in range(N_SEQS)] 20 | Xs = [random_seq(L, 100) for L in seq_lengths] 21 | Ys = [random_seq(L, NCLASSS) for L in seq_lengths] 22 | 23 | m = dy.Model() 24 | trainer = dy.SimpleSGDTrainer(m) 25 | 26 | E = m.add_lookup_parameters((1000, EMBED_SIZE)) 27 | fwR = dy.VanillaLSTMBuilder(1, EMBED_SIZE, HIDDEN, m) 28 | bwR = dy.VanillaLSTMBuilder(1, EMBED_SIZE, HIDDEN, m) 29 | T_= m.add_parameters((HIDDEN, HIDDEN*2)) 30 | fwR2 = dy.VanillaLSTMBuilder(1, EMBED_SIZE, HIDDEN, m) 31 | bwR2 = dy.VanillaLSTMBuilder(1, EMBED_SIZE, HIDDEN, m) 32 | W_= m.add_parameters((NCLASSS, HIDDEN*1)) 33 | 34 | total_time = 0.0 35 | 36 | def transduce(seq,Y): 37 | seq = [E[i] for i in seq] 38 | fw = fwR.initial_state().transduce(seq) 39 | 40 | # this UNUSED part affects strategy 2 41 | XXX = fwR2.initial_state().transduce([E[3],E[5]]) 42 | 43 | W = W_.expr() 44 | outs = [W*z for z in fw] 45 | losses = [dy.pickneglogsoftmax(o,y) for o,y in zip(outs,Y)] 46 | s = dy.esum(losses) 47 | return s 48 | 49 | batch=[] 50 | start = time.time() 51 | for X,Y in zip(Xs,Ys): 52 | loss = transduce(X,Y) 53 | batch.append(loss) 54 | if len(batch)==BATCH_SIZE: 55 | s = dy.esum(batch) 56 | s_ = time.time() 57 | s.forward() 58 | total_time = total_time + time.time() - s_ 59 | print s.npvalue() 60 | sys.exit() 61 | #break 62 | s.backward() 63 | trainer.update() 64 | batch = [] 65 | dy.renew_cg() 66 | print "total time:",time.time() - start, len(Xs) / (time.time() - start) 67 | print "forward time:",total_time, len(Xs) / total_time 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /cmake/FindDyNet.cmake: -------------------------------------------------------------------------------- 1 | 2 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7 FATAL_ERROR) 3 | 4 | INCLUDE(FindPackageHandleStandardArgs) 5 | 6 | FIND_LIBRARY(TH_LIBRARY TH) 7 | FIND_PATH(TH_INCLUDE_DIR "TH.h" PATHS "${CMAKE_PREFIX_PATH}/include/TH") 8 | 9 | SET(TH_LIBRARIES ${TH_LIBRARY}) 10 | 11 | FIND_PACKAGE_HANDLE_STANDARD_ARGS( 12 | TH 13 | REQUIRED_ARGS 14 | TH_INCLUDE_DIR 15 | TH_LIBRARY) 16 | -------------------------------------------------------------------------------- /config.h.cmake: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_CONFIG_H_ 2 | #define DYNET_CONFIG_H_ 3 | 4 | #cmakedefine WITH_MINERVA_BACKEND @WITH_MINERVA_BACKEND@ 5 | #cmakedefine WITH_THPP_BACKEND @WITH_THPP_BACKEND@ 6 | #cmakedefine WITH_EIGEN_BACKEND @WITH_EIGEN_BACKEND@ 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /contrib/c/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(dynet_c) 3 | 4 | if (WITH_CUDA_BACKEND) 5 | set(DYNET_C_USE_CUDA 1) 6 | endif() 7 | 8 | include_directories( 9 | ${PROJECT_SOURCE_DIR} 10 | ${PROJECT_BINARY_DIR} 11 | "../../dynet") 12 | add_subdirectory(dynet_c) 13 | 14 | if(ENABLE_BOOST) 15 | add_subdirectory(test) 16 | enable_testing() 17 | endif() 18 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | configure_file( 2 | ${CMAKE_CURRENT_SOURCE_DIR}/config.h.in 3 | ${CMAKE_CURRENT_BINARY_DIR}/config.h 4 | ) 5 | install( 6 | FILES 7 | ${CMAKE_CURRENT_BINARY_DIR}/config.h 8 | DESTINATION include/dynet_c 9 | ) 10 | 11 | set(dynet_c_HDRS 12 | api.h 13 | define.h 14 | devices.h 15 | dim.h 16 | expr.h 17 | graph.h 18 | init.h 19 | internal.h 20 | io.h 21 | model.h 22 | param-init.h 23 | rnn-builder.h 24 | softmax-builder.h 25 | status.h 26 | tensor.h 27 | training.h 28 | ) 29 | set( 30 | dynet_c_SRCS 31 | devices.cc 32 | dim.cc 33 | expr.cc 34 | graph.cc 35 | init.cc 36 | internal.cc 37 | io.cc 38 | model.cc 39 | param-init.cc 40 | rnn-builder.cc 41 | softmax-builder.cc 42 | status.cc 43 | tensor.cc 44 | training.cc 45 | ) 46 | 47 | set(BUILD_SHARED_LIBS ON) 48 | add_library(dynet_c ${dynet_c_HDRS} ${dynet_c_SRCS}) 49 | target_link_libraries(dynet_c dynet) 50 | install(FILES ${dynet_c_HDRS} DESTINATION include/dynet_c) 51 | install(TARGETS dynet_c DESTINATION lib) 52 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/api.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_API_H_ 2 | #define DYNET_C_API_H_ 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #endif // DYNET_C_API_H_ 21 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_CONFIG_H_ 2 | #define DYNET_C_CONFIG_H_ 3 | 4 | #cmakedefine DYNET_C_USE_CUDA 5 | 6 | #endif // DYNET_C_CONFIG_H_ 7 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/define.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_DEFINE_H_ 2 | #define DYNET_C_DEFINE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __cplusplus 8 | #define DYNET_C_EXTERN extern "C" 9 | #else 10 | #define DYNET_C_EXTERN extern 11 | #endif // __cplusplus 12 | 13 | #if defined(__GNUC__) || defined(__clang__) 14 | #define DYNET_C_EXPORT __attribute__((visibility("default"))) 15 | #elif defined(_MSC_VER) 16 | #ifdef DYNET_C_DLLEXPORT 17 | #define DYNET_C_EXPORT __declspec(dllexport) 18 | #else 19 | #define DYNET_C_EXPORT __declspec(dllimport) 20 | #endif // DYNET_C_DLLEXPORT 21 | #else 22 | #define DYNET_C_EXPORT 23 | #endif // __GNUC__, __clang__, _MSC_VER 24 | 25 | #define DYNET_C_API DYNET_C_EXTERN DYNET_C_EXPORT 26 | 27 | /* 28 | * Boolean type. 29 | */ 30 | typedef uint32_t DYNET_C_BOOL; 31 | 32 | /* 33 | * Boolean values. 34 | * `DYNET_C_TRUE` can not be compared with any `DYNET_C_BOOL` values. 35 | * Only substituting `DYNET_C_TRUE` to `DYNET_C_BOOL` variables is 36 | * allowed. 37 | */ 38 | #define DYNET_C_FALSE 0 39 | #define DYNET_C_TRUE 1 40 | 41 | /* 42 | * Return codes. 43 | */ 44 | typedef uint32_t DYNET_C_STATUS; 45 | #define DYNET_C_OK 0 46 | #define DYNET_C_ERROR -1 47 | 48 | #endif // DYNET_C_DEFINE_H_ 49 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/devices.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | using dynet_c::internal::to_c_ptr; 8 | 9 | DYNET_C_STATUS dynetGetGlobalDevice( 10 | const char *name, dynetDevice_t **newobj) try { 11 | DYNET_C_CHECK_NOT_NULL(newobj); 12 | if (name) { 13 | *newobj = to_c_ptr(dynet::get_device_manager()->get_global_device(name)); 14 | } else { 15 | *newobj = to_c_ptr(dynet::get_device_manager()->get_global_device("")); 16 | } 17 | return DYNET_C_OK; 18 | } DYNET_C_HANDLE_EXCEPTIONS 19 | 20 | DYNET_C_STATUS dynetGetNumDevices(uint32_t *retval) try { 21 | DYNET_C_CHECK_NOT_NULL(retval); 22 | *retval = dynet::get_device_manager()->num_devices(); 23 | return DYNET_C_OK; 24 | } DYNET_C_HANDLE_EXCEPTIONS 25 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/devices.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_DEVICES_H_ 2 | #define DYNET_C_DEVICES_H_ 3 | 4 | #include 5 | 6 | /** 7 | * Opaque type of Device. 8 | */ 9 | typedef struct dynetDevice dynetDevice_t; 10 | 11 | /** 12 | * Retrieves a global device. 13 | * @param newobj Pointer to receive a handler. 14 | * @return Status code. 15 | */ 16 | DYNET_C_API DYNET_C_STATUS dynetGetGlobalDevice( 17 | const char *name, dynetDevice_t **newobj); 18 | 19 | /** 20 | * Returns the number of global devices. 21 | * @param retval Pointer to receive the number, 22 | * @return Status code. 23 | */ 24 | DYNET_C_API DYNET_C_STATUS dynetGetNumDevices(uint32_t *retval); 25 | 26 | #endif // DYNET_C_DEVICES_H_ 27 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/internal.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | namespace dynet_c { 6 | 7 | namespace internal { 8 | 9 | static thread_local ErrorHandler error_handler; 10 | 11 | ErrorHandler &ErrorHandler::get_instance() { 12 | return error_handler; 13 | } 14 | 15 | } // namespace internal 16 | 17 | } // namespace dynet_c 18 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/status.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | using dynet_c::internal::ErrorHandler; 9 | 10 | DYNET_C_STATUS dynetResetStatus() try { 11 | ErrorHandler::get_instance().reset(); 12 | return DYNET_C_OK; 13 | } DYNET_C_HANDLE_EXCEPTIONS 14 | 15 | DYNET_C_STATUS dynetGetMessage(char *retval, size_t *size) try { 16 | DYNET_C_CHECK_NOT_NULL(size); 17 | dynet_c::internal::copy_string_to_array( 18 | ErrorHandler::get_instance().get_message(), retval, size); 19 | return DYNET_C_OK; 20 | } DYNET_C_HANDLE_EXCEPTIONS 21 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/status.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_STATUS_H_ 2 | #define DYNET_C_STATUS_H_ 3 | 4 | #include 5 | 6 | DYNET_C_API DYNET_C_STATUS dynetResetStatus(); 7 | 8 | DYNET_C_API DYNET_C_STATUS dynetGetMessage(char *retval, size_t *size); 9 | 10 | #endif // DYNET_C_STATUS_H_ 11 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/tensor.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | using dynet_c::internal::to_c_ptr; 10 | using dynet_c::internal::to_cpp_ptr; 11 | 12 | DYNET_C_STATUS dynetDeleteTensor(dynetTensor_t *tensor) try { 13 | DYNET_C_CHECK_NOT_NULL(tensor); 14 | delete to_cpp_ptr(tensor); 15 | return DYNET_C_OK; 16 | } DYNET_C_HANDLE_EXCEPTIONS 17 | 18 | DYNET_C_STATUS dynetGetTensorDim( 19 | const dynetTensor_t *tensor, dynetDim_t **newobj) try { 20 | DYNET_C_CHECK_NOT_NULL(tensor); 21 | DYNET_C_CHECK_NOT_NULL(newobj); 22 | dynet::Dim *d = new dynet::Dim(to_cpp_ptr(tensor)->d); 23 | *newobj = to_c_ptr(d); 24 | return DYNET_C_OK; 25 | } DYNET_C_HANDLE_EXCEPTIONS 26 | 27 | DYNET_C_STATUS dynetEvaluateTensorAsScalar( 28 | const dynetTensor_t *tensor, float *retval) try { 29 | DYNET_C_CHECK_NOT_NULL(tensor); 30 | DYNET_C_CHECK_NOT_NULL(retval); 31 | *retval = dynet::as_scalar(*to_cpp_ptr(tensor)); 32 | return DYNET_C_OK; 33 | } DYNET_C_HANDLE_EXCEPTIONS 34 | 35 | DYNET_C_STATUS dynetEvaluateTensorAsArray( 36 | const dynetTensor_t *tensor, float *retval, size_t *size) try { 37 | DYNET_C_CHECK_NOT_NULL(tensor); 38 | DYNET_C_CHECK_NOT_NULL(size); 39 | dynet_c::internal::copy_vector_to_array( 40 | dynet::as_vector(*to_cpp_ptr(tensor)), retval, size); 41 | return DYNET_C_OK; 42 | } DYNET_C_HANDLE_EXCEPTIONS 43 | 44 | DYNET_C_API DYNET_C_STATUS dynetRepresentTensorAsString( 45 | const dynetTensor_t *tensor, char *retval, size_t *size) try { 46 | DYNET_C_CHECK_NOT_NULL(tensor); 47 | DYNET_C_CHECK_NOT_NULL(size); 48 | std::stringstream ss; 49 | dynet::operator<<(ss, *to_cpp_ptr(tensor)); 50 | dynet_c::internal::copy_string_to_array(ss.str(), retval, size); 51 | return DYNET_C_OK; 52 | } DYNET_C_HANDLE_EXCEPTIONS 53 | -------------------------------------------------------------------------------- /contrib/c/dynet_c/tensor.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_TENSOR_H_ 2 | #define DYNET_C_TENSOR_H_ 3 | 4 | #include 5 | #include 6 | 7 | /** 8 | * Opaque type of Tensor. 9 | */ 10 | typedef struct dynetTensor dynetTensor_t; 11 | 12 | /** 13 | * Deletes the Tensor object. 14 | * @param shape Pointer of a handler. 15 | * @return Status code. 16 | */ 17 | DYNET_C_API DYNET_C_STATUS dynetDeleteTensor(dynetTensor_t *tensor); 18 | 19 | /** 20 | * Returns dim of the tensor. 21 | * @param tensor Pointer of a handler. 22 | * @param newobj Pointer to receive a Dim object. 23 | * @return Status code. 24 | */ 25 | DYNET_C_API DYNET_C_STATUS dynetGetTensorDim( 26 | const dynetTensor_t *tensor, dynetDim_t **newobj); 27 | 28 | /** 29 | * Retrieves one internal value in the tensor. 30 | * @param tensor Pointer of a handler. 31 | * @param retval Pointer to receive an internal float value. 32 | * @return Status code. 33 | */ 34 | DYNET_C_API DYNET_C_STATUS dynetEvaluateTensorAsScalar( 35 | const dynetTensor_t *tensor, float *retval); 36 | 37 | /** 38 | * Retrieves internal values in the tensor as a vector. 39 | * @param tensor Pointer of a handler. 40 | * @param retval Pointer to receive a list of the internal values. 41 | * @param size Pointer to receive the length of the array. 42 | * @return Status code. 43 | */ 44 | DYNET_C_API DYNET_C_STATUS dynetEvaluateTensorAsArray( 45 | const dynetTensor_t *tensor, float *retval, size_t *size); 46 | 47 | /** 48 | * Returns a string representation of the tensor. 49 | * @param tensor Pointer of a handler. 50 | * @param retval Pointer to receive the encoded string. 51 | * @param size Pointer to receive a length of the char sequence. 52 | * @return Status code. 53 | */ 54 | DYNET_C_API DYNET_C_STATUS dynetRepresentTensorAsString( 55 | const dynetTensor_t *tensor, char *retval, size_t *size); 56 | 57 | #endif // DYNET_C_TENSOR_H_ 58 | -------------------------------------------------------------------------------- /contrib/c/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(CTest) 2 | 3 | find_package(Boost COMPONENTS system filesystem unit_test_framework REQUIRED) 4 | include_directories(${TEST_SOURCE_DIR}/src ${Boost_INCLUDE_DIRS}) 5 | add_definitions(-DBOOST_TEST_DYN_LINK) 6 | 7 | function(dynet_c_test name) 8 | add_executable(test-c-${name} ${name}_test.cc) 9 | target_link_libraries(test-c-${name} dynet_c 10 | ${LIBS} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) 11 | add_test( 12 | NAME test-c-${name} 13 | COMMAND test-c-${name} 14 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 15 | ) 16 | endfunction() 17 | 18 | dynet_c_test(dim) 19 | dynet_c_test(exec) 20 | dynet_c_test(params) 21 | -------------------------------------------------------------------------------- /contrib/c/test/dim_test.cc: -------------------------------------------------------------------------------- 1 | #define BOOST_TEST_MODULE C_DIM_TEST 2 | 3 | #include 4 | #include 5 | #include "test_utils.h" 6 | 7 | struct CDimTest { 8 | CDimTest() {} 9 | }; 10 | 11 | BOOST_FIXTURE_TEST_SUITE(c_dim_test, CDimTest); 12 | 13 | BOOST_AUTO_TEST_CASE(test_dim_truncate_no_trailing_one) { 14 | ::dynetDim_t *d1; 15 | uint32_t dims[] = {1, 3, 4}; 16 | BOOST_CHECK_EQUAL(DYNET_C_OK, 17 | ::dynetCreateDimWithDimensions(dims, 3, &d1)); 18 | ::dynetDim_t *t1; 19 | BOOST_CHECK_EQUAL(DYNET_C_OK, ::dynetTruncateDim(d1, &t1)); 20 | uint32_t nd; 21 | BOOST_CHECK_EQUAL(DYNET_C_OK, ::dynetGetDimNDimensions(t1, &nd)); 22 | BOOST_CHECK_EQUAL(nd, 3); 23 | BOOST_CHECK_EQUAL(DYNET_C_OK, ::dynetDeleteDim(d1)); 24 | BOOST_CHECK_EQUAL(DYNET_C_OK, ::dynetDeleteDim(t1)); 25 | } 26 | 27 | BOOST_AUTO_TEST_SUITE_END() 28 | -------------------------------------------------------------------------------- /contrib/c/test/test_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C_TEST_UTILS_H_ 2 | #define DYNET_C_TEST_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | namespace test_utils { 11 | 12 | void init_dynet() { 13 | const char *argv[] = {"DynetCTest", "--dynet-mem", "512"}; 14 | ::dynetDynetParams_t *params; 15 | ::dynetExtractDynetParams(3, const_cast(argv), false, ¶ms); 16 | ::dynetInitialize(params); 17 | ::dynetDeleteDynetParams(params); 18 | } 19 | 20 | void show_message() { 21 | std::size_t length = 0u; 22 | ::dynetGetMessage(nullptr, &length); 23 | char str[length]; 24 | ::dynetGetMessage(str, &length); 25 | std::cout << str << std::endl; 26 | ::dynetResetStatus(); 27 | } 28 | 29 | } // namespace test_utils 30 | 31 | #endif // DYNET_C_TEST_UTILS_H_ 32 | -------------------------------------------------------------------------------- /contrib/csharp/dynetsharp/dynetsharp.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.27428.2011 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dynetsharp", "dynetsharp\dynetsharp.vcxproj", "{FF5FE224-135E-4D68-9B34-E65109360BA1}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|x64 = Debug|x64 11 | Debug|x86 = Debug|x86 12 | MinSizeRel|x64 = MinSizeRel|x64 13 | MinSizeRel|x86 = MinSizeRel|x86 14 | Release|x64 = Release|x64 15 | Release|x86 = Release|x86 16 | RelWithDebInfo|x64 = RelWithDebInfo|x64 17 | RelWithDebInfo|x86 = RelWithDebInfo|x86 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Debug|x64.ActiveCfg = Debug|x64 21 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Debug|x64.Build.0 = Debug|x64 22 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Debug|x86.ActiveCfg = Debug|Win32 23 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Debug|x86.Build.0 = Debug|Win32 24 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.MinSizeRel|x64.ActiveCfg = MinSizeRel|x64 25 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.MinSizeRel|x64.Build.0 = MinSizeRel|x64 26 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Win32 27 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.MinSizeRel|x86.Build.0 = MinSizeRel|Win32 28 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Release|x64.ActiveCfg = Release|x64 29 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Release|x64.Build.0 = Release|x64 30 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Release|x86.ActiveCfg = Release|Win32 31 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.Release|x86.Build.0 = Release|Win32 32 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.RelWithDebInfo|x64.ActiveCfg = RelWithDebInfo|x64 33 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.RelWithDebInfo|x64.Build.0 = RelWithDebInfo|x64 34 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Win32 35 | {FF5FE224-135E-4D68-9B34-E65109360BA1}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Win32 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | GlobalSection(ExtensibilityGlobals) = postSolution 41 | SolutionGuid = {C963D2AE-0072-43C2-B8D6-6248294EC5B3} 42 | EndGlobalSection 43 | EndGlobal 44 | -------------------------------------------------------------------------------- /contrib/csharp/dynetsharp/dynetsharp/dynetsharp.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Header Files 20 | 21 | 22 | 23 | 24 | Source Files 25 | 26 | 27 | -------------------------------------------------------------------------------- /contrib/csharp/dynetsharp/dynetsharp/dynetsharp.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /contrib/rust/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /dynet-sys/target/ 3 | **/*.rs.bk 4 | Cargo.lock 5 | *.model 6 | /examples/*/data 7 | -------------------------------------------------------------------------------- /contrib/rust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dynet" 3 | version = "0.1.0" 4 | authors = ["Hiroki Teranishi "] 5 | description = "DyNet: The Dynamic Neural Network Toolkit" 6 | keywords = ["deep-learning", "neural-networks", "machine-learning"] 7 | license = "Apache-2.0" 8 | repository = "https://github.com/clab/dynet" 9 | readme = "README.md" 10 | 11 | [dependencies] 12 | backtrace = "0.3" 13 | dynet-sys = { path = "dynet-sys" } 14 | libc = "0.2" 15 | 16 | [dev-dependencies] 17 | rand = "0.6" 18 | 19 | [features] 20 | cuda = ["dynet-sys/cuda"] 21 | 22 | [[example]] 23 | name = "xor" 24 | path = "examples/xor/xor.rs" 25 | 26 | [[example]] 27 | name = "mnist" 28 | path = "examples/mnist/mnist.rs" 29 | 30 | [[example]] 31 | name = "rnnlm" 32 | path = "examples/rnnlm/rnnlm.rs" 33 | -------------------------------------------------------------------------------- /contrib/rust/README.md: -------------------------------------------------------------------------------- 1 | # Rust bindings for DyNet 2 | 3 | This package provides the Rust bindings for [DyNet](https://github.com/clab/dynet). 4 | 5 | ## Building the Rust bindings 6 | 7 | To build the Rust bindings, just run `cargo build` command: 8 | 9 | ``` 10 | $ cargo build [--features cuda] 11 | ``` 12 | 13 | You can specify the library directory using `DYNET_C_LIBRARY_DIR` containing `libdynet_c.so`. 14 | 15 | ## Running the Examples 16 | 17 | You can run examples with `cargo run` command: 18 | 19 | ``` 20 | $ cargo run --example xor 21 | ``` 22 | 23 | ## Usage 24 | 25 | The current Rust API works mostly like the C++ API. 26 | 27 | ## Acknowledgements 28 | 29 | The DyNet Rust bindings are designed in reference to [primitiv](https://github.com/primitiv/primitiv-rust). 30 | -------------------------------------------------------------------------------- /contrib/rust/dynet-sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dynet-sys" 3 | version = "0.1.0" 4 | authors = ["Hiroki Teranishi "] 5 | description = "The package provides bindings to dynet." 6 | keywords = ["deep-learning", "neural-networks", "machine-learning", "dynet"] 7 | license = "Apache-2.0" 8 | repository = "https://github.com/clab/dynet" 9 | readme = "README.md" 10 | 11 | build = "build.rs" 12 | 13 | [dependencies] 14 | 15 | [build-dependencies] 16 | bindgen = "0.42" 17 | 18 | [features] 19 | cuda = [] 20 | -------------------------------------------------------------------------------- /contrib/rust/dynet-sys/README.md: -------------------------------------------------------------------------------- 1 | dynet-sys 2 | ============= 3 | 4 | The package provides bindings to [dynet](https://github.com/clab/dynet). 5 | 6 | Prerequisites 7 | ------------- 8 | 9 | * Rust (1.26 or later) 10 | * Clang (3.9 or later) 11 | -------------------------------------------------------------------------------- /contrib/rust/dynet-sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_upper_case_globals)] 2 | #![allow(non_camel_case_types)] 3 | #![allow(non_snake_case)] 4 | 5 | include!(concat!(env!("OUT_DIR"), "/bindings.rs")); 6 | -------------------------------------------------------------------------------- /contrib/rust/examples/mnist/download_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | rm -rf data 3 | mkdir -p data 4 | wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz -P data 5 | wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz -P data 6 | wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz -P data 7 | wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz -P data 8 | gunzip data/* 9 | -------------------------------------------------------------------------------- /contrib/rust/examples/rnnlm/install-examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | data_version=ptb-mikolov-01.tar.gz 5 | 6 | ROOTDIR=`dirname $0` 7 | cd $ROOTDIR 8 | 9 | rm -f $data_version 10 | rm -rf data && mkdir data 11 | curl -f http://demo.clab.cs.cmu.edu/cdyer/$data_version -o $data_version 12 | tar xzf $data_version -C data --strip-components 1 13 | rm -f $data_version 14 | 15 | echo SUCCESS. 1>&2 16 | -------------------------------------------------------------------------------- /contrib/rust/examples/xor/xor.rs: -------------------------------------------------------------------------------- 1 | extern crate dynet; 2 | 3 | use std::env; 4 | 5 | use dynet::*; 6 | 7 | fn main() { 8 | let args: Vec = env::args().collect(); 9 | dynet::initialize(&mut DynetParams::from_args(false)); 10 | 11 | let ITERATIONS = 30; 12 | 13 | let mut m = ParameterCollection::new(); 14 | let mut trainer = SimpleSGDTrainer::default(&mut m); 15 | 16 | let HIDDEN_SIZE = 8; 17 | let initializer = ParameterInitGlorot::default(); 18 | let mut p_W = m.add_parameters([HIDDEN_SIZE, 2], &initializer); 19 | let mut p_b = m.add_parameters([HIDDEN_SIZE], &initializer); 20 | let mut p_V = m.add_parameters([1, HIDDEN_SIZE], &initializer); 21 | let mut p_a = m.add_parameters([1], &initializer); 22 | if args.len() == 2 { 23 | m.load(&args[1]); 24 | } 25 | 26 | let mut cg = ComputationGraph::new(); 27 | for iter in 0..ITERATIONS { 28 | let mut loss = 0.; 29 | for mi in 0..4 { 30 | let x1 = mi % 2 == 1; 31 | let x2 = (mi / 2) % 2 == 1; 32 | let x_values = [if x1 { 1. } else { -1. }, if x2 { 1. } else { -1. }]; 33 | let y_value = if x1 != x2 { 1. } else { -1. }; 34 | 35 | cg.clear(); 36 | let W = parameter(&mut cg, &mut p_W); 37 | let b = parameter(&mut cg, &mut p_b); 38 | let V = parameter(&mut cg, &mut p_V); 39 | let a = parameter(&mut cg, &mut p_a); 40 | let x = input(&mut cg, [2], &x_values); 41 | let y = input_scalar(&mut cg, y_value); 42 | let h = tanh(W * x + b); 43 | let y_pred = V * h + a; 44 | let loss_expr = squared_distance(y_pred, y); 45 | 46 | if iter == 0 && mi == 0 { 47 | cg.print_graphviz(); 48 | } 49 | 50 | loss += cg.forward(&loss_expr).as_scalar(); 51 | cg.backward(&loss_expr); 52 | trainer.update(); 53 | } 54 | loss /= 4.; 55 | println!("E = {}", loss); 56 | } 57 | m.save("xor.model"); 58 | } 59 | -------------------------------------------------------------------------------- /contrib/rust/src/device.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::CString; 2 | use std::ptr::{self, NonNull}; 3 | 4 | use dynet_sys; 5 | 6 | use super::{ApiResult, Result, Wrap}; 7 | 8 | /// A struct to represent a device. 9 | /// 10 | /// # Examples 11 | /// 12 | /// ``` 13 | /// # use dynet::{Device, DynetParams}; 14 | /// dynet::initialize(&mut DynetParams::default()); 15 | /// let device = Device::default(); 16 | /// ``` 17 | // TODO(chantera): write example with the function that takes `device` as an argument. 18 | #[derive(Debug)] 19 | pub struct Device { 20 | inner: NonNull, 21 | owned: bool, 22 | } 23 | 24 | impl_wrap!(Device, dynetDevice_t); 25 | 26 | impl Drop for Device { 27 | fn drop(&mut self) {} 28 | } 29 | 30 | impl Device { 31 | /// Retrieves a global device. 32 | /// 33 | /// Before calling this method, `dynet::initialize` must be executed to initialize devices. 34 | /// 35 | /// If `name` is empty, this returns the default device. 36 | /// 37 | /// # Panics 38 | /// 39 | /// Panics if the device `name` does not exist. 40 | /// 41 | /// # Examples 42 | /// 43 | /// ```no_run 44 | /// # use dynet::{Device, DynetParams}; 45 | /// // Run your program with the argument `--dynet-devices CPU,GPU:0,GPU:1`. 46 | /// dynet::initialize(&mut DynetParams::from_args(false)); 47 | /// let dev0 = Device::global_device("GPU:0"); 48 | /// let dev1 = Device::global_device("GPU:1"); 49 | /// ``` 50 | pub fn global_device(name: &str) -> Device { 51 | unsafe { 52 | let mut device_ptr: *mut dynet_sys::dynetDevice_t = ptr::null_mut(); 53 | let name_c = CString::new(name).unwrap(); 54 | check_api_status!(dynet_sys::dynetGetGlobalDevice( 55 | name_c.as_ptr(), 56 | &mut device_ptr, 57 | )); 58 | Device::from_raw(device_ptr, false) 59 | } 60 | } 61 | 62 | /// Returns the number of global devices. 63 | pub fn num_devices() -> u32 { 64 | unsafe { 65 | let mut retval: u32 = 0; 66 | check_api_status!(dynet_sys::dynetGetNumDevices(&mut retval)); 67 | retval 68 | } 69 | } 70 | } 71 | 72 | impl Default for Device { 73 | fn default() -> Device { 74 | Device::global_device("") 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /contrib/rust/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The `dynet` crate provides Rust interfaces for DyNet. 2 | //! 3 | //! DyNet is a neural network library developed by Carnegie Mellon University and many others. It 4 | //! is written in C++ and is designed to be efficient when run on either CPU or GPU, and to work 5 | //! well with networks that have dynamic structures that change for every training instance. For 6 | //! example, these kinds of networks are particularly important in natural language processing 7 | //! tasks, and DyNet has been used to build state-of-the-art systems for syntactic parsing, machine 8 | //! translation, morphological inflection, and many other application areas. 9 | 10 | #![deny(missing_docs)] 11 | #![warn( 12 | missing_copy_implementations, missing_debug_implementations, trivial_casts, 13 | trivial_numeric_casts, unused_extern_crates, unused_import_braces, unused_qualifications 14 | )] 15 | 16 | extern crate backtrace; 17 | extern crate dynet_sys; 18 | extern crate libc; 19 | 20 | #[macro_use] 21 | mod status; 22 | pub(crate) use status::*; 23 | 24 | #[macro_use] 25 | mod util; 26 | pub use util::*; 27 | 28 | mod device; 29 | pub use device::Device; 30 | mod dim; 31 | pub use dim::Dim; 32 | mod expr; 33 | pub use expr::*; 34 | mod graph; 35 | pub use graph::*; 36 | mod init; 37 | pub use init::{initialize, reset_rng, DynetParams}; 38 | mod io; 39 | pub use io::*; 40 | mod model; 41 | pub use model::{LookupParameter, Parameter, ParameterCollection}; 42 | mod param_init; 43 | pub use param_init::*; 44 | mod rnn_builder; 45 | pub use rnn_builder::*; 46 | mod softmax_builder; 47 | pub use softmax_builder::*; 48 | mod tensor; 49 | pub use tensor::Tensor; 50 | mod training; 51 | pub use training::*; 52 | -------------------------------------------------------------------------------- /contrib/swig/project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5") 2 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Dim.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /** Represents a "dimension", which you should think of as the dimension of a tensor. Can only be 4 | * constructed using factory methods in the companion object. 5 | * 6 | * @param dim 7 | */ 8 | class Dim private[dynet] (private[dynet] val dim: internal.Dim) { 9 | 10 | def size(): Long = dim.size 11 | def batchSize(): Long = dim.batch_size() 12 | def sumDims(): Long = dim.sum_dims() 13 | 14 | def truncate(): Dim = new Dim(dim.truncate()) 15 | def singleBatch(): Dim = new Dim(dim.single_batch()) 16 | 17 | def resize(i: Long) = dim.resize(i) 18 | def nDims(): Long = dim.ndims() 19 | def rows(): Long = dim.rows() 20 | def cols(): Long = dim.cols() 21 | def batchElems(): Long = dim.batch_elems() 22 | 23 | def set(i: Long, s: Long): Unit = dim.set(i, s) 24 | def get(i: Long): Long = dim.get(i) 25 | def size(i: Long): Long = dim.size(i) 26 | 27 | def deleteDim(i: Long): Unit = dim.delete_dim(i) 28 | 29 | def transpose(): Dim = new Dim(dim.transpose()) 30 | 31 | /** We override `equals` so that `Dim` objects should be equal whenever all of their dimension 32 | * sizes match. 33 | */ 34 | override def equals(that: Any) = that match { 35 | case that: Dim => dim == that.dim 36 | case _ => false 37 | } 38 | override def hashCode(): Int = dim.hashCode() 39 | 40 | override def toString: String = "Dim(" + (0 until nDims.toInt).map(get(_)).mkString(", ") + ")" 41 | 42 | def debugString(): String = s"(Dim: ${size} ${nDims} ${(0 until nDims.toInt).map(get(_))} )" 43 | } 44 | 45 | /** Factory for [[edu.cmu.dynet.Dim]] instances. */ 46 | object Dim { 47 | /** Creates a Dim object from a `Seq` of dimensions and a batch size 48 | * 49 | * @param values a `Seq` of dimensions 50 | * @param b the batch size (zero by default) 51 | */ 52 | def apply(values: Seq[Int], b: Long = 0): Dim = { 53 | val lv = new internal.LongVector() 54 | values.foreach(lv.add) 55 | val dim: internal.Dim = if (b > 0) new internal.Dim(lv, b) else new internal.Dim(lv) 56 | new Dim(dim) 57 | } 58 | 59 | /** Creates a Dim object from a list of the dimensions 60 | * 61 | * @param values a list of the dimensions 62 | */ 63 | def apply(values: Int*): Dim = apply(values) 64 | } 65 | 66 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/FastLstmBuilder.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | class FastLstmBuilder private[dynet](private[dynet] val builder: internal.FastLSTMBuilder) 4 | extends RnnBuilder(builder) { 5 | 6 | def this() { this(new internal.FastLSTMBuilder()) } 7 | 8 | def this(layers: Long, inputDim: Long, hiddenDim: Long, model: ParameterCollection) { 9 | this(new internal.FastLSTMBuilder(layers, inputDim, hiddenDim, model.model)) 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/GRUBuilder.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /** Builder method for creating GRUs, as in the C++ code. For its public methods see 4 | * [[edu.cmu.dynet.RnnBuilder]]. 5 | */ 6 | class GruBuilder private[dynet](private[dynet] val builder: internal.GRUBuilder) 7 | extends RnnBuilder(builder) { 8 | 9 | /** Create a new, empty GruBuilder. */ 10 | def this() { this(new internal.GRUBuilder()) } 11 | 12 | /** Create a GruBuilder with the specified parameters. 13 | */ 14 | def this(layers: Long, inputDim: Long, hiddenDim: Long, model: ParameterCollection) { 15 | this(new internal.GRUBuilder(layers, inputDim, hiddenDim, model.model)) 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Initialize.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /** Contains methods for initializing DyNet */ 4 | object Initialize { 5 | private def initialize(params: internal.DynetParams): Unit = { 6 | internal.dynet_swig.initialize(params) 7 | } 8 | 9 | /** Initializes DyNet using the specified args 10 | * 11 | * @param args a Map of key-value pairs indicating any args for initialization 12 | */ 13 | def initialize(args: Map[String, Any] = Map.empty): Unit = { 14 | val params = new internal.DynetParams() 15 | 16 | args.get("dynet-mem") 17 | .foreach(arg => params.setMem_descriptor(arg.asInstanceOf[String])) 18 | 19 | args.get("random-seed") 20 | .foreach(arg => params.setRandom_seed(arg.asInstanceOf[Long])) 21 | 22 | args.get("weight-decay") 23 | .foreach(arg => params.setWeight_decay(arg.asInstanceOf[Float])) 24 | 25 | args.get("shared-parameters") 26 | .foreach(arg => params.setShared_parameters(arg.asInstanceOf[Boolean])) 27 | 28 | args.get("autobatch") 29 | .foreach(arg => params.setAutobatch(arg.asInstanceOf[Int])) 30 | 31 | args.get("profiling") 32 | .foreach(arg => params.setProfiling(arg.asInstanceOf[Int])) 33 | 34 | initialize(params) 35 | } 36 | } -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/ParameterCollection.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | class ParameterCollection private[dynet] (private[dynet] val model: internal.ParameterCollection) { 4 | 5 | def this() { this( new internal.ParameterCollection ) } 6 | 7 | def gradientL2Norm(): Float = model.gradient_l2_norm() 8 | def resetGradient(): Unit = model.reset_gradient() 9 | 10 | /* 11 | def getParameter(index: Long): Parameter = 12 | new Parameter(new internal.Parameter(this.model, index)) 13 | def getLookupParameter(index: Long): LookupParameter = { 14 | new LookupParameter(new internal.LookupParameter(this.model, index)) 15 | } 16 | */ 17 | 18 | def addParameters(d: Dim, scale: Float = 0.0f): Parameter = 19 | new Parameter(model.add_parameters(d.dim, scale)) 20 | def addParameters(d: Dim, init: ParameterInit): Parameter = 21 | new Parameter(model.add_parameters(d.dim, init.parameterInit)) 22 | 23 | def addLookupParameters(n: Long, d: Dim): LookupParameter = 24 | new LookupParameter(model.add_lookup_parameters(n, d.dim)) 25 | def addLookupParameters(n: Long, d: Dim, init: ParameterInit) = 26 | new LookupParameter(model.add_lookup_parameters(n, d.dim, init.parameterInit)) 27 | 28 | def projectWeights(radius: Float = 0.0f) = model.project_weights(radius) 29 | def setWeightDecayLambda(lambda: Float) = model.set_weight_decay_lambda(lambda) 30 | 31 | def parameterCount(): Long = model.parameter_count() 32 | def updatedParameterCount(): Long = model.updated_parameter_count() 33 | 34 | /* 35 | def setUpdatedParam(p: Parameter, status: Boolean): Unit = 36 | model.set_updated_param(p.parameter, status) 37 | def setUpdatedLookupParam(p: LookupParameter, status: Boolean): Unit = 38 | model.set_updated_lookup_param(p.lookupParameter, status) 39 | 40 | def isUpdatedParam(p: Parameter) = model.is_updated_param(p.parameter) 41 | def isUpdatedLookupParam(p: LookupParameter) = model.is_updated_lookup_param(p.lookupParameter) 42 | */ 43 | 44 | def parametersList(): Seq[ParameterStorage] = { 45 | val params = model.parameters_list 46 | for (i <- 0 until params.size.toInt) yield new ParameterStorage(params.get(i)) 47 | } 48 | 49 | def lookupParametersList(): Seq[LookupParameterStorage] = { 50 | val params = model.lookup_parameters_list 51 | for (i <- 0 until params.size.toInt) yield new LookupParameterStorage(params.get(i)) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Pointer.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import internal.{dynet_swig => dn} 4 | 5 | /** For the most part, SWIG makes it so you just use Scala classes where you would use C++ pointers 6 | * to class instances. The exception is for pointers to primitives. The SWIG wrappers for those are 7 | * not Scala-like at all, so here are wrapper that are easier to work with. 8 | */ 9 | class FloatPointer { 10 | val floatp = dn.new_floatp 11 | set(0f) 12 | 13 | def set(value: Float): Unit = dn.floatp_assign(floatp, value) 14 | 15 | def value(): Float = dn.floatp_value(floatp) 16 | 17 | override protected def finalize(): Unit = { 18 | dn.delete_floatp(floatp) 19 | } 20 | } 21 | 22 | class IntPointer { 23 | val intp = dn.new_intp 24 | set(0) 25 | 26 | def set(value: Int): Unit = dn.intp_assign(intp, value) 27 | 28 | def value(): Int = dn.intp_value(intp) 29 | 30 | def increment(by: Int = 1) = set(value + by) 31 | 32 | override protected def finalize(): Unit = { 33 | dn.delete_intp(intp) 34 | } 35 | } 36 | 37 | class UnsignedPointer { 38 | val uintp = dn.new_uintp() 39 | set(0) 40 | 41 | def set(value: Int): Unit = dn.uintp_assign(uintp, value) 42 | 43 | def value(): Int = dn.uintp_value(uintp).toInt 44 | 45 | override protected def finalize(): Unit = { 46 | dn.delete_uintp(uintp) 47 | } 48 | } -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Serialization.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /** New serialization, much less featureful than old serialization. */ 4 | 5 | class ModelSaver private[dynet](saver: internal.TextFileSaver) { 6 | def this(filename: String) { this(new internal.TextFileSaver(filename))} 7 | 8 | def addModel(model: ParameterCollection, key: String = ""): Unit = saver.save(model.model, key) 9 | def addParameter(p: Parameter, key: String = ""): Unit = saver.save(p.parameter, key) 10 | def addLookupParameter(p: LookupParameter, key: String = ""): Unit = saver.save(p.lookupParameter, key) 11 | 12 | def done(): Unit = saver.delete() 13 | } 14 | 15 | class ModelLoader private[dynet](loader: internal.TextFileLoader) { 16 | def this(filename: String) { this(new internal.TextFileLoader(filename))} 17 | 18 | def populateModel(model: ParameterCollection, key: String = ""): Unit = loader.populate(model.model, key) 19 | def populateParameter(p: Parameter, key: String = ""): Unit = loader.populate(p.parameter, key) 20 | def populateLookupParameter(p: LookupParameter, key: String = ""): Unit = loader.populate(p.lookupParameter, key) 21 | 22 | def done(): Unit = loader.delete() 23 | } 24 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Tensor.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /** DyNet operates on `Tensor` objects. You should never need to construct them directly; 4 | * however, here are some methods for getting values out of them. 5 | */ 6 | class Tensor private[dynet] (private[dynet] val tensor: internal.Tensor) { 7 | def toFloat(): Float = internal.dynet_swig.as_scalar(tensor) 8 | def toVector(): FloatVector = new FloatVector(internal.dynet_swig.as_vector(tensor)) 9 | def toSeq(): Seq[Float] = toVector() 10 | def getD(): Dim = new Dim(tensor.getD) 11 | } 12 | 13 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/TreeLSTMBuilder.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | 4 | abstract class TreeLSTMBuilder[A<: internal.TreeLSTMBuilder] private[dynet](private[dynet] val builder: A) 5 | extends RnnBuilder(builder) { 6 | 7 | def addInput(id: Int, children: IntVector, x: Expression): Expression = { 8 | val expr = builder.add_input(id, children.vector, x.expr) 9 | new Expression(expr) 10 | } 11 | 12 | def setNumElements(num: Int) = { 13 | builder.set_num_elements(num) 14 | } 15 | } 16 | 17 | 18 | class BidirectionalTreeLSTMBuilder private[dynet](private[dynet] builder: internal.BidirectionalTreeLSTMBuilder) 19 | extends TreeLSTMBuilder[internal.BidirectionalTreeLSTMBuilder](builder) { 20 | 21 | def this(layers: Long, inputDim: Long, hiddenDim: Long, model: ParameterCollection) { 22 | this(new internal.BidirectionalTreeLSTMBuilder(layers, inputDim, hiddenDim, model.model)) 23 | } 24 | } 25 | 26 | class UnidirectionalTreeLSTMBuilder private[dynet](private[dynet] builder: internal.UnidirectionalTreeLSTMBuilder) 27 | extends TreeLSTMBuilder[internal.UnidirectionalTreeLSTMBuilder](builder) { 28 | 29 | def this(layers: Long, inputDim: Long, hiddenDim: Long, model: ParameterCollection) { 30 | this(new internal.UnidirectionalTreeLSTMBuilder(layers, inputDim, hiddenDim, model.model)) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/Utilities.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import scala.language.implicitConversions 4 | 5 | /** Helpers that don't fit anywhere else but that might make your life easier */ 6 | object Utilities { 7 | 8 | /** Shuffle indices (for, say, training in random order) */ 9 | def shuffle(vs: IntVector): Unit = { 10 | val values = for (i <- 0 until vs.size) yield vs(i) 11 | scala.util.Random.shuffle(values).zipWithIndex.foreach { case (v, i) => vs.update(i, v) } 12 | } 13 | 14 | /** Sample from a discrete distribution */ 15 | def sample(v: FloatVector): Int = { 16 | // random pick 17 | val p = scala.util.Random.nextFloat 18 | 19 | // Seq(0f, p(0), p(0) + p(1), .... ) 20 | val cumulative = v.scanLeft(0f)(_ + _) 21 | 22 | // Return the largest index where the cumulative probability is <= p. 23 | // Since cumulative(0) is 0f, there's always at least one element in the 24 | // takeWhile, so it's ok to use .last 25 | val i = cumulative.zipWithIndex 26 | .takeWhile { case (c, i) => c <= p } 27 | .last 28 | ._2 29 | if(i == v.length) i - 1 else i 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/VariableIndex.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | /* You shouldn't need to use this. */ 4 | private[dynet] class VariableIndex(private[dynet] val index: internal.SWIGTYPE_p_dynet__VariableIndex) {} -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/examples/LinearRegression.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet.examples 2 | 3 | import edu.cmu.dynet._ 4 | 5 | import scala.language.implicitConversions 6 | 7 | object LinearRegression { 8 | def main(args: Array[String]) { 9 | Initialize.initialize() 10 | 11 | // x from 0.0 to 10.0 12 | val xs = (0 until 100).map(_.toFloat / 10) 13 | 14 | // y_i = 2 * x_i - 5 + epsilon 15 | val ys = for { 16 | x <- xs 17 | r = scala.util.Random.nextGaussian() 18 | } yield (-5.0 + 2.0 * x + 0.0033 * r).toFloat 19 | 20 | val model = new ParameterCollection 21 | val trainer = new SimpleSGDTrainer(model, 0.01f) 22 | ComputationGraph.renew() 23 | 24 | val p_W = model.addParameters(Dim(1)) 25 | val W = Expression.parameter(p_W) 26 | 27 | val p_b = model.addParameters(Dim(1)) 28 | val b = Expression.parameter(p_b) 29 | 30 | for (iter <- 1 to 20) { 31 | // track the total error for each iteration 32 | var iterLoss = 0f 33 | for ((x, y) <- xs.zip(ys)) { 34 | val prediction = W * x + b 35 | val loss = Expression.square(prediction - y) 36 | iterLoss += ComputationGraph.forward(loss).toFloat 37 | ComputationGraph.backward(loss) 38 | trainer.update() 39 | } 40 | 41 | // print the current parameter values 42 | val W_ = p_W.values.toFloat 43 | val b_ = p_b.values.toFloat 44 | println(s"(iter $iter) y = $W_ x + $b_ (error: $iterLoss)") 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/examples/WordDict.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet.examples 2 | 3 | import edu.cmu.dynet.IntVector 4 | 5 | // Stripped-down replacement for dynet/dict.h 6 | class WordDict { 7 | val mapping = new scala.collection.mutable.HashMap[String, Int] 8 | val words = new scala.collection.mutable.ArrayBuffer[String] 9 | 10 | var frozen = false 11 | var mapUnk = false 12 | var unkId = -1 13 | 14 | def size(): Int = words.size 15 | def freeze(): Unit = { frozen = true } 16 | def is_frozen(): Boolean = frozen 17 | 18 | def contains(word: String): Boolean = words.contains(word) 19 | 20 | def convert(word: String): Int = mapping.get(word) match { 21 | case Some(i) => i 22 | case None if frozen && mapUnk => unkId 23 | case None if frozen => throw new RuntimeException("unknown word in frozen dict") 24 | case None => { 25 | val index = mapping.size 26 | mapping.put(word, index) 27 | words.append(word) 28 | index 29 | } 30 | } 31 | 32 | def convert(i: Int): String = words(i) 33 | 34 | def set_unk(s: String) = { 35 | if (!frozen) throw new RuntimeException("called set_unk on unfrozen dict") 36 | if (mapUnk) throw new RuntimeException("called set_unk more than once") 37 | 38 | frozen = false 39 | unkId = convert(s) 40 | frozen = true 41 | mapUnk = true 42 | } 43 | 44 | def getUnkId(): Int = unkId 45 | } 46 | 47 | object WordDict { 48 | def read_sentence(line: String, sd: WordDict): IntVector = { 49 | new IntVector(line.split(" ").map(sd.convert).toSeq) 50 | } 51 | 52 | def read_sentence_pair(line: String, sd: WordDict, td: WordDict): (IntVector, Int) = { 53 | val Array(before, after) = line.split(""" \|\|\| """) 54 | val tokens = read_sentence(before, sd) 55 | val count = td.convert(read_sentence(after, td)(0)).toInt 56 | (tokens, count) 57 | } 58 | } -------------------------------------------------------------------------------- /contrib/swig/src/main/scala/edu/cmu/dynet/examples/XorScala.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet.examples 2 | 3 | import edu.cmu.dynet._ 4 | 5 | object XorScala { 6 | val HIDDEN_SIZE = 8 7 | val ITERATIONS = 30 8 | 9 | def main(args: Array[String]) { 10 | println("Running XOR example") 11 | Initialize.initialize() 12 | println("Dynet initialized!") 13 | val m = new ParameterCollection 14 | val sgd = new SimpleSGDTrainer(m) 15 | ComputationGraph.renew() 16 | 17 | val p_W = m.addParameters(Dim(HIDDEN_SIZE, 2)) 18 | val p_b = m.addParameters(Dim(HIDDEN_SIZE)) 19 | val p_V = m.addParameters(Dim(1, HIDDEN_SIZE)) 20 | val p_a = m.addParameters(Dim(1)) 21 | 22 | val W = Expression.parameter(p_W) 23 | val b = Expression.parameter(p_b) 24 | val V = Expression.parameter(p_V) 25 | val a = Expression.parameter(p_a) 26 | 27 | val x_values = new FloatVector(2) 28 | val x = Expression.input(Dim(2), x_values) 29 | 30 | // Need a pointer representation of scalars so updates are tracked 31 | val y_value = new FloatPointer 32 | y_value.set(0) 33 | val y = Expression.input(y_value) 34 | 35 | val h = Expression.tanh(W * x + b) 36 | val y_pred = V * h + a 37 | val loss_expr = Expression.squaredDistance(y_pred, y) 38 | 39 | println() 40 | println("Computation graphviz structure:") 41 | ComputationGraph.printGraphViz() 42 | println() 43 | println("Training...") 44 | 45 | for (iter <- 0 to ITERATIONS - 1) { 46 | var loss: Float = 0 47 | for (mi <- 0 to 3) { 48 | val x1: Boolean = mi % 2 > 0 49 | val x2: Boolean = (mi / 2) % 2 > 0 50 | x_values.update(0, if (x1) 1 else -1) 51 | x_values.update(1, if (x2) 1 else -1) 52 | y_value.set(if (x1 != x2) 1 else -1) 53 | loss += ComputationGraph.forward(loss_expr).toFloat 54 | ComputationGraph.backward(loss_expr) 55 | sgd.update() 56 | } 57 | sgd.learningRate *= 0.998f 58 | loss /= 4 59 | println("iter = " + iter + ", loss = " + loss) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /contrib/swig/src/test/scala/edu/cmu/dynet/ComputationGraphSpec.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import org.scalatest._ 4 | 5 | class ComputationGraphSpec extends FlatSpec with Matchers { 6 | 7 | "ComputationGraph" should "allow repeated calls to renew" in { 8 | for (_ <- 1 to 100) { 9 | ComputationGraph.renew() 10 | ComputationGraph.addInput(10) 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /contrib/swig/src/test/scala/edu/cmu/dynet/DimSpec.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import org.scalatest._ 4 | 5 | class DimSpec extends FlatSpec with Matchers { 6 | 7 | import Utilities._ 8 | 9 | "Dims" should "get constructed properly with the varargs constructor" in { 10 | val dim123 = Dim(1, 2, 3) 11 | 12 | dim123.nDims() shouldBe 3 13 | dim123.get(0) shouldBe 1 14 | dim123.get(1) shouldBe 2 15 | dim123.get(2) shouldBe 3 16 | } 17 | 18 | "Dims" should "be equal when they're the same" in { 19 | val dim123 = Dim(1, 2, 3) 20 | val dim12 = Dim(1, 2) 21 | val dim123_ = Dim(1, 2, 3) 22 | val dim1234 = Dim(1, 2, 3, 4) 23 | val dim124 = Dim(1, 2, 4) 24 | 25 | dim123.dim == dim123_.dim shouldBe true 26 | dim123 == dim123_ shouldBe true 27 | dim123 == dim12 shouldBe false 28 | dim123_ == dim12 shouldBe false 29 | dim123 == dim1234 shouldBe false 30 | dim123 == dim124 shouldBe false 31 | 32 | dim123.hashCode() == dim123_.hashCode() shouldBe true 33 | dim123.hashCode() == dim12.hashCode() shouldBe false 34 | dim123_.hashCode() == dim12.hashCode() shouldBe false 35 | } 36 | } -------------------------------------------------------------------------------- /contrib/swig/src/test/scala/edu/cmu/dynet/LinearRegressionSpec.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import org.scalatest._ 4 | import Matchers._ 5 | 6 | case class RegressionLine(slope: Float, intercept: Float) {} 7 | 8 | /* An end-to-end example for sanity check purposes. */ 9 | class LinearRegressionSpec extends FlatSpec with Matchers { 10 | 11 | import Utilities._ 12 | 13 | Initialize.initialize() 14 | 15 | def regress(xs: Seq[Float], ys: Seq[Float], numIterations: Int = 20): RegressionLine = { 16 | assert(xs.size > 0) 17 | assert(xs.size == ys.size) 18 | 19 | val model = new ParameterCollection 20 | val trainer = new SimpleSGDTrainer(model, 0.01f) 21 | 22 | val p_W = model.addParameters(Dim(1)) 23 | val p_b = model.addParameters(Dim(1)) 24 | 25 | val examples = xs.zip(ys) 26 | 27 | for (iter <- 1 to numIterations) { 28 | for ((x, y) <- examples) { 29 | ComputationGraph.renew() 30 | val W = Expression.parameter(p_W) 31 | val b = Expression.parameter(p_b) 32 | 33 | val prediction = W * x + b 34 | val loss = Expression.square(prediction - y) 35 | ComputationGraph.forward(loss) 36 | ComputationGraph.backward(loss) 37 | trainer.update() 38 | } 39 | } 40 | 41 | RegressionLine(p_W.values.toFloat, p_b.values.toFloat) 42 | } 43 | 44 | "regression" should "learn the correct model" in { 45 | // x from 0.0 to 10.0 46 | val xs = (0 until 100).map(_.toFloat / 10) 47 | 48 | // y_i = 2 * x_i 49 | val ys = for { 50 | x <- xs 51 | } yield (-5.0 + 2.0 * x).toFloat 52 | 53 | val result = regress(xs, ys, 20) 54 | 55 | // These are very weak bounds, 20 iterations should always get this close. 56 | result.slope shouldBe 2f +- 1f 57 | result.intercept shouldBe -5f +- 1f 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /contrib/swig/src/test/scala/edu/cmu/dynet/SampleSpec.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import org.scalatest._ 4 | 5 | class SampleSpec extends FlatSpec with Matchers { 6 | 7 | import Utilities._ 8 | 9 | "Sample" should "do the right thing" in { 10 | 11 | val probs = new FloatVector(Seq(0.1f, 0.1f, 0.7f, 0.1f)) 12 | 13 | val samples = for (_ <- 1 to 10000) yield sample(probs) 14 | 15 | // should take on all values 16 | samples.distinct.toSet shouldBe Set(0, 1, 2, 3) 17 | 18 | // at least half should be in the 70% class 19 | samples.filter(_ == 2).size > 5000 shouldBe true 20 | } 21 | } -------------------------------------------------------------------------------- /contrib/swig/src/test/scala/edu/cmu/dynet/VectorSpec.scala: -------------------------------------------------------------------------------- 1 | package edu.cmu.dynet 2 | 3 | import org.scalatest._ 4 | 5 | class VectorSpec extends FlatSpec with Matchers { 6 | 7 | import Utilities._ 8 | 9 | "FloatVector" should "construct when given a Seq[Float]" in { 10 | val fv = new FloatVector(Seq(2.3f, 4.5f)) 11 | 12 | fv.size shouldBe 2 13 | fv(0) shouldBe 2.3f 14 | fv(1) shouldBe 4.5f 15 | } 16 | 17 | /* 18 | "FloatVector" should "construct when given a Seq[Double]" in { 19 | val fv = FloatVector(Seq(2.3, 4.5)) 20 | 21 | fv.size shouldBe 2 22 | fv(0) shouldBe 2.3f 23 | fv(1) shouldBe 4.5f 24 | } 25 | */ 26 | 27 | /* 28 | "DoubleVector" should "construct when given a Seq[Double]" in { 29 | val dv = DoubleVector(Seq(2.3, 4.5, -10.2)) 30 | 31 | dv.size shouldBe 3 32 | dv.get(0) shouldBe 2.3 33 | dv.get(1) shouldBe 4.5 34 | dv.get(2) shouldBe -10.2 35 | } 36 | */ 37 | 38 | "IntVector" should "construct when given a Seq[Int]" in { 39 | val iv = new IntVector(Seq(23, 45, -102)) 40 | 41 | iv.size shouldBe 3 42 | iv(0) shouldBe 23 43 | iv(1) shouldBe 45 44 | iv(2) shouldBe -102 45 | } 46 | } -------------------------------------------------------------------------------- /doc/build_doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run Doxygen 4 | cd doxygen 5 | doxygen 6 | cd .. 7 | 8 | # Run sphinx to generate text and html doc 9 | make html 10 | -------------------------------------------------------------------------------- /doc/doxygen/references.bib: -------------------------------------------------------------------------------- 1 | @article{cho2014learning, 2 | title={Learning phrase representations using RNN encoder-decoder for statistical machine translation}, 3 | author={Cho, Kyunghyun and Van Merri{\"e}nboer, Bart and Gulcehre, Caglar and Bahdanau, Dzmitry and Bougares, Fethi and Schwenk, Holger and Bengio, Yoshua}, 4 | journal={arXiv preprint arXiv:1406.1078}, 5 | year={2014} 6 | } 7 | @inproceedings{sutskever2014sequence, 8 | title={Sequence to sequence learning with neural networks}, 9 | author={Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V}, 10 | booktitle={Advances in neural information processing systems}, 11 | pages={3104--3112}, 12 | year={2014} 13 | } 14 | @article{zeiler2012adadelta, 15 | title={ADADELTA: an adaptive learning rate method}, 16 | author={Zeiler, Matthew D}, 17 | journal={arXiv preprint arXiv:1212.5701}, 18 | year={2012} 19 | } -------------------------------------------------------------------------------- /doc/source/builders.rst: -------------------------------------------------------------------------------- 1 | Builders 2 | -------- 3 | 4 | Builders combine together various operations to implement more 5 | complicated things such as recurrent, LSTM networks or hierarchical softmax 6 | 7 | RNN Builders 8 | ~~~~~~~~~~~~ 9 | 10 | .. doxygengroup:: rnnbuilders 11 | :members: 12 | :content-only: 13 | 14 | Softmax Builders 15 | ~~~~~~~~~~~~~~~~ 16 | 17 | .. doxygengroup:: softmaxbuilders 18 | :members: 19 | :content-only: 20 | -------------------------------------------------------------------------------- /doc/source/citing.rst: -------------------------------------------------------------------------------- 1 | .. _citing: 2 | 3 | Citing/Logos 4 | ============ 5 | 6 | If you use DyNet for research, please cite our `technical report `_ as follows:: 7 | 8 | @article{dynet, 9 | title={DyNet: The Dynamic Neural Network Toolkit}, 10 | author={Graham Neubig and Chris Dyer and Yoav Goldberg and Austin Matthews and Waleed Ammar and Antonios Anastasopoulos and Miguel Ballesteros and David Chiang and Daniel Clothiaux and Trevor Cohn and Kevin Duh and Manaal Faruqui and Cynthia Gan and Dan Garrette and Yangfeng Ji and Lingpeng Kong and Adhiguna Kuncoro and Gaurav Kumar and Chaitanya Malaviya and Paul Michel and Yusuke Oda and Matthew Richardson and Naomi Saphra and Swabha Swayamdipta and Pengcheng Yin}, 11 | journal={arXiv preprint arXiv:1701.03980}, 12 | year={2017} 13 | } 14 | 15 | If you want to talk about DyNet in presentations, blog posts, etc., feel free to use one of the logos below! 16 | 17 | .. image:: images/dynet_logo.png 18 | :align: center 19 | 20 | .. image:: images/dynet_logo_bg.png 21 | :align: center 22 | 23 | .. image:: images/dynet_logomark.png 24 | :align: center 25 | -------------------------------------------------------------------------------- /doc/source/commandline.rst: -------------------------------------------------------------------------------- 1 | .. _command-line-options: 2 | 3 | Command Line Options 4 | ==================== 5 | 6 | All programs using DyNet have a few command line options. These must be 7 | specified at the very beginning of the command line, before other 8 | options. 9 | 10 | - ``--dynet-mem NUMBER``: DyNet runs by default with 512MB of memory, 11 | which is split evenly for the forward and backward steps, parameter 12 | storage as well as scratch use. This will be expanded automatically every 13 | time one of the pools runs out of memory. By setting NUMBER here, DyNet 14 | will allocate more memory immediately at the initialization stage. 15 | Note that you can also individually set the amount of memory for 16 | forward calculation, backward calculation, parameters, and scratch use by 17 | using comma separated variables ``--dynet-mem FOR,BACK,PARAM,SCRATCH``. This is 18 | useful if, for example, you are performing testing and don't need to 19 | allocate any memory for backward calculation. 20 | - ``--dynet-weight-decay NUMBER``: Adds weight decay to the parameters, 21 | which modifies each parameter w such that `w *= (1-weight_decay)` after 22 | every update. This is similar to L2 regularization, but different in a 23 | couple ways, which are noted in detail in the "Unorthodox Design" 24 | section. 25 | - ``--dynet-autobatch NUMBER``: Turns on DyNet's automatic operation 26 | batching capability. This makes it possible to speed up computation with 27 | a minimum of work. More information about this functionality can be found 28 | `here `_. 29 | - ``--dynet-gpus NUMBER``: Specify how many GPUs you want to use, if 30 | DyNet is compiled with CUDA. 31 | - ``--dynet-gpu``: Specify whether to use GPU or not. Note that it is an option for Python programs. 32 | - ``--dynet-devices CPU,GPU:1,GPU:3,GPU:0``: Specify the CPU/GPU devices that you 33 | want to use. You can the physical ID for GPU and can not specify the ID for CPU. 34 | This is an useful option working together with your multi-device code. 35 | Currently, DyNet needs you to specify the device ID explictly. 36 | The option ``--dynet-gpu-ids`` is deprecated. 37 | - ``--dynet-profiling NUMBER``: Will output information about the amount of 38 | time/memory used by each node in the graph. Profile level with ``0, 1`` and ``2``. 39 | -------------------------------------------------------------------------------- /doc/source/contributing.rst: -------------------------------------------------------------------------------- 1 | .. _contributing: 2 | 3 | Contributing to DyNet 4 | ==================== 5 | 6 | DyNet is an open source project that is only possible because of contributions from users like you! 7 | We greatly welcome any problems, whether they are to correct a bug or add a feature, and these should be made through a pull request on the `DyNet github page `_. 8 | 9 | Below are some guidelines to guarantee consistency. 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | code_style 15 | doc_style 16 | contributors 17 | -------------------------------------------------------------------------------- /doc/source/contributors.rst: -------------------------------------------------------------------------------- 1 | Contributors 2 | ============ 3 | 4 | DyNet was started at `Carnegie Mellon University `_ by Chris Dyer (now at `DeepMind `_), and the project is now led by Graham Neubig (`CMU `_) and Yoav Goldberg (`Bar Ilan University `_). 5 | It relies `contributors `_ from a wide variety of institutions, including: 6 | 7 | .. image:: images/cmu.png 8 | :target: http://www.cmu.edu 9 | 10 | .. image:: images/biu.jpg 11 | :target: https://www1.biu.ac.il/indexE.php 12 | 13 | .. image:: images/ai2.png 14 | :target: http://allenai.org 15 | 16 | .. image:: images/naist.png 17 | :target: http://www.naist.jp/en/ 18 | 19 | .. image:: images/petuum.png 20 | :target: http://petuum.com 21 | 22 | .. image:: images/uw.jpg 23 | :target: http://washington.edu 24 | 25 | and many others! 26 | -------------------------------------------------------------------------------- /doc/source/cpp_ref.rst: -------------------------------------------------------------------------------- 1 | C++ Reference manual 2 | =================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | core 8 | operations 9 | builders 10 | optimizers -------------------------------------------------------------------------------- /doc/source/examples.rst: -------------------------------------------------------------------------------- 1 | .. mdinclude:: ../../examples/README.md 2 | 3 | -------------------------------------------------------------------------------- /doc/source/images/ai2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/ai2.png -------------------------------------------------------------------------------- /doc/source/images/autobatch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/autobatch.gif -------------------------------------------------------------------------------- /doc/source/images/biu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/biu.jpg -------------------------------------------------------------------------------- /doc/source/images/cmu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/cmu.png -------------------------------------------------------------------------------- /doc/source/images/dynet_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/dynet_logo.png -------------------------------------------------------------------------------- /doc/source/images/dynet_logo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/dynet_logo_bg.png -------------------------------------------------------------------------------- /doc/source/images/dynet_logo_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/dynet_logo_white.png -------------------------------------------------------------------------------- /doc/source/images/dynet_logo_white_and_color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/dynet_logo_white_and_color.png -------------------------------------------------------------------------------- /doc/source/images/dynet_logomark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/dynet_logomark.png -------------------------------------------------------------------------------- /doc/source/images/minibatch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/minibatch.png -------------------------------------------------------------------------------- /doc/source/images/naist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/naist.png -------------------------------------------------------------------------------- /doc/source/images/petuum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/petuum.png -------------------------------------------------------------------------------- /doc/source/images/uw.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/doc/source/images/uw.jpg -------------------------------------------------------------------------------- /doc/source/multiprocessing.rst: -------------------------------------------------------------------------------- 1 | Multi-processing 2 | ================ 3 | 4 | In addition to minibatch support, the DyNet C++ API also supports training models using many CPU cores (Python support is pending). 5 | This is particularly useful when performing training of networks that are not conducive to simple mini-batching, such as tree-structured networks. 6 | 7 | DyNet abstracts most of the behind-the-scenes grit from the user. 8 | The user defines a function to be called for each datum in the training data set, and passes this function, along with an array of data, to DyNet. 9 | Internally, DyNet launches a pool of training processes and automatically handles passing data examples to each worker. 10 | Each worker process individually processes a datum, computing the results of the forward and backward passes, computes gradients with respect to each parameter, and passes these results back to the parent process via a shared memory variable. 11 | Whenever the parent process, which is also processing data, completes a gradient computation, it averages all of the gradients currently in the shared memory gradient storage and updates all parameters with respect to that average gradient. 12 | In this way running training on ``n`` cores is similar to training with a stochastic minibatch size with expected value of approximately ``n``. 13 | This method is quite efficient, achieving nearly linear speedups with increasing numbers of cores, due to its lockless nature. 14 | 15 | Examples of how to use the multi-processing API can be found in the ``xor-mp`` and ``rnnlm-mp`` sections of the ``examples/cpp`` directory. 16 | -------------------------------------------------------------------------------- /doc/source/optimizers.rst: -------------------------------------------------------------------------------- 1 | Optimizers 2 | ---------- 3 | 4 | The various optimizers that you can use to tune your parameters 5 | 6 | .. doxygengroup:: optimizers 7 | :members: 8 | :content-only: 9 | -------------------------------------------------------------------------------- /doc/source/other_languages.rst: -------------------------------------------------------------------------------- 1 | Installing/Using in Other Languages 2 | =================================== 3 | 4 | DyNet mainly supports the C++ and Python bindings, but there are also bindings for 5 | other languages that have been contributed by the community. 6 | 7 | C APIs 8 | ---------- 9 | 10 | DyNet provides the `C APIs `_ that can be used to build bindings for other languages. Please see the README linked above for details. 11 | 12 | Rust 13 | ---------- 14 | 15 | DyNet has `Rust Bindings `_ 16 | developed by Hiroki Teranishi at Nara Institute of Science and Technology. Please see 17 | the README linked above for details. 18 | 19 | Scala/Java 20 | ---------- 21 | 22 | DyNet has `Scala/Java Bindings `_ 23 | developed by Joel Grus at the Allen Institute for Artificial Intelligence. Please see 24 | the README linked above for details. 25 | 26 | The `CLU Lab `_ at the University of Arizona has packaged 27 | the Scala/Java bindings into a single, multi-platform jar file that can be incorporated 28 | into a project as a simple library dependency. This 29 | `fatdynet `_ is also able to read models directly 30 | from jar or zip files so that they can be similarly deployed. 31 | -------------------------------------------------------------------------------- /doc/source/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=1.4 2 | breathe>=4.5 3 | nbsphinx 4 | ipykernel 5 | sphinxcontrib-napoleon 6 | m2r 7 | -------------------------------------------------------------------------------- /doc/source/tutorial.rst: -------------------------------------------------------------------------------- 1 | DyNet Tutorial 2 | ============== 3 | 4 | C++ Tutorial 5 | ------------ 6 | 7 | See the tutorials for the C++ version of DyNet 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | cpp_basic_tutorial 13 | cpp_saving_tutorial 14 | 15 | Python Tutorial 16 | --------------- 17 | 18 | Guided examples in Python can be found below : 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | :titlesonly: 23 | 24 | tutorials_notebooks/tutorial-1-xor.ipynb 25 | tutorials_notebooks/API.ipynb 26 | tutorials_notebooks/RNNs.ipynb 27 | tutorials_notebooks/Autobatching.ipynb 28 | python_saving_tutorial 29 | 30 | A more comprehensive tutorial can be found here_ (EMNLP 2016 tutorial). 31 | 32 | 33 | .. _here: https://github.com/clab/dynet_tutorial_examples 34 | -------------------------------------------------------------------------------- /docker/Dockerfile-i686: -------------------------------------------------------------------------------- 1 | FROM quay.io/pypa/manylinux1_i686 2 | MAINTAINER Daniel Hershcovich 3 | ADD .travis/ /root/.travis 4 | WORKDIR /root/.travis 5 | -------------------------------------------------------------------------------- /docker/Dockerfile-x86_64: -------------------------------------------------------------------------------- 1 | FROM quay.io/pypa/manylinux1_x86_64 2 | MAINTAINER Daniel Hershcovich 3 | ADD .travis/ /root/.travis 4 | WORKDIR /root/.travis 5 | -------------------------------------------------------------------------------- /dynet/aligned-mem-pool.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_ALIGNED_MEM_POOL_H 2 | #define DYNET_ALIGNED_MEM_POOL_H 3 | 4 | #include 5 | #include "dynet/mem.h" 6 | #include "dynet/globals.h" 7 | #include "dynet/except.h" 8 | 9 | namespace dynet { 10 | 11 | class InternalMemoryPool { 12 | public: 13 | explicit InternalMemoryPool(const std::string & name, size_t cap, MemAllocator* a) : name(name), a(a) { 14 | sys_alloc(cap); 15 | zero_all(); 16 | } 17 | 18 | ~InternalMemoryPool() { 19 | a->free(mem); 20 | } 21 | 22 | void* allocate(size_t n); 23 | 24 | void free() { 25 | //std::cerr << "freeing " << used << " bytes\n"; 26 | used = 0; 27 | } 28 | // zeros out the amount of allocations 29 | void zero_allocated_memory() { 30 | if (used == 0) return; 31 | a->zero(mem, used); 32 | } 33 | 34 | size_t used; 35 | private: 36 | void sys_alloc(size_t cap); 37 | 38 | void zero_all() { 39 | a->zero(mem, capacity); 40 | } 41 | std::string name; 42 | size_t capacity; 43 | MemAllocator* a; 44 | void* mem; 45 | }; 46 | 47 | class AlignedMemoryPool { 48 | public: 49 | explicit AlignedMemoryPool(const std::string &name, size_t initial_cap, MemAllocator *a, size_t expanding_unit = 1<<24); 50 | ~AlignedMemoryPool(); 51 | 52 | void* allocate(size_t n); 53 | 54 | void free(); 55 | 56 | void zero_allocated_memory(); 57 | 58 | size_t used(); 59 | void set_used(size_t s); 60 | size_t get_cap(); 61 | 62 | private: 63 | std::string name; 64 | std::vector pools; 65 | size_t cap; 66 | int current; 67 | MemAllocator* a; 68 | size_t expanding_unit; 69 | }; 70 | 71 | } // namespace dynet 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /dynet/c2w.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_C2W_H_ 2 | #define DYNET_C2W_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "dynet/dynet.h" 8 | #include "dynet/model.h" 9 | #include "dynet/lstm.h" 10 | 11 | namespace dynet { 12 | 13 | // computes a representation of a word by reading characters 14 | // one at a time 15 | struct C2WBuilder { 16 | LSTMBuilder fc2w; 17 | LSTMBuilder rc2w; 18 | LookupParameter p_lookup; 19 | std::vector words; 20 | std::map wordid2vi; 21 | ParameterCollection local_model; 22 | explicit C2WBuilder(int vocab_size, 23 | unsigned layers, 24 | unsigned input_dim, 25 | unsigned hidden_dim, 26 | ParameterCollection& m) : 27 | p_lookup(m->add_lookup_parameters(vocab_size, {input_dim})) { 28 | local_model = m.add_subcollection("--c2w-builder"); 29 | fc2w = LSTMBuilder(layers, input_dim, hidden_dim, local_model); 30 | rc2w = LSTMBuilder(layers, input_dim, hidden_dim, local_model); 31 | } 32 | void new_graph(ComputationGraph* cg) { 33 | words.clear(); 34 | fc2w.new_graph(cg); 35 | rc2w.new_graph(cg); 36 | } 37 | // compute a composed representation of a word out of characters 38 | // wordid should be a unique index for each word *type* in the graph being built 39 | VariableIndex add_word(int word_id, const std::vector& chars, ComputationGraph* cg) { 40 | auto it = wordid2vi.find(word_id); 41 | if (it == wordid2vi.end()) { 42 | fc2w.start_new_sequence(cg); 43 | rc2w.start_new_sequence(cg); 44 | std::vector ins(chars.size()); 45 | std::map c2i; 46 | for (unsigned i = 0; i < ins.size(); ++i) { 47 | VariableIndex& v = c2i[chars[i]]; 48 | if (!v) v = cg->add_lookup(p_lookup, chars[i]); 49 | ins[i] = v; 50 | fc2w.add_input(v, cg); 51 | } 52 | for (int i = ins.size() - 1; i >= 0; --i) 53 | rc2w.add_input(ins[i], cg); 54 | VariableIndex i_concat = cg->add_function({fc2w.back(), rc2w.back()}); 55 | it = wordid2vi.insert(std::make_pair(word_id, i_concat)).first; 56 | } 57 | return it->second; 58 | } 59 | ParameterCollection & get_parameter_collection() { 60 | return local_model; 61 | } 62 | }; 63 | 64 | } // namespace dynet 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /dynet/deep-lstm.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_DEEP_LSTM_H_ 2 | #define DYNET_DEEP_LSTM_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/rnn.h" 6 | #include "dynet/expr.h" 7 | 8 | namespace dynet { 9 | 10 | class ParameterCollection; 11 | 12 | struct DeepLSTMBuilder : public RNNBuilder { 13 | DeepLSTMBuilder() = default; 14 | explicit DeepLSTMBuilder(unsigned layers, 15 | unsigned input_dim, 16 | unsigned hidden_dim, 17 | ParameterCollection& model); 18 | 19 | Expression back() const override { return h.back().back(); } 20 | std::vector final_h() const override { return (h.size() == 0 ? h0 : h.back()); } 21 | std::vector final_s() const override { 22 | std::vector ret = (c.size() == 0 ? c0 : c.back()); 23 | for(auto my_h : final_h()) ret.push_back(my_h); 24 | return ret; 25 | } 26 | /** 27 | * \brief Get parameters in DeepLSTMBuilder 28 | * \return list of points to ParameterStorage object 29 | */ 30 | ParameterCollection & get_parameter_collection() override; 31 | protected: 32 | void new_graph_impl(ComputationGraph& cg, bool update) override; 33 | void start_new_sequence_impl(const std::vector& h0) override; 34 | Expression add_input_impl(int prev, const Expression& x) override; 35 | 36 | public: 37 | ParameterCollection local_model; 38 | // first index is layer, then ... 39 | std::vector> params; 40 | 41 | // first index is layer, then ... 42 | std::vector> param_vars; 43 | 44 | // first index is time, second is layer 45 | std::vector> h, c; 46 | std::vector o; 47 | 48 | // initial values of h and c at each layer 49 | // - both default to zero matrix input 50 | bool has_initial_state; // if this is false, treat h0 and c0 as 0 51 | std::vector h0; 52 | std::vector c0; 53 | unsigned layers; 54 | }; 55 | 56 | } // namespace dynet 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /dynet/device-structs.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_DEVICE_STRUCTS_H 2 | #define DYNET_DEVICE_STRUCTS_H 3 | 4 | namespace dynet { 5 | 6 | enum class DeviceType {CPU, GPU}; 7 | 8 | /* 9 | * FXS -> forward pass memory 10 | * DEDFS -> backward pass memory 11 | * PS -> parameter memory 12 | * SCS -> scratch memory (for use in temporary calculations) 13 | * NONE -> when a memory pool has not been assigned yet 14 | */ 15 | enum class DeviceMempool {FXS = 0, DEDFS = 1, PS = 2, SCS = 3, NONE = 4}; 16 | 17 | struct ComputationGraph; // TODO is there a nicer way to resolve this cyclic dependency? 18 | struct Tensor; 19 | 20 | struct DeviceMempoolSizes { 21 | size_t used[4]; 22 | DeviceMempoolSizes() = default; 23 | DeviceMempoolSizes(size_t total_s); 24 | DeviceMempoolSizes(size_t fxs_s, size_t dEdfs_s, size_t ps_s, size_t sc_s); 25 | DeviceMempoolSizes(const std::string & descriptor); 26 | }; 27 | 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /dynet/dict.cc: -------------------------------------------------------------------------------- 1 | #include "dict.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | 9 | namespace dynet { 10 | 11 | std::vector read_sentence(const std::string& line, Dict& sd) { 12 | std::istringstream in(line); 13 | std::string word; 14 | std::vector res; 15 | while(in) { 16 | in >> word; 17 | if (!in || word.empty()) break; 18 | res.push_back(sd.convert(word)); 19 | } 20 | return res; 21 | } 22 | 23 | void read_sentence_pair(const std::string& line, std::vector& s, Dict& sd, std::vector& t, Dict& td) { 24 | std::istringstream in(line); 25 | std::string word; 26 | std::string sep = "|||"; 27 | Dict* d = &sd; 28 | std::vector* v = &s; 29 | while(in) { 30 | in >> word; 31 | if (!in) break; 32 | if (word == sep) { d = &td; v = &t; continue; } 33 | v->push_back(d->convert(word)); 34 | } 35 | } 36 | 37 | } // namespace dynet 38 | 39 | -------------------------------------------------------------------------------- /dynet/dim.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/dim.h" 2 | 3 | #include 4 | 5 | using namespace std; 6 | 7 | namespace dynet { 8 | 9 | ostream& operator<<(ostream& os, const Dim& d) { 10 | os << '{'; 11 | for (unsigned i = 0; i < d.nd; ++i) { 12 | if (i) os << ','; 13 | os << d.d[i]; 14 | } 15 | if(d.bd != 1) os << 'X' << d.bd; 16 | return os << '}'; 17 | } 18 | 19 | void Dim::print_profile(ostream& os) const { 20 | os << '{'; 21 | for (unsigned i = 0; i < nd; ++i) { 22 | if (i) os << ','; 23 | os << d[i]; 24 | } 25 | os << '}'; 26 | } 27 | 28 | ostream& operator<<(ostream& os, const vector& ds) { 29 | os << '['; 30 | for (unsigned i = 0; i < ds.size(); ++i) 31 | os << (i ? " " : "") << ds[i]; 32 | return os << ']'; 33 | } 34 | 35 | istream& operator>>(istream& is, Dim& d) { 36 | char place_holder; 37 | is >> place_holder; 38 | d.resize(DYNET_MAX_TENSOR_DIM); 39 | bool batch_flag = false; 40 | unsigned i = 0; 41 | for (; i < DYNET_MAX_TENSOR_DIM + 1; ++i) { 42 | if (i) { 43 | is >> place_holder; 44 | if (place_holder == 'X') { 45 | batch_flag = true; 46 | break; 47 | } else if (place_holder == '}') { 48 | break; 49 | } 50 | } 51 | is >> d.d[i]; 52 | } 53 | d.resize(i); 54 | if (batch_flag) { 55 | is >> d.bd >> place_holder; 56 | } 57 | return is; 58 | } 59 | 60 | } // namespace dynet 61 | 62 | -------------------------------------------------------------------------------- /dynet/dynet-helper.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_HELPER_H_ 2 | #define DYNET_HELPER_H_ 3 | 4 | #include 5 | 6 | /// helper functions 7 | 8 | namespace dynet { 9 | 10 | /** 11 | this fix a compilation problem in cygwin 12 | */ 13 | #if defined(__CYGWIN__) 14 | template 15 | inline std::string to_string(T value) 16 | { 17 | std::ostringstream os; 18 | os << value; 19 | return os.str(); 20 | } 21 | #endif 22 | 23 | } // namespace dynet 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /dynet/globals.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/globals.h" 2 | #include "dynet/devices.h" 3 | #include "dynet/timing.h" 4 | 5 | #ifdef HAVE_CUDA 6 | #include "dynet/cuda.h" 7 | #endif 8 | 9 | namespace dynet { 10 | 11 | std::mt19937* rndeng = nullptr; 12 | Device* default_device = nullptr; 13 | float default_weight_decay_lambda; 14 | int autobatch_flag; 15 | int profiling_flag = 0; 16 | NamedTimer timer; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /dynet/globals.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GLOBALS_H 2 | #define DYNET_GLOBALS_H 3 | 4 | #include 5 | 6 | namespace dynet { 7 | 8 | class Device; 9 | class NamedTimer; 10 | 11 | extern std::mt19937* rndeng; 12 | extern Device* default_device; 13 | extern NamedTimer timer; // debug timing in executors. 14 | 15 | } // namespace dynet 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /dynet/gpu-kernels.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GPU_KERNELS_H 2 | #define DYNET_GPU_KERNELS_H 3 | 4 | #include "dynet/cuda.h" 5 | 6 | namespace dynet { 7 | namespace gpu { 8 | 9 | template 10 | __global__ void unaryExprKernel(int n, const float* x, float* y, Func func) { 11 | int i = threadIdx.x + blockIdx.x * blockDim.x; 12 | while (i < n) { 13 | y[i] = func(x[i]); 14 | i += gridDim.x * blockDim.x; 15 | } 16 | } 17 | 18 | template 19 | __global__ void accUnaryExprKernel(int n, const float* x, float* y, Func func) { 20 | int i = threadIdx.x + blockIdx.x * blockDim.x; 21 | while (i < n) { 22 | y[i] += func(x[i]); 23 | i += gridDim.x * blockDim.x; 24 | } 25 | } 26 | 27 | template 28 | __global__ void binaryExprKernel(int n, const float* x0, const float* x1, float* y, Func func) { 29 | int i = threadIdx.x + blockIdx.x * blockDim.x; 30 | while (i < n) { 31 | y[i] = func(x0[i], x1[i]); 32 | i += gridDim.x * blockDim.x; 33 | } 34 | } 35 | 36 | template 37 | __global__ void accBinaryExprKernel(int n, const float* x0, const float* x1, float* y, Func func) { 38 | int i = threadIdx.x + blockIdx.x * blockDim.x; 39 | while (i < n) { 40 | y[i] += func(x0[i], x1[i]); 41 | i += gridDim.x * blockDim.x; 42 | } 43 | } 44 | 45 | template 46 | __global__ void slowReduceKernel(int n, const float* x0, const float* x1, float* y, Func func) { 47 | float ty = 0; 48 | // THIS IS BAD - FIX THIS TO MAKE IT FAST 49 | for (int i = 0; i < n; ++i) 50 | ty += func(x0[i], x1[i]); 51 | y[0] = ty; 52 | } 53 | 54 | } // namespace gpu 55 | } // namespace dynet 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /dynet/gpu-ops.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GPU_OPS_H 2 | #define DYNET_GPU_OPS_H 3 | 4 | #include 5 | 6 | namespace dynet { 7 | namespace gpu { 8 | 9 | void dense_to_sparse_assign(int n, const unsigned int* ids, float* src, float* trg); 10 | void sparse_to_dense_assign(int n, const unsigned int* ids, float* src, float* trg); 11 | void dense_to_sparse_subtract(int n, const unsigned int* ids, float* src, float* trg); 12 | void sparse_to_dense_block_assign_and_multiply(int n, const unsigned *idx, int bsize, float mult, float *src, float *trg); 13 | void dense_to_sparse_block_add(int n, const unsigned* ids, int bsize, float* src, float* trg); 14 | void parallel_memcpy(int num_seqs, int max_len, float **src, float **trg, std::size_t *len); 15 | void parallel_accumulate(int num_seqs, int max_len, float **src, float **trg, std::size_t *len); 16 | } // namespace gpu 17 | } // namespace dynet 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /dynet/grad-check.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GRAD_CHECK_H 2 | #define DYNET_GRAD_CHECK_H 3 | 4 | #include "dynet/expr.h" 5 | 6 | namespace dynet { 7 | 8 | class ParameterCollection; 9 | struct ComputationGraph; 10 | 11 | // verbosity is zero for silence, one for only printing errors, two for everything 12 | bool check_grad(ParameterCollection& m, Expression& expr, int verbosity = 1); 13 | 14 | } // namespace dynet 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /dynet/graph.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/graph.h" 2 | #include "dynet/dynet.h" 3 | #include 4 | #include "dynet/dynet-helper.h" 5 | 6 | using namespace std; 7 | 8 | namespace dynet { 9 | 10 | void graph_optimize(ComputationGraph* cg) { 11 | // topo sort 12 | vector& nodes = cg->nodes; 13 | vector longest_paths(nodes.size()); 14 | for (unsigned i = 0; i < nodes.size(); ++i) { 15 | auto& v = *nodes[i]; // vertex v_i 16 | auto& lp = longest_paths[i]; // distance to v_i 17 | for (auto e : v.args) { 18 | int weight = 0; 19 | if (v.args.size() == 7) weight = 1; 20 | int pte = longest_paths[e] + weight; 21 | if (pte > lp) lp = pte; 22 | } 23 | } 24 | for (unsigned i = 0; i < nodes.size(); ++i) { 25 | vector x; 26 | for (auto e : nodes[i]->args) { 27 | x.push_back(string("x") + to_string(e)); 28 | } 29 | cerr << "LONGEST PATH: " << longest_paths[i] << "\tx" << i << " = " << nodes[i]->as_string(x) << endl; 30 | } 31 | throw std::runtime_error("Failure in graph optimization");// DEBUGGING 32 | } 33 | 34 | } // namespaiice dynet 35 | -------------------------------------------------------------------------------- /dynet/graph.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GRAPH_H 2 | #define DYNET_GRAPH_H 3 | 4 | namespace dynet { 5 | struct ComputationGraph; 6 | void graph_optimize(ComputationGraph* cg); 7 | } // namespace dynet 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /dynet/gru.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_GRU_H_ 2 | #define DYNET_GRU_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/rnn.h" 6 | 7 | namespace dynet { 8 | 9 | class ParameterCollection; 10 | 11 | struct GRUBuilder : public RNNBuilder { 12 | GRUBuilder() = default; 13 | explicit GRUBuilder(unsigned layers, 14 | unsigned input_dim, 15 | unsigned hidden_dim, 16 | ParameterCollection& model); 17 | Expression back() const override { return (cur == -1? h0.back() : h[cur].back()); } 18 | std::vector final_h() const override { return (h.size() == 0 ? h0 : h.back()); } 19 | std::vector final_s() const override { return final_h(); } 20 | std::vector get_h(RNNPointer i) const override { return (i == -1 ? h0 : h[i]); } 21 | std::vector get_s(RNNPointer i) const override { return get_h(i); } 22 | unsigned num_h0_components() const override { return layers; } 23 | void copy(const RNNBuilder & params) override; 24 | ParameterCollection & get_parameter_collection() override; 25 | 26 | 27 | // first index is layer, then ... 28 | std::vector> params; 29 | // first index is layer, then ... 30 | std::vector> param_vars; 31 | 32 | protected: 33 | void new_graph_impl(ComputationGraph& cg, bool update) override; 34 | void start_new_sequence_impl(const std::vector& h0) override; 35 | Expression add_input_impl(int prev, const Expression& x) override; 36 | Expression set_h_impl(int prev, const std::vector& h_new) override; 37 | Expression set_s_impl(int prev, const std::vector& s_new) override; 38 | 39 | ParameterCollection local_model; 40 | 41 | // first index is time, second is layer 42 | std::vector> h; 43 | 44 | // initial values of h at each layer 45 | // - default to zero matrix input 46 | std::vector h0; 47 | 48 | unsigned hidden_dim; 49 | unsigned layers; 50 | }; 51 | 52 | } // namespace dynet 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /dynet/init.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_EIGEN_INIT_H 2 | #define DYNET_EIGEN_INIT_H 3 | 4 | #include 5 | #include 6 | 7 | namespace dynet { 8 | 9 | extern float default_weight_decay_lambda; 10 | extern int autobatch_flag; 11 | extern int profiling_flag; 12 | 13 | /** 14 | * \brief Represents general parameters for dynet 15 | * 16 | */ 17 | struct DynetParams { 18 | DynetParams(); 19 | ~DynetParams(); 20 | unsigned random_seed; /**< The seed for random number generation */ 21 | std::string mem_descriptor; /**< Total memory to be allocated for Dynet */ 22 | float weight_decay; /**< Weight decay rate for L2 regularization */ 23 | int autobatch; /**< Whether to autobatch or not */ 24 | int profiling; /**< Whether to show autobatch debug info or not */ 25 | bool shared_parameters; /**< TO DOCUMENT */ 26 | bool ngpus_requested; /**< GPUs requested by number */ 27 | bool ids_requested; /**< GPUs requested by ids */ 28 | bool cpu_requested; /**< CPU requested in multi-device case */ 29 | int requested_gpus; /**< Number of requested GPUs */ 30 | std::vector gpu_mask; /**< List of required GPUs by ids */ 31 | }; 32 | 33 | DynetParams extract_dynet_params(int& argc, char**& argv, bool shared_parameters = false); 34 | void initialize(DynetParams& params); 35 | void initialize(int& argc, char**& argv, bool shared_parameters = false); 36 | void cleanup(); 37 | 38 | /** 39 | * \brief Resets random number generators 40 | * 41 | */ 42 | void reset_rng(unsigned seed); 43 | 44 | } // namespace dynet 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /dynet/mem.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/mem.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #if !_WINDOWS 7 | #include 8 | #include 9 | #endif 10 | 11 | #include 12 | #if !_WINDOWS 13 | #include 14 | #endif 15 | #include "dynet/except.h" 16 | #include "dynet/devices.h" 17 | #if HAVE_CUDA 18 | #include "dynet/cuda.h" 19 | #include 20 | #include 21 | #endif 22 | 23 | using namespace std; 24 | 25 | namespace dynet { 26 | 27 | MemAllocator::~MemAllocator() {} 28 | 29 | void* CPUAllocator::malloc(size_t n) { 30 | void* ptr = _mm_malloc(n, align); 31 | if (!ptr) { 32 | show_pool_mem_info(); 33 | cerr << "CPU memory allocation failed n=" << n << " align=" << align << endl; 34 | throw dynet::out_of_memory("CPU memory allocation failed"); 35 | } 36 | return ptr; 37 | } 38 | 39 | void CPUAllocator::free(void* mem) { 40 | _mm_free(mem); 41 | } 42 | 43 | void CPUAllocator::zero(void* p, size_t n) { 44 | memset(p, 0, n); 45 | } 46 | 47 | void* SharedAllocator::malloc(size_t n) { 48 | #if _WINDOWS 49 | cerr << "Shared memory not supported in Windows" << endl; 50 | throw dynet::out_of_memory("Shared memory allocation failed"); 51 | #else 52 | void* ptr = mmap(NULL, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_SHARED, -1, 0); 53 | if (ptr == MAP_FAILED) { 54 | show_pool_mem_info(); 55 | cerr << "Shared memory allocation failed n=" << n << endl; 56 | throw dynet::out_of_memory("Shared memory allocation failed"); 57 | } 58 | return ptr; 59 | #endif 60 | } 61 | 62 | void SharedAllocator::free(void* mem) { 63 | // munmap(mem, n); 64 | } 65 | 66 | void SharedAllocator::zero(void* p, size_t n) { 67 | memset(p, 0, n); 68 | } 69 | 70 | #if HAVE_CUDA 71 | void* GPUAllocator::malloc(size_t n) { 72 | void* ptr = nullptr; 73 | CUDA_CHECK(cudaSetDevice(devid)); 74 | CUDA_CHECK(cudaMalloc(&ptr, n)); 75 | if (!ptr) { 76 | show_pool_mem_info(); 77 | cerr << "GPU memory allocation failed n=" << n << endl; 78 | throw dynet::out_of_memory("GPU memory allocation failed"); 79 | } 80 | return ptr; 81 | } 82 | 83 | void GPUAllocator::free(void* mem) { 84 | CUDA_CHECK(cudaFree(mem)); 85 | } 86 | 87 | void GPUAllocator::zero(void* p, size_t n) { 88 | CUDA_CHECK(cudaSetDevice(devid)); 89 | CUDA_CHECK(cudaMemsetAsync(p, 0, n)); 90 | } 91 | 92 | #endif 93 | 94 | } // namespace dynet 95 | -------------------------------------------------------------------------------- /dynet/mem.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_MEM_H 2 | #define DYNET_MEM_H 3 | 4 | #include 5 | 6 | namespace dynet { 7 | 8 | // allocates memory from the device (CPU, GPU) 9 | // only used to create the memory pools 10 | // creates alignment appropriate for that device 11 | struct MemAllocator { 12 | explicit MemAllocator(int align) : align(align) {} 13 | MemAllocator(const MemAllocator&) = delete; 14 | MemAllocator& operator=(const MemAllocator&) = delete; 15 | virtual ~MemAllocator(); 16 | virtual void* malloc(std::size_t n) = 0; 17 | virtual void free(void* mem) = 0; 18 | virtual void zero(void* p, std::size_t n) = 0; 19 | inline std::size_t round_up_align(std::size_t n) const { 20 | if (align < 2) return n; 21 | return ((n + align - 1) / align) * align; 22 | } 23 | const int align; 24 | }; 25 | 26 | struct CPUAllocator : public MemAllocator { 27 | CPUAllocator() : MemAllocator(32) {} 28 | void* malloc(std::size_t n) override; 29 | void free(void* mem) override; 30 | void zero(void* p, std::size_t n) override; 31 | }; 32 | 33 | struct SharedAllocator : public MemAllocator { 34 | SharedAllocator() : MemAllocator(32) {} 35 | void* malloc(std::size_t n) override; 36 | void free(void* mem) override; 37 | void zero(void* p, std::size_t n) override; 38 | }; 39 | 40 | #if HAVE_CUDA 41 | struct GPUAllocator : public MemAllocator { 42 | explicit GPUAllocator(int devid) : MemAllocator(256), devid(devid) {} 43 | void* malloc(std::size_t n) override; 44 | void free(void* mem) override; 45 | void zero(void* p, std::size_t n) override; 46 | const int devid; 47 | }; 48 | #endif 49 | 50 | } // namespace dynet 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /dynet/nodes-affinetransform.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_AFFINETRANSFORM_H_ 2 | #define DYNET_NODES_AFFINETRANSFORM_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // fx = xs[0] + \sum_{i=1, 3 ...} xs[i] * xs[i+1] 10 | struct AffineTransform : public Node { 11 | template explicit AffineTransform(const T& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 14 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 15 | virtual void autobatch_reshape(const ComputationGraph & cg, 16 | const std::vector & batch_ids, 17 | const std::vector & concat, 18 | std::vector& xs, 19 | Tensor& fx) const override { 20 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 21 | } 22 | DYNET_NODE_DEFINE_DEV_IMPL() 23 | mutable float* dEdf_mem; 24 | }; 25 | 26 | } // namespace dynet 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /dynet/nodes-argmax.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_ARGMAX_H_ 2 | #define DYNET_NODES_ARGMAX_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y_i = 1 if i = argmax(x) else 0 10 | struct Argmax : public Node { 11 | explicit Argmax(const std::initializer_list& a, unsigned d, bool straight_through=false) : Node(a), d(d), straight_through(straight_through) {} 12 | DYNET_NODE_DEFINE_DEV_IMPL() 13 | virtual bool supports_multibatch() const override { return true; } 14 | size_t aux_storage_size() const override; 15 | unsigned d; 16 | bool straight_through; 17 | }; 18 | 19 | } // namespace dynet 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /dynet/nodes-arith-const.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_ARITH_CONST_H_ 2 | #define DYNET_NODES_ARITH_CONST_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = c + x_1 10 | // (c is a std::vector or matrix of the constant, usually 1, but can be configured) 11 | struct ConstantPlusX : public Node { 12 | explicit ConstantPlusX(const std::initializer_list& a, real o) : Node(a), c(o) {} 13 | virtual bool supports_multibatch() const override { return true; } 14 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::plus_const); s.add_float(c); return sm.get_idx(s); } 15 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 16 | DYNET_NODE_DEFINE_DEV_IMPL() 17 | real c; 18 | }; 19 | 20 | // y = c - x_1 21 | // (c is a std::vector or matrix of the constant, usually 1, but can be configured) 22 | struct ConstantMinusX : public Node { 23 | explicit ConstantMinusX(const std::initializer_list& a, real o) : Node(a), c(o) {} 24 | virtual bool supports_multibatch() const override { return true; } 25 | DYNET_NODE_DEFINE_DEV_IMPL() 26 | real c; 27 | }; 28 | 29 | // y = alpha * x_1 30 | struct ConstScalarMultiply : public Node { 31 | explicit ConstScalarMultiply(const std::initializer_list& a, float alpha) : Node(a), alpha(alpha) {} 32 | virtual bool supports_multibatch() const override { return true; } 33 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::scalar_mult); s.add_float(alpha); return sm.get_idx(s); } 34 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 35 | DYNET_NODE_DEFINE_DEV_IMPL() 36 | float alpha; 37 | }; 38 | 39 | } // namespace dynet 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /dynet/nodes-arith-sum.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_ARITH_SUM_H_ 2 | #define DYNET_NODES_ARITH_SUM_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = \sum_i x_i 10 | struct Sum : public Node { 11 | template explicit Sum(const T& a) : Node(a) {} 12 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 13 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 14 | virtual void autobatch_reshape(const ComputationGraph & cg, 15 | const std::vector & batch_ids, 16 | const std::vector & concat, 17 | std::vector& xs, 18 | Tensor& fx) const override { 19 | if(dim.bd != 1) 20 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 21 | } 22 | DYNET_NODE_DEFINE_DEV_IMPL() 23 | virtual bool supports_multibatch() const override { return true; } 24 | }; 25 | 26 | // y = \sum_i,j,... x[i,j,...] 27 | struct SumElements : public Node { 28 | template explicit SumElements(const T& a) : Node(a) {} 29 | DYNET_NODE_DEFINE_DEV_IMPL() 30 | virtual bool supports_multibatch() const override { return true; } 31 | }; 32 | 33 | //y = \sum_i x_i 34 | struct SumDimension : public Node { 35 | template explicit SumDimension(const T& a, const std::vector & d, bool b=false) : Node(a), dims(d), include_batch_dim(b) {} 36 | DYNET_NODE_DEFINE_DEV_IMPL() 37 | virtual bool supports_multibatch() const override { return true; } 38 | private: 39 | std::vector dims; 40 | bool include_batch_dim; 41 | }; 42 | 43 | // M = x_0, v = x_1 44 | // y = M + v (broadcasting over columns) 45 | struct AddVectorToAllColumns : public Node { 46 | explicit AddVectorToAllColumns(const std::initializer_list& a) : Node(a) {} 47 | virtual bool supports_multibatch() const override { return true; } 48 | DYNET_NODE_DEFINE_DEV_IMPL() 49 | }; 50 | 51 | } // namespace dynet 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /dynet/nodes-concat.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CONCAT_H_ 2 | #define DYNET_NODES_CONCAT_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // concatenate along a particular dimension 10 | struct Concatenate : public Node { 11 | template explicit Concatenate(const T& a, unsigned d) : Node(a), dimension(d) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 14 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(args.size(), 1); } 15 | virtual void autobatch_reshape(const ComputationGraph & cg, 16 | const std::vector & batch_ids, 17 | const std::vector & concat, 18 | std::vector& xs, 19 | Tensor& fx) const override { 20 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 21 | } 22 | DYNET_NODE_DEFINE_DEV_IMPL() 23 | // src_row_indices[i] says what row in fx the ith x std::vector was assigned to 24 | // used to simplify backprop 25 | mutable std::vector src_indices; 26 | unsigned dimension; 27 | }; 28 | 29 | // concatenate different batched experssions into one single batched tensor 30 | struct ConcatenateToBatch : public Node { 31 | template explicit ConcatenateToBatch(const T& a) : Node(a) {} 32 | DYNET_NODE_DEFINE_DEV_IMPL() 33 | virtual bool supports_multibatch() const override {return true;} 34 | mutable std::vector src_element_indices; 35 | }; 36 | 37 | } // namespace dynet 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /dynet/nodes-const.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/tensor-eigen.h" 2 | #include "dynet/nodes-const.h" 3 | 4 | #include "dynet/nodes-impl-macros.h" 5 | 6 | using namespace std; 7 | 8 | namespace dynet { 9 | 10 | // ************* Constant ************* 11 | 12 | #ifndef __CUDACC__ 13 | 14 | string Constant::as_string(const vector& arg_names) const { 15 | ostringstream s; 16 | s << "constant(" << dim << ',' << value << ')'; 17 | return s.str(); 18 | } 19 | 20 | Dim Constant::dim_forward(const vector& xs) const { 21 | return dim; 22 | } 23 | 24 | #endif 25 | 26 | template 27 | void Constant::forward_dev_impl(const MyDevice & dev, const vector& xs, Tensor& fx) const { 28 | DYNET_ASSERT(xs.size() == 0, "Failed dimension check in Constant::forward"); 29 | if (value == 0.f) 30 | TensorTools::zero(fx); 31 | else 32 | TensorTools::constant(fx, value); 33 | } 34 | 35 | template 36 | void Constant::backward_dev_impl(const MyDevice & dev, 37 | const vector& xs, 38 | const Tensor& fx, 39 | const Tensor& dEdf, 40 | unsigned i, 41 | Tensor& dEdxi) const { 42 | DYNET_RUNTIME_ERR("Called backward() on an arity 0 node"); 43 | } 44 | DYNET_NODE_INST_DEV_IMPL(Constant) 45 | 46 | } 47 | -------------------------------------------------------------------------------- /dynet/nodes-const.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CONST_H_ 2 | #define DYNET_NODES_CONST_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // represents a simple std::vector of 0s 10 | struct Constant : public Node { 11 | explicit Constant(const Dim& d, float val=0.f) : dim(d), value(val) {} 12 | DYNET_NODE_DEFINE_DEV_IMPL() 13 | virtual bool supports_multibatch() const override { return true; } 14 | Dim dim; 15 | float value; 16 | }; 17 | 18 | } // namespace dynet 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /dynet/nodes-contract.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CONTRACT_H_ 2 | #define DYNET_NODES_CONTRACT_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // Y_i = A_ijk * B_k 10 | struct InnerProduct3D_1D : public Node { 11 | InnerProduct3D_1D(const std::initializer_list& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | }; 15 | 16 | // Y_i = A_ijk * B_k * C_j 17 | struct InnerProduct3D_1D_1D : public Node { 18 | InnerProduct3D_1D_1D(const std::initializer_list& a) : Node(a) {} 19 | DYNET_NODE_DEFINE_DEV_IMPL() 20 | }; 21 | 22 | } // namespace dynet 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /dynet/nodes-conv2d.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CONV2D_H_ 2 | #define DYNET_NODES_CONV2D_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | #if HAVE_CUDNN 8 | #include "dynet/cudnn-ops.h" 9 | #include 10 | #endif 11 | 12 | namespace dynet { 13 | 14 | // conv2d 15 | // y = x_1 *conv2d x_2 16 | // x_1 \in R^{H x W x Ci x N} (input) 17 | // x_2 \in R^{H x W x Ci x Co} (filter) 18 | // stride[0] corresponds to H 19 | // stride[1] corresponds to W 20 | // is_valid: true for 'VALID' and false for 'SAME' 21 | struct Conv2D: public Node { 22 | explicit Conv2D(const std::initializer_list& a, const std::vector& s, 23 | const bool padding_type = true) 24 | : Node(a), stride(s), is_valid(padding_type) {} 25 | virtual bool supports_multibatch() const override { return true; } 26 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 27 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 28 | virtual void autobatch_reshape(const ComputationGraph & cg, 29 | const std::vector & batch_ids, 30 | const std::vector & concat, 31 | std::vector& xs, 32 | Tensor& fx) const override { 33 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 34 | } 35 | DYNET_NODE_DEFINE_DEV_IMPL() 36 | const std::vector stride; 37 | const bool is_valid; 38 | 39 | private: 40 | #if HAVE_CUDNN 41 | mutable std::unique_ptr cudnn_conv_op_; 42 | #endif 43 | }; 44 | 45 | } // namespace dynet 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /dynet/nodes-cumulative.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/tensor-eigen.h" 2 | #include "dynet/nodes-cumulative.h" 3 | 4 | #include "dynet/nodes-impl-macros.h" 5 | 6 | using namespace std; 7 | 8 | namespace dynet { 9 | 10 | // ************* CumulativeSum ************* 11 | 12 | #ifndef __CUDACC__ 13 | 14 | string CumulativeSum::as_string(const vector& arg_names) const { 15 | ostringstream s; 16 | s << "cumsum(expression=" << arg_names[0] << ',' << d << ')'; 17 | return s.str(); 18 | } 19 | 20 | Dim CumulativeSum::dim_forward(const vector& xs) const { 21 | DYNET_ASSERT(xs.size() == 1, "Failed input count check in CumulativeSum"); 22 | DYNET_ARG_CHECK(xs[0].nd <= 3, "CumulativeSum implemented up to tensors of order 3 for now") 23 | DYNET_ARG_CHECK(d <= xs[0].nd, "dimension " << d << " is out of bounds of tensor of order " << xs[0].nd << " in CumulativeSum" ) 24 | Dim ret(xs[0]); 25 | return ret; 26 | } 27 | 28 | size_t CumulativeSum::aux_storage_size() const { 29 | return dim.size() * sizeof(float); 30 | } 31 | 32 | #endif 33 | 34 | template 35 | void CumulativeSum::forward_dev_impl(const MyDevice & dev, const vector& xs, Tensor& fx) const { 36 | DYNET_ASSERT(xs.size() == 1, "Failed input count check in CumulativeSum"); 37 | tb<3>(fx).device(*dev.edevice) = tb<3>(*(xs[0])).cumsum(d); 38 | } 39 | 40 | template 41 | void CumulativeSum::backward_dev_impl(const MyDevice & dev, 42 | const vector& xs, 43 | const Tensor& fx, 44 | const Tensor& dEdf, 45 | unsigned i, 46 | Tensor& dEdxi) const { 47 | DYNET_ARG_CHECK(i == 0, "Failed dimension check in CumulativeSum::backward"); 48 | Eigen::array reverse_dim = {false, false, false, false}; 49 | reverse_dim[d] = true; 50 | // First reverse the gradient 51 | Tensor dEdf_reversed(dim, (float*)aux_mem, fx.device, DeviceMempool::FXS); 52 | tb<3>(dEdf_reversed).device(*dev.edevice) = tb<3>(dEdf).reverse(reverse_dim); 53 | // Then accumulate and reverse 54 | tb<3>(dEdxi).device(*dev.edevice) += tb<3>(dEdf_reversed).cumsum(d).reverse(reverse_dim); 55 | } 56 | DYNET_NODE_INST_DEV_IMPL(CumulativeSum) 57 | 58 | } 59 | -------------------------------------------------------------------------------- /dynet/nodes-cumulative.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CUMULATIVE_H_ 2 | #define DYNET_NODES_CUMULATIVE_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | 10 | //y_i = \sum_{j explicit CumulativeSum(const T& a, unsigned d) : Node(a), d(d){} 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | size_t aux_storage_size() const override; 15 | virtual bool supports_multibatch() const override { return true; } 16 | private: 17 | unsigned d; 18 | }; 19 | } 20 | #endif 21 | -------------------------------------------------------------------------------- /dynet/nodes-def-macros.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODE_DEF_MACROS_H_ 2 | #define DYNET_NODE_DEF_MACROS_H_ 3 | 4 | #include "dynet/dim.h" 5 | #include "dynet/except.h" 6 | #include 7 | 8 | namespace dynet { 9 | 10 | inline bool LooksLikeVector(const Dim& d) { 11 | if (d.ndims() == 1) return true; 12 | if (d.ndims() > 1) { 13 | for (unsigned i = 1; i < d.ndims(); ++i) 14 | if (d[i] != 1) return false; 15 | } 16 | return true; 17 | } 18 | 19 | template 20 | inline std::string print_vec(const std::vector & vec) { 21 | std::string sep = "["; 22 | std::ostringstream oss; 23 | for(auto f : vec) { 24 | oss << sep << f; sep = ","; 25 | } 26 | oss << "]"; 27 | return oss.str(); 28 | } 29 | 30 | template 31 | inline std::string print_vecs(const std::vector > & vec) { 32 | std::string sep = "["; 33 | std::ostringstream oss; 34 | for(auto & f : vec) { 35 | oss << sep << print_vec(f); sep = ","; 36 | } 37 | oss << "]"; 38 | return oss.str(); 39 | } 40 | 41 | } 42 | 43 | // A macro to dispatch things to the appropriate device 44 | #define DYNET_NODE_DEFINE_DEV_IMPL() \ 45 | std::string as_string(const std::vector& arg_names) const override; \ 46 | Dim dim_forward(const std::vector& xs) const override; \ 47 | void forward_impl(const std::vector& xs, Tensor& fx) const override; \ 48 | template \ 49 | void forward_dev_impl(const MyDevice & dev, const std::vector& xs, Tensor& fx) const; \ 50 | void backward_impl(const std::vector& xs, \ 51 | const Tensor& fx, \ 52 | const Tensor& dEdf, \ 53 | unsigned i, \ 54 | Tensor& dEdxi) const override; \ 55 | template \ 56 | void backward_dev_impl( \ 57 | const MyDevice & dev, \ 58 | const std::vector& xs, \ 59 | const Tensor& fx, \ 60 | const Tensor& dEdf, \ 61 | unsigned i, \ 62 | Tensor& dEdxi) const; 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /dynet/nodes-dropout.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_DROPOUT_H_ 2 | #define DYNET_NODES_DROPOUT_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = dropout(x,p) where p specifies the dropout probability 10 | struct Dropout : public Node { 11 | explicit Dropout(const std::initializer_list& a, real p) : Node(a), p(p) {} 12 | DYNET_NODE_DEFINE_DEV_IMPL() 13 | size_t aux_storage_size() const override; 14 | virtual bool supports_multibatch() const override { return true; } 15 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::dropout); s.add_float(p); return sm.get_idx(s); } 16 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 17 | real p; 18 | }; 19 | 20 | // y = dropout(x,p) where p specifies the dropout probability 21 | struct DropoutDim : public Node { 22 | explicit DropoutDim(const std::initializer_list& a, unsigned d,real p) : Node(a), dimension(d), p(p) {} 23 | DYNET_NODE_DEFINE_DEV_IMPL() 24 | size_t aux_storage_size() const override; 25 | virtual bool supports_multibatch() const override { return true; } 26 | unsigned dimension; 27 | real p; 28 | }; 29 | 30 | // y = dropout(x,p) where p specifies the dropout probability 31 | struct DropoutBatch : public Node { 32 | explicit DropoutBatch(const std::initializer_list& a, real p) : Node(a), p(p) {} 33 | DYNET_NODE_DEFINE_DEV_IMPL() 34 | size_t aux_storage_size() const override; 35 | virtual bool supports_multibatch() const override { return true; } 36 | real p; 37 | }; 38 | 39 | // y = block_dropout(x,p) where p specifies the probability for dropping-out the entire block 40 | struct BlockDropout : public Node { 41 | explicit BlockDropout(const std::initializer_list& a, real p) : Node(a), dropout_probability(p) {} 42 | DYNET_NODE_DEFINE_DEV_IMPL() 43 | size_t aux_storage_size() const override; 44 | real dropout_probability; 45 | }; 46 | 47 | } // namespace dynet 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /dynet/nodes-linalg.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_LINALG_H_ 2 | #define DYNET_NODES_LINALG_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = x_1^T 10 | // NOTE: if you have a column or row std::vector as input, runtime is constant 11 | // if you have a matrix as input, the runtime is O(mn) - try to avoid using this 12 | struct Transpose : public Node { 13 | explicit Transpose(const std::initializer_list& a, 14 | const std::vector& dims) 15 | : Node(a), dims(dims) {} 16 | DYNET_NODE_DEFINE_DEV_IMPL() 17 | virtual bool supports_multibatch() const override { return true; } 18 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 19 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 20 | virtual void autobatch_reshape(const ComputationGraph & cg, 21 | const std::vector & batch_ids, 22 | const std::vector & concat, 23 | std::vector& xs, 24 | Tensor& fx) const override { 25 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 26 | } 27 | std::vector dims; 28 | }; 29 | 30 | // y = inv(x) 31 | // x = an invertible matrix 32 | struct MatrixInverse : public Node { 33 | explicit MatrixInverse(const std::initializer_list& a) 34 | : Node(a) { 35 | this->has_cuda_implemented = false; 36 | } 37 | DYNET_NODE_DEFINE_DEV_IMPL() 38 | }; 39 | 40 | // y = log det(x) 41 | struct LogDet : public Node { 42 | template 43 | explicit LogDet(const T& a) : Node(a) { 44 | this->has_cuda_implemented = false; 45 | } 46 | DYNET_NODE_DEFINE_DEV_IMPL() 47 | }; 48 | 49 | // y = Tr(x_1 * x_2^T) 50 | struct TraceOfProduct : public Node { 51 | explicit TraceOfProduct(const std::initializer_list& a) 52 | : Node(a) { 53 | this->has_cuda_implemented = false; 54 | } 55 | DYNET_NODE_DEFINE_DEV_IMPL() 56 | }; 57 | 58 | } // namespace dynet 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /dynet/nodes-logsumexp.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_LOGSUMEXP_H_ 2 | #define DYNET_NODES_LOGSUMEXP_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = \log \sum_i \exp x_i 10 | // done in log space carefully to avoid over/underflow issues 11 | struct LogSumExp : public Node { 12 | template explicit LogSumExp(const T& a) : Node(a) {} 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | virtual bool supports_multibatch() const override { return true; } 15 | }; 16 | 17 | struct LogSumExpDimension : public Node { 18 | template explicit LogSumExpDimension(const T& a, unsigned d = 0) : Node(a), dimension(d) {} 19 | DYNET_NODE_DEFINE_DEV_IMPL() 20 | virtual bool supports_multibatch() const override { return true; } 21 | private: 22 | unsigned dimension; 23 | }; 24 | 25 | } // namespace dynet 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /dynet/nodes-losses.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_LOSSES_H_ 2 | #define DYNET_NODES_LOSSES_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // x_1 is a scalar (or row std::vector) 10 | // x_2 is a scalar (or row std::vector) 11 | // y = max(0, margin - x_1 + x_2) 12 | struct PairwiseRankLoss : public Node { 13 | explicit PairwiseRankLoss(const std::initializer_list& a, real m = 1.0) : Node(a), margin(m) {} 14 | virtual bool supports_multibatch() const override { return true; } 15 | DYNET_NODE_DEFINE_DEV_IMPL() 16 | real margin; 17 | }; 18 | 19 | // you could do this with LogisticSigmoid, Softmax or a variety of other 20 | // functions, but this is often useful. 21 | // x_1 must be a std::vector with values between 0 and 1 22 | // target_y is an equivalently sized std::vector w values between 0 and 1 23 | // y = ty * log(x_1) + (1 - ty) * log(x_1) 24 | struct BinaryLogLoss : public Node { 25 | BinaryLogLoss(const std::initializer_list& a) : Node(a) {} 26 | DYNET_NODE_DEFINE_DEV_IMPL() 27 | }; 28 | 29 | // this is used to implement poisson regression 30 | // x_1 = log predicted mean 31 | // ty = true y (this is not a VariableIndex since it has to be a nonnegative integer and 32 | // is therefore nondifferentiable. There are various continuous extensions 33 | // using the incomplete gamma function that could be used, but meh) 34 | // y = log Poisson(ty; \lambda = \exp x_1) 35 | // = ty*x_1 - exp(x_1) - log(ty!) 36 | struct PoissonRegressionLoss : public Node { 37 | explicit PoissonRegressionLoss(const std::initializer_list& a, unsigned true_y) : Node(a), ty(true_y), pty(&ty) {} 38 | explicit PoissonRegressionLoss(const std::initializer_list& a, const unsigned* ptrue_y) : Node(a), ty(), pty(ptrue_y) {} 39 | DYNET_NODE_DEFINE_DEV_IMPL() 40 | private: 41 | unsigned ty; 42 | const unsigned* pty; 43 | }; 44 | 45 | } // namespace dynet 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /dynet/nodes-matrixmultiply.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_MATRIXMULTIPLY_H_ 2 | #define DYNET_NODES_MATRIXMULTIPLY_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = x_1 * x_2 10 | struct MatrixMultiply : public Node { 11 | explicit MatrixMultiply(const std::initializer_list& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 14 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 15 | virtual void autobatch_reshape(const ComputationGraph & cg, 16 | const std::vector & batch_ids, 17 | const std::vector & concat, 18 | std::vector& xs, 19 | Tensor& fx) const override { 20 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 21 | } 22 | DYNET_NODE_DEFINE_DEV_IMPL() 23 | }; 24 | 25 | } // namespace dynet 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /dynet/nodes-maxpooling2d.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_MAXPOOLING2D_H_ 2 | #define DYNET_NODES_MAXPOOLING2D_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | #if HAVE_CUDNN 8 | #include "dynet/cudnn-ops.h" 9 | #include 10 | #endif 11 | 12 | namespace dynet { 13 | 14 | // maxpooling2d 15 | // y = x_1 * maxpooling2d 16 | // x_1 \in R^{H x W x Ci x N} (input) 17 | // ksize[0] corresponds to H 18 | // ksize[1] corresponds to W 19 | // stride[0] corresponds to H 20 | // stride[1] corresponds to W 21 | // is_valid: true for 'VALID' and false for 'SAME' 22 | struct MaxPooling2D: public Node { 23 | explicit MaxPooling2D(const std::initializer_list& a, const std::vector& k, const std::vector& s, 24 | const bool padding_type = true) 25 | : Node(a), ksize(k), stride(s), is_valid(padding_type) {} 26 | virtual bool supports_multibatch() const override { return true; } 27 | DYNET_NODE_DEFINE_DEV_IMPL() 28 | const std::vector ksize; 29 | const std::vector stride; 30 | const bool is_valid; 31 | 32 | private: 33 | #if HAVE_CUDNN 34 | mutable std::unique_ptr cudnn_maxpool_op_; 35 | #endif 36 | }; 37 | 38 | } // namespace dynet 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /dynet/nodes-minmax.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_MINMAX_H_ 2 | #define DYNET_NODES_MINMAX_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = min{x_1, x_2} 10 | struct Min : public Node { 11 | explicit Min(const std::initializer_list& a) : Node(a) {} 12 | DYNET_NODE_DEFINE_DEV_IMPL() 13 | virtual bool supports_multibatch() const override { return true; } 14 | size_t aux_storage_size() const override; 15 | }; 16 | 17 | // y = max{x_1, x_2} 18 | struct Max : public Node { 19 | template explicit Max(const T& a) : Node(a) {} 20 | DYNET_NODE_DEFINE_DEV_IMPL() 21 | virtual bool supports_multibatch() const override { return true; } 22 | size_t aux_storage_size() const override; 23 | }; 24 | 25 | struct MinDimension : public Node { 26 | explicit MinDimension(const std::initializer_list& a, unsigned dimension = 0) : Node(a), reduced_dim(dimension) { 27 | first_dim = reduced_dim == 0 ? 1 : 0; 28 | second_dim = first_dim + 1 == reduced_dim ? first_dim + 2 : first_dim + 1; 29 | } 30 | DYNET_NODE_DEFINE_DEV_IMPL() 31 | virtual bool supports_multibatch() const override { return true; } 32 | size_t aux_storage_size() const override; 33 | unsigned reduced_dim; 34 | unsigned first_dim; 35 | unsigned second_dim; 36 | }; 37 | 38 | struct MaxDimension : public Node { 39 | explicit MaxDimension(const std::initializer_list& a, unsigned dimension = 0) : Node(a), reduced_dim(dimension) { 40 | first_dim = reduced_dim == 0 ? 1 : 0; 41 | second_dim = first_dim + 1 == reduced_dim ? first_dim + 2 : first_dim + 1; 42 | } 43 | DYNET_NODE_DEFINE_DEV_IMPL() 44 | virtual bool supports_multibatch() const override { return true; } 45 | size_t aux_storage_size() const override; 46 | unsigned reduced_dim; 47 | unsigned first_dim; 48 | unsigned second_dim; 49 | }; 50 | 51 | } // namespace dynet 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /dynet/nodes-normalization.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_NORMALIZATION_H_ 2 | #define DYNET_NODES_NORMALIZATION_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = x_1 * x_2 10 | struct WeightNormalization : public Node { 11 | explicit WeightNormalization(const std::initializer_list& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return false; } 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | }; 15 | 16 | } // namespace dynet 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /dynet/nodes-norms.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_NORMS_H_ 2 | #define DYNET_NODES_NORMS_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = || x_1 ||^2 10 | struct SquaredNorm : public Node { 11 | explicit SquaredNorm(const std::initializer_list& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | }; 15 | 16 | // y = || x_1 || 17 | struct L2Norm : public Node { 18 | explicit L2Norm(const std::initializer_list& a) : Node(a) {} 19 | virtual bool supports_multibatch() const override { return true; } 20 | DYNET_NODE_DEFINE_DEV_IMPL() 21 | }; 22 | 23 | } // namespace dynet 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /dynet/nodes-pickneglogsoftmax.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_PICKNEGLOGSOFTMAX_H_ 2 | #define DYNET_NODES_PICKNEGLOGSOFTMAX_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // z = \sum_j \exp (x_i)_j 10 | // y = (x_1)_element - \log z 11 | struct PickNegLogSoftmax : public Node { 12 | explicit PickNegLogSoftmax(const std::initializer_list& a, unsigned v) : Node(a), val(v), pval(&val), vals(), pvals() {} 13 | // use this constructor if you want to perform mini-batching 14 | explicit PickNegLogSoftmax(const std::initializer_list& a, const std::vector& v) : Node(a), val(), pval(), vals(v), pvals(&vals) {} 15 | // use these constructors if you want to change the value after the graph is constructed 16 | explicit PickNegLogSoftmax(const std::initializer_list& a, const unsigned* pv) : Node(a), val(), pval(pv), vals(), pvals() {} 17 | explicit PickNegLogSoftmax(const std::initializer_list& a, const std::vector* pv) : Node(a), val(), pval(), vals(), pvals(pv) {} 18 | DYNET_NODE_DEFINE_DEV_IMPL() 19 | virtual bool supports_multibatch() const override { return true; } 20 | size_t aux_storage_size() const override; 21 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 22 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 23 | virtual Node* autobatch_pseudo_node(const ComputationGraph & cg, 24 | const std::vector & batch_ids) const override; 25 | virtual void autobatch_reshape(const ComputationGraph & cg, 26 | const std::vector & batch_ids, 27 | const std::vector & concat, 28 | std::vector& xs, 29 | Tensor& fx) const override { 30 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 31 | } 32 | unsigned val; 33 | const unsigned* pval; 34 | std::vector vals; 35 | const std::vector* pvals; 36 | }; 37 | 38 | } // namespace dynet 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /dynet/nodes-random.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_RANDOM_H_ 2 | #define DYNET_NODES_RANDOM_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // n_{i,j} ~ N(0,stddev) 10 | // y = x + n 11 | struct GaussianNoise : public Node { 12 | explicit GaussianNoise(const std::initializer_list& a, real stddev) : Node(a), stddev(stddev) {} 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | virtual bool supports_multibatch() const override { return true; } 15 | real stddev; 16 | }; 17 | 18 | // draw random noise from Normal(0, 1) 19 | struct RandomNormal : public Node { 20 | explicit RandomNormal(const Dim& d, float m=0.f, float s=1.f) : dim(d), mean(m), stddev(s) {} 21 | DYNET_NODE_DEFINE_DEV_IMPL() 22 | virtual bool supports_multibatch() const override { return true; } 23 | Dim dim; 24 | float mean, stddev; 25 | }; 26 | 27 | // draw from Bernoulli(p) 28 | struct RandomBernoulli : public Node { 29 | explicit RandomBernoulli(const std::initializer_list& a, const Dim& d, real p, real scale = 1.0f) : dim(d), p(p), scale(scale) { 30 | DYNET_ASSERT(a.size() == 0, "RandomBernoulli doesn't accept nodes as input"); 31 | } 32 | DYNET_NODE_DEFINE_DEV_IMPL() 33 | virtual bool supports_multibatch() const override { return true; } 34 | Dim dim; 35 | real p; 36 | real scale; 37 | }; 38 | 39 | // draw a random real from Uniform(left, right) 40 | struct RandomUniform : public Node { 41 | explicit RandomUniform(const std::initializer_list& a, const Dim& d, real left, real right) : dim(d), left(left), right(right) { 42 | DYNET_ASSERT(a.size() == 0, "RandomUniform doesn't accept nodes as input"); 43 | } 44 | DYNET_NODE_DEFINE_DEV_IMPL() 45 | virtual bool supports_multibatch() const override { return true; } 46 | Dim dim; 47 | real left, right; 48 | }; 49 | 50 | // draw a random real from Gubmel(mu, beta) 51 | struct RandomGumbel : public Node { 52 | explicit RandomGumbel(const std::initializer_list& a, const Dim& d, real mu, real beta) : dim(d), mu(mu), beta(beta) { 53 | DYNET_ASSERT(a.size() == 0, "RandomGumbel doesn't accept nodes as input"); 54 | } 55 | DYNET_NODE_DEFINE_DEV_IMPL() 56 | virtual bool supports_multibatch() const override { return true; } 57 | Dim dim; 58 | real mu, beta; 59 | }; 60 | 61 | } // namespace dynet 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /dynet/nodes-rounding.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_ROUNDING_H_ 2 | #define DYNET_NODES_ROUNDING_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // round to nearest int 10 | struct Round : public Node { 11 | explicit Round(const std::initializer_list& a, bool straight_through=false) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::round); s.add_int((int)straight_through); return sm.get_idx(s); } 14 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 15 | DYNET_NODE_DEFINE_DEV_IMPL() 16 | bool straight_through; 17 | }; 18 | 19 | // round up to int 20 | struct Ceil : public Node { 21 | explicit Ceil(const std::initializer_list& a, bool straight_through=false) : Node(a) {} 22 | virtual bool supports_multibatch() const override { return true; } 23 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::ceiling); s.add_int((int)straight_through); return sm.get_idx(s); } 24 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 25 | DYNET_NODE_DEFINE_DEV_IMPL() 26 | bool straight_through; 27 | }; 28 | 29 | // round down to int 30 | struct Floor : public Node { 31 | explicit Floor(const std::initializer_list& a, bool straight_through=false) : Node(a) {} 32 | virtual bool supports_multibatch() const override { return true; } 33 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override { Sig s(nt::floor); s.add_int((int)straight_through); return sm.get_idx(s); } 34 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override { return std::vector(1, 1); } 35 | DYNET_NODE_DEFINE_DEV_IMPL() 36 | bool straight_through; 37 | }; 38 | 39 | } // namespace dynet 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /dynet/nodes-similarities.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_SIMILARITIES_H_ 2 | #define DYNET_NODES_SIMILARITIES_H_ 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | // y = x_1^T . x_2 10 | struct DotProduct : public Node { 11 | explicit DotProduct(const std::initializer_list& a) : Node(a) {} 12 | virtual bool supports_multibatch() const override { return true; } 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | }; 15 | 16 | // y = || x_1 - x_2 ||_H(d) 17 | struct HuberDistance : public Node { 18 | explicit HuberDistance(const std::initializer_list& a, float d = 1.345f) : Node(a), d(d) {} 19 | DYNET_NODE_DEFINE_DEV_IMPL() 20 | float d; 21 | }; 22 | 23 | // y = || x_1 - x_2 ||_1 24 | struct L1Distance : public Node { 25 | explicit L1Distance(const std::initializer_list& a) : Node(a) {} 26 | DYNET_NODE_DEFINE_DEV_IMPL() 27 | }; 28 | 29 | // y = || x_1 - x_2 ||^2 30 | struct SquaredEuclideanDistance : public Node { 31 | explicit SquaredEuclideanDistance(const std::initializer_list& a) : Node(a) {} 32 | virtual bool supports_multibatch() const override { return true; } 33 | virtual int autobatch_sig(const ComputationGraph &cg, SigMap &sm) const override; 34 | virtual std::vector autobatch_concat(const ComputationGraph & cg) const override; 35 | virtual void autobatch_reshape(const ComputationGraph & cg, 36 | const std::vector & batch_ids, 37 | const std::vector & concat, 38 | std::vector& xs, 39 | Tensor& fx) const override { 40 | autobatch_reshape_concatonly(cg, batch_ids, concat, xs, fx); 41 | } 42 | DYNET_NODE_DEFINE_DEV_IMPL() 43 | }; 44 | 45 | } // namespace dynet 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /dynet/nodes-to-device.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/tensor-eigen.h" 2 | #include 3 | #include 4 | 5 | #include "dynet/nodes-to-device.h" 6 | #include "dynet/tensor.h" 7 | #include "dynet/dim.h" 8 | #include "dynet/nodes-impl-macros.h" 9 | #include "dynet/functors.h" 10 | 11 | using namespace std; 12 | 13 | namespace dynet { 14 | 15 | #ifndef __CUDACC__ 16 | string ToDevice::as_string(const vector& arg_names) const { 17 | return "copy " + arg_names[0] + " between devices"; 18 | } 19 | 20 | Dim ToDevice::dim_forward(const vector & xs) const { 21 | DYNET_ASSERT(xs.size() == 1, "Failed dimension check in ToDevice::dim_forward"); 22 | return xs[0]; 23 | } 24 | #endif 25 | 26 | template 27 | void ToDevice::forward_dev_impl(const MyDevice & dev, 28 | const vector & xs, 29 | Tensor & fx) const { 30 | #ifdef HAVE_CUDA 31 | DYNET_ASSERT(xs.size() == 1, "Failed dimension check in ToDevice::forward"); 32 | TensorTools::copy_elements(fx, *xs[0]); 33 | #endif 34 | } 35 | 36 | template 37 | void ToDevice::backward_dev_impl(const MyDevice & dev, 38 | const vector & xs, 39 | const Tensor & fx, 40 | const Tensor & dEdf, 41 | unsigned i, 42 | Tensor & dEdxi) const { 43 | #ifdef HAVE_CUDA 44 | Tensor tmp; 45 | tmp.d = dEdxi.d; 46 | tmp.device = dEdxi.device; 47 | tmp.mem_pool = dEdxi.mem_pool; 48 | tmp.v = static_cast(dEdxi.device->pools[(int)dEdxi.mem_pool] 49 | ->allocate(dEdxi.d.size() * sizeof(float))); 50 | if (!tmp.v) DYNET_RUNTIME_ERR("out of memory while attempting to allocate space for aggregate_elements."); 51 | TensorTools::copy_elements(tmp, dEdf); 52 | TensorTools::accumulate(dEdxi, tmp); 53 | #endif 54 | } 55 | DYNET_NODE_INST_DEV_IMPL(ToDevice) 56 | 57 | } // namespace dynet 58 | -------------------------------------------------------------------------------- /dynet/nodes-to-device.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_NODES_CHANGE_DEVICES_H 2 | #define DYNET_NODES_CHANGE_DEVICES_H 3 | 4 | #include "dynet/dynet.h" 5 | #include "dynet/nodes-def-macros.h" 6 | 7 | namespace dynet { 8 | 9 | struct ToDevice : public Node { 10 | explicit ToDevice(const std::initializer_list& a, Device *device) : Node(a, device) {} 11 | virtual bool supports_multibatch() const override { return true; } 12 | virtual bool supports_multidevice() const override { return true; } 13 | DYNET_NODE_DEFINE_DEV_IMPL() 14 | }; 15 | 16 | } // namespace dynet 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /dynet/nodes.h: -------------------------------------------------------------------------------- 1 | #include "dynet/nodes-activations.h" 2 | #include "dynet/nodes-affinetransform.h" 3 | #include "dynet/nodes-argmax.h" 4 | #include "dynet/nodes-arith-const.h" 5 | #include "dynet/nodes-arith-cwise.h" 6 | #include "dynet/nodes-arith-sum.h" 7 | #include "dynet/nodes-arith-unary.h" 8 | #include "dynet/nodes-concat.h" 9 | #include "dynet/nodes-const.h" 10 | #include "dynet/nodes-contract.h" 11 | #include "dynet/nodes-conv.h" 12 | #include "dynet/nodes-conv2d.h" 13 | #include "dynet/nodes-dropout.h" 14 | #include "dynet/nodes-flow.h" 15 | #include "dynet/nodes-hinge.h" 16 | #include "dynet/nodes-linalg.h" 17 | #include "dynet/nodes-logsumexp.h" 18 | #include "dynet/nodes-losses.h" 19 | #include "dynet/nodes-matrixmultiply.h" 20 | #include "dynet/nodes-maxpooling2d.h" 21 | #include "dynet/nodes-minmax.h" 22 | #include "dynet/nodes-moments.h" 23 | #include "dynet/nodes-cumulative.h" 24 | #include "dynet/nodes-normalization.h" 25 | #include "dynet/nodes-norms.h" 26 | #include "dynet/nodes-pickneglogsoftmax.h" 27 | #include "dynet/nodes-random.h" 28 | #include "dynet/nodes-rounding.h" 29 | #include "dynet/nodes-select.h" 30 | #include "dynet/nodes-similarities.h" 31 | #include "dynet/nodes-softmaxes.h" 32 | #include "dynet/nodes-trig.h" 33 | #include "dynet/nodes-to-device.h" 34 | #include "dynet/nodes-lstm.h" 35 | -------------------------------------------------------------------------------- /dynet/param-init.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/param-init.h" 2 | #include "dynet/tensor.h" 3 | #include "dynet/except.h" 4 | 5 | using namespace dynet; 6 | using namespace std; 7 | 8 | #include 9 | 10 | void ParameterInitNormal::initialize_params(Tensor & values) const { 11 | TensorTools::randomize_normal(values, mean, sqrt(var)); 12 | } 13 | 14 | void ParameterInitUniform::initialize_params(Tensor & values) const { 15 | TensorTools::randomize_uniform(values, left, right); 16 | } 17 | 18 | void ParameterInitConst::initialize_params(Tensor & values) const { 19 | TensorTools::constant(values, cnst); 20 | } 21 | 22 | void ParameterInitIdentity::initialize_params(Tensor & values) const { 23 | TensorTools::identity(values); 24 | } 25 | 26 | void ParameterInitGlorot::initialize_params(Tensor & values) const { 27 | int dims = 0, dim_len = values.d.nd - (lookup ? 1 : 0); 28 | float my_scale = 0.0; 29 | if (dim_len == 4) { 30 | // When doing a Conv the parameters is (H, W, In, Out) 31 | int receptive_field = values.d[0] * values.d[1]; 32 | // Other framework m + n are calculated by multiplying by the kernel size. 33 | dims = values.d[2] * receptive_field + values.d[3] * receptive_field; 34 | my_scale = gain * sqrt(6) / sqrt(dims); 35 | } else { 36 | for (int i = 0; i < dim_len; ++i) dims += values.d[i]; 37 | my_scale = gain * sqrt(3 * dim_len) / sqrt(dims); 38 | } 39 | TensorTools::randomize_uniform(values, -my_scale, my_scale); 40 | } 41 | 42 | void ParameterInitSaxe::initialize_params(Tensor & values) const { 43 | TensorTools::randomize_orthonormal(values, gain); 44 | } 45 | 46 | 47 | void ParameterInitFromVector::initialize_params(Tensor & values) const { 48 | TensorTools::set_elements(values, vec); 49 | } 50 | 51 | void ParameterInitFromFile::initialize_params(Tensor & values) const { 52 | ifstream is(filename); 53 | istream_iterator start(is), end; 54 | vector param_vector(start, end); 55 | TensorTools::set_elements(values, param_vector); 56 | } 57 | -------------------------------------------------------------------------------- /dynet/pretrain.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/pretrain.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "dynet/dict.h" 9 | #include "dynet/model.h" 10 | #include "dynet/tensor-eigen.h" 11 | 12 | using namespace std; 13 | 14 | namespace dynet { 15 | 16 | void save_pretrained_embeddings(const std::string& fname, 17 | const Dict& d, 18 | const LookupParameter& lp) { 19 | cerr << "Writing word vectors to " << fname << " ...\n"; 20 | ofstream out(fname); 21 | if(!out) 22 | DYNET_INVALID_ARG("Could not save embeddings to " << fname); 23 | auto& m = lp.get_storage(); 24 | for (unsigned i = 0; i < d.size(); ++i) { 25 | out << d.convert(i) << ' ' << mat(m.values[i]).transpose() << endl; 26 | } 27 | } 28 | 29 | void read_pretrained_embeddings(const std::string& fname, 30 | Dict& d, 31 | std::unordered_map>& vectors) { 32 | int unk = -1; 33 | if (d.is_frozen()) unk = d.get_unk_id(); 34 | cerr << "Loading word vectors from " << fname << " ...\n"; 35 | ifstream in(fname); 36 | if(!in) 37 | DYNET_INVALID_ARG("Could not load embeddings from " << fname); 38 | string line; 39 | string word; 40 | vector v; 41 | getline(in, line); 42 | istringstream lin(line); 43 | lin >> word; 44 | while(lin) { 45 | float x; 46 | lin >> x; 47 | if (!lin) break; 48 | v.push_back(x); 49 | } 50 | unsigned vec_size = v.size(); 51 | int wid = d.convert(word); 52 | if (wid != unk) vectors[wid] = v; 53 | while(getline(in, line)) { 54 | istringstream lin(line); 55 | lin >> word; 56 | int w = d.convert(word); 57 | if (w != unk) { 58 | for (unsigned i = 0; i < vec_size; ++i) lin >> v[i]; 59 | vectors[w] = v; 60 | } 61 | } 62 | } 63 | 64 | } // dynet 65 | -------------------------------------------------------------------------------- /dynet/pretrain.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_PRETRAIN_H 2 | #define DYNET_PRETRAIN_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "dynet/dict.h" 8 | #include "dynet/model.h" 9 | 10 | namespace dynet { 11 | 12 | void save_pretrained_embeddings(const std::string& fname, 13 | const Dict& d, 14 | const LookupParameter& lp); 15 | 16 | void read_pretrained_embeddings(const std::string& fname, 17 | Dict& d, 18 | std::unordered_map>& vectors); 19 | 20 | } // namespace dynet 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /dynet/rnn-state-machine.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/rnn-state-machine.h" 2 | 3 | #include 4 | 5 | #include "dynet/dynet.h" 6 | 7 | using namespace std; 8 | 9 | namespace dynet { 10 | 11 | void RNNStateMachine::failure(RNNOp op) { 12 | ostringstream oss; oss << "State transition error: currently in state " << q_ << " but received operation " << op; 13 | throw std::invalid_argument(oss.str()); 14 | } 15 | 16 | } // namespace dynet 17 | 18 | -------------------------------------------------------------------------------- /dynet/rnn-state-machine.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_RNN_STATE_MACHINE_H_ 2 | #define DYNET_RNN_STATE_MACHINE_H_ 3 | 4 | namespace dynet { 5 | 6 | // CURRENT STATE | ACTION | NEXT STATE 7 | // --------------+---------------------+----------------- 8 | // CREATED | new_graph | GRAPH_READY 9 | // GRAPH_READY | start_new_sequence | READING_INPUT 10 | // READING_INPUT | add_input | READING_INPUT 11 | // READING_INPUT | start_new_seqeunce | READING_INPUT 12 | // READING_INPUT | new_graph | GRAPH_READY 13 | 14 | enum RNNState {CREATED, GRAPH_READY, READING_INPUT}; 15 | enum RNNOp {new_graph, start_new_sequence, add_input}; 16 | 17 | class RNNStateMachine { 18 | public: 19 | RNNStateMachine() : q_(RNNState::CREATED) {} 20 | void failure(RNNOp op); 21 | void transition(RNNOp op) { 22 | switch (q_) { 23 | case RNNState::CREATED: 24 | if (op == RNNOp::new_graph) { q_ = RNNState::GRAPH_READY; break; } 25 | failure(op); 26 | case RNNState::GRAPH_READY: 27 | if (op == RNNOp::new_graph) { break; } 28 | if (op == RNNOp::start_new_sequence) { q_ = RNNState::READING_INPUT; break; } 29 | failure(op); 30 | case RNNState::READING_INPUT: 31 | if (op == RNNOp::add_input) { break; } 32 | if (op == RNNOp::start_new_sequence) { break; } 33 | if (op == RNNOp::new_graph) { q_ = RNNState::GRAPH_READY; break; } 34 | failure(op); 35 | } 36 | } 37 | private: 38 | RNNState q_; 39 | }; 40 | 41 | } // namespace dynet 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /dynet/saxe-init.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/saxe-init.h" 2 | #include "dynet/tensor.h" 3 | #include "dynet/tensor-eigen.h" 4 | #include "dynet/globals.h" 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | using namespace std; 12 | 13 | namespace dynet { 14 | 15 | void orthonormal_random(unsigned dd, float g, Tensor& x) { 16 | Tensor t; 17 | t.d = Dim({dd, dd}); 18 | t.v = new float[dd * dd]; 19 | normal_distribution distribution(0, 0.01); 20 | auto b = [&] () {return distribution(*rndeng);}; 21 | generate(t.v, t.v + dd*dd, b); 22 | Eigen::JacobiSVD svd(mat(t), Eigen::ComputeFullU); 23 | mat(x) = svd.matrixU(); 24 | delete[] t.v; 25 | } 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /dynet/saxe-init.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_SAXE_INIT_H_ 2 | #define DYNET_SAXE_INIT_H_ 3 | 4 | namespace dynet { 5 | 6 | struct Tensor; 7 | 8 | void orthonormal_random(unsigned dim, float g, Tensor& x); 9 | 10 | } 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /dynet/shadow-params.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/dynet.h" 2 | 3 | #include "dynet/shadow-params.h" 4 | #include "dynet/tensor.h" 5 | #include "dynet/aligned-mem-pool.h" 6 | #include "dynet/model.h" 7 | #include "dynet/devices.h" 8 | 9 | #define LOAD_INIT_FUNC() initialize_lookups() 10 | 11 | using namespace std; 12 | 13 | namespace dynet { 14 | 15 | ShadowParameters::ShadowParameters(const ParameterStorage& p) : h(p.values) { 16 | p.device->allocate_tensor(DeviceMempool::PS, h); 17 | TensorTools::zero(h); 18 | } 19 | 20 | ShadowLookupParameters::ShadowLookupParameters(const LookupParameterStorage& lp) : all_h(lp.all_values) { 21 | lp.device->allocate_tensor(DeviceMempool::PS, all_h); 22 | TensorTools::zero(all_h); 23 | initialize_lookups(); 24 | } 25 | 26 | void ShadowLookupParameters::initialize_lookups() { 27 | int num = all_h.d[all_h.d.nd-1]; 28 | Dim dim = all_h.d; dim.nd--; 29 | int dim_size = dim.size(); 30 | if(h.size() == 0) { 31 | h.resize(num); 32 | for(int i = 0; i < num; ++i) 33 | h[i] = Tensor(dim, all_h.v + i*dim_size, all_h.device, all_h.mem_pool); 34 | } 35 | } 36 | 37 | void allocate_shadow_parameters(const ParameterCollection& m, unsigned allocated, vector& target) { 38 | auto& params = m.parameters_list(); 39 | vector> to_allocate(params.begin() + allocated, params.end()); 40 | vector v; 41 | target.reserve(params.size()); 42 | for (auto& p : to_allocate) 43 | target.emplace_back(*p); 44 | } 45 | 46 | void allocate_shadow_lookup_parameters(const ParameterCollection& m, unsigned allocated, vector& target) { 47 | auto& params = m.lookup_parameters_list(); 48 | vector> to_allocate(params.begin() + allocated, params.end()); 49 | target.reserve(params.size()); 50 | for (auto& p : to_allocate) 51 | target.emplace_back(*p); 52 | } 53 | 54 | } // namespace dynet 55 | 56 | -------------------------------------------------------------------------------- /dynet/shadow-params.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_SHADOW_PARAMS_H 2 | #define DYNET_SHADOW_PARAMS_H 3 | 4 | #include 5 | #include "dynet/tensor.h" 6 | 7 | // if your learner needs to keep track of an extra set of values (one per 8 | // parameter), use the Shadow classes. this can be used to implement, e.g., 9 | // momentum or adagrad 10 | 11 | namespace dynet { 12 | 13 | class ParameterCollection; 14 | struct ParameterStorage; 15 | struct LookupParameterStorage; 16 | 17 | struct ShadowParameters { 18 | ShadowParameters() {} 19 | explicit ShadowParameters(const ParameterStorage& p); 20 | Tensor h; 21 | }; 22 | 23 | struct ShadowLookupParameters { 24 | ShadowLookupParameters() {} 25 | explicit ShadowLookupParameters(const LookupParameterStorage& lp); 26 | Tensor all_h; 27 | std::vector h; 28 | private: 29 | void initialize_lookups(); 30 | }; 31 | 32 | // one per element in model.parameters_list 33 | void allocate_shadow_parameters(const ParameterCollection& model, unsigned allocated, std::vector& target); 34 | // one per element in model.lookup_parameters_list 35 | void allocate_shadow_lookup_parameters(const ParameterCollection& model, unsigned allocated, std::vector& target); 36 | 37 | } // namespace dynet 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /dynet/str-util.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_STR_UTIL_H_ 2 | #define DYNET_STR_UTIL_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace dynet { 8 | 9 | inline bool startswith(const std::string & str, const std::string & key) { 10 | return str.find(key) == 0; 11 | } 12 | 13 | inline bool endswith(const std::string & str, const std::string & key) { 14 | if (str.size() < key.size()) return false; 15 | return str.rfind(key) == (str.size() - key.size()); 16 | } 17 | 18 | inline std::vector str_split(const std::string & str, const char sep) { 19 | std::vector lst; 20 | size_t st = 0, en = 0; 21 | while (1) { 22 | en = str.find(sep, st); 23 | auto s = str.substr(st, en - st); 24 | if(s.size()) lst.push_back(s); 25 | if(en == std::string::npos) break; 26 | st = en + 1; 27 | } 28 | return lst; 29 | } 30 | 31 | } // namespace dynet 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /dynet/weight-decay.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/weight-decay.h" 2 | 3 | namespace dynet {} 4 | -------------------------------------------------------------------------------- /dynet/weight-decay.h: -------------------------------------------------------------------------------- 1 | #ifndef DYNET_WEIGHT_DECAY_H 2 | #define DYNET_WEIGHT_DECAY_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace dynet { 9 | 10 | // I don't bother with learning rates when computing how much the weight 11 | // decay changes- those are hard to define in the adaptive update rules. 12 | // So, we do something simple that works with everything. 13 | // 14 | // Note: you may want to discount lambda as you learn if your eta is on a 15 | // decaying schedule. 16 | struct L2WeightDecay { 17 | explicit L2WeightDecay(float lambda = 1e-6) : weight_decay(1) { set_lambda(lambda); } 18 | void set_lambda(float lam) { 19 | if (lam < 0) throw std::domain_error("Bad value of lambda in set_lambda"); 20 | lambda = lam; 21 | } 22 | float get_lambda() const { return lambda; } 23 | void update_weight_decay(unsigned num_updates = 1) { 24 | if (num_updates == 0) return; 25 | if (num_updates == 1) 26 | weight_decay -= weight_decay * lambda; 27 | else weight_decay = (float)(weight_decay * std::pow(1 - lambda, num_updates)); 28 | } 29 | float current_weight_decay() const { return weight_decay; } 30 | bool parameters_need_rescaled() const { 31 | return (weight_decay < 0.25f); 32 | } 33 | void reset_weight_decay() { 34 | std::cerr << "RESCALE WEIGHT DECAY FROM " << weight_decay << " to 1.0\n"; 35 | weight_decay = 1.0f; 36 | } 37 | private: 38 | float weight_decay; 39 | float lambda; 40 | }; 41 | 42 | } // namespace dynet 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8) 2 | 3 | macro(ADD_EXAMPLE FOLDER TARGET) 4 | include_directories(${PROJECT_SOURCE_DIR}/examples/cpp-utils) 5 | if(EXISTS ${PROJECT_SOURCE_DIR}/examples/${FOLDER}/train_${TARGET}.cc) 6 | add_executable(${TARGET} ${FOLDER}/train_${TARGET}.cc) 7 | elseif(EXISTS ${PROJECT_SOURCE_DIR}/examples/${FOLDER}/${TARGET}.cc) 8 | add_executable(${TARGET} ${FOLDER}/${TARGET}.cc) 9 | else() 10 | message( SEND_ERROR "Neither ${FOLDER}/train_${TARGET}.cc or ${FOLDER}/${TARGET}.cc does exist!" ) 11 | endif() 12 | target_link_libraries(${TARGET} dynet ${LIBS}) 13 | if (WITH_CUDA_BACKEND) 14 | CUDA_ADD_CUBLAS_TO_TARGET(${TARGET}) 15 | endif (WITH_CUDA_BACKEND) 16 | if(UNIX AND NOT APPLE) 17 | target_link_libraries(${TARGET} rt) 18 | endif() 19 | endmacro(ADD_EXAMPLE) 20 | 21 | ADD_EXAMPLE(autobatch rnn-autobatch) 22 | ADD_EXAMPLE(autobatch xor-autobatch) 23 | ADD_EXAMPLE(batching rnnlm-batch) 24 | ADD_EXAMPLE(batching xor-batch) 25 | ADD_EXAMPLE(document-classification imdb) 26 | ADD_EXAMPLE(fflm fflm) 27 | ADD_EXAMPLE(mnist mnist) 28 | ADD_EXAMPLE(noise-contrastive-estimation rnnlm-batch-nce) 29 | ADD_EXAMPLE(poisson-regression poisson-regression) 30 | ADD_EXAMPLE(read-write read-write) 31 | ADD_EXAMPLE(rnnlm rnnlm) 32 | # This is commented out because it was out of date and needs to be updated to work with recent DyNet 33 | # ADD_EXAMPLE(segmental-rnn segrnn-sup) 34 | ADD_EXAMPLE(sentence-embedding embed-cl) 35 | ADD_EXAMPLE(sequence-to-sequence attention) 36 | ADD_EXAMPLE(sequence-to-sequence encdec) 37 | ADD_EXAMPLE(softmax-builders rnnlm-cfsm) 38 | ADD_EXAMPLE(tagger tag-bilstm) 39 | # This is commented out because it was out of date and needs to be updated to work with recent DyNet 40 | # ADD_EXAMPLE(textcat textcat) 41 | ADD_EXAMPLE(variational-autoencoder rnnlm-aevb) 42 | ADD_EXAMPLE(word-embedding tok-embed) 43 | ADD_EXAMPLE(xor xor) 44 | ADD_EXAMPLE(devices/xor-multidevice xor-multidevice) 45 | 46 | if(ENABLE_BOOST) 47 | if(NOT MSVC) 48 | ADD_EXAMPLE(multiprocessing rnnlm-mp) 49 | ADD_EXAMPLE(multiprocessing xor-mp) 50 | ADD_EXAMPLE(multiprocessing xor-simple-mp) 51 | endif() 52 | ADD_EXAMPLE(transformer transformer-train) 53 | ADD_EXAMPLE(transformer transformer-decode) 54 | endif() 55 | -------------------------------------------------------------------------------- /examples/autobatch/train_xor-autobatch.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/training.h" 2 | #include "dynet/expr.h" 3 | 4 | #include 5 | #include 6 | 7 | using namespace std; 8 | using namespace dynet; 9 | 10 | int main(int argc, char** argv) { 11 | dynet::initialize(argc, argv); 12 | 13 | // parameters 14 | const unsigned ITERATIONS = 30; 15 | ParameterCollection m; 16 | SimpleSGDTrainer trainer(m); 17 | //MomentumSGDTrainer trainer(m); 18 | 19 | Parameter p_W, p_b, p_V, p_a; 20 | const unsigned HIDDEN_SIZE = 3; 21 | p_W = m.add_parameters({HIDDEN_SIZE, 2}); 22 | p_b = m.add_parameters({HIDDEN_SIZE}); 23 | p_V = m.add_parameters({1, HIDDEN_SIZE}); 24 | p_a = m.add_parameters({1}); 25 | 26 | // train the parameters 27 | for (unsigned iter = 0; iter < ITERATIONS; ++iter) { 28 | 29 | ComputationGraph cg; 30 | Expression W = parameter(cg, p_W); 31 | Expression b = parameter(cg, p_b); 32 | Expression V = parameter(cg, p_V); 33 | Expression a = parameter(cg, p_a); 34 | 35 | vector losses; 36 | 37 | for (unsigned mi = 0; mi < 4; ++mi) { 38 | 39 | bool x1 = mi % 2; 40 | bool x2 = (mi / 2) % 2; 41 | vector x_values(2); 42 | x_values[0] = x1 ? 1 : -1; 43 | x_values[1] = x2 ? 1 : -1; 44 | float y_value = (x1 != x2) ? 1 : -1; 45 | 46 | Expression x = input(cg, {2}, x_values); 47 | Expression y = input(cg, y_value); 48 | 49 | //Expression h = tanh(W*x + b); 50 | Expression h = tanh(affine_transform({b, W, x})); 51 | //Expression h = softsign(W*x + b); 52 | Expression y_pred = affine_transform({a, V, h}); 53 | losses.push_back(squared_distance(y_pred, y)); 54 | 55 | } 56 | 57 | Expression loss_expr = sum(losses); 58 | 59 | // Print the graph, just for fun. 60 | if(iter == 0) { 61 | cg.print_graphviz(); 62 | } 63 | 64 | // Calculate the loss. Batching will automatically be done here. 65 | float loss = as_scalar(cg.forward(loss_expr)) / 4; 66 | cg.backward(loss_expr); 67 | trainer.update(); 68 | 69 | cerr << "E = " << loss << endl; 70 | } 71 | 72 | } 73 | 74 | -------------------------------------------------------------------------------- /examples/batching/minibatch.py: -------------------------------------------------------------------------------- 1 | import dynet as dy 2 | import numpy as np 3 | 4 | m = dy.Model() 5 | lp = m.add_lookup_parameters((100,10)) 6 | 7 | # regular lookup 8 | a = lp[1].npvalue() 9 | b = lp[2].npvalue() 10 | c = lp[3].npvalue() 11 | 12 | # batch lookup instead of single elements. 13 | # two ways of doing this. 14 | abc1 = dy.lookup_batch(lp, [1,2,3]) 15 | print(abc1.npvalue()) 16 | 17 | abc2 = lp.batch([1,2,3]) 18 | print(abc2.npvalue()) 19 | 20 | print(np.hstack([a,b,c])) 21 | 22 | 23 | # use pick and pickneglogsoftmax in batch mode 24 | # (must be used in conjunction with lookup_batch): 25 | print("\nPick") 26 | W = dy.parameter( m.add_parameters((5, 10)) ) 27 | h = W * lp.batch([1,2,3]) 28 | print(h.npvalue()) 29 | print(dy.pick_batch(h,[1,2,3]).npvalue()) 30 | print(dy.pick(W*lp[1],1).value(), dy.pick(W*lp[2],2).value(), dy.pick(W*lp[3],3).value()) 31 | 32 | # using pickneglogsoftmax_batch 33 | print("\nPick neg log softmax") 34 | print((-dy.log(dy.softmax(h))).npvalue()) 35 | print(dy.pickneglogsoftmax_batch(h,[1,2,3]).npvalue()) 36 | -------------------------------------------------------------------------------- /examples/batching/train_xor-batch.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/training.h" 2 | #include "dynet/expr.h" 3 | #include "dynet/io.h" 4 | 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | using namespace dynet; 10 | 11 | int main(int argc, char** argv) { 12 | dynet::initialize(argc, argv); 13 | 14 | // parameters 15 | const unsigned HIDDEN_SIZE = 8; 16 | const unsigned ITERATIONS = 200; 17 | ParameterCollection m; 18 | SimpleSGDTrainer trainer(m); 19 | 20 | ComputationGraph cg; 21 | Parameter p_W, p_b, p_V, p_a; 22 | 23 | p_W = m.add_parameters({HIDDEN_SIZE, 2}); 24 | p_b = m.add_parameters({HIDDEN_SIZE}); 25 | p_V = m.add_parameters({1, HIDDEN_SIZE}); 26 | p_a = m.add_parameters({1}); 27 | 28 | if (argc == 2) { 29 | // Load the model and parameters from file if given. 30 | TextFileLoader loader(argv[1]); 31 | loader.populate(m); 32 | } 33 | 34 | Expression W = parameter(cg, p_W); 35 | Expression b = parameter(cg, p_b); 36 | Expression V = parameter(cg, p_V); 37 | Expression a = parameter(cg, p_a); 38 | 39 | // set x_values to change the inputs to the network 40 | Dim x_dim({2}, 4), y_dim({1}, 4); 41 | cerr << "x_dim=" << x_dim << ", y_dim=" << y_dim << endl; 42 | vector x_values = {1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0}; 43 | Expression x = input(cg, x_dim, &x_values); 44 | // set y_values expressing the output 45 | vector y_values = {-1.0, 1.0, 1.0, -1.0}; 46 | Expression y = input(cg, y_dim, &y_values); 47 | 48 | Expression h = tanh(W*x + b); 49 | //Expression h = tanh(affine_transform({b, W, x})); 50 | //Expression h = softsign(W*x + b); 51 | Expression y_pred = V*h + a; 52 | Expression loss = squared_distance(y_pred, y); 53 | Expression sum_loss = sum_batches(loss); 54 | 55 | // train the parameters 56 | for (unsigned iter = 0; iter < ITERATIONS; ++iter) { 57 | float my_loss = as_scalar(cg.forward(sum_loss)) / 4; 58 | cg.backward(sum_loss); 59 | trainer.update(); 60 | cerr << "E = " << my_loss << endl; 61 | } 62 | 63 | // Output the model and parameter objects 64 | // to a cout. 65 | TextFileSaver saver("xor-batch.model"); 66 | saver.save(m); 67 | } 68 | -------------------------------------------------------------------------------- /examples/cpp-utils/getpid.h: -------------------------------------------------------------------------------- 1 | // Platform-independent header to allow calls to getpid() 2 | // 3 | #if _WINDOWS 4 | #include 5 | #else 6 | #include 7 | #endif 8 | -------------------------------------------------------------------------------- /examples/devices/cpu_vs_gpu.py: -------------------------------------------------------------------------------- 1 | # Usage: python cpu_vs_gpu.py 2 | 3 | import time 4 | from multiprocessing import Process 5 | 6 | def do_cpu(): 7 | import _dynet as C 8 | C.init() 9 | cm = C.Model() 10 | cpW = cm.add_parameters((1000,1000)) 11 | s = time.time() 12 | C.renew_cg() 13 | W = C.parameter(cpW) 14 | W = W*W*W*W*W*W*W 15 | z = C.squared_distance(W,W) 16 | z.value() 17 | z.backward() 18 | print("CPU time:",time.time() - s) 19 | 20 | 21 | def do_gpu(): 22 | import _dynet as G 23 | import sys 24 | sys.argv.append('--dynet-devices') 25 | sys.argv.append('GPU:0') 26 | G.init() 27 | gm = G.Model() 28 | gpW = gm.add_parameters((1000,1000)) 29 | s = time.time() 30 | G.renew_cg() 31 | W = G.parameter(gpW) 32 | W = W*W*W*W*W*W*W 33 | z = G.squared_distance(W,W) 34 | z.value() 35 | z.backward() 36 | print("GPU time:",time.time() - s) 37 | 38 | if __name__ == '__main__': 39 | procs1 = Process(target=do_cpu) 40 | procs1.start() 41 | procs2 = Process(target=do_gpu) 42 | procs2.start() 43 | procs1.join() 44 | procs2.join() 45 | -------------------------------------------------------------------------------- /examples/example-data/dev-hsm.txt: -------------------------------------------------------------------------------- 1 | i think this is wrong . can you check it ? 2 | fine . 3 | i went back to my hotel room and found a here . 4 | it would be one hundred sixty- eight dollars per night for a twin , with tax and service charge . 5 | i would like to sit down and take a rest for a little while . 6 | i am . 7 | number three . 8 | who are you ? 9 | which is the biggest department store in this city ? 10 | can i see your ticket , sir ? 11 | -------------------------------------------------------------------------------- /examples/example-data/dev-poi.txt: -------------------------------------------------------------------------------- 1 | i 'll take it . ||| 5 2 | we want to have a table near the window . ||| 10 3 | how can i help you ? ||| 6 4 | would you , please ? ||| 5 5 | -------------------------------------------------------------------------------- /examples/example-data/fin-toy.txt: -------------------------------------------------------------------------------- 1 | 21 11 7 6 7 22 22 1 24 1 21 22 11 2 | 21 3 13 21 3 14 3 11 21 14 7 10 22 11 7 16 3 | 24 3 14 11 22 22 3 24 3 21 22 3 4 | 12 17 10 3 16 16 3 3 5 | 14 3 11 15 7 16 22 3 15 11 21 7 13 21 11 6 | 20 23 23 13 11 16 13 23 12 3 14 14 3 7 | 27 6 11 16 24 3 13 3 24 3 20 3 11 21 23 23 6 7 16 8 | 8 17 14 13 10 1 14 21 3 16 11 16 9 | 27 17 11 9 17 3 10 | 20 3 21 11 22 22 11 24 3 22 11 | 14 3 23 14 3 15 11 16 7 16 12 | 22 23 17 10 11 14 3 15 15 7 14 22 3 13 | 10 7 14 21 11 16 13 11 14 1 11 21 13 7 11 14 3 3 12 3 14 | 15 3 13 21 3 16 7 11 6 7 16 15 | 21 23 18 7 20 27 13 21 11 13 2 16 16 | 21 27 27 22 22 1 12 1 21 22 1 17 | 18 1 11 10 6 7 17 16 9 7 14 15 11 11 16 18 | 21 11 11 22 17 21 7 16 19 | 13 11 10 3 20 17 11 22 3 20 | 22 23 10 13 3 11 21 7 16 21 | 20 23 15 3 3 16 22 | 21 23 15 15 11 21 21 3 23 | 18 1 11 10 6 7 3 21 11 17 11 21 22 3 24 | 7 21 11 17 22 22 7 14 23 11 21 21 3 3 16 25 | 24 3 11 24 3 23 22 23 16 23 22 22 3 26 | 15 7 16 11 13 2 27 | 4 3 5 13 7 11 14 14 3 28 | 15 11 9 20 7 7 16 11 16 29 | 20 3 13 13 3 23 21 21 23 10 22 7 11 21 21 3 30 | 17 4 12 7 13 22 11 21 22 3 31 | 21 17 21 11 3 14 11 21 22 11 18 20 7 21 11 6 7 16 22 22 11 32 | 22 27 10 12 11 7 16 33 | 21 10 7 16 28 10 7 16 11 14 1 11 16 7 16 34 | 3 14 3 13 3 21 22 11 11 16 35 | 10 3 23 21 13 3 16 18 11 22 17 3 36 | 15 23 17 22 22 11 11 16 37 | 3 20 17 15 3 3 22 22 11 21 7 22 38 | 12 1 20 24 7 16 22 11 7 39 | 17 15 11 21 22 7 11 21 7 14 14 7 40 | 21 23 24 3 11 22 21 7 24 3 11 16 7 16 41 | 10 7 20 13 11 21 22 27 11 16 42 | 22 17 11 15 7 22 22 17 15 3 16 3 43 | 22 3 13 7 20 22 23 23 44 | 22 23 17 22 3 16 22 17 24 3 11 10 7 7 21 21 3 45 | 24 3 18 3 3 13 3 18 18 3 14 7 17 11 13 7 23 21 46 | 22 11 7 22 7 7 16 22 7 13 11 12 1 22 47 | 15 3 23 14 3 24 11 20 22 3 48 | 27 2 18 3 13 13 3 21 11 3 49 | 18 7 14 11 3 12 3 21 21 3 50 | 15 1 16 16 11 13 2 11 21 22 1 51 | -------------------------------------------------------------------------------- /examples/example-data/seg-sup.dev.txt: -------------------------------------------------------------------------------- 1 | a a 0 1 a ||| O:1 O:1 N:2 O:1 2 | x y 1 a ||| O:1 O:1 N:1 O:1 3 | a a a ||| O:1 O:1 O:1 4 | 1 0 1 a ||| N:3 O:1 5 | a 1 a 1 ||| O:1 N:1 O:1 N:1 6 | 0 0 a a 0 0 a a ||| N:2 O:1 O:1 N:2 O:1 O:1 7 | -------------------------------------------------------------------------------- /examples/example-data/textcat.txt: -------------------------------------------------------------------------------- 1 | what is your name ? ||| Q 2 | where do you live ? ||| Q 3 | i live in pittsburgh . ||| D 4 | my name is bond . ||| D 5 | wtf ? she asked . ||| D 6 | did she say " yes . " ? ||| Q 7 | -------------------------------------------------------------------------------- /examples/jupyter-tutorials/imgs/autobatch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/jupyter-tutorials/imgs/autobatch.gif -------------------------------------------------------------------------------- /examples/jupyter-tutorials/imgs/bist-autobatch-speed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/jupyter-tutorials/imgs/bist-autobatch-speed.png -------------------------------------------------------------------------------- /examples/jupyter-tutorials/imgs/treelstm-autobatch-speed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/jupyter-tutorials/imgs/treelstm-autobatch-speed.png -------------------------------------------------------------------------------- /examples/mnist/README.md: -------------------------------------------------------------------------------- 1 | # MNIST example 2 | 3 | Here's an example usage of dynet for the "Hello World" of deep learning : MNIST digit classification 4 | 5 | ## Usage 6 | 7 | First, download the MNIST dataset from the [official website](http://yann.lecun.com/exdb/mnist/) and decompress it. 8 | 9 | wget -O - http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz | gunzip > train-images.idx3-ubyte 10 | wget -O - http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz | gunzip > train-labels.idx1-ubyte 11 | wget -O - http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz | gunzip > t10k-images.idx3-ubyte 12 | wget -O - http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz | gunzip > t10k-labels.idx1-ubyte 13 | 14 | Then, run the training (here for a batch size of 128 and 20 epochs) : 15 | 16 | ./train_mnist \ 17 | --train train-images.idx3-ubyte \ 18 | --train_labels train-labels.idx1-ubyte \ 19 | --dev t10k-images.idx3-ubyte \ 20 | --dev_labels t10k-labels.idx1-ubyte \ 21 | --batch_size 128 \ 22 | --num_epochs 20 23 | 24 | ## Benchmark 25 | 26 | System | Speed | Test accuracy (after 20 epochs) 27 | ------------ | ------------- | ------------- 28 | Intel® Core™ i5-4200H CPU @ 2.80GHz × 4 | ~7±0.5 s per epoch| 97.84 % 29 | -------------------------------------------------------------------------------- /examples/python-utils/util.py: -------------------------------------------------------------------------------- 1 | import mmap 2 | 3 | class Vocab: 4 | def __init__(self, w2i): 5 | self.w2i = dict(w2i) 6 | self.i2w = {i:w for w,i in w2i.items()} 7 | 8 | @classmethod 9 | def from_corpus(cls, corpus): 10 | w2i = {} 11 | for sent in corpus: 12 | for word in sent: 13 | w2i.setdefault(word, len(w2i)) 14 | return Vocab(w2i) 15 | 16 | def size(self): 17 | return len(self.w2i.keys()) 18 | 19 | #This corpus reader can be used when reading large text file into a memory can solve IO bottleneck of training. 20 | #Use it exactly as the regular CorpusReader from the rnnlm.py 21 | class FastCorpusReader: 22 | def __init__(self, fname): 23 | self.fname = fname 24 | self.f = open(fname, 'rb') 25 | def __iter__(self): 26 | #This usage of mmap is for a Linux\OS-X 27 | #For Windows replace prot=mmap.PROT_READ with access=mmap.ACCESS_READ 28 | m = mmap.mmap(self.f.fileno(), 0, prot=mmap.PROT_READ) 29 | data = m.readline() 30 | while data: 31 | line = data 32 | data = m.readline() 33 | line = line.lower() 34 | line = line.strip().split() 35 | yield line 36 | 37 | class CorpusReader: 38 | def __init__(self, fname): 39 | self.fname = fname 40 | def __iter__(self): 41 | for line in file(self.fname): 42 | line = line.strip().split() 43 | #line = [' ' if x == '' else x for x in line] 44 | yield line 45 | 46 | class CharsCorpusReader: 47 | def __init__(self, fname, begin=None): 48 | self.fname = fname 49 | self.begin = begin 50 | 51 | def __iter__(self): 52 | begin = self.begin 53 | with open(self.fname) as f: 54 | for line in f: 55 | line = list(line) 56 | if begin: 57 | line = [begin] + line 58 | yield line 59 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/README.md: -------------------------------------------------------------------------------- 1 | # Reinforcement Learning using DyNet 2 | We provide DyNet example implementations of two commonly-adopted reinforcement learning (RL) algorithms 3 | * [Deep Q-Network (DQN)](https://arxiv.org/abs/1312.5602) 4 | * [Deep Deterministic Policy Gradient (DDPG)](https://arxiv.org/abs/1509.02971) 5 | 6 | In order to obtain good training performance, we also (optionally) incorporate some de facto techniques for RL, including [Double Q-Learning](https://arxiv.org/abs/1509.06461), [Prioritized Replay](https://arxiv.org/abs/1511.05952) and [Dueling Network Architectures](https://arxiv.org/abs/1511.06581). 7 | 8 | ## Requirements 9 | * Python (>= 3.5) 10 | * [DyNet (Python)](http://dynet.readthedocs.io/en/latest/python.html#manual-installation) 11 | * [OpenAI Gym](https://github.com/openai/gym) 12 | 13 | ## Classical Control with Discrete Action Space using DQN 14 | ### Run training 15 | ``` 16 | python main_dqn.py --env_id 0 17 | ``` 18 | where `env_id` chooses one of the classic control environment from OpenAI Gym, which includes [CartPole](https://gym.openai.com/envs/CartPole-v1/), [MountainCar](https://gym.openai.com/envs/MountainCar-v0/), and [Acrobot](https://gym.openai.com/envs/Acrobot-v1/), all with **discrete action space**. 19 | 20 | To enable Double Q-Learning, Prioritized Replay and Dueling Network Architecture, we can add the following tags, respectively: 21 | ``` 22 | python main_dqn.py --env_id 0 --double --dueling --prioritized 23 | ``` 24 | ### Demo results 25 | 26 | 27 | ## Mujoco (Continuous Control) using DDPG: 28 | ### Run training 29 | ``` 30 | python main_ddpg.py 31 | ``` 32 | ### Demo results 33 | 34 | 35 | In the Mujoco game, the action space is **continuous**. The two-leg robot is supposed to be trained not to fall down and to move forward as fast as possible. The above three different strategies learned by the bot are *standing still*, *moving slowly* and *hopping forward*. The corresponding rewards of these strategies in the 'Walker2d-v2' environment is around 1050, 1200 and 1600, respectively, compared to random policies which get 1.7 in average. 36 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/main_ddpg.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gym 3 | from ddpg import DDPG 4 | from train_test_utils import train_pipeline_conservative, test 5 | 6 | 7 | def establish_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--env_name", default="Walker2d-v2", type=str) 10 | parser.add_argument("--memory_size", default=1e6, type=float) 11 | parser.add_argument("--batch_size", default=64, type=int) 12 | parser.add_argument("--n_epoch", default=500, type=int) 13 | parser.add_argument("--rollout_per_epoch", default=100, type=int) 14 | parser.add_argument("--train_per_epoch", default=100, type=int) 15 | args = parser.parse_args() 16 | return args 17 | 18 | 19 | args = establish_args() 20 | env = gym.make(args.env_name) 21 | player = DDPG(env.observation_space.shape[0], env.action_space.shape[0], hiddens_actor=[64, 64], 22 | hiddens_critic=[64, 64], memory_size=int(args.memory_size)) 23 | 24 | train_pipeline_conservative(env, player, score_threshold=999, batch_size=args.batch_size, n_epoch=args.n_epoch, 25 | n_rollout=args.rollout_per_epoch, n_train=args.train_per_epoch) 26 | test(env, player, n_turns=10, render=True) 27 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/main_dqn.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import gym 3 | 4 | from dqn import DeepQNetwork 5 | from network import MLP, Header 6 | from train_test_utils import train_pipeline_progressive, train_pipeline_conservative, test 7 | 8 | 9 | def establish_args(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('--dynet-gpus', default=0, type=int) 12 | 13 | parser.add_argument('--env_id', default=0, type=int) 14 | parser.add_argument('--double', default=False, action='store_true') 15 | parser.add_argument('--dueling', default=False, action='store_true') 16 | parser.add_argument('--prioritized', default=False, action='store_true') 17 | args = parser.parse_args() 18 | return args 19 | 20 | 21 | # ==== args ==== 22 | args = establish_args() 23 | 24 | # ==== environment ===== 25 | ENVs = ['CartPole-v1', 'Acrobot-v1', 'MountainCar-v0'] 26 | score_thresholds = [499, -100, -100] 27 | 28 | env_id = args.env_id 29 | ENV, score_threshold = ENVs[env_id], score_thresholds[env_id] 30 | env = gym.make(ENV) 31 | 32 | # ==== DQN ==== 33 | MEMORY_SIZE = 50000 34 | 35 | HIDDENS = [128] 36 | 37 | network = Header(inpt_shape=env.observation_space.shape, hiddens=HIDDENS, opt_size=env.action_space.n, 38 | network=MLP, dueling=args.dueling) 39 | target_network = Header(inpt_shape=env.observation_space.shape, hiddens=HIDDENS, 40 | opt_size=env.action_space.n, network=MLP, dueling=args.dueling) if args.double else None 41 | 42 | dqn = DeepQNetwork(network=network, memory_size=MEMORY_SIZE, use_double_dqn=args.double, target_network=target_network, 43 | dueling=args.dueling, prioritized=args.prioritized) 44 | 45 | # ==== train & test ==== 46 | # choose one of the two pipelines 47 | if env_id == 0: 48 | train_pipeline_conservative(env, dqn, score_threshold, n_epoch=500, n_rollout=100, n_train=1000, batch_size=256) 49 | if env_id == 1 or env_id == 2: 50 | train_pipeline_progressive(env, dqn, score_threshold, batch_size=32, n_episode=2000) 51 | 52 | test(env, dqn, n_turns=10, render=True) 53 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/memory.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import log, ceil 3 | 4 | from reduce_tree import ReduceTree, SumTree 5 | 6 | 7 | # A simple memory to store and sample experiences. 8 | class Memory(object): 9 | def __init__(self, size): 10 | self.size = size 11 | self.idx = 0 12 | self.memory = np.zeros(size, dtype=object) 13 | 14 | def store(self, exp): 15 | self.memory[self.idx % self.size] = exp 16 | self.idx += 1 17 | 18 | def sample(self, batch_size): 19 | indices = np.random.choice(min(self.idx, self.size), batch_size) 20 | return self.memory[indices] 21 | 22 | 23 | # Prioritized Replay: https://arxiv.org/abs/1511.05952 24 | class PrioritizedMemory(object): 25 | def __init__(self, size, alpha=0.6): 26 | self.size = int(2 ** ceil(log(size, 2))) 27 | self.memory = np.zeros(self.size, dtype=object) 28 | self.sum_tree = SumTree(self.size) 29 | self.min_tree = ReduceTree(self.size, min) 30 | self.idx = 0 31 | self.max_value = 1. 32 | self.max_value_upper = 1000. 33 | self.alpha = alpha 34 | 35 | def store(self, exp): 36 | idx = self.idx % self.size 37 | self.memory[idx] = exp 38 | self.sum_tree[idx] = self.max_value 39 | self.min_tree[idx] = self.max_value 40 | self.idx += 1 41 | 42 | def sample(self, batch_size, beta): 43 | indices = [] 44 | max_value = self.sum_tree.root 45 | for _ in range(batch_size): 46 | value = np.random.uniform(0, max_value) 47 | idx = self.sum_tree.sample(value) 48 | indices.append(idx) 49 | min_value = self.min_tree.root 50 | return indices, self.memory[indices], (self.sum_tree[indices] / (min_value + 1e-4)) ** (-beta) 51 | 52 | def update(self, indices, values): 53 | values = np.array(values) 54 | values_modified = values ** self.alpha 55 | self.sum_tree[indices] = values_modified 56 | self.min_tree[indices] = values_modified 57 | self.max_value = max(self.max_value, np.max(values)) 58 | self.max_value = min(self.max_value, self.max_value_upper) 59 | 60 | def is_full(self): 61 | return self.idx >= self.size 62 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/reduce_tree.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import numpy as np 3 | 4 | 5 | # A simple binary tree structure to calculate some statistics from leaves. 6 | class ReduceTree(object): 7 | def __init__(self, size, op): 8 | if size & (size - 1) != 0: 9 | raise ValueError("size mush be a power of 2.") 10 | self.size = size 11 | self.values = np.zeros(2 * self.size) 12 | self.op = op 13 | 14 | @property 15 | def root(self): 16 | return self.values[1] 17 | 18 | def __setitem__(self, idx, val): 19 | if isinstance(idx, int): 20 | idx += self.size 21 | self.values[idx] = val 22 | self._percolate_up(idx, val) 23 | elif isinstance(idx, list): 24 | for i, idxx in enumerate(idx): 25 | idxx += self.size 26 | self.values[idxx] = val[i] 27 | self._percolate_up(idxx, val[i]) 28 | else: 29 | raise RuntimeError("Not indexable type") 30 | 31 | def __getitem__(self, idx): 32 | if isinstance(idx, int): 33 | return self.values[idx + self.size] 34 | elif isinstance(idx, list): 35 | return self.values[np.array(idx) + self.size] 36 | else: 37 | raise RuntimeError("Not indexable type") 38 | 39 | def _percolate_up(self, idx, val): 40 | idx //= 2 41 | while idx > 0: 42 | self.values[idx] = self.op(self.values[2 * idx], self.values[2 * idx + 1]) 43 | idx //= 2 44 | 45 | 46 | class SumTree(ReduceTree): 47 | def __init__(self, size): 48 | super().__init__(size, operator.add) 49 | 50 | def sample(self, value): 51 | idx = 1 52 | while idx < self.size: 53 | child = 2 * idx 54 | if value <= self.values[child]: 55 | idx = child 56 | else: 57 | value -= self.values[child] 58 | idx = child + 1 59 | 60 | return idx - self.size 61 | -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/Acrobot.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/Acrobot.gif -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/Cartpole.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/Cartpole.gif -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/MountainCar.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/MountainCar.gif -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/hopforward.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/hopforward.gif -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/move.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/move.gif -------------------------------------------------------------------------------- /examples/reinforcement-learning/results/stand_still.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/reinforcement-learning/results/stand_still.gif -------------------------------------------------------------------------------- /examples/rnnlm/README.md: -------------------------------------------------------------------------------- 1 | ### Obtaining LM data 2 | 3 | ./install-examples.sh 4 | 5 | This downloads the data used in this tutorial. 6 | 7 | ### Training example 8 | 9 | Train an LSTM LM using a class-factor softmax: 10 | 11 | ./train_rnnlm -s -t ../rnnlm/ptb-mikolov/train.txt -d ../rnnlm/ptb-mikolov/valid.txt \ 12 | -c ../rnnlm/ptb-mikolov/clusters-mkcls.txt -D 0.3 --hidden_size 256 --eta_decay_onset_epoch 10 --eta_decay_rate 0.5 13 | 14 | Train an LSTM LM with a standard softmax: 15 | 16 | ./train_rnnlm -s -t ../rnnlm/ptb-mikolov/train.txt -d ../rnnlm/ptb-mikolov/valid.txt \ 17 | -D 0.3 --hidden_size 256 --eta_decay_onset_epoch 10 --eta_decay_rate 0.5 18 | 19 | ### Evaluation example 20 | 21 | Evaluate a trained model: 22 | 23 | ./train_rnnlm -t ../rnnlm/ptb-mikolov/train.txt -c ../rnnlm/ptb-mikolov/clusters-mkcls.txt \ 24 | -m lm_0.3_2_128_256-pid7865.params --hidden_size 256 -p ../rnnlm/ptb-mikolov/test.txt 25 | 26 | ### PTB Baselines 27 | 28 | | Model | dev | test | 29 | | ----- | ---:| ----:| 30 | | 5-gram KN | 188.0 | 178.9 | 31 | | 2x128, dropout=0.3, class-factored softmax | 164.4 | 157.7 | 32 | | 2x256, dropout=0.3, CFSM, decay 0.5@>10 | 129.7 | 125.4 | 33 | -------------------------------------------------------------------------------- /examples/rnnlm/install-examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | data_version=ptb-mikolov-01.tar.gz 5 | 6 | ROOTDIR=`dirname $0` 7 | cd $ROOTDIR 8 | 9 | rm -f $data_version 10 | rm -rf ptb-mikolov 11 | curl -f http://demo.clab.cs.cmu.edu/cdyer/$data_version -o $data_version 12 | tar xzf $data_version 13 | rm -f $data_version 14 | 15 | echo SUCCESS. 1>&2 16 | 17 | -------------------------------------------------------------------------------- /examples/tensorboard/runcrayonserver.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | # This script makes the assumption that there is only one image with name alband/crayon, which should always be true, but also it assumes the crayon container name is crayon and that there is only one docker container running from the already mentioned image. 4 | 5 | 6 | use strict; 7 | use encoding 'utf8'; 8 | 9 | binmode(STDIN, ":utf8"); 10 | binmode(STDOUT, ":utf8"); 11 | binmode(STDERR, ":utf8"); 12 | 13 | my ($p1, $p2) = @ARGV; 14 | 15 | $p1 = 8888 if not defined($p1); 16 | $p2 = 8889 if not defined($p2); 17 | 18 | print "[+] Checking for docker images.\n"; 19 | my $docker_images = `sudo docker images`; 20 | 21 | if($docker_images =~ m/alband\/crayon/){ 22 | print "Docker image already exists. If for some reason you want to pull a different version you have to do it manually\n"; 23 | } else { 24 | print "[+] Downloading docker image.\n"; 25 | `sudo docker pull alband/crayon`; 26 | } 27 | 28 | print "[+] Checking for docker containers.\n"; 29 | my $docker_containers = `sudo docker ps -a`; 30 | 31 | if($docker_containers =~ m/alband\/crayon/){ 32 | print "[+] Restarting docker\n"; 33 | `sudo docker restart crayon`; 34 | } else { 35 | print "[+] Docker container does not exist, running for the first time."; 36 | my $dock_run = "sudo docker run -d -p $p1:$p1 -p $p2:$p2 --name crayon alband/crayon"; 37 | print "Running $dock_run\n"; 38 | `$dock_run`; 39 | } 40 | 41 | 42 | 1; 43 | -------------------------------------------------------------------------------- /examples/tensorboard/rundocker.perl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | use strict; 4 | use encoding 'utf8'; 5 | 6 | binmode(STDIN, ":utf8"); 7 | binmode(STDOUT, ":utf8"); 8 | binmode(STDERR, ":utf8"); 9 | 10 | 11 | print "[+] Checking whether docker is running.\n"; 12 | my $service_status = `systemctl status docker.service`; 13 | #print $service_status; 14 | 15 | if($service_status =~ m/Active: inactive/) { 16 | print "[+] Docker was inactive, starting it now.\n"; 17 | `sudo systemctl start docker.service`; 18 | } else { 19 | print "[+] Docker is already active.\n"; 20 | 21 | } 22 | 23 | 1; 24 | -------------------------------------------------------------------------------- /examples/tensorboard/tensorboardexample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/tensorboard/tensorboardexample.png -------------------------------------------------------------------------------- /examples/tensorboard/tensorboardexample2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clab/dynet/22b44958476731af4d1fdbc553626eca3a27e808/examples/tensorboard/tensorboardexample2.png -------------------------------------------------------------------------------- /examples/tensorboard/util.py: -------------------------------------------------------------------------------- 1 | import mmap 2 | 3 | class Vocab: 4 | def __init__(self, w2i): 5 | self.w2i = dict(w2i) 6 | self.i2w = {i:w for w,i in w2i.items()} 7 | 8 | @classmethod 9 | def from_corpus(cls, corpus): 10 | w2i = {} 11 | for sent in corpus: 12 | for word in sent: 13 | w2i.setdefault(word, len(w2i)) 14 | return Vocab(w2i) 15 | 16 | def size(self): 17 | return len(self.w2i.keys()) 18 | 19 | #This corpus reader can be used when reading large text file into a memory can solve IO bottleneck of training. 20 | #Use it exactly as the regular CorpusReader from the rnnlm.py 21 | class FastCorpusReader: 22 | def __init__(self, fname): 23 | self.fname = fname 24 | self.f = open(fname, 'rb') 25 | def __iter__(self): 26 | #This usage of mmap is for a Linux\OS-X 27 | #For Windows replace prot=mmap.PROT_READ with access=mmap.ACCESS_READ 28 | m = mmap.mmap(self.f.fileno(), 0, prot=mmap.PROT_READ) 29 | data = m.readline() 30 | while data: 31 | line = data 32 | data = m.readline() 33 | line = line.lower() 34 | line = line.strip().split() 35 | yield line 36 | 37 | class CorpusReader: 38 | def __init__(self, fname): 39 | self.fname = fname 40 | def __iter__(self): 41 | for line in file(self.fname): 42 | line = line.strip().split() 43 | #line = [' ' if x == '' else x for x in line] 44 | yield line 45 | 46 | class CharsCorpusReader: 47 | def __init__(self, fname, begin=None): 48 | self.fname = fname 49 | self.begin = begin 50 | 51 | def __iter__(self): 52 | begin = self.begin 53 | with open(self.fname) as f: 54 | for line in f: 55 | line = list(line) 56 | if begin: 57 | line = [begin] + line 58 | yield line 59 | -------------------------------------------------------------------------------- /examples/transformer/def.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | using namespace std; 6 | 7 | typedef int WordId;// word Id 8 | typedef std::vector WordIdSentence;// word Id sentence 9 | typedef std::vector WordIdSentences;// batches of sentences 10 | typedef tuple WordIdSentencePair; // Note: can be extended to include additional information (e.g., document ID) 11 | typedef vector WordIdCorpus; 12 | 13 | 14 | -------------------------------------------------------------------------------- /examples/transformer/dict-utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "dynet/dict.h" 9 | 10 | using namespace std; 11 | using namespace dynet; 12 | 13 | inline void load_vocabs(const string& src_vocab_file, const string& trg_vocab_file 14 | , dynet::Dict& sd, dynet::Dict& td, bool freeze=true); 15 | 16 | inline void load_vocabs(const string& src_vocab_file, const string& trg_vocab_file 17 | , dynet::Dict& sd, dynet::Dict& td, bool freeze) 18 | { 19 | if ("" == src_vocab_file || "" == trg_vocab_file) return; 20 | 21 | cerr << endl << "Loading vocabularies from files..." << endl; 22 | cerr << "Source vocabulary file: " << src_vocab_file << endl; 23 | cerr << "Target vocabulary file: " << trg_vocab_file << endl; 24 | ifstream if_src_vocab(src_vocab_file), if_trg_vocab(trg_vocab_file); 25 | string sword, tword; 26 | while (getline(if_src_vocab, sword)) sd.convert(sword); 27 | while (getline(if_trg_vocab, tword)) td.convert(tword); 28 | 29 | cerr << "Source vocabluary: " << sd.size() << endl; 30 | cerr << "Target vocabluary: " << td.size() << endl; 31 | 32 | if (freeze){ 33 | sd.freeze(); 34 | td.freeze(); 35 | } 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /examples/transformer/mask-utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // DyNet 4 | #include "dynet/nodes.h" 5 | #include "dynet/dynet.h" 6 | #include "dynet/expr.h" 7 | 8 | // STL 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | using namespace dynet; 14 | 15 | #define PSEUDO_MIN_VALUE -999999.f 16 | 17 | dynet::Expression create_triangle_mask(dynet::ComputationGraph &cg, unsigned length, bool upper=true/*false for lower*/); 18 | 19 | dynet::Expression create_triangle_mask(dynet::ComputationGraph &cg, unsigned length, bool upper) { 20 | // fill triangle mask 21 | std::vector vMask(length * length, 0.f); 22 | for(unsigned i = 0; i < length; ++i){ 23 | for(unsigned j = 0; j <= i; ++j){ 24 | if (!upper)// lower 25 | vMask[i * length + j] = 1.f; 26 | else// upper 27 | vMask[j * length + i] = 1.f; 28 | } 29 | } 30 | 31 | dynet::Expression i_mask = dynet::input(cg, {length, length}, vMask); 32 | 33 | i_mask = (1.f - i_mask) * PSEUDO_MIN_VALUE;// convert 0/1 mask to transformer style -inf mask 34 | 35 | return i_mask; 36 | } 37 | 38 | -------------------------------------------------------------------------------- /examples/transformer/math-utils.h: -------------------------------------------------------------------------------- 1 | #include // DBL_MAX 2 | #include // numeric_limits 3 | 4 | template 5 | inline bool is_infinite( const T &value ) 6 | { 7 | // Since we're a template, it's wise to use std::numeric_limits 8 | // 9 | // Note: std::numeric_limits::min() behaves like DBL_MIN, and is the smallest absolute value possible. 10 | // 11 | 12 | T max_value = std::numeric_limits::max(); 13 | T min_value = - max_value; 14 | 15 | return !( min_value <= value && value <= max_value ); 16 | } 17 | 18 | template 19 | inline bool is_nan( const T &value ) 20 | { 21 | // True if NAN 22 | return value != value; 23 | } 24 | 25 | template 26 | inline bool is_valid( const T &value ) 27 | { 28 | return !is_infinite(value) && !is_nan(value); 29 | } 30 | 31 | -------------------------------------------------------------------------------- /examples/transformer/str-utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | using namespace std; 7 | 8 | std::vector split_words(const std::string & str); 9 | 10 | std::vector split_words(const std::string &line) { 11 | std::istringstream in(line); 12 | std::string word; 13 | std::vector res; 14 | while(in) { 15 | in >> word; 16 | if (!in || word.empty()) break; 17 | res.push_back(word); 18 | } 19 | return res; 20 | } 21 | 22 | -------------------------------------------------------------------------------- /examples/transformer/timer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | struct MyTimer { 10 | MyTimer(const std::string& msg) : msg(msg), start(std::chrono::high_resolution_clock::now()) {} 11 | ~MyTimer() { 12 | } 13 | void show(){ 14 | auto stop = std::chrono::high_resolution_clock::now(); 15 | std::cerr << '[' << msg << ' ' << std::chrono::duration(stop-start).count() << " ms]\n"; 16 | } 17 | void reset(){ 18 | start = std::chrono::high_resolution_clock::now(); 19 | } 20 | double elapsed() { 21 | auto stop = std::chrono::high_resolution_clock::now(); 22 | return std::chrono::duration(stop-start).count();// in ms 23 | } 24 | std::string msg; 25 | std::chrono::high_resolution_clock::time_point start; 26 | }; 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/transformer/utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "def.h" 4 | 5 | #include "str-utils.h" 6 | #include "dict-utils.h" 7 | #include "math-utils.h" 8 | #include "batch-utils.h" 9 | #include "data-utils.h" 10 | #include "mask-utils.h" 11 | 12 | #include "expr-xtra.h" 13 | 14 | #include "timer.h" 15 | 16 | -------------------------------------------------------------------------------- /examples/treelstm/filter_glove.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import re 3 | import os 4 | 5 | data_dir = 'trees' 6 | datasets = ['train', 'dev', 'test'] 7 | glove_origin_path = 'glove.840B.300d.txt' 8 | glove_filtered_path = 'glove_filtered.txt' 9 | 10 | 11 | def get_vocab(file_path): 12 | vocab = set() 13 | tokker = re.compile(r'([^ ()]+)\)') 14 | with codecs.open(file_path) as f: 15 | for line in f: 16 | for match in tokker.finditer(line.strip()): 17 | vocab.add(match.group(1)) 18 | return vocab 19 | 20 | 21 | vocab = set() 22 | for dataset in datasets: 23 | tem_set = get_vocab(os.path.join(data_dir, dataset + '.txt')) 24 | vocab.update(tem_set) 25 | 26 | total = cnt = 0 27 | with codecs.open(glove_origin_path) as fin: 28 | with codecs.open(glove_filtered_path, 'w') as fout: 29 | for line in fin: 30 | total += 1 31 | word = line.split(' ', 1)[0] 32 | if word in vocab or word == '(' or word == ')': 33 | cnt += 1 34 | fout.write(line) 35 | print('total: {}, after filtering: {}'.format(total, cnt)) 36 | 37 | -------------------------------------------------------------------------------- /examples/treelstm/utils.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import numpy as np 3 | import dynet as dy 4 | 5 | 6 | def acc_eval(dataset, model): 7 | dataset.reset(shuffle=False) 8 | good = bad = 0.0 9 | for tree in dataset: 10 | dy.renew_cg() 11 | pred = np.argmax(model.predict_for_tree(tree, decorate=False, training=False)) 12 | if pred == tree.label: 13 | good += 1 14 | else: 15 | bad += 1 16 | acc = good / (good + bad) 17 | return acc 18 | 19 | 20 | def get_embeds(embed_path): 21 | word_embeds, w2i = [np.random.randn(300)], {'_UNK_': 0} 22 | with codecs.open(embed_path) as f: 23 | for line in f: 24 | line = line.strip().split(' ') 25 | word, embed = line[0], line[1:] 26 | w2i[word] = len(word_embeds) 27 | word_embeds.append(np.array(embed, dtype=np.float32)) 28 | w2i['-LRB-'] = w2i['('] 29 | w2i['-RRB-'] = w2i[')'] 30 | return np.array(word_embeds), w2i 31 | -------------------------------------------------------------------------------- /examples/variational-autoencoder/basic-image-recon/README.md: -------------------------------------------------------------------------------- 1 | # Basic VAE Example: MNIST Images 2 | 3 | This example is a DyNet version of the same example as [a PyTorch one](https://github.com/pytorch/examples/tree/master/vae). 4 | Benchmarks are attached below. 5 | 6 | > This is an improved implementation of the paper Stochastic Gradient VB and the Variational Auto-Encoder by Kingma and Welling. It uses ReLUs and the adam optimizer, instead of sigmoids and adagrad. These changes make the network converge much faster. 7 | 8 | ### 1. Prepare Data 9 | 10 | ```sh 11 | mkdir data 12 | wget -O - http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz | gunzip > data/train-images-idx3-ubyte 13 | wget -O - http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz | gunzip > data/train-labels-idx1-ubyte 14 | wget -O - http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz | gunzip > data/t10k-images-idx3-ubyte 15 | wget -O - http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz | gunzip > data/t10k-labels-idx1-ubyte 16 | ``` 17 | 18 | ### 2. Run 19 | 20 | ```sh 21 | pip install -r requirements.txt 22 | python vae.py 23 | 24 | # To support gpu, please refer to the DyNet document, 25 | # install a gpu version of DyNet with pip, 26 | # and run >>> python vae.py --dynet-gpu 27 | ``` 28 | 29 | ### Benchmarks 30 | | OS | Device | Framework | Speed | Test Loss (After 10 Epochs)| 31 | | --- | --- | --- | --- | --- | 32 | | MAC OS 10.13.3 | 2.3 GHz Intel Core i5 | PyTorch | ~ 12.5±0.3 s per epoch | 97.0215 | 33 | | MAC OS 10.13.3 | 2.3 GHz Intel Core i5 | DyNet | ~ 19.8±0.3 s per epoch | 97.4233 | 34 | | Ubuntu 16.04 | 3.40 GHz Intel Core i7-6800K | PyTorch | ~ 13.38±0.05 s per epoch | 97.1197 | 35 | | Ubuntu 16.04 | 3.40 GHz Intel Core i7-6800K | DyNet | ~ 8.73±0.01 s per epoch | 97.3211 | 36 | | Ubuntu 16.04 | GeForce GTX 1080 Ti | PyTorch | ~ 4.23±0.01 s per epoch | 97.5330 | 37 | | Ubuntu 16.04 | GeForce GTX 1080 Ti | DyNet | ~ 2.91±0.01 s per epoch | 97.6848 | 38 | -------------------------------------------------------------------------------- /examples/variational-autoencoder/basic-image-recon/requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.28 2 | dyNET==2.0.3 3 | numpy==1.14.2 4 | Pillow==9.0.0 5 | -------------------------------------------------------------------------------- /examples/xor/xor.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import dynet as dy 3 | 4 | #xsent = True 5 | xsent = False 6 | 7 | HIDDEN_SIZE = 8 8 | ITERATIONS = 2000 9 | 10 | m = dy.Model() 11 | trainer = dy.SimpleSGDTrainer(m) 12 | 13 | W = m.add_parameters((HIDDEN_SIZE, 2)) 14 | b = m.add_parameters(HIDDEN_SIZE) 15 | V = m.add_parameters((1, HIDDEN_SIZE)) 16 | a = m.add_parameters(1) 17 | 18 | if len(sys.argv) == 2: 19 | m.populate_from_textfile(sys.argv[1]) 20 | 21 | x = dy.vecInput(2) 22 | y = dy.scalarInput(0) 23 | h = dy.tanh((W*x) + b) 24 | if xsent: 25 | y_pred = dy.logistic((V*h) + a) 26 | loss = dy.binary_log_loss(y_pred, y) 27 | T = 1 28 | F = 0 29 | else: 30 | y_pred = (V*h) + a 31 | loss = dy.squared_distance(y_pred, y) 32 | T = 1 33 | F = -1 34 | 35 | 36 | for iter in range(ITERATIONS): 37 | mloss = 0.0 38 | for mi in range(4): 39 | x1 = mi % 2 40 | x2 = (mi // 2) % 2 41 | x.set([T if x1 else F, T if x2 else F]) 42 | y.set(T if x1 != x2 else F) 43 | mloss += loss.scalar_value() 44 | loss.backward() 45 | trainer.update() 46 | mloss /= 4. 47 | print("loss: %0.9f" % mloss) 48 | 49 | x.set([F,T]) 50 | z = -(-y_pred) 51 | print(z.scalar_value()) 52 | 53 | m.save("xor.pymodel") 54 | 55 | dy.renew_cg() 56 | 57 | x = dy.vecInput(2) 58 | y = dy.scalarInput(0) 59 | h = dy.tanh((W*x) + b) 60 | if xsent: 61 | y_pred = dy.logistic((V*h) + a) 62 | else: 63 | y_pred = (V*h) + a 64 | x.set([T,F]) 65 | print("TF",y_pred.scalar_value()) 66 | x.set([F,F]) 67 | print("FF",y_pred.scalar_value()) 68 | x.set([T,T]) 69 | print("TT",y_pred.scalar_value()) 70 | x.set([F,T]) 71 | print("FT",y_pred.scalar_value()) 72 | 73 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "cython"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /python/dynet.py.in: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | import dynet_config 4 | 5 | HAVE_CUDA = ${HAVE_CUDA} 6 | 7 | _CONF = dynet_config.get() 8 | 9 | if '--dynet-viz' in sys.argv: 10 | sys.argv.remove('--dynet-viz') 11 | from dynet_viz import * 12 | else: 13 | def print_graphviz(**kwarge): 14 | print("Run with --dynet-viz to get the visualization behavior.") 15 | from _dynet import * 16 | 17 | __version__ = "2.1" 18 | 19 | __gitversion__ = "${GIT_DESCRIBE}" 20 | 21 | # check HAVE_CUDA 22 | if not HAVE_CUDA: 23 | ERRMSG = 'DyNet was not installed with GPU support. Please see the installation instructions for how to make it possible to use GPUs.' 24 | if '--dynet-gpu' in sys.argv or '--dynet-gpus' in sys.argv: 25 | raise RuntimeError(ERRMSG) 26 | elif '--dynet-devices' in sys.argv: 27 | if 'GPU' in sys.argv[sys.argv.index('--dynet-devices')+1]: 28 | raise RuntimeError(ERRMSG) 29 | 30 | if _CONF is None: 31 | init() 32 | else: 33 | _params = DynetParams() 34 | _params.from_config(_CONF) 35 | _params.init() 36 | -------------------------------------------------------------------------------- /python/dynet_config.py: -------------------------------------------------------------------------------- 1 | def set(mem="512", random_seed=0, autobatch=0, 2 | profiling=0, weight_decay=0, shared_parameters=0, 3 | requested_gpus=0, gpu_mask=None): 4 | 5 | if "__DYNET_CONFIG" in __builtins__: 6 | (mem, random_seed, auto_batch, profiling) = ( 7 | __builtins__["__DYNET_CONFIG"]["mem"] if __builtins__["__DYNET_CONFIG"].get("mem") == mem else mem, 8 | __builtins__["__DYNET_CONFIG"]["seed"] if __builtins__["__DYNET_CONFIG"].get("seed") == random_seed else random_seed, 9 | __builtins__["__DYNET_CONFIG"]["autobatch"] if __builtins__["__DYNET_CONFIG"].get("autobatch") == autobatch else autobatch, 10 | __builtins__["__DYNET_CONFIG"]["profiling"] if __builtins__["__DYNET_CONFIG"].get("profiling") == profiling else profiling) 11 | (weight_decay, shared_parameters, requested_gpus, gpu_mask) = ( 12 | __builtins__["__DYNET_CONFIG"]["weight_decay"] if __builtins__["__DYNET_CONFIG"].get("weight_decay") == weight_decay else weight_decay, 13 | __builtins__["__DYNET_CONFIG"]["shared_params"] if __builtins__["__DYNET_CONFIG"].get("shared_params") == shared_parameters else shared_parameters, 14 | __builtins__["__DYNET_CONFIG"]["requested_gpus"] if __builtins__["__DYNET_CONFIG"].get("requested_gpus") == requested_gpus else requested_gpus, 15 | __builtins__["__DYNET_CONFIG"]["gpu_mask"] if __builtins__["__DYNET_CONFIG"].get("gpu_mask") == gpu_mask else gpu_mask) 16 | 17 | # TODO read "gpu_mask" from list of IDs? 18 | __builtins__["__DYNET_CONFIG"] = { 19 | "mem":mem, "seed": random_seed, "autobatch": autobatch, 20 | "profiling":profiling, "weight_decay": weight_decay, 21 | "shared_params": shared_parameters, 22 | "requested_gpus": requested_gpus, 23 | "gpu_mask": gpu_mask if gpu_mask else list(), 24 | } 25 | 26 | def set_gpu(flag=True): 27 | __builtins__["__DYNET_GPU"]=flag 28 | if "__DYNET_CONFIG" in __builtins__: 29 | __builtins__["__DYNET_CONFIG"]["requested_gpus"] = 1 30 | else: 31 | set(requested_gpus=1) 32 | 33 | def gpu(): 34 | if "__DYNET_GPU" in __builtins__: 35 | return __builtins__["__DYNET_GPU"] 36 | return None 37 | 38 | def get(): 39 | if "__DYNET_CONFIG" in __builtins__: 40 | return __builtins__["__DYNET_CONFIG"] 41 | return None 42 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(CTest) 2 | enable_testing() 3 | 4 | find_package(Boost COMPONENTS system filesystem unit_test_framework REQUIRED) 5 | include_directories(${TEST_SOURCE_DIR}/src ${Boost_INCLUDE_DIRS}) 6 | add_definitions(-DBOOST_TEST_DYN_LINK) 7 | 8 | if (DEFINED ENV{DYNET_TEST_DEVICES}) # use env variable as preprocessor macro 9 | add_definitions(-DDYNET_TEST_DEVICES=$ENV{DYNET_TEST_DEVICES}) 10 | endif() 11 | 12 | foreach(TESTNAME dim dynet exec io mem nodes params tensor trainers trainers-io rnn softmax) 13 | add_executable(test-${TESTNAME} test-${TESTNAME}.cc) 14 | if (NOT MSVC) 15 | target_link_libraries(test-${TESTNAME} ${Boost_LIBRARIES} ${CMAKE_DL_LIBS}) 16 | endif() 17 | target_link_libraries(test-${TESTNAME} dynet ${LIBS} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY}) 18 | if (WITH_CUDA_BACKEND) 19 | CUDA_ADD_CUBLAS_TO_TARGET(test-${TESTNAME}) 20 | endif(WITH_CUDA_BACKEND) 21 | add_test(NAME test-${TESTNAME} COMMAND test-${TESTNAME}) 22 | endforeach() 23 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | Use `test-dynet.cc` as a reference for how to set up subsequent tests. 2 | -------------------------------------------------------------------------------- /tests/test-dim.cc: -------------------------------------------------------------------------------- 1 | #define BOOST_TEST_MODULE TEST_DIM 2 | 3 | #include 4 | 5 | #include "test.h" 6 | 7 | using namespace dynet; 8 | 9 | struct DimTest { 10 | 11 | }; 12 | 13 | BOOST_FIXTURE_TEST_SUITE(dim_test, DimTest); 14 | 15 | BOOST_AUTO_TEST_CASE( test_dim_truncate_no_trailing_one ) { 16 | Dim d1({1,3,4}); 17 | Dim t1 = d1.truncate(); 18 | BOOST_CHECK_EQUAL(t1.nd, 3); 19 | } 20 | 21 | BOOST_AUTO_TEST_CASE( test_dim_truncate_all_one ) { 22 | Dim d1({1,1,1}); 23 | Dim t1 = d1.truncate(); 24 | BOOST_CHECK_EQUAL(t1.nd, 1); 25 | } 26 | 27 | BOOST_AUTO_TEST_CASE( test_dim_truncate_trailing_one ) { 28 | Dim d1({4,3,1}); 29 | Dim t1 = d1.truncate(); 30 | BOOST_CHECK_EQUAL(t1.nd, 2); 31 | } 32 | 33 | BOOST_AUTO_TEST_CASE( test_dim_truncate_multiple_one ) { 34 | Dim d1({4,1,1}); 35 | Dim t1 = d1.truncate(); 36 | BOOST_CHECK_EQUAL(t1.nd, 1); 37 | } 38 | 39 | BOOST_AUTO_TEST_CASE( test_dim_truncate_a_one ) { 40 | Dim d1({1}); 41 | Dim t1 = d1.truncate(); 42 | BOOST_CHECK_EQUAL(t1.nd, 1); 43 | } 44 | 45 | BOOST_AUTO_TEST_SUITE_END() 46 | 47 | -------------------------------------------------------------------------------- /tests/test-dynet.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #define BOOST_TEST_MODULE DYNETBasicTest 3 | #include 4 | #include "test.h" 5 | 6 | struct ConfigureDYNETTest { 7 | ConfigureDYNETTest() { 8 | // set up some dummy arguments to dynet 9 | for (auto x : {"ConfigureDYNETTest", "--dynet-mem", "10"}) { 10 | av.push_back(strdup(x)); 11 | } 12 | ADD_EXTRA_ARGUMENTS(av) 13 | char **argv = &av[0]; 14 | int argc = av.size(); 15 | dynet::initialize(argc, argv); 16 | } 17 | ~ConfigureDYNETTest() { 18 | for (auto x : av) free(x); 19 | } 20 | std::vector av; 21 | }; 22 | 23 | // configure DYNET 24 | BOOST_GLOBAL_FIXTURE(ConfigureDYNETTest); 25 | 26 | BOOST_AUTO_TEST_CASE( aligned_allocator ) { 27 | dynet::CPUAllocator a; 28 | void* mem = a.malloc(1024); 29 | BOOST_CHECK_EQUAL(((uintptr_t)(mem) & 0x1f), 0); 30 | ((char*)mem)[0] = 99; 31 | a.zero(mem, 1024); 32 | BOOST_CHECK_EQUAL(((char*)mem)[0], 0); 33 | a.free(mem); 34 | } 35 | 36 | -------------------------------------------------------------------------------- /tests/test-mem.cc: -------------------------------------------------------------------------------- 1 | #define BOOST_TEST_MODULE TEST_MEM 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "test.h" 9 | #include 10 | 11 | using namespace dynet; 12 | using namespace std; 13 | 14 | 15 | struct MemTest { 16 | MemTest() { 17 | // initialize if necessary 18 | for (auto x : {"MemTest", "--dynet-mem", "4"}) { 19 | av.push_back(strdup(x)); 20 | } 21 | ADD_EXTRA_ARGUMENTS(av) 22 | char **argv = &av[0]; 23 | int argc = av.size(); 24 | dynet::initialize(argc, argv); 25 | } 26 | ~MemTest() { 27 | for (auto x : av) free(x); 28 | } 29 | 30 | std::vector av; 31 | }; 32 | 33 | // define the test suite 34 | BOOST_FIXTURE_TEST_SUITE(mem_test, MemTest); 35 | 36 | BOOST_AUTO_TEST_CASE( expand_test ) { 37 | if(!autobatch_flag) { 38 | dynet::ParameterCollection mod; 39 | dynet::Parameter param = mod.add_parameters({1024,1024}); 40 | SimpleSGDTrainer trainer(mod); 41 | dynet::ComputationGraph cg; 42 | Expression x = parameter(cg, param); 43 | Expression z = sum_elems(x); 44 | cg.forward(z); 45 | cg.backward(z); 46 | trainer.update(); 47 | } 48 | } 49 | 50 | BOOST_AUTO_TEST_SUITE_END(); 51 | -------------------------------------------------------------------------------- /tutorial/0_multiply.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/dynet.h" 2 | #include "dynet/training.h" 3 | #include "dynet/expr.h" 4 | 5 | #include 6 | 7 | using namespace std; 8 | using namespace dynet; 9 | 10 | int main(int argc, char** argv) { 11 | dynet::initialize(argc, argv); 12 | 13 | ComputationGraph cg; 14 | float ia, ib; 15 | Expression a = input(cg, &ia); 16 | Expression b = input(cg, &ib); 17 | Expression y = a * b; 18 | 19 | ia = 1; 20 | ib = 2; 21 | cout << as_scalar(cg.forward(y)) << endl; // 2 22 | 23 | ia = 3; 24 | ib = 3; 25 | cout << as_scalar(cg.forward(y)) << endl; // 9 26 | } 27 | -------------------------------------------------------------------------------- /tutorial/1_linear_regression.cc: -------------------------------------------------------------------------------- 1 | #include "dynet/dynet.h" 2 | #include "dynet/training.h" 3 | #include "dynet/expr.h" 4 | 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | using namespace dynet; 10 | 11 | int main(int argc, char** argv) { 12 | dynet::initialize(argc, argv); 13 | 14 | default_random_engine rng; 15 | normal_distribution normal(0.0f, 1.0f); 16 | 17 | vector xs; 18 | vector ys; 19 | for (unsigned i = 0; i < 100; ++i) { 20 | float x = -1 + 2.0 / 100; 21 | float y = 2 * x + normal(rng) * 0.33f; 22 | xs.push_back(x); 23 | ys.push_back(y); 24 | } 25 | 26 | ParameterCollection model; 27 | Parameter pW = model.add_parameters({1}); 28 | 29 | SimpleSGDTrainer trainer(model, 0.1); 30 | 31 | ComputationGraph cg; 32 | Expression W = parameter(cg, pW); 33 | 34 | for (unsigned i = 0; i < xs.size(); ++i) { 35 | Expression pred = W * xs[i]; 36 | Expression loss = square(pred - ys[i]); 37 | cg.forward(loss); 38 | cg.backward(loss); 39 | trainer.update(); 40 | } 41 | 42 | cout << as_scalar(W.value()) << endl; // something around 2 43 | } 44 | -------------------------------------------------------------------------------- /tutorial/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8) 2 | 3 | foreach(TARGET 0_multiply 1_linear_regression) 4 | add_executable(${TARGET} ${TARGET}.cc) 5 | target_link_libraries(${TARGET} dynet ${LIBS}) 6 | if(UNIX AND NOT APPLE) 7 | target_link_libraries(${TARGET} rt) 8 | endif() 9 | if (WITH_CUDA_BACKEND) 10 | CUDA_ADD_CUBLAS_TO_TARGET(${TARGET}) 11 | endif (WITH_CUDA_BACKEND) 12 | endforeach() 13 | 14 | --------------------------------------------------------------------------------