├── .gitignore ├── .travis.yml ├── LICENSE.md ├── Makefile ├── Pioneer.ipynb ├── README.md ├── cython └── dali │ ├── .gitignore │ ├── core.pyx.pre │ └── core │ ├── .gitignore │ ├── data_processing │ └── Batch.pyx │ ├── layers │ ├── GRU.pyx.pre │ ├── LSTM.pyx.pre │ └── Layers.pyx.pre │ ├── math │ ├── TensorInternal.pyx │ ├── memory_bank │ │ ├── MemoryBankWrapper.cpp │ │ └── MemoryBankWrapper.h │ ├── memory_status.cpp │ └── memory_status.h │ ├── tensor │ ├── Mat.pyx.pre │ ├── MatOps.pyx.pre │ ├── Solver.pyx.pre │ ├── Tape.pyx │ ├── matrix_initializations.cpp │ ├── matrix_initializations.h │ ├── python_tape.cpp │ ├── python_tape.h │ └── random.pyx.pre │ └── utils │ ├── config.pyx │ ├── core_utils.pyx │ ├── cpp_utils.cpp │ └── cpp_utils.h ├── dali ├── __init__.py ├── activation.py ├── beam_search.py ├── data │ ├── __init__.py │ ├── batch.py │ ├── process.py │ ├── translation.py │ └── utils.py ├── models │ ├── __init__.py │ └── mlp.py └── utils │ ├── __init__.py │ ├── capture.py │ ├── misc.py │ ├── scoring.py │ ├── solver.py │ ├── throttled.py │ └── vocab.py ├── examples ├── mlbasics_learn_to_add.py ├── mlbasics_rnn_binary_addition.py └── translation │ ├── predict.py │ ├── train.py │ └── translation.py ├── libcpp11 ├── __init__.pxd ├── memory.pxd ├── stringstream.pxd ├── unordered_map.pxd └── vector.pxd ├── modern_numpy └── __init__.pxd ├── notebooks ├── LSTM.ipynb ├── Machine Translation.ipynb ├── NTM.ipynb └── language_model.ipynb ├── preprocessor_utils.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── test.txt ├── tests_beam_search.py ├── tests_data.py └── tests_vocab.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | *.dali 10 | MANIFEST 11 | 12 | # Compiled protobuffs # 13 | *.pb.cc 14 | *.pb.h 15 | *_pb2.py 16 | 17 | # Outputs of Cython 18 | cython/dali/core.cpp 19 | cython/dali/layers/Layer.pyx 20 | cython/dali/layers/LSTM.pyx 21 | cython/dali/tensor/Mat.pyx 22 | cython/dali/tensor/Solver.pyx 23 | cython/dali/tensor/random.pyx 24 | 25 | # Packages # 26 | ############ 27 | # it's better to unpack these files and commit the raw source 28 | # git has its own built in compression methods 29 | *.7z 30 | *.dmg 31 | *.gz 32 | *.iso 33 | *.jar 34 | *.rar 35 | *.tar 36 | *.zip 37 | *.gem 38 | *.pem 39 | 40 | # Special redis file (???) 41 | *.rdb 42 | 43 | .ipynb_checkpoints/ 44 | 45 | # Saves # 46 | ######### 47 | *.npy 48 | *.mat 49 | *.vocab 50 | *.svocab 51 | text8 52 | __pycache__/* 53 | *.pyc 54 | *.egg-info 55 | 56 | # NPM modules 57 | node_modules 58 | 59 | # Logs and databases # 60 | ###################### 61 | *.log 62 | *.sql 63 | *.sqlite 64 | 65 | # OS generated files # 66 | ###################### 67 | .DS_Store 68 | .DS_Store? 69 | ._* 70 | .Spotlight-V100 71 | .Trashes 72 | ehthumbs.db 73 | Thumbs.db 74 | .DS_Store 75 | build/ 76 | 77 | # file manifest is auto-generated by setup.py 78 | MANIFEST.in 79 | MANIFEST 80 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: trusty 3 | sudo: required 4 | python: 5 | - '3.3' 6 | - '3.4' 7 | - '3.5' 8 | os: 9 | - linux 10 | before_install: 11 | - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y 12 | - sudo apt-get update 13 | - sudo apt-get install -y git 14 | - sudo apt-get install -y cmake 15 | - sudo apt-get install -y gcc 16 | - sudo apt-get install -y zlib1g-dev 17 | - sudo apt-get install -y libgflags2 libgflags-dev 18 | - sudo apt-get install -y build-essential 19 | - sudo apt-get install -y libblas-dev 20 | - sudo apt-get install -y gcc-4.9 g++-4.9 cpp-4.9 21 | - sudo apt-get install -y libgtest-dev 22 | - export URL='https://github.com/JonathanRaiman/Dali/releases/download/v1.0.0/dali-1.0.0-Linux-x86_64-cpu.deb' 23 | - export FILE=`mktemp` 24 | - wget "$URL" -qO $FILE 25 | - sudo dpkg -i $FILE 26 | - rm $FILE 27 | install: 28 | - pip3 install -r requirements.txt 29 | - python3 setup.py install 30 | - pip3 install nose2 31 | - cp $(find . -name "core*.so") dali/ 32 | script: nose2 33 | notifications: 34 | email: 35 | recipients: 36 | - szymon.sidor@gmail.com 37 | - jonathanraiman@gmail.com 38 | on_success: change 39 | on_failure: always 40 | slack: 41 | secure: UEkhyrjDPiwRYwLJ9yRNfH8iq57YdUKrQ2kY/trSaZTRv5cPGiUfoLYo1Vj65d8aGwyPeyWd/1/z4KOJ1+qWCIcqTLDrdrjig64v3qVfJt/054eEEUSGjMQgR9ureQ2DjR3fzbyJJLxt7IZJl8ytZtXLp13LpWBPJJOhKHcK1OLG70jUqPRj/cQ2RSFVLfcpvls+uJ8NHSJpM1434ML8DI/0Ucifp8j3NMi71odrTMcRl5AAouFgG5U5A9FvIKIIGAomArwaEKIIr+O/i10Gy6vKZxb82Hq+VQx6yow6mNEkvV40w9QZUmmTgXgsEgILcdMMeOw5iifPBg7wD1dEIBTN3PO+l+JXSv+CTx3sQdSWZb66ZSybWc6m8z5LnEGa1+ygagWBPs5CgzAXkmLQSkIdgocPCAMBSPxuxj2Hi+7s7SF8INut/aeP5/mCVxKRfcJo4Za56QTExXYqQPTQobdAKgoNOcwzK7nh2K2JDfh0qA8kaE59AZaQKnyZBLnFrQ4wSXHzcZ3kjF+nlQx/oFwQetFgu5ocNZQ5OFzToV+iLKBz+seoU0L1/e4Eweds+EjYtBHfqVB1hHmk4piFHgzQ8mC9WuL3Z9LZVRp4vykUb8yZrxbGB5Xc5WQfiAzXzsDYWDia4ZLKqcnPYq5fFw86gSf0gWCVZidVK33Yj3g= 42 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jonathan Raiman, Szymon Sidor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: 2 | python3 setup.py clean 3 | python3 setup.py build_ext --inplace 4 | 5 | cuda: 6 | python3 setup.py clean cuda 7 | python3 setup.py build_ext --inplace cuda 8 | 9 | test: 10 | nose2 11 | 12 | clean: 13 | python3 setup.py clean 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dali in Python 2 | 3 | [![PyPI version](https://badge.fury.io/py/dali.svg)](https://badge.fury.io/py/dali) 4 | [![Build Status](https://travis-ci.org/dali-ml/dali-cython.svg?branch=master)](https://travis-ci.org/dali-ml/dali-cython) 5 | [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md) 6 | 7 | This repository contains an early version of Python bindings to [Dali](https://github.com/dali-ml/Dali), the automatic differentiation framework. 8 | 9 | With this module you can construct arbitrary mathematical models, including differentiating through control code and loops, optimize and run them using your GPU or CPU. 10 | 11 | The bindings are light-weight, and the resulting module retains about 90%-95% of the C++ performance in most use-cases (with this performance going up for larger models and GPU intensive computations). 12 | 13 | [![Jonathan Raiman, author](https://img.shields.io/badge/Author-Jonathan%20Raiman%20-blue.svg)](https://github.com/JonathanRaiman/) [![Szymon Sidor, author](https://img.shields.io/badge/Author-Szymon%20Sidor%20-blue.svg)](https://github.com/nivwusquorum) 14 | 15 | ### Installation 16 | 17 | 1. Install [Dali](https://github.com/dali-ml/Dali#installation) using `homebrew`, `yum`, or `apt-get`. 18 | 19 | 2. `pip3 install dali` 20 | 21 | ### Usage 22 | 23 | See examples under notebooks. 24 | -------------------------------------------------------------------------------- /cython/dali/.gitignore: -------------------------------------------------------------------------------- 1 | core.pyx 2 | -------------------------------------------------------------------------------- /cython/dali/core.pyx.pre: -------------------------------------------------------------------------------- 1 | pyp 2 | from preprocessor_utils import ( 3 | type_repeat, 4 | ) 5 | ypy 6 | 7 | 8 | from libcpp.string cimport string 9 | from libcpp11.vector cimport vector 10 | from libcpp11.memory cimport shared_ptr 11 | # Import the Python-level symbols of numpy 12 | import numpy as np 13 | # Import the C-level symbols of numpy 14 | cimport modern_numpy as np 15 | 16 | cdef string normalize_s(s): 17 | if type(s) is str: 18 | return s.encode("utf-8") 19 | elif type(s) is bytes: 20 | return s 21 | else: 22 | raise TypeError("Must pass a str or bytes object.") 23 | 24 | cdef bint is_fdtype(np.NPY_TYPES type_id) nogil: 25 | return type_id == np.NPY_FLOAT32 or \ 26 | type_id == np.NPY_FLOAT64 27 | 28 | cdef inline void ensure_fdtype(np.NPY_TYPES type_id): 29 | if not is_fdtype(type_id): 30 | raise ValueError( 31 | "Invalid dtype: " + 32 | str(np.PyArray_DescrFromType(type_id)) + 33 | " (should be one of float32, float64)") 34 | 35 | include "core/utils/config.pyx" 36 | 37 | # File IO, save / load, etc... 38 | include "core/utils/core_utils.pyx" 39 | 40 | # Used for storing Tensor buffers 41 | include "core/math/TensorInternal.pyx" 42 | 43 | # Matrix class 44 | include "core/tensor/Mat.pyx" 45 | 46 | pypinline type_repeat(pyp, 47 | cdef TYPED(copy_name)(const CMat[TYPE_NAME]& source, const CMat[TYPE_NAME]& dest): 48 | if source.name == NULL: 49 | (source).set_name('') 50 | (dest).name = source.name 51 | ypy 52 | 53 | # Matrix initialization with random numbers. 54 | include "core/tensor/random.pyx" 55 | 56 | # Softmax, crossentropy etc.... 57 | include "core/tensor/MatOps.pyx" 58 | 59 | # Related to backpropagation. 60 | include "core/tensor/Tape.pyx" 61 | 62 | # Layer, RNN, StackedInputLayer, etc... 63 | include "core/layers/Layers.pyx" 64 | 65 | include "core/layers/GRU.pyx" 66 | 67 | include "core/layers/LSTM.pyx" 68 | 69 | # # Matrix class 70 | include "core/data_processing/Batch.pyx" 71 | 72 | # SGD, Adagrad, Adadelta, etc... 73 | include "core/tensor/Solver.pyx" 74 | -------------------------------------------------------------------------------- /cython/dali/core/.gitignore: -------------------------------------------------------------------------------- 1 | tensor/Mat.pyx 2 | tensor/MatOps.pyx 3 | tensor/random.pyx 4 | 5 | layers/Layers.pyx 6 | layers/LSTM.pyx 7 | layers/GRU.pyx 8 | 9 | tensor/Solver.pyx 10 | layers/Layers.pyx 11 | -------------------------------------------------------------------------------- /cython/dali/core/data_processing/Batch.pyx: -------------------------------------------------------------------------------- 1 | cdef extern from "dali/data_processing/Batch.h": 2 | cdef cppclass CBatch "Batch" [T]: 3 | CMat[int] data 4 | CMat[int] target 5 | CMat[T] mask 6 | vector[int] code_lengths 7 | int total_codes 8 | CBatch() 9 | size_t size() const 10 | size_t max_length() const 11 | void insert_example( 12 | const vector[string]& example, 13 | const CVocab& vocab, 14 | size_t example_idx, 15 | int offset) 16 | int example_length(const int& idx) const 17 | -------------------------------------------------------------------------------- /cython/dali/core/layers/GRU.pyx.pre: -------------------------------------------------------------------------------- 1 | pyp 2 | from preprocessor_utils import ( 3 | typed_fexpression, 4 | typed_fexpression_args, 5 | typed_fexpression_list, 6 | type_frepeat 7 | ) 8 | ypy 9 | 10 | cdef extern from "dali/layers/GRU.h" nogil: 11 | cdef cppclass CGRU "GRU" [T]: 12 | CStackedInputLayer[T] reset_layer 13 | CStackedInputLayer[T] memory_interpolation_layer 14 | CStackedInputLayer[T] memory_to_memory_layer 15 | int input_size 16 | int hidden_size 17 | 18 | CGRU() 19 | CGRU(int _input_size, int _hidden_size) 20 | CGRU(const CGRU[T]&, bint copy_w, bint copy_dw) 21 | 22 | CGRU[T] shallow_copy() const 23 | CMat[T] activate(CMat[T] input_vector, CMat[T] previous_state) except + 24 | CMat[T] activate_sequence( 25 | const vector[CMat[T]]& input_sequence) except + 26 | CMat[T] activate_sequence( 27 | const vector[CMat[T]]& input_sequence, 28 | CMat[T] state) except + 29 | 30 | vector[CMat[T]] parameters() const 31 | CMat[T] initial_states() const 32 | 33 | cdef class GRU: 34 | cdef void* layerinternal 35 | cdef np.NPY_TYPES dtypeinternal 36 | 37 | def __cinit__(GRU self, int input_size, int hidden_size, dtype=np.float32): 38 | self.layerinternal = NULL 39 | self.dtypeinternal = np.NPY_NOTYPE 40 | self.dtypeinternal = np.dtype(dtype).num 41 | ensure_fdtype(self.dtypeinternal) 42 | 43 | pypinline typed_fexpression(pyp, 44 | self.layerinternal = new CGRU[TYPE_NAME](input_size, hidden_size) 45 | ypy 46 | 47 | def __dealloc__(GRU self): 48 | self.free_internal() 49 | 50 | cdef free_internal(GRU self): 51 | pypinline type_frepeat(pyp, 52 | cdef CGRU[TYPE_NAME]* TYPED(ptr_internal) 53 | ypy 54 | if self.layerinternal != NULL: 55 | pypinline typed_fexpression(pyp, 56 | TYPED(ptr_internal) = PTR_GRU(self) 57 | with nogil: 58 | del TYPED(ptr_internal) 59 | ypy 60 | self.layerinternal = NULL 61 | 62 | property dtype: 63 | def __get__(GRU self): 64 | return np.PyArray_DescrFromType(self.dtypeinternal) 65 | 66 | pyprepeat PROPERTY_NAME in ['input_size', 'hidden_size',] 67 | property PROPERTY_NAME: 68 | def __get__(GRU self): 69 | pypinline typed_fexpression(pyp, 70 | return DEREF_GRU(self).PROPERTY_NAME 71 | ypy 72 | ypyrepeat 73 | 74 | pyprepeat INTERNAL_LAYER_NAME in ['reset_layer', 'memory_interpolation_layer', 'memory_to_memory_layer'] 75 | property INTERNAL_LAYER_NAME: 76 | def __get__(GRU self): 77 | pypinline typed_fexpression(pyp, 78 | return WRAP_STACKEDLAYER(DEREF_GRU(self).INTERNAL_LAYER_NAME) 79 | ypy 80 | ypyrepeat 81 | 82 | def activate(GRU self, Mat input_vector, Mat previous_state): 83 | """ 84 | Activate 85 | -------- 86 | 87 | Inputs 88 | ------ 89 | 90 | Mat input_vector 91 | Mat previous_state 92 | 93 | Outputs 94 | ------- 95 | 96 | Mat next_state 97 | """ 98 | assert( 99 | (self.dtypeinternal == input_vector.dtypeinternal) and 100 | (previous_state.dtypeinternal == self.dtypeinternal) 101 | ), "All arguments must be of the same type" 102 | 103 | pypinline type_frepeat(pyp, 104 | cdef CMat[TYPE_NAME] TYPED(out) 105 | ypy 106 | 107 | pypinline typed_fexpression(pyp, 108 | with nogil: 109 | TYPED(out) = DEREF_GRU(self).activate(DEREF_MAT(input_vector), DEREF_MAT(previous_state)) 110 | return WRAP_MAT(TYPED(out)) 111 | ypy 112 | 113 | def initial_states(GRU self): 114 | pypinline typed_fexpression(pyp, 115 | return WRAP_MAT(DEREF_GRU(self).initial_states()) 116 | ypy 117 | 118 | def parameters(GRU self): 119 | params = [] 120 | pypinline type_frepeat(pyp, 121 | cdef CMat[TYPE_NAME] TYPED(param) 122 | cdef vector[CMat[TYPE_NAME]] TYPED(param_vec) 123 | 124 | ypy 125 | pypinline typed_fexpression(pyp, 126 | TYPED(param_vec) = DEREF_GRU(self).parameters() 127 | for TYPED(param) in TYPED(param_vec): 128 | params.append(WRAP_MAT(TYPED(param))) 129 | ypy 130 | return params 131 | 132 | def __setstate__(GRU self, state): 133 | for param, saved_param in zip(self.parameters(), state["parameters"]): 134 | param.w = saved_param.w 135 | self.dtypeinternal = state["dtype"].num 136 | 137 | def __getstate__(self): 138 | return { 139 | "parameters" : self.parameters(), 140 | "dtype" : self.dtype 141 | } 142 | 143 | def __reduce__(self): 144 | return ( 145 | self.__class__, 146 | ( 147 | self.input_size, 148 | self.hidden_size, 149 | ), self.__getstate__(), 150 | ) 151 | 152 | def activate_sequence(GRU self, list input_sequence, initial_state = None): 153 | pypinline type_frepeat(pyp, 154 | cdef vector[CMat[TYPE_NAME]] TYPED(c_input_sequence) 155 | cdef CMat[TYPE_NAME] TYPED(out) 156 | ypy 157 | 158 | if initial_state is None: 159 | pypinline typed_fexpression_list(pyp, "input_sequence", "Mat", 160 | TYPED(c_input_sequence) = TYPED(mats_to_vec)(input_sequence) 161 | if self.dtypeinternal != TYPE_NPYINTERNAL: 162 | raise ValueError("Invalid dtype for input_sequence: " + str(input_sequence[0].dtype) + ", when GRU is " + str(self.dtype)) 163 | with nogil: 164 | TYPED(out) = DEREF_GRU(self).activate_sequence(TYPED(c_input_sequence)) 165 | return WRAP_MAT(TYPED(out)) 166 | ypy 167 | else: 168 | if type(initial_state) is not Mat: 169 | raise ValueError("initial_state must be a Mat") 170 | pypinline typed_fexpression_list(pyp, "input_sequence", "Mat", 171 | TYPED(c_input_sequence) = TYPED(mats_to_vec)(input_sequence) 172 | if self.dtypeinternal != TYPE_NPYINTERNAL: 173 | raise ValueError("Invalid dtype for input_sequence: " + str(input_sequence[0].dtype) + ", when GRU is " + str(self.dtype)) 174 | if (initial_state).dtypeinternal != self.dtypeinternal: 175 | raise ValueError("Invalid dtype for initial_state: " + str(initial_state.dtype) + ", when GRU is " + str(self.dtype)) 176 | with nogil: 177 | TYPED(out) = DEREF_GRU(self).activate_sequence(TYPED(c_input_sequence), DEREF_MAT(initial_state)) 178 | return WRAP_MAT(TYPED(out)) 179 | ypy 180 | 181 | def __str__(GRU self): 182 | return "