├── .gitignore ├── Auto-Encoder-Animate.ipynb ├── Auto-Encoder.ipynb ├── CNN with MNIST.ipynb ├── Caffe Practice 1-1.ipynb ├── Caffe Practice 1-2.ipynb ├── Caffe Practice 1.ipynb ├── Caffe Practice 2.ipynb ├── Caffe Practice 3.ipynb ├── Dockerfile_ttskc ├── Logistic Regression_Python.ipynb ├── NN with Numpy Weight Matrix.ipynb ├── NN with Shared Variables.ipynb ├── README.md ├── RNN-story.ipynb ├── Simple RNN.ipynb ├── TensorFlow.pdf ├── Tensorflow_RL ├── 1000_index_x.txt ├── 1000_index_y.txt ├── Checkpoint │ ├── checkpoint │ ├── mlp-0 │ ├── mlp-0.data-00000-of-00001 │ ├── mlp-0.index │ ├── mlp-0.meta │ ├── mlp-1000 │ ├── mlp-1000.data-00000-of-00001 │ ├── mlp-1000.index │ ├── mlp-1000.meta │ ├── mlp-1500 │ ├── mlp-1500.data-00000-of-00001 │ ├── mlp-1500.index │ ├── mlp-1500.meta │ ├── mlp-2000 │ ├── mlp-2000.data-00000-of-00001 │ ├── mlp-2000.index │ ├── mlp-2000.meta │ ├── mlp-2500 │ ├── mlp-2500.data-00000-of-00001 │ ├── mlp-2500.index │ ├── mlp-2500.meta │ ├── mlp-3000 │ ├── mlp-3000.data-00000-of-00001 │ ├── mlp-3000.index │ ├── mlp-3000.meta │ ├── mlp-3500 │ ├── mlp-3500.data-00000-of-00001 │ ├── mlp-3500.index │ ├── mlp-3500.meta │ ├── mlp-4000 │ ├── mlp-4000.data-00000-of-00001 │ ├── mlp-4000.index │ ├── mlp-4000.meta │ ├── mlp-4500 │ ├── mlp-4500.data-00000-of-00001 │ ├── mlp-4500.index │ ├── mlp-4500.meta │ ├── mlp-500 │ ├── mlp-500.data-00000-of-00001 │ ├── mlp-500.index │ ├── mlp-500.meta │ ├── mlp-5000 │ ├── mlp-5000.data-00000-of-00001 │ ├── mlp-5000.index │ └── mlp-5000.meta ├── MNIST_data │ ├── t10k-images-idx3-ubyte.gz │ ├── t10k-labels-idx1-ubyte.gz │ ├── train-images-idx3-ubyte.gz │ └── train-labels-idx1-ubyte.gz ├── RNN_Homework_Automated_trading.ipynb ├── Reinforcement_Learning_Tutorial.ipynb ├── TensorFlow_Tutorial_MLP_CNN.ipynb ├── TensorFlow_Tutorial_RNN.ipynb ├── dbpedia_data │ └── dbpedia_csv │ │ ├── classes.txt │ │ └── readme.txt ├── images │ ├── Cnn_layer.png │ ├── SimpleRNN01.png │ ├── apache2.0_license.png │ ├── big_cnn.png │ ├── cnn.png │ ├── deeprl.png │ ├── discounted_factor.png │ ├── eth.png │ ├── example.gif │ ├── excel.png │ ├── gridworld2.png │ ├── longterm.png │ ├── mlp.png │ ├── mlp_total.png │ ├── mnist.png │ ├── news.png │ ├── q_learning3.jpg │ ├── q_table.png │ ├── qlearning.png │ ├── rl.png │ ├── rnn.png │ ├── rnn_seq2seq.jpg │ ├── rnn_word.png │ ├── tensorboard_mlp.png │ ├── tf_license.png │ ├── unroll_rnn.png │ └── word_embedding.png ├── rnn_word_embedding_skflowipynb ├── tf_rl │ ├── __init__.py │ ├── controller │ │ ├── __init__.py │ │ ├── discrete_deepq.py │ │ └── human_controller.py │ ├── models.py │ ├── simulate.py │ ├── simulation │ │ ├── __init__.py │ │ └── karpathy_game.py │ └── utils │ │ ├── __init__.py │ │ ├── event_queue.py │ │ ├── geometry.py │ │ ├── getch.py │ │ └── svg.py └── tinyshakespeare.txt ├── Theano Tutorial Basic1.ipynb ├── Theano_deeplearing_tutorial_1.pdf ├── Theano_eskim.pdf ├── Theano_talk160630.pdf ├── example.ipynb ├── examples ├── lady.jpg ├── moheo.jpg └── mom.jpg ├── images ├── ae_single.png ├── ae_stacked.png ├── higher_layer.png ├── lower_layer.png └── sae.jpg ├── mnist.hdf5 ├── setup_nbserver.py ├── sshd_config ├── stories.txt ├── trained_ae.hdf5 └── trained_ae.pkl /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | .ipynb_checkpoints 4 | *.tar.gz 5 | *.csv 6 | 7 | -------------------------------------------------------------------------------- /Dockerfile_ttskc: -------------------------------------------------------------------------------- 1 | #docker build --rm -t imcomking/ttskc -f Dockerfile_ttskc . 2 | 3 | #docker run -it --device /dev/nvidiactl --device /dev/nvidia-uvm --device /dev/nvidia0 -p 22 -p 6006 -p 8888 imcomking/ttskc 4 | 5 | #if you want to solve the bug "libdc1394 error: Failed to initialize libdc1394" 6 | #ln /dev/null /dev/raw1394 7 | 8 | ################################## 9 | 10 | # Start with cuda7.5 base image 11 | # NVIDIA-SMI 352.39 Driver Version: 352.39 12 | FROM kaixhin/cuda:latest 13 | MAINTAINER Dong-Hyun Kwak 14 | 15 | # Install [cudnn 7.0 (v4)] 16 | RUN wget --quiet https://www.dropbox.com/s/dcux1l862p4ml4m/cudnn-7.0-linux-x64-v4.0-rc.tar?dl=0 -O cudnn-7.0-linux-x64-v4.0-rc.tar 17 | RUN tar -xvf cudnn-7.0-linux-x64-v4.0-rc.tar && \ 18 | rm cudnn-7.0-linux-x64-v4.0-rc.tar 19 | 20 | RUN ls /cuda/include/cudnn.h 21 | RUN cp /cuda/include/cudnn.h /usr/local/cuda/include/ 22 | RUN cp /cuda/lib64/libcudnn.so* /usr/local/cuda/lib64/ 23 | 24 | 25 | 26 | # install [caffe] 27 | RUN apt-get update && apt-get install -y --no-install-recommends \ 28 | build-essential \ 29 | cmake \ 30 | git \ 31 | wget \ 32 | libatlas-base-dev \ 33 | libboost-all-dev \ 34 | libgflags-dev \ 35 | libgoogle-glog-dev \ 36 | libhdf5-serial-dev \ 37 | libleveldb-dev \ 38 | liblmdb-dev \ 39 | libopencv-dev \ 40 | libprotobuf-dev \ 41 | libsnappy-dev \ 42 | protobuf-compiler \ 43 | python-dev \ 44 | python-numpy \ 45 | python-pip \ 46 | python-scipy && \ 47 | rm -rf /var/lib/apt/lists/* 48 | 49 | ENV CAFFE_ROOT=/opt/caffe 50 | WORKDIR $CAFFE_ROOT 51 | 52 | # FIXME: clone a specific git tag and use ARG instead of ENV once DockerHub supports this. 53 | ENV CLONE_TAG=master 54 | 55 | RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/BVLC/caffe.git . && \ 56 | for req in $(cat python/requirements.txt) pydot; do pip install $req; done && \ 57 | mkdir build && cd build && \ 58 | cmake -DUSE_CUDNN=1 .. && \ 59 | make -j"$(nproc)" 60 | 61 | ENV PYCAFFE_ROOT $CAFFE_ROOT/python 62 | ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH 63 | ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH 64 | RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | # install [tensorflow] 74 | RUN apt-get update && apt-get install -y \ 75 | # Tensorflow. this is come from the official tensorflow docker file 76 | build-essential \ 77 | curl \ 78 | git \ 79 | libfreetype6-dev \ 80 | libpng12-dev \ 81 | libzmq3-dev \ 82 | pkg-config \ 83 | python-dev \ 84 | python-numpy \ 85 | python-pip \ 86 | software-properties-common \ 87 | swig \ 88 | zip \ 89 | zlib1g-dev \ 90 | 91 | # for etc 92 | python-scipy \ 93 | python-nose \ 94 | python-setuptools \ 95 | python-h5py \ 96 | python-matplotlib \ 97 | python-yaml \ 98 | libopenblas-dev \ 99 | screen \ 100 | vim \ 101 | unzip \ 102 | libatlas-dev \ 103 | libhdf5-dev \ 104 | libatlas3gf-base \ 105 | openssh-server \ 106 | mcrypt \ 107 | graphviz \ 108 | && \ 109 | apt-get clean && \ 110 | rm -rf /var/lib/apt/lists/* 111 | 112 | 113 | 114 | # Install recent pip # this must be done before tensorflow 115 | RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ 116 | python get-pip.py && \ 117 | rm get-pip.py 118 | 119 | 120 | # Install Tensorflow 121 | # Set up Bazel. 122 | 123 | # We need to add a custom PPA to pick up JDK8, since trusty doesn't 124 | # have an openjdk8 backport. openjdk-r is maintained by a reliable contributor: 125 | # Matthias Klose (https://launchpad.net/~doko). It will do until 126 | # we either update the base image beyond 14.04 or openjdk-8 is 127 | # finally backported to trusty; see e.g. 128 | # https://bugs.launchpad.net/trusty-backports/+bug/1368094 129 | RUN add-apt-repository -y ppa:openjdk-r/ppa && \ 130 | apt-get update && \ 131 | apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ 132 | apt-get clean && \ 133 | rm -rf /var/lib/apt/lists/* 134 | 135 | # Running bazel inside a `docker build` command causes trouble, cf: 136 | # https://github.com/bazelbuild/bazel/issues/134 137 | # The easiest solution is to set up a bazelrc file forcing --batch. 138 | RUN echo "startup --batch" >>/root/.bazelrc 139 | # Similarly, we need to workaround sandboxing issues: 140 | # https://github.com/bazelbuild/bazel/issues/418 141 | RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ 142 | >>/root/.bazelrc 143 | ENV BAZELRC /root/.bazelrc 144 | # Install the most recent bazel release. 145 | ENV BAZEL_VERSION 0.2.1 146 | WORKDIR / 147 | RUN mkdir /bazel && \ 148 | cd /bazel && \ 149 | curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ 150 | curl -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE.txt && \ 151 | chmod +x bazel-*.sh && \ 152 | ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ 153 | cd / && \ 154 | rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh 155 | 156 | 157 | # Download and build TensorFlow. 158 | 159 | RUN git clone --recursive https://github.com/tensorflow/tensorflow.git && \ 160 | cd tensorflow && \ 161 | git checkout r0.7 162 | WORKDIR /tensorflow 163 | 164 | # Configure the build for our CUDA configuration. 165 | ENV CUDA_TOOLKIT_PATH /usr/local/cuda 166 | ENV CUDNN_INSTALL_PATH /usr/local/cuda 167 | ENV TF_NEED_CUDA 1 168 | 169 | 170 | # change the setting of protobuf 171 | RUN sed -i "s/kDefaultTotalBytesLimit = 64/kDefaultTotalBytesLimit = 1024/g" google/protobuf/src/google/protobuf/io/coded_stream.h 172 | RUN sed -i "s/kDefaultTotalBytesWarningThreshold = 32/kDefaultTotalBytesWarningThreshold = 1024/g" google/protobuf/src/google/protobuf/io/coded_stream.h 173 | 174 | 175 | RUN ./configure && \ 176 | bazel build -c opt --config=cuda tensorflow/tools/pip_package:build_pip_package && \ 177 | bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ 178 | pip install --upgrade /tmp/pip/tensorflow-*.whl 179 | 180 | # Set up CUDA variables 181 | ENV CUDA_PATH /usr/local/cuda 182 | 183 | 184 | 185 | 186 | # SSH settings 187 | #RUN apt-get install -y openssh-server mcrypt && \ 188 | # mkdir /var/run/sshd && chmod 0755 /var/run/sshd 189 | 190 | RUN mkdir /var/run/sshd && chmod 0755 /var/run/sshd 191 | 192 | #ADD https://raw.githubusercontent.com/GeographicaGS/Docker-SFTP/master/sshd_config /etc/ssh/sshd_config 193 | #ADD https://raw.githubusercontent.com/GeographicaGS/Docker-SFTP/master/start.sh /usr/local/bin/start.sh 194 | 195 | #my custom files 196 | #ADD https://www.dropbox.com/s/ufu6ckktl4q2vnj/sshd_config?dl=0 /etc/ssh/sshd_config 197 | RUN wget --quiet https://www.dropbox.com/s/ufu6ckktl4q2vnj/sshd_config?dl=0 -O /etc/ssh/sshd_config 198 | #ADD https://www.dropbox.com/s/u94mdss02amvjsz/start.sh?dl=0 /usr/local/bin/start.sh 199 | 200 | #VOLUME ["/data"] 201 | 202 | #ENTRYPOINT ["/bin/bash"] 203 | #ENTRYPOINT ["/bin/bash", "/usr/local/bin/start.sh"] 204 | 205 | 206 | 207 | 208 | 209 | # pip install 210 | #RUN pip install --upgrade pip 211 | #RUN pip --no-cache-dir install matplotlib 212 | RUN pip install --upgrade Request 213 | 214 | #pydot is not supported under python 3 and pydot2 doesn't work properly. pydotplus works nicely (pip install pydotplus) 215 | RUN pip install pydotplus 216 | 217 | # Install [Ipython 3.2.1] 218 | RUN pip install "ipython[notebook]==3.2.1" 219 | 220 | # IPython nbserver 221 | RUN ipython profile create nbserver 222 | #RUN cd /root && wget --quiet https://www.dropbox.com/s/7z43hnuza0eb8ba/setup_nbserver_v2.py?dl=0 -O setup_nbserver.py 223 | 224 | 225 | 226 | 227 | # Install [Theano] 228 | ENV CUDA_ROOT /usr/local/cuda/bin 229 | RUN pip install --upgrade --no-deps git+git://github.com/Theano/Theano.git 230 | RUN printf "[global]\ndevice=gpu0\nfloatX=float32\ncuda.root=/usr/local/cuda/bin\n[nvcc]\nfastmath=True" > /root/.theanorc 231 | 232 | # Gitclone Deeplearning tutorial(theano) 233 | RUN cd /root && git clone git://github.com/lisa-lab/DeepLearningTutorials.git 234 | 235 | 236 | # Install [Keras] 237 | 238 | # Upgrade six 239 | RUN pip install --upgrade six 240 | 241 | # Clone Keras repo and move into it 242 | RUN cd /root && git clone https://github.com/fchollet/keras.git && cd keras && \ 243 | python setup.py install 244 | 245 | 246 | # Install [bleeding-edge Lasagne] # this is from Kaixhin's docker file 247 | RUN pip install --upgrade https://github.com/Lasagne/Lasagne/archive/master.zip 248 | 249 | # Install [mpld3] 250 | RUN pip install mpld3 251 | 252 | 253 | # Install [Scikit-learn] 254 | RUN update-alternatives --set libblas.so.3 \ 255 | /usr/lib/atlas-base/atlas/libblas.so.3; \ 256 | update-alternatives --set liblapack.so.3 \ 257 | /usr/lib/atlas-base/atlas/liblapack.so.3 258 | 259 | RUN pip install -U scikit-learn 260 | 261 | 262 | 263 | 264 | # Install [bleeding-edge JSAnimation] 265 | WORKDIR $HOME 266 | RUN git clone https://github.com/jakevdp/JSAnimation.git 267 | RUN python JSAnimation/setup.py install 268 | ENV PYTHONPATH $PYTHONPATH:$HOME/JSAnimation/: 269 | 270 | 271 | #RUN apt-get clean && \ 272 | #rm -rf /var/lib/apt/lists/* 273 | 274 | 275 | # screen setting 276 | RUN printf "\nexport PATH=/usr/local/cuda/bin:$PATH\nexport LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH\n" >> /root/.bashrc 277 | RUN printf "\nshell -/bin/bash\n" >> /root/.screenrc 278 | 279 | 280 | 281 | 282 | # TensorBoard 283 | #EXPOSE 6006 284 | 285 | # IPython 286 | #EXPOSE 8888 287 | 288 | # sftp 289 | #EXPOSE 22 290 | 291 | 292 | #port open for variable purpose. to use these, you should run with '-P' option which opens all exposed port randomly 293 | 294 | #EXPOSE 32000 32001 32002 32003 32004 295 | #32005 32006 32007 32008 32009 32010 32011 32012 296 | 297 | WORKDIR /root 298 | RUN ["/bin/bash"] 299 | 300 | -------------------------------------------------------------------------------- /NN with Numpy Weight Matrix.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stderr", 12 | "output_type": "stream", 13 | "text": [ 14 | "Using gpu device 0: GeForce GTX TITAN X (CNMeM is disabled)\n" 15 | ] 16 | } 17 | ], 18 | "source": [ 19 | "#http://cs231n.stanford.edu/slides/winter1516_lecture12.pdf\n", 20 | "\n", 21 | "import theano\n", 22 | "import theano.tensor as T\n", 23 | "\n", 24 | "num_data = 64\n", 25 | "num_dim = 1000\n", 26 | "num_hid1 = 100\n", 27 | "num_output = 10\n", 28 | "\n", 29 | "x = T.matrix('x')\n", 30 | "y = T.vector('y', dtype = 'int64')\n", 31 | "w1 = T.matrix('w1')\n", 32 | "w2 = T.matrix('w2')" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "metadata": { 39 | "collapsed": true 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "hid1 = x.dot(w1)\n", 44 | "hid1_relu = T.nnet.relu(hid1)\n", 45 | "output = hid1_relu.dot(w2)\n", 46 | "\n", 47 | "output_prob = T.nnet.softmax(output)\n", 48 | "loss = T.nnet.categorical_crossentropy(output_prob, y).mean()\n" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 11, 54 | "metadata": { 55 | "collapsed": true 56 | }, 57 | "outputs": [], 58 | "source": [ 59 | "dw1, dw2 = T.grad(loss, [w1, w2])" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 12, 65 | "metadata": { 66 | "collapsed": false 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "f = theano.function(\n", 71 | " inputs = [x, y, w1, w2],\n", 72 | " outputs = [loss, output, dw1, dw2]\n", 73 | ")" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 13, 79 | "metadata": { 80 | "collapsed": false 81 | }, 82 | "outputs": [], 83 | "source": [ 84 | "import numpy as np\n", 85 | "\n", 86 | "xx = np.random.randn(num_data, num_dim).astype('float32')\n", 87 | "yy = np.random.randint(num_output, size=num_data)\n", 88 | "ww1 = 1e-3 * np.random.randn(num_dim, num_hid1).astype('float32')\n", 89 | "ww2 = 1e-3 * np.random.randn(num_hid1, num_output).astype('float32')" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 15, 95 | "metadata": { 96 | "collapsed": false 97 | }, 98 | "outputs": [ 99 | { 100 | "name": "stdout", 101 | "output_type": "stream", 102 | "text": [ 103 | "2.284917593\n", 104 | "2.28074860573\n", 105 | "2.2755279541\n", 106 | "2.26898193359\n", 107 | "2.26076078415\n", 108 | "2.25044536591\n", 109 | "2.23754167557\n", 110 | "2.22135019302\n", 111 | "2.20115947723\n", 112 | "2.17598557472\n", 113 | "2.14468932152\n", 114 | "2.10625219345\n", 115 | "2.05970168114\n", 116 | "2.0042757988\n", 117 | "1.94025707245\n", 118 | "1.86889958382\n", 119 | "1.79337239265\n", 120 | "1.71750867367\n", 121 | "1.64509725571\n", 122 | "1.57823824883\n" 123 | ] 124 | } 125 | ], 126 | "source": [ 127 | "learning_rate = 1e-1\n", 128 | "num_epoch = 20\n", 129 | "\n", 130 | "for t in xrange(num_epoch):\n", 131 | " loss_val, output_val, dw1_val, dw2_val = f(xx, yy, ww1, ww2)\n", 132 | " print loss_val\n", 133 | " ww1 -= learning_rate * dw1_val\n", 134 | " ww2 -= learning_rate * dw2_val" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "collapsed": true 142 | }, 143 | "outputs": [], 144 | "source": [] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "Python 2", 150 | "language": "python", 151 | "name": "python2" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 2 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython2", 163 | "version": "2.7.6" 164 | } 165 | }, 166 | "nbformat": 4, 167 | "nbformat_minor": 0 168 | } 169 | -------------------------------------------------------------------------------- /NN with Shared Variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 8, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "#http://cs231n.stanford.edu/slides/winter1516_lecture12.pdf\n", 12 | "\n", 13 | "import theano\n", 14 | "import theano.tensor as T\n", 15 | "import numpy as np\n", 16 | "\n", 17 | "num_data = 64\n", 18 | "num_dim = 1000\n", 19 | "num_hid1 = 100\n", 20 | "num_output = 10\n", 21 | "\n", 22 | "x = T.matrix('x')\n", 23 | "y = T.vector('y', dtype = 'int64')\n", 24 | "w1 = theano.shared(1e-3 * np.random.randn(num_dim, num_hid1).astype('float32'), name='w1')\n", 25 | "w2 = theano.shared(1e-3 * np.random.randn(num_hid1, num_output).astype('float32'), name='w2')\n" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 9, 31 | "metadata": { 32 | "collapsed": true 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "hid1 = x.dot(w1)\n", 37 | "hid1_relu = T.nnet.relu(hid1)\n", 38 | "output = hid1_relu.dot(w2)\n", 39 | "\n", 40 | "output_prob = T.nnet.softmax(output)\n", 41 | "loss = T.nnet.categorical_crossentropy(output_prob, y).mean()\n" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 10, 47 | "metadata": { 48 | "collapsed": true 49 | }, 50 | "outputs": [], 51 | "source": [ 52 | "dw1, dw2 = T.grad(loss, [w1, w2])" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 14, 58 | "metadata": { 59 | "collapsed": true 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "learning_rate = 1e-1\n", 64 | "num_epoch = 20" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 15, 70 | "metadata": { 71 | "collapsed": false 72 | }, 73 | "outputs": [], 74 | "source": [ 75 | "train = theano.function(\n", 76 | " inputs = [x, y],\n", 77 | " outputs = loss,\n", 78 | " updates=(\n", 79 | " (w1, w1 - learning_rate * dw1),\n", 80 | " (w2, w2 - learning_rate * dw2)\n", 81 | " )\n", 82 | ")" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 16, 88 | "metadata": { 89 | "collapsed": true 90 | }, 91 | "outputs": [], 92 | "source": [ 93 | "xx = np.random.randn(num_data, num_dim).astype('float32')\n", 94 | "yy = np.random.randint(num_output, size=num_data)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 17, 100 | "metadata": { 101 | "collapsed": false 102 | }, 103 | "outputs": [ 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "2.30256009102\n", 109 | "2.30241537094\n", 110 | "2.302267313\n", 111 | "2.30210924149\n", 112 | "2.3019349575\n", 113 | "2.30173707008\n", 114 | "2.30150723457\n", 115 | "2.30123519897\n", 116 | "2.30090904236\n", 117 | "2.30051469803\n", 118 | "2.30003404617\n", 119 | "2.29944396019\n", 120 | "2.2987165451\n", 121 | "2.2978143692\n", 122 | "2.29668974876\n", 123 | "2.2952837944\n", 124 | "2.29351425171\n", 125 | "2.29129290581\n", 126 | "2.28848266602\n", 127 | "2.28491687775\n" 128 | ] 129 | } 130 | ], 131 | "source": [ 132 | "for i in xrange(num_epoch):\n", 133 | " loss_val = train(xx, yy)\n", 134 | " print loss_val" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "collapsed": true 142 | }, 143 | "outputs": [], 144 | "source": [] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "Python 2", 150 | "language": "python", 151 | "name": "python2" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 2 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython2", 163 | "version": "2.7.6" 164 | } 165 | }, 166 | "nbformat": 4, 167 | "nbformat_minor": 0 168 | } 169 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BI Lab Deep Learning Tutorial 2 | This repository consists of IPython notebooks of basic and advanced examples of deep learning tools such as Caffe, Tensorflow and Theano. 3 | 4 | ## MNIST data 5 | MNIST data is not ours. You can find the license info and the original data uploads here: http://yann.lecun.com/exdb/mnist/ 6 | 7 | ## Extra materials 8 | https://github.com/nzer0/theano_short_demo/ 9 | -------------------------------------------------------------------------------- /RNN-story.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import tensorflow as tf\n", 13 | "from tensorflow.python.ops import rnn_cell\n", 14 | "import math" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "metadata": { 21 | "collapsed": true 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "#weight 매트릭스 초기화 함수\n", 26 | "#https://github.com/therne/dmn-tensorflow 참고\n", 27 | "def weight(name, shape, init='he', range=None):\n", 28 | " initializer = tf.constant_initializer()\n", 29 | " if init == 'xavier':\n", 30 | " fan_in, fan_out = _get_dims(shape)\n", 31 | " range = math.sqrt(6.0 / (fan_in + fan_out))\n", 32 | " initializer = tf.random_uniform_initializer(-range, range)\n", 33 | "\n", 34 | " elif init == 'he':\n", 35 | " fan_in, _ = shape\n", 36 | " std = math.sqrt(2.0 / fan_in)\n", 37 | " initializer = tf.random_normal_initializer(stddev=std)\n", 38 | "\n", 39 | " elif init == 'normal':\n", 40 | " initializer = tf.random_normal_initializer(stddev=0.1)\n", 41 | "\n", 42 | " elif init == 'uniform':\n", 43 | " if range is None:\n", 44 | " raise ValueError(\"range must not be None if uniform init is used.\")\n", 45 | " initializer = tf.random_uniform_initializer(-range, range)\n", 46 | "\n", 47 | " var = tf.get_variable(name, shape, initializer=initializer)\n", 48 | " tf.add_to_collection('l2', tf.nn.l2_loss(var)) # Add L2 Loss\n", 49 | " return var" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "##문자열 처리 함수" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 3, 62 | "metadata": { 63 | "collapsed": false 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "#문장 부호 제거\n", 68 | "def remove_marks(strIn):\n", 69 | " marks = ['.', ',', '-', '!', '?', '\\\"']\n", 70 | " \n", 71 | " result = strIn\n", 72 | " for mark in marks:\n", 73 | " result = result.replace(mark, '')\n", 74 | " \n", 75 | " result = result.strip()\n", 76 | " return result\n", 77 | "\n", 78 | "\n", 79 | "#데이터 읽어오기\n", 80 | "def read_story():\n", 81 | " sentences = []\n", 82 | " labels = []\n", 83 | " with open('stories.txt', 'r') as f:\n", 84 | " for line in f: #문장을 한 줄씩 읽어서\n", 85 | " chunks = line.split('\\t') #탭을 기준으로 분리. 앞쪽이 문장, 뒤쪽은 클래스 레이블\n", 86 | " sentences.append( remove_marks(chunks[0]) ) #문장에서 문장부호 제거\n", 87 | " labels.append(int(chunks[1]))\n", 88 | " \n", 89 | " return sentences, labels\n", 90 | "\n", 91 | "#본 예제에서는 GloVe 벡터를 단어 representation로 사용\n", 92 | "#http://nlp.stanford.edu/projects/glove/ 에서 다운\n", 93 | "def load_glove(dim):\n", 94 | " word2vec = {} \n", 95 | " with open( \"glove.6B.\" + str(dim) + \"d.txt\") as f: \n", 96 | " for line in f:\n", 97 | " l = line.split()\n", 98 | " word2vec [l[0]] = map( float, l[1:])\n", 99 | " return word2vec\n", 100 | "\n", 101 | "#GloVe에 미리 정의되지 않은 단어에 대해서는, 랜덤하게 representation 생성\n", 102 | "def create_vector(word, word2vec, word_vector_size):\n", 103 | " vector = np.random.uniform( 0.0, 1.0, (word_vector_size,) )\n", 104 | " word2vec[word] = vector\n", 105 | " return vector\n", 106 | "\n", 107 | "#문장에서 하나의 단어를 입력으로 받아 GloVe vector로 바꾸는 함수\n", 108 | "def process_word(word, word2vec, vocab, ivocab, word_vector_size):\n", 109 | " if not word in word2vec:\n", 110 | " create_vector(word, word2vec, word_vector_size)\n", 111 | " if not word in vocab:\n", 112 | " next_index = len(vocab)\n", 113 | " vocab[word] = next_index\n", 114 | " ivocab[next_index] = word\n", 115 | " \n", 116 | " return word2vec[word]\n", 117 | "\n", 118 | "#문장을 받아서, 띄어쓰기 단위로 분리하고 GloVe representation으로 바꾸어 전체 입력 데이터를 만드는 함수\n", 119 | "#각 문장의 단어수도 저장\n", 120 | "def make_input_vector(sentences, word2vec, vocab, ivocab, word_vector_size):\n", 121 | " inputs = []\n", 122 | " num_words = []\n", 123 | " for line in sentences:\n", 124 | " inp = line.lower().split(' ') #띄어쓰기 기준으로 문장 나누기\n", 125 | " inp = [ w for w in inp if len(w) > 0] \n", 126 | " \n", 127 | " inp_vector = [process_word( word = w,\n", 128 | " word2vec = word2vec,\n", 129 | " vocab = vocab,\n", 130 | " ivocab = ivocab,\n", 131 | " word_vector_size = word_vector_size) for w in inp]\n", 132 | " inp_vector = np.vstack((inp_vector))\n", 133 | " #print inp_vector.shape\n", 134 | " inputs.append( inp_vector )\n", 135 | " num_words.append(len(inp))\n", 136 | " #print len(vocab), len(ivocab)\n", 137 | " \n", 138 | " return inputs, num_words" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 4, 144 | "metadata": { 145 | "collapsed": false 146 | }, 147 | "outputs": [ 148 | { 149 | "name": "stdout", 150 | "output_type": "stream", 151 | "text": [ 152 | "load_glove\n" 153 | ] 154 | } 155 | ], 156 | "source": [ 157 | "#GloVe representation 의 벡터 사이즈 지정\n", 158 | "word_vector_size = 100\n", 159 | "assert word_vector_size in [50, 100, 200, 300]\n", 160 | "\n", 161 | "#GloVe 로딩\n", 162 | "print 'load_glove'\n", 163 | "word2vec = load_glove(word_vector_size)\n" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": 5, 169 | "metadata": { 170 | "collapsed": true 171 | }, 172 | "outputs": [], 173 | "source": [ 174 | "#Data Padding, 가장 긴 문장을 기준으로 zero padding\n", 175 | "def process_batch():\n", 176 | " start_index = 0\n", 177 | " end_index = 200\n", 178 | " \n", 179 | " inp = inputs[start_index:end_index]\n", 180 | " num_word = num_words[start_index:end_index]\n", 181 | " label = labels[start_index:end_index]\n", 182 | " \n", 183 | " B, V = batch_size, word_vector_size\n", 184 | "\n", 185 | " L = 0\n", 186 | " for n in range(B):\n", 187 | " sent_len = len(inp[n])\n", 188 | " L = max(L, sent_len)\n", 189 | "\n", 190 | "\n", 191 | " new_input = np.zeros([B, L, V]) # zero padding\n", 192 | " new_labels = []\n", 193 | " new_num_words = []\n", 194 | " for n in range(B):\n", 195 | " new_input[n, :num_word[n]] = inp[n]\n", 196 | " new_labels.append(label[n])\n", 197 | " new_num_words.append(num_word[n])\n", 198 | "\n", 199 | "\n", 200 | " return new_input, new_labels, new_num_words" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "##모델 parameter define" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 6, 213 | "metadata": { 214 | "collapsed": false 215 | }, 216 | "outputs": [], 217 | "source": [ 218 | "vocab = {}\n", 219 | "ivocab = {}\n", 220 | "\n", 221 | "batch_size = 200 \n", 222 | "hidden_size = 50 #GRU hidden layer의 노드 수\n", 223 | "num_classes = 4\n", 224 | "learning_rate = 0.001\n", 225 | "sentences, labels = read_story()\n", 226 | "inputs, num_words = make_input_vector(sentences, word2vec, vocab, ivocab, word_vector_size)\n", 227 | "vocab_size = len(vocab)\n", 228 | "\n" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "##RNN 모델 정의" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 7, 241 | "metadata": { 242 | "collapsed": false 243 | }, 244 | "outputs": [], 245 | "source": [ 246 | "#B: batch size, L: 가장 긴 문장의 길이(=가장 긴 문장에 있는 단어의 수)\n", 247 | "#V: GloVe word vector size, h: GRU Hidden Node의 수, A: 데이터에 포함된 단어의 총 갯수\n", 248 | "B, L = batch_size, max(num_words)\n", 249 | "V, h, A = word_vector_size, hidden_size, vocab_size\n", 250 | "\n", 251 | "inp, len_sentences, label = process_batch()\n", 252 | "inp = np.float32(inp)\n", 253 | "#feed_dict={inp:new_input, len_sentences:new_num_words, label:new_labels}\n", 254 | "\"\"\"\n", 255 | "inp = tf.placeholder('float32', shape = [B, L, V], name = 'x')\n", 256 | "len_sentences = tf.placeholder('int32', shape = [B], name = 'x_len')\n", 257 | "label = tf.placeholder('int32', shape=[B], name = 'y')\n", 258 | "\"\"\"\n", 259 | "\n", 260 | "#GRU Cell 정의\n", 261 | "gru = rnn_cell.GRUCell(h)\n", 262 | "outputs, final_states = tf.nn.dynamic_rnn(cell = gru, \n", 263 | " inputs = inp, \n", 264 | " sequence_length = len_sentences, \n", 265 | " dtype = tf.float32) \n", 266 | "\n", 267 | "#GRU hidden 으로부터 클래스를 예측하기 위한 추가 layer\n", 268 | "with tf.name_scope('answer'):\n", 269 | " w_a = weight('answer_weight', [h, num_classes] )\n", 270 | " logits = tf.matmul(final_states, w_a)\n", 271 | "\n", 272 | "#Cross entropy로 loss function 정의\n", 273 | "with tf.name_scope('loss'):\n", 274 | " cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, label)\n", 275 | " loss = tf.reduce_mean(cross_entropy)\n", 276 | "\n", 277 | "#정답과 모델이 예측한 답을 비교하여 정확도 측정\n", 278 | "with tf.name_scope('acc'):\n", 279 | " predicts = tf.cast(tf.argmax(logits, 1), 'int32')\n", 280 | " corrects = tf.equal(predicts, labels[:batch_size])\n", 281 | " num_corrects = tf.reduce_sum(tf.cast(corrects, tf.float32))\n", 282 | " accuracy = tf.reduce_mean(tf.cast(corrects, tf.float32))\n", 283 | "\n", 284 | "optimizer = tf.train.AdamOptimizer(learning_rate)\n", 285 | "opt_op = optimizer.minimize(loss) #, global_step = self.global_step)" 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "##Computational Graph Launching, 실행" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": 11, 298 | "metadata": { 299 | "collapsed": false 300 | }, 301 | "outputs": [], 302 | "source": [ 303 | "\n", 304 | "with tf.Session() as sess:\n", 305 | " sess.run(tf.initialize_all_variables())\n", 306 | " \n", 307 | "\n", 308 | " for i in range(10):\n", 309 | " sess.run(opt_op)\n", 310 | " a = sess.run(final_states)\n", 311 | " acc_print = sess.run(accuracy)\n", 312 | " loss_print = sess.run(loss)\n", 313 | " " 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": null, 319 | "metadata": { 320 | "collapsed": true 321 | }, 322 | "outputs": [], 323 | "source": [] 324 | } 325 | ], 326 | "metadata": { 327 | "kernelspec": { 328 | "display_name": "Python 2", 329 | "language": "python", 330 | "name": "python2" 331 | }, 332 | "language_info": { 333 | "codemirror_mode": { 334 | "name": "ipython", 335 | "version": 2 336 | }, 337 | "file_extension": ".py", 338 | "mimetype": "text/x-python", 339 | "name": "python", 340 | "nbconvert_exporter": "python", 341 | "pygments_lexer": "ipython2", 342 | "version": "2.7.6" 343 | } 344 | }, 345 | "nbformat": 4, 346 | "nbformat_minor": 0 347 | } 348 | -------------------------------------------------------------------------------- /TensorFlow.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/TensorFlow.pdf -------------------------------------------------------------------------------- /Tensorflow_RL/1000_index_x.txt: -------------------------------------------------------------------------------- 1 | 43564 12720 7726 44378 2 | 48370 35352 26687 628 3 | 41220 25779 41220 4923 4 | 49847 47535 49832 49809 5 | 41188 41188 41185 41185 6 | 26210 13205 13205 23548 7 | 42111 3953 44400 46967 8 | 47237 45465 45467 48379 9 | 546 14144 2710 14144 10 | 49847 49832 49663 49663 11 | 2640 2583 163 4904 12 | 50199 41854 2336 1727 13 | 41226 159 11824 11826 14 | 48746 47553 48549 42160 15 | 4980 4980 46959 49023 16 | 48572 28449 50412 31842 17 | 30945 25478 27488 47554 18 | 46947 46940 10575 46940 19 | 50470 51499 51499 51300 20 | 49355 49355 49430 49430 21 | 26505 43227 42153 42155 22 | 25274 11622 749 749 23 | 42934 40006 41324 41324 24 | 44417 48147 42133 18098 25 | 43252 44036 41845 44036 26 | 26029 26029 26029 26029 27 | 41183 42010 43682 10886 28 | 29018 29017 43199 1475 29 | 48995 48966 48966 48949 30 | 48632 48652 48553 48643 31 | 28926 47237 13374 13257 32 | 49578 28956 49479 49511 33 | 43232 43232 1917 12198 34 | 49262 48209 48208 24665 35 | 11124 11338 41847 23602 36 | 22591 22591 49511 49879 37 | 50411 50271 49214 50330 38 | 49656 48442 42121 29461 39 | 3211 27141 571 571 40 | 43250 43250 11016 30688 41 | 47537 47537 20819 42094 42 | 50548 44782 26844 50424 43 | 30697 49918 49949 12953 44 | 27404 20990 24250 25574 45 | 41993 41993 41890 41892 46 | 44421 44417 20421 43741 47 | 11529 10281 24302 47376 48 | 13257 43931 42125 11338 49 | 41993 41994 41854 41183 50 | 23556 23645 46624 23331 51 | 47547 47237 48713 31842 52 | 444 41183 43684 42004 53 | 50156 49362 49208 49665 54 | 42156 39503 42158 42160 55 | 43626 23165 23174 23165 56 | 12889 45112 12888 45101 57 | 14065 41188 41185 29342 58 | 45727 46310 45694 46583 59 | 48715 31842 47558 48715 60 | 15048 4867 2404 4869 61 | 24701 24701 18385 24701 62 | 49452 49478 49478 48853 63 | 18098 48781 48781 48781 64 | 47521 48715 47513 47548 65 | 49605 49542 5189 49488 66 | 45755 23648 23552 23553 67 | 50398 50398 48757 27268 68 | 50280 50462 50423 50347 69 | 23012 10701 47390 42156 70 | 45458 45458 45458 45458 71 | 41874 41874 28926 50092 72 | 39719 29158 45314 44785 73 | 41264 43737 43737 43737 74 | 44421 5666 5368 44785 75 | 11338 11338 42982 42982 76 | 24665 20410 24665 24665 77 | 45537 45509 41996 44036 78 | 43744 43743 44037 23293 79 | 47390 39460 46967 44272 80 | 29015 49403 49841 20031 81 | 46563 46563 5521 15920 82 | 10457 47632 48684 49826 83 | 43679 23788 29200 49338 84 | 23543 23558 45754 23602 85 | 39935 29386 39936 43159 86 | 6685 5439 22535 41209 87 | 42133 42150 9199 41189 88 | 42157 27220 17522 48706 89 | 44421 44885 43776 41319 90 | 23192 23192 23165 23192 91 | 48771 419 48771 419 92 | 25183 45353 41890 42083 93 | 39654 41849 41849 41293 94 | 22452 22452 12329 12329 95 | 52735 52735 52735 52735 96 | 46855 4870 16633 11065 97 | 48721 1811 51289 24665 98 | 46177 46872 46871 29457 99 | 22782 22753 3845 12863 100 | 24309 27434 24309 24309 101 | 28749 28750 28754 28751 102 | 14053 28494 14053 10728 103 | 43181 43181 43181 5305 104 | 11541 11541 16364 16364 105 | 49816 49582 49816 49818 106 | 44377 12394 44377 44377 107 | 419 420 421 48771 108 | 25478 30697 43779 47362 109 | 16482 42937 45481 11852 110 | 42156 12965 12965 31842 111 | 41252 41848 13413 31381 112 | 20795 20792 20799 20795 113 | 5357 28768 13621 15850 114 | 45233 20259 20259 45592 115 | 6045 49211 41989 3963 116 | 41870 39997 50036 49591 117 | 41989 39640 26233 2082 118 | 49458 49458 47375 47375 119 | 9193 4656 47655 9193 120 | 43722 28999 954 1190 121 | 41990 45233 10844 15681 122 | 27215 27143 41310 22548 123 | 25609 45227 25938 6045 124 | 49821 52664 50455 52664 125 | 24414 49823 49545 28412 126 | 22452 45238 22452 45236 127 | 41850 39652 39652 39667 128 | 49847 49847 49824 49824 129 | 49046 43181 49503 49479 130 | 39934 43793 40000 47549 131 | 50174 50074 50078 50074 132 | 12329 25590 16381 20801 133 | 24290 25564 20211 15588 134 | 40002 40002 41330 4923 135 | 49433 50448 47313 49433 136 | 29311 45765 546 1289 137 | 11534 39973 39973 39968 138 | 19988 41186 12783 41893 139 | 11438 2060 45345 47247 140 | 42076 42009 42080 43738 141 | 28897 49457 28930 28923 142 | 49429 49346 49431 49306 143 | 42150 42150 45466 45468 144 | 49355 41974 11124 29332 145 | 1261 1385 1261 1385 146 | 14718 41849 40011 40011 147 | 43685 43685 43684 41183 148 | 5451 20985 5451 5157 149 | 48684 26917 44141 44140 150 | 41260 44432 44832 47346 151 | 13973 46956 46967 39984 152 | 13299 1919 15618 45352 153 | 29382 49367 29382 20708 154 | 28428 28428 39998 39998 155 | 41805 49849 1687 41805 156 | 45310 39983 46973 4884 157 | 47557 46966 46966 19499 158 | 49341 49343 49341 47690 159 | 48842 51130 51510 30203 160 | 12408 12593 45086 12591 161 | 29416 42927 42951 45330 162 | 49497 49544 49503 49544 163 | 49159 48578 29401 42157 164 | 983 31830 41786 26197 165 | 42150 41314 28763 12965 166 | 49164 48872 49430 49360 167 | 49171 49173 49168 42156 168 | 27591 17895 7013 39367 169 | 17517 43158 43158 7611 170 | 42156 10279 43863 43861 171 | 41893 41890 41893 41890 172 | 45664 46872 11223 46872 173 | 49307 49307 49307 49307 174 | 29167 49267 49229 24300 175 | 44616 45056 44615 44607 176 | 43970 43961 41878 41878 177 | 39957 39664 39665 39652 178 | 9451 29474 9827 884 179 | 41202 41204 41203 41202 180 | 39359 39359 39721 39721 181 | 3726 27254 20632 20766 182 | 2939 42113 41853 42113 183 | 47718 48464 47718 45468 184 | 44023 49458 12957 43847 185 | 41219 41219 7969 7969 186 | 50344 50279 50278 50261 187 | 45103 12570 2299 39981 188 | 41202 41204 41202 41202 189 | 13153 29135 13153 27357 190 | 27187 27187 42130 42984 191 | 26844 26844 42108 42108 192 | 42122 42122 42124 42124 193 | 49582 50063 47494 27308 194 | 5194 5194 5194 5194 195 | 49870 27647 443 10651 196 | 44037 12723 41167 44037 197 | 1298 11983 44301 44329 198 | 41183 41183 43685 43685 199 | 26160 11671 27764 11675 200 | 15544 15541 15541 15542 201 | 2017 43194 43194 23087 202 | 28749 28753 28754 28753 203 | 39716 10932 41208 41310 204 | 44040 42922 44040 44040 205 | 29469 42929 42160 42160 206 | 2407 2407 22994 45510 207 | 41327 41324 43678 41324 208 | 50066 50062 50118 50278 209 | 42140 42140 41334 42140 210 | 49872 49879 49865 49834 211 | 43684 43684 40013 40013 212 | 49819 49819 49814 49814 213 | 39669 39669 39666 39649 214 | 50275 50275 50275 50333 215 | 25478 6770 39360 39703 216 | 18671 5758 41203 26490 217 | 26377 26377 25274 25274 218 | 48438 47659 47659 48438 219 | 7939 10897 20808 39708 220 | 47530 47530 3203 47530 221 | 39738 10643 39738 10624 222 | 49827 50278 50362 50343 223 | 46660 46650 45755 549 224 | 49486 49512 49512 49513 225 | 1553 1554 3243 44191 226 | 4874 4874 4873 4873 227 | 49429 47665 49208 49665 228 | 2027 47545 22987 22831 229 | 4867 4867 4867 4867 230 | 11332 44938 29329 42107 231 | 3810 3811 3810 3808 232 | 49159 49164 49162 48872 233 | 25284 5196 10156 5176 234 | 49808 49808 49808 49808 235 | 41338 25609 45226 39973 236 | 11354 10874 2677 41207 237 | 41213 3006 41213 24366 238 | 24242 20977 25523 44929 239 | 49955 50150 49405 49946 240 | 49682 26933 49684 45453 241 | 5387 43735 41895 42935 242 | 39957 39957 39957 39651 243 | 9862 1126 44272 1126 244 | 42115 4537 7948 27079 245 | 44580 44580 44580 44580 246 | 9063 50348 50348 22451 247 | 44432 44831 44421 46644 248 | 46966 42153 42155 42155 249 | 50280 50446 48572 49811 250 | 49545 27169 23515 23552 251 | 43684 43647 47491 47491 252 | 49432 19477 49432 51360 253 | 49963 49948 49950 49433 254 | 48771 420 422 422 255 | 42973 42969 43780 43775 256 | 49601 49824 49908 41875 257 | 51374 51504 51558 52028 258 | 49842 49843 49947 49934 259 | 43476 8705 43476 41990 260 | 44607 44607 44616 44616 261 | 48875 48875 1822 1821 262 | 43738 42077 42097 42151 263 | 12965 49207 47512 42149 264 | 45320 42057 45236 49408 265 | 48733 48572 48723 19809 266 | 51507 50470 51374 48712 267 | 49821 49821 50460 50446 268 | 14064 41874 41338 41874 269 | 49955 49955 49955 50118 270 | 39724 39359 41275 41275 271 | 29166 49823 18393 18385 272 | 46980 26862 28876 44277 273 | 46959 49024 49159 49045 274 | 829 7828 829 829 275 | 39961 39651 39961 42097 276 | 2691 2748 44278 17525 277 | 45506 22946 22946 23038 278 | 43255 13220 41303 1996 279 | 38493 39446 43159 39725 280 | 42010 40013 41176 41297 281 | 50073 50063 9257 50077 282 | 19805 48643 48553 48632 283 | 22697 22697 22697 22697 284 | 29035 29035 29019 29050 285 | 50435 50280 50462 46946 286 | 49600 22591 49511 28956 287 | 49344 49344 47196 49436 288 | 45234 22621 11403 11403 289 | 13990 46565 46564 46564 290 | 48557 47396 43255 47620 291 | 42113 552 752 9188 292 | 3964 835 28514 46984 293 | 530 39446 47531 47632 294 | 48571 48571 48571 46948 295 | 41329 14065 41225 2731 296 | 798 49367 49601 49568 297 | 46958 46960 43776 42983 298 | 46963 5758 26383 5758 299 | 41540 41540 41681 30407 300 | 20818 42953 20818 11796 301 | 43227 43227 11356 43227 302 | 51760 44023 51760 51760 303 | 26377 41839 26377 43683 304 | 29027 29055 18158 18174 305 | 44270 42156 44271 41314 306 | 43744 43744 828 22710 307 | 22727 22731 27510 16381 308 | 26380 47375 26380 20574 309 | 26117 16080 48739 49829 310 | 43744 43744 43785 43785 311 | 46203 48718 47527 48684 312 | 12579 12579 12580 12580 313 | 25479 30697 49910 25479 314 | 49827 50344 50279 39442 315 | 49024 49032 49030 49023 316 | 48721 25286 763 10299 317 | 48419 46867 11945 11946 318 | 50350 50328 47995 50348 319 | 41260 42157 42950 41260 320 | 50174 29409 50174 50174 321 | 22294 48860 49302 75 322 | 10932 2060 47235 10728 323 | 47555 49948 49948 47554 324 | 31834 7987 11016 13260 325 | 7789 8483 24340 25664 326 | 43682 43682 41843 41843 327 | 19767 19767 20823 26862 328 | 51506 51507 51506 51573 329 | 49835 49879 49879 49879 330 | 45664 46872 45664 29421 331 | 26380 49839 49838 43847 332 | 49487 49501 46936 41319 333 | 18098 9080 18098 31158 334 | 18969 18897 18969 18969 335 | 44837 44836 42952 42926 336 | 49819 49816 49582 49582 337 | 48599 39993 48659 28472 338 | 24721 24721 9951 9951 339 | 4992 8415 8417 8419 340 | 27187 49047 42130 27220 341 | 41993 41992 41992 41992 342 | 47329 15635 22149 25696 343 | 45461 13257 45449 45467 344 | 52280 50439 50224 51798 345 | 42153 44353 42150 30688 346 | 854 854 27670 45218 347 | 47531 47577 42109 47535 348 | 11332 22533 24576 5237 349 | 2334 2339 2333 3862 350 | 48699 49457 49160 49160 351 | 49607 49565 49514 783 352 | 3961 3962 3961 22291 353 | 6331 43776 43684 46933 354 | 15048 15048 15049 44598 355 | 50202 50196 50202 50149 356 | 45467 47237 47548 45467 357 | 48599 48599 48602 48602 358 | 21553 25478 25479 25479 359 | 46896 46202 46867 45664 360 | 24274 29190 24274 24272 361 | 11326 2199 22621 11404 362 | 41849 41849 41293 9889 363 | 41176 40012 40012 41183 364 | 49480 49342 49967 21547 365 | 41875 41875 41875 45459 366 | 43932 41307 43932 47555 367 | 15398 15400 15400 15398 368 | 44956 28918 16296 1190 369 | 47513 47548 47547 43744 370 | 48465 47717 9257 48443 371 | 1178 31413 1145 31413 372 | 14850 26457 26436 47993 373 | 20376 20376 20376 20376 374 | 13619 46184 46876 46185 375 | 49822 49822 49928 28449 376 | 18555 48245 18555 44982 377 | 31822 29475 27643 28412 378 | 3417 3417 527 2418 379 | 50246 8350 22569 50075 380 | 15919 15919 15919 15919 381 | 27220 41316 29050 29040 382 | 44885 39932 43776 44885 383 | 39398 871 30697 43779 384 | 20930 47513 48715 47551 385 | 42923 3145 43928 42123 386 | 4973 8411 23027 15048 387 | 124 39419 1993 38491 388 | 41320 41302 42011 42076 389 | 13183 13180 13188 13183 390 | 49948 52090 49822 47555 391 | 29428 46198 29428 46198 392 | 49832 47527 49888 49845 393 | 26931 26931 26931 26931 394 | 5387 3598 41186 29342 395 | 2122 45665 11214 46203 396 | 30992 32184 44269 45451 397 | 5918 41837 29327 41837 398 | 9503 2976 22954 31399 399 | 49816 49582 49582 49882 400 | 459 43779 12951 3597 401 | 50127 50127 50127 49595 402 | 9994 41183 43684 43647 403 | 39970 39970 39970 46976 404 | 7121 25674 14743 44428 405 | 709 8487 4862 4862 406 | 39981 39981 42011 42012 407 | 46195 46202 46203 15346 408 | 48556 48643 48652 48461 409 | 41174 44578 22832 22987 410 | 41253 26824 26824 5925 411 | 48554 48444 47718 48443 412 | 49436 3186 3187 24416 413 | 44616 44617 45243 10023 414 | 43684 43684 43684 43685 415 | 41870 3890 26080 25893 416 | 41325 30660 41262 41291 417 | 15588 23041 41991 43781 418 | 18535 39994 6686 17548 419 | 24390 11534 11535 42127 420 | 42004 42004 42004 42012 421 | 43685 41183 42004 41322 422 | 47661 47661 47661 47661 423 | 50201 50201 50044 50044 424 | 26560 26560 26560 26560 425 | 29386 39935 29386 20458 426 | 49888 49805 48718 49196 427 | 29415 49037 6411 49037 428 | 43737 52286 39508 5698 429 | 47393 47394 19818 47394 430 | 42119 44885 44885 43776 431 | 11466 45367 45583 46292 432 | 3598 41186 41185 29342 433 | 2425 10182 2425 2446 434 | 13034 50505 50505 51798 435 | 41989 41989 3953 41288 436 | 48554 48706 48706 48578 437 | 49430 49376 48736 49211 438 | 7507 19684 24275 29190 439 | 1027 1026 1026 41212 440 | 44124 44349 44124 44126 441 | 49457 28923 28942 28956 442 | 16057 20213 16126 16057 443 | 45329 42132 42132 42132 444 | 50463 50461 50411 51504 445 | 30688 29228 31842 45308 446 | 7743 45242 42156 46964 447 | 28925 42015 23104 23104 448 | 50055 50117 49563 29382 449 | 49544 28956 28930 49500 450 | 39398 9078 7743 26885 451 | 20029 5179 5175 44423 452 | 29382 50062 29386 49361 453 | 48780 48780 48780 48780 454 | 41220 41220 45451 45451 455 | 47362 29219 42131 22533 456 | 9337 39904 29475 15077 457 | 2277 24838 2277 26581 458 | 39727 41875 35338 25590 459 | 18319 18319 18315 18319 460 | 15474 15463 15466 14450 461 | 14366 44246 26949 29435 462 | 50243 50243 50244 50244 463 | 15588 9467 41895 51736 464 | 47995 44421 44618 10434 465 | 1714 1711 49884 48863 466 | 52694 46777 46762 47582 467 | 35338 41334 35338 22994 468 | 46963 44831 44831 44831 469 | 42115 23087 29279 7599 470 | 6044 6044 44784 364 471 | 31841 32175 30688 31398 472 | 44573 44574 44573 44570 473 | 46893 46856 46893 46858 474 | 50344 48902 41895 50331 475 | 41183 43931 43786 41339 476 | 43250 43250 20265 43682 477 | 39742 26084 16169 39742 478 | 11804 11816 4907 11816 479 | 20349 4188 26318 26318 480 | 2577 2577 50280 50462 481 | 49867 49883 49867 49883 482 | 23530 45745 23465 23465 483 | 45479 15967 48608 16058 484 | 20349 4188 20349 39931 485 | 49429 49355 49429 20458 486 | 42152 42151 43228 42151 487 | 29386 29386 29386 29386 488 | 12508 15597 12508 15597 489 | 7770 17896 17896 8376 490 | 41991 41991 41991 41991 491 | 41841 41841 41841 24934 492 | 10724 45665 45665 26952 493 | 11467 46263 44105 45580 494 | 45053 45315 26806 45054 495 | 16252 45103 12571 12570 496 | 546 13037 546 41232 497 | 39742 47553 41260 8765 498 | 24414 15093 14332 14248 499 | 41246 39716 39716 39716 500 | 44280 46967 39714 44571 501 | 44887 38491 38492 38491 502 | 5251 41304 4469 47223 503 | 47993 47995 47995 47993 504 | 49501 48727 49580 48877 505 | 41849 41293 8595 41293 506 | 9755 20328 9755 39973 507 | 49429 49429 20379 20379 508 | 42013 5427 44270 39977 509 | 440 10660 39662 10660 510 | 22559 8350 50075 50150 511 | 49879 49666 49872 49818 512 | 50150 50333 50204 50078 513 | 31823 16075 41255 47537 514 | 12705 25272 12705 12705 515 | 8352 50333 8355 22559 516 | 50440 50426 48771 50404 517 | 42983 46934 43776 44850 518 | 39935 29386 49355 20458 519 | 45306 46928 27423 46929 520 | 43778 21553 49909 25771 521 | 27489 26380 45312 42980 522 | 41217 2710 41217 2760 523 | 29081 22452 45238 22452 524 | 29476 2429 2420 10636 525 | 49479 28956 49879 49834 526 | 29050 29050 29050 29040 527 | 49595 10718 24669 52122 528 | 39904 39904 22452 22452 529 | 43979 43980 45437 39410 530 | 15681 48380 49440 47334 531 | 49841 50198 29289 49375 532 | 43921 43921 43921 1657 533 | 49433 50243 49208 50261 534 | 2454 2454 15618 11447 535 | 42156 42158 42156 42159 536 | 49834 49859 49879 49326 537 | 49208 49603 10037 9155 538 | 46936 29422 45663 13619 539 | 45317 50047 6045 35328 540 | 41874 28926 19066 22379 541 | 1066 18338 1063 2528 542 | 50076 50344 49349 50118 543 | 35334 16534 10897 10883 544 | 14729 22813 49162 26369 545 | 26915 26915 26915 26915 546 | 49818 49802 49818 49814 547 | 6611 6611 43777 43777 548 | 49948 29386 50055 50185 549 | 41326 8595 41327 41326 550 | 49567 49510 49576 43238 551 | 42148 42153 42154 42153 552 | 47678 48436 47627 47655 553 | 51760 25771 26030 51796 554 | 48711 47675 10281 41290 555 | 23530 23483 23507 12394 556 | 48773 48773 48773 29677 557 | 11191 39966 15541 15542 558 | 48722 48695 48694 48723 559 | 29337 22603 39399 24117 560 | 39523 27491 27492 27491 561 | 12335 44884 43250 30660 562 | 50092 50077 50170 29339 563 | 49297 39407 49297 39407 564 | 743 461 742 4652 565 | 8310 50122 44486 44488 566 | 49562 49562 18419 30948 567 | 39993 29171 23027 23027 568 | 41209 41252 41209 39965 569 | 1605 26887 1605 26887 570 | 49858 47658 27169 49817 571 | 41293 25161 44617 44615 572 | 29246 29245 29279 29245 573 | 48572 48651 48733 48733 574 | 22395 11170 11542 11171 575 | 48572 50340 29388 50424 576 | 26295 43681 43681 43681 577 | 50361 8490 22696 50337 578 | 49818 49810 49818 49818 579 | 5251 690 5251 5251 580 | 42010 41183 43647 41176 581 | 32174 39959 39976 42153 582 | 11224 128 46893 3888 583 | 49545 49547 29041 49830 584 | 49487 49487 49487 49487 585 | 22452 13292 45236 13292 586 | 14448 14439 15505 14253 587 | 48706 49436 49436 48746 588 | 43684 43684 41185 41183 589 | 46967 4884 46975 46978 590 | 44380 49504 12330 49334 591 | 48839 44275 48839 49303 592 | 50398 27492 42552 30889 593 | 49547 49547 49547 39997 594 | 43781 12965 31842 31122 595 | 49802 49802 49803 49810 596 | 43684 43684 43684 43685 597 | 9889 33524 11153 44469 598 | 75 26560 48860 75 599 | 3441 18810 22468 18811 600 | 43229 43157 43157 43738 601 | 42160 42158 42935 43781 602 | 31374 28429 31374 15934 603 | 47639 48438 884 6212 604 | 22395 13297 22395 22395 605 | 41187 26415 9258 44616 606 | 48464 48465 48447 48465 607 | 30697 30697 43778 24433 608 | 42120 7757 26415 26415 609 | 9137 22544 29390 14060 610 | 41850 10912 10840 11333 611 | 39969 39969 13980 39969 612 | 45238 22452 19988 9198 613 | 28930 28956 49578 28467 614 | 44418 44418 44418 44418 615 | 44884 9754 49373 23557 616 | 39740 39740 39740 40000 617 | 2335 39733 39704 39527 618 | 49355 49355 39935 49032 619 | 50333 22569 22559 50075 620 | 29346 29382 20376 7854 621 | 47567 28429 24304 28429 622 | 44278 42953 5815 43785 623 | 43227 44138 4884 1292 624 | 10728 15597 29014 33672 625 | 44215 44215 8516 41869 626 | 12711 12712 39460 1657 627 | 27488 21553 27488 13222 628 | 745 4648 25277 11610 629 | 50175 50221 6581 50226 630 | 30945 25478 8377 27489 631 | 31825 31825 31825 31825 632 | 3869 23574 23576 46921 633 | 43684 43685 42010 43647 634 | 49487 46872 23527 49872 635 | 10897 45588 10897 10892 636 | 13038 33103 30862 33019 637 | 13299 13298 13299 13299 638 | 128 46855 128 46179 639 | 7621 9199 29404 48277 640 | 49497 49501 47660 49502 641 | 45717 50533 50129 50533 642 | 28923 28930 49479 43164 643 | 49879 49877 49872 49865 644 | 47527 2027 16635 16725 645 | 1838 1799 16706 1838 646 | 43647 43685 41183 42010 647 | 2549 18315 44570 44570 648 | 18162 20708 7854 39934 649 | 25609 25609 39498 45506 650 | 21910 47328 26533 26533 651 | 44037 12723 44037 25154 652 | 14694 45227 47235 45226 653 | 16534 19852 16534 19852 654 | 49802 49666 49802 49802 655 | 43162 43162 7661 29413 656 | 3103 41874 25609 4681 657 | 21553 572 27488 3622 658 | 29022 29057 39904 42121 659 | 44038 44038 12965 12965 660 | 50278 50279 49880 49880 661 | 48030 48035 48033 48030 662 | 43659 43660 45590 11450 663 | 48758 48877 9155 49488 664 | 42985 42985 22298 48847 665 | 43660 43660 41992 41992 666 | 50063 50092 40006 17519 667 | 51298 51264 51296 51262 668 | 45450 45482 2894 46932 669 | 49578 49457 49511 49512 670 | 49835 31120 49665 49875 671 | 49582 49582 28919 49543 672 | 41205 5758 41205 42148 673 | 50460 50410 50408 50409 674 | 50204 50074 50079 50079 675 | 42936 42934 12965 43777 676 | 44418 44417 44421 43098 677 | 25594 25594 16053 42149 678 | 49880 50344 49827 50279 679 | 12248 12232 12248 12239 680 | 2933 2933 44340 42112 681 | 39643 20930 39643 48853 682 | 25884 15919 27529 15919 683 | 50280 48572 49842 49843 684 | 49354 49208 49208 49208 685 | 48695 48695 48859 48691 686 | 50278 49880 49827 50261 687 | 39996 49967 47563 49939 688 | 47717 39724 47684 49452 689 | 50386 29015 29014 50386 690 | 29073 48143 26868 47669 691 | 43685 41183 43684 42010 692 | 25891 25891 25891 27529 693 | 13973 46967 44400 44273 694 | 39964 46559 39874 36078 695 | 42079 42079 41206 41206 696 | 28930 49479 49879 49877 697 | 15323 11412 141 142 698 | 24755 27447 3534 11016 699 | 41245 41287 39873 27559 700 | 46867 45665 46867 15051 701 | 49838 49838 43782 41318 702 | 47697 47548 29082 29082 703 | 45047 15964 28429 41253 704 | 24721 49830 49830 24722 705 | 35335 35335 25609 25609 706 | 49478 49308 49501 49487 707 | 43783 43783 43783 43783 708 | 48599 48599 48598 48599 709 | 49508 49507 49508 49507 710 | 28916 10974 30945 30945 711 | 13969 39666 39665 52735 712 | 42157 44038 43743 44038 713 | 14872 43926 14872 43926 714 | 25479 48707 49374 49190 715 | 41336 41988 41851 41854 716 | 43679 43679 43679 43679 717 | 49810 28930 49819 30198 718 | 49865 49879 49872 49877 719 | 45216 29045 44898 29339 720 | 43684 43685 41183 42004 721 | 42076 43738 42076 11356 722 | 42113 42113 42113 42113 723 | 50463 50437 50439 52280 724 | 48997 41805 49849 41805 725 | 44266 44266 49458 49458 726 | 31045 44648 39654 48617 727 | 41284 27189 25944 26376 728 | 42151 17892 17671 38490 729 | 9065 47993 50280 50462 730 | 44885 46958 46958 42929 731 | 39720 50119 8353 22559 732 | 45497 49830 49835 49873 733 | 26383 44216 6045 27486 734 | 29037 29037 29037 29037 735 | 44834 44834 39370 41213 736 | 45640 22341 22300 22742 737 | 44064 44064 44064 44064 738 | 45461 45461 45461 45461 739 | 39367 49435 49918 21553 740 | 52735 45691 44027 44027 741 | 48378 45461 42149 31842 742 | 42129 41313 44884 42926 743 | 48733 47512 48733 48443 744 | 12335 42134 12335 44840 745 | 51730 51659 51659 52019 746 | 49364 49364 49359 49364 747 | 15588 24284 20990 24246 748 | 43779 572 572 3612 749 | 28956 28930 49602 49606 750 | 1912 1918 11450 11450 751 | 14257 14257 50047 44323 752 | 46273 46273 11403 2199 753 | 50191 50190 50129 46618 754 | 39967 39967 39968 39968 755 | 45665 46864 46867 11214 756 | 20828 41214 41216 41216 757 | 48081 16952 1834 11076 758 | 7609 46938 46938 46938 759 | 27489 42131 27489 30697 760 | 41202 41202 41204 22197 761 | 50411 50407 50407 50409 762 | 9257 9258 9862 41226 763 | 39714 49440 45460 48710 764 | 15346 15346 45665 11214 765 | 48464 43776 48444 48721 766 | 50077 50063 50073 50066 767 | 43685 43685 43685 43684 768 | 45024 45024 45022 45022 769 | 47551 47551 5708 29305 770 | 47335 20756 3858 10685 771 | 10640 448 448 10652 772 | 42156 41246 25674 45590 773 | 50062 49488 40012 31381 774 | 45215 45215 43985 43985 775 | 49511 49512 49602 28930 776 | 29824 3104 29824 9968 777 | 47555 49948 42158 49949 778 | 43682 43682 43682 26928 779 | 45467 45314 45314 47237 780 | 39361 39360 39363 20303 781 | 48571 48571 43684 46935 782 | 50092 49955 49376 50202 783 | 5427 48819 24649 5427 784 | 6770 29196 43931 39504 785 | 13114 13114 41311 41311 786 | 50411 50411 50462 50462 787 | 45497 49830 15552 45497 788 | 50406 49812 49813 49669 789 | 24637 24793 26058 18443 790 | 4784 32198 32194 13841 791 | 28895 20324 29087 4985 792 | 50411 50446 50278 50344 793 | 50279 22569 22559 50075 794 | 12881 12881 8690 12881 795 | 46870 4870 44602 4870 796 | 42001 42001 42001 42001 797 | 43164 49479 49511 49872 798 | 49818 49818 49819 49882 799 | 49430 49430 8027 8027 800 | 26304 44402 46360 44402 801 | 31822 36532 48436 48436 802 | 3064 44362 51320 51326 803 | 48417 47352 47521 48713 804 | 3941 3942 3941 22745 805 | 12724 12724 12724 39738 806 | 12951 12951 24577 3597 807 | 48734 48734 16382 49829 808 | 26377 39569 43162 9146 809 | 18554 24706 18555 22961 810 | 31380 44831 41833 890 811 | 419 9980 420 48771 812 | 43793 42120 44417 44398 813 | 44134 44016 44097 44002 814 | 25660 25661 25660 25658 815 | 39961 10882 31844 46287 816 | 7933 49334 24307 24307 817 | 426 50404 27491 50398 818 | 47421 46676 47204 45386 819 | 49206 4991 49206 50197 820 | 42084 19765 14921 13084 821 | 1457 15367 4460 27189 822 | 44040 42150 42156 31824 823 | 39376 49545 39997 49565 824 | 23518 23518 23518 23518 825 | 44832 41253 46928 42156 826 | 7854 47651 29392 39510 827 | 42131 30945 42131 42131 828 | 49429 49429 49343 49451 829 | 42935 42934 41314 39383 830 | 39503 31842 42150 44217 831 | 44428 44428 42129 44428 832 | 35552 35553 35217 26140 833 | 1190 16222 1190 29296 834 | 45249 45330 45249 20288 835 | 52664 50407 49811 22802 836 | 15989 40006 41315 41315 837 | 47657 26867 26867 47657 838 | 31819 42002 31819 47334 839 | 48611 51504 26946 48842 840 | 23448 43932 13257 44884 841 | 27489 30179 30179 30179 842 | 45306 41287 45306 41287 843 | 28923 49513 28956 49512 844 | 49568 47511 49668 47535 845 | 49639 13583 19947 20501 846 | 49458 47375 47375 47375 847 | 49190 49189 49191 43793 848 | 28763 47580 20931 26986 849 | 25154 26285 31842 39485 850 | 50149 50149 50149 50070 851 | 49872 49835 50396 50396 852 | 29029 29339 31838 29047 853 | 41976 39956 41976 41287 854 | 42139 42139 42139 42139 855 | 49818 49818 49818 49818 856 | 49847 49826 26852 49832 857 | 27445 26534 27445 44840 858 | 25869 43733 49917 25869 859 | 42934 42934 28763 28763 860 | 6787 47576 20884 49848 861 | 10806 459 25479 49458 862 | 5698 29050 29015 29041 863 | 11307 11307 44972 19574 864 | 8355 8352 8355 50333 865 | 28096 20609 27843 27926 866 | 15345 29457 46866 2122 867 | 21590 13785 13797 8857 868 | 41176 41176 41176 40013 869 | 10780 6685 6685 6685 870 | 1475 29017 29017 29018 871 | 50563 50563 29018 29018 872 | 49941 49941 50066 50066 873 | 41275 41262 41262 41275 874 | 22579 19490 46871 46872 875 | 5439 25165 5698 39398 876 | 32215 39640 39636 43159 877 | 22389 11168 42012 12480 878 | 11852 2749 20828 11855 879 | 43782 47373 47372 43782 880 | 29415 49037 29415 49037 881 | 42018 42017 42018 44895 882 | 49159 22539 49083 49458 883 | 13222 27488 3597 459 884 | 12965 50403 50462 48572 885 | 29015 29014 29051 29050 886 | 44831 44831 44831 47390 887 | 48771 50404 50426 48771 888 | 2713 2737 14011 11858 889 | 29192 47237 26818 39934 890 | 14337 22298 16761 22546 891 | 36144 36144 6459 6459 892 | 49159 48823 49026 49159 893 | 39669 25753 25755 18285 894 | 50271 50326 51507 51506 895 | 11338 13219 41848 18684 896 | 28930 28923 28930 41326 897 | 27171 41175 5704 41175 898 | 24838 2540 7799 18340 899 | 49046 49029 6410 6410 900 | 28895 50227 28895 50227 901 | 26560 22294 26560 49302 902 | 937 30862 13260 7611 903 | 41805 41805 41805 41805 904 | 22569 50078 50281 50505 905 | 44599 4873 45665 46874 906 | 31383 3953 31383 22760 907 | 41222 45451 24639 13630 908 | 49031 49355 49359 20708 909 | 7811 7811 44390 11542 910 | 5698 29040 29038 43198 911 | 42138 42138 39655 39655 912 | 42160 22539 49920 49434 913 | 51736 50131 28931 23593 914 | 44793 9146 44217 44404 915 | 4564 7782 41205 44428 916 | 3321 13781 13819 13792 917 | 44404 41189 42956 44418 918 | 3145 5647 20360 3043 919 | 45461 42150 48379 45468 920 | 19810 48847 19802 41183 921 | 25363 49497 49544 49603 922 | 8999 8999 11826 11817 923 | 45498 1285 25508 25509 924 | 41252 41976 39956 41191 925 | 41293 41849 11436 2068 926 | 26886 40010 26886 13115 927 | 28930 49578 47655 47539 928 | 47339 47346 46964 44217 929 | 49205 6765 4988 49205 930 | 50460 50436 48757 27492 931 | 1012 47361 47360 44266 932 | 43743 44038 44038 43743 933 | 44421 41319 44417 44417 934 | 41207 2070 15594 45399 935 | 49175 48863 10281 49171 936 | 44581 44595 26857 29085 937 | 44382 44382 35552 22994 938 | 49822 50505 50463 50446 939 | 42936 39503 41314 43777 940 | 41336 39968 39968 39983 941 | 22452 22452 45236 45236 942 | 48848 48814 7854 49827 943 | 15635 47329 15635 47329 944 | 9808 9808 47236 47236 945 | 44841 24728 13619 46184 946 | 49274 48710 12724 39738 947 | 28438 41275 41327 29347 948 | 41327 41326 41324 41298 949 | 41988 20825 28714 45201 950 | 33018 32367 33112 36531 951 | 43227 43227 43227 43227 952 | 46958 46959 46957 46960 953 | 13965 13965 13972 13965 954 | 48793 48798 48864 49159 955 | 46360 46360 46360 41226 956 | 42932 22759 42930 42930 957 | 49833 49832 49847 49832 958 | 12499 12501 11532 12590 959 | 50261 50261 50363 50343 960 | 43744 41167 43743 44038 961 | 49367 18162 7854 39934 962 | 48771 48754 27492 422 963 | 10705 26202 31403 26226 964 | 47657 26867 26868 8252 965 | 10882 2471 10882 39957 966 | 5441 44273 44273 5441 967 | 26377 26377 35337 27533 968 | 12724 21558 39738 9296 969 | 44280 49925 45317 49819 970 | 23515 11534 29040 29041 971 | 50316 49477 48727 48727 972 | 49834 49871 49879 49855 973 | 28763 32175 44270 2713 974 | 16301 28931 43655 29003 975 | 48690 48695 48694 48814 976 | 44142 44790 44221 44791 977 | 742 26090 26090 10505 978 | 41310 41310 41310 41181 979 | 43684 12723 30198 10750 980 | 42973 43928 43928 43928 981 | 3869 3870 42005 49907 982 | 43744 42934 42936 42150 983 | 8025 49306 49354 31122 984 | 48442 15902 16075 16074 985 | 41849 41849 41293 41293 986 | 8740 16035 39862 39862 987 | 8451 27277 47404 11958 988 | 21553 27488 21553 25771 989 | 9446 30953 30953 46966 990 | 49812 49811 49813 49669 991 | 44394 19832 48713 44884 992 | 42120 41319 43776 44885 993 | 46968 46968 46968 46968 994 | 48684 48684 48684 48684 995 | 41183 43683 25594 10886 996 | 29042 29033 43198 29033 997 | 48728 48727 48727 48728 998 | 15557 42156 41310 42156 999 | 22579 41849 43678 41298 1000 | 25478 27488 25478 30945 1001 | -------------------------------------------------------------------------------- /Tensorflow_RL/1000_index_y.txt: -------------------------------------------------------------------------------- 1 | 47174 2 | 8740 3 | 4926 4 | 16605 5 | 41187 6 | 29150 7 | 3851 8 | 45464 9 | 11231 10 | 49809 11 | 163 12 | 354 13 | 11826 14 | 48549 15 | 49024 16 | 48822 17 | 47554 18 | 43912 19 | 51300 20 | 20708 21 | 27445 22 | 22253 23 | 41324 24 | 9077 25 | 44036 26 | 26030 27 | 26232 28 | 46974 29 | 48949 30 | 48632 31 | 45467 32 | 49567 33 | 11497 34 | 18555 35 | 23589 36 | 49865 37 | 50425 38 | 18172 39 | 27141 40 | 42149 41 | 42094 42 | 50423 43 | 20977 44 | 44855 45 | 41893 46 | 26430 47 | 24302 48 | 13219 49 | 43684 50 | 23589 51 | 47513 52 | 42010 53 | 22606 54 | 42160 55 | 23431 56 | 12538 57 | 31121 58 | 46582 59 | 47521 60 | 4872 61 | 24698 62 | 41841 63 | 48781 64 | 44217 65 | 49510 66 | 48468 67 | 27492 68 | 50334 69 | 47312 70 | 24391 71 | 47561 72 | 44418 73 | 43737 74 | 44417 75 | 44618 76 | 24665 77 | 41845 78 | 41991 79 | 43706 80 | 29154 81 | 5521 82 | 1818 83 | 3068 84 | 23574 85 | 39442 86 | 42081 87 | 45310 88 | 49159 89 | 42119 90 | 23414 91 | 48771 92 | 46258 93 | 41293 94 | 14916 95 | 52735 96 | 21549 97 | 48431 98 | 46867 99 | 44569 100 | 31903 101 | 28753 102 | 10728 103 | 43181 104 | 11541 105 | 49816 106 | 13031 107 | 422 108 | 47359 109 | 2731 110 | 31842 111 | 9153 112 | 20798 113 | 4548 114 | 44369 115 | 44843 116 | 49596 117 | 22695 118 | 47375 119 | 9193 120 | 16296 121 | 48380 122 | 22546 123 | 10728 124 | 50455 125 | 13611 126 | 22452 127 | 39667 128 | 49847 129 | 49578 130 | 1811 131 | 50074 132 | 20985 133 | 24286 134 | 11354 135 | 26117 136 | 29311 137 | 39968 138 | 41892 139 | 47257 140 | 40006 141 | 28942 142 | 49306 143 | 45467 144 | 42956 145 | 41329 146 | 30689 147 | 42004 148 | 3045 149 | 16919 150 | 47494 151 | 39968 152 | 45240 153 | 7854 154 | 48710 155 | 49849 156 | 39983 157 | 47557 158 | 47690 159 | 27503 160 | 31844 161 | 45330 162 | 49478 163 | 49045 164 | 31830 165 | 43781 166 | 31122 167 | 49948 168 | 38494 169 | 6131 170 | 23553 171 | 41893 172 | 45664 173 | 49307 174 | 26295 175 | 7532 176 | 41878 177 | 39957 178 | 7927 179 | 26383 180 | 41275 181 | 20632 182 | 41853 183 | 49374 184 | 12957 185 | 48468 186 | 50261 187 | 39981 188 | 41204 189 | 44948 190 | 39665 191 | 26844 192 | 41303 193 | 39723 194 | 10156 195 | 20223 196 | 11469 197 | 44303 198 | 43647 199 | 17610 200 | 15485 201 | 47912 202 | 28751 203 | 45784 204 | 44040 205 | 29469 206 | 39640 207 | 41327 208 | 50349 209 | 42140 210 | 49405 211 | 43685 212 | 49823 213 | 39666 214 | 22559 215 | 17522 216 | 18671 217 | 12267 218 | 47693 219 | 48840 220 | 47530 221 | 12724 222 | 50344 223 | 14054 224 | 49486 225 | 3244 226 | 44600 227 | 49409 228 | 22832 229 | 4874 230 | 44599 231 | 3810 232 | 49171 233 | 4740 234 | 49822 235 | 39656 236 | 41245 237 | 24366 238 | 20204 239 | 50116 240 | 26933 241 | 39398 242 | 39957 243 | 20134 244 | 4544 245 | 44580 246 | 50348 247 | 23560 248 | 42153 249 | 49669 250 | 23515 251 | 47484 252 | 43737 253 | 49948 254 | 39486 255 | 42974 256 | 47655 257 | 52026 258 | 49934 259 | 23589 260 | 44617 261 | 45537 262 | 42096 263 | 42150 264 | 26505 265 | 44885 266 | 26946 267 | 50329 268 | 42955 269 | 50118 270 | 37109 271 | 18384 272 | 44276 273 | 49038 274 | 829 275 | 31844 276 | 20819 277 | 22946 278 | 1994 279 | 26430 280 | 43684 281 | 50198 282 | 7076 283 | 22697 284 | 29019 285 | 51375 286 | 49479 287 | 49438 288 | 22621 289 | 2161 290 | 43912 291 | 752 292 | 39 293 | 47535 294 | 46948 295 | 5521 296 | 49367 297 | 46935 298 | 5758 299 | 41604 300 | 20818 301 | 42140 302 | 29247 303 | 29323 304 | 18178 305 | 27486 306 | 43744 307 | 25590 308 | 47373 309 | 43206 310 | 41306 311 | 48431 312 | 12583 313 | 45664 314 | 50261 315 | 49022 316 | 796 317 | 11945 318 | 50350 319 | 44832 320 | 50169 321 | 49302 322 | 41253 323 | 47390 324 | 17521 325 | 25663 326 | 43682 327 | 4551 328 | 50470 329 | 49879 330 | 46874 331 | 26380 332 | 49576 333 | 31157 334 | 18897 335 | 15588 336 | 49816 337 | 4780 338 | 24722 339 | 8420 340 | 29204 341 | 41994 342 | 25700 343 | 16919 344 | 50462 345 | 32175 346 | 46900 347 | 48718 348 | 5237 349 | 3932 350 | 47554 351 | 783 352 | 3961 353 | 6331 354 | 15048 355 | 50070 356 | 47542 357 | 48599 358 | 25478 359 | 11750 360 | 24276 361 | 22694 362 | 41293 363 | 39963 364 | 2341 365 | 45469 366 | 48715 367 | 15403 368 | 27400 369 | 42109 370 | 48444 371 | 1143 372 | 44125 373 | 5979 374 | 46185 375 | 50460 376 | 18555 377 | 29050 378 | 2416 379 | 50344 380 | 15919 381 | 29022 382 | 41319 383 | 24737 384 | 34186 385 | 560 386 | 389 387 | 7519 388 | 41206 389 | 13183 390 | 42156 391 | 46195 392 | 49832 393 | 2310 394 | 752 395 | 45665 396 | 23788 397 | 5918 398 | 31400 399 | 49816 400 | 459 401 | 50127 402 | 42010 403 | 46975 404 | 44428 405 | 42156 406 | 42012 407 | 46182 408 | 7076 409 | 20348 410 | 5925 411 | 48551 412 | 24417 413 | 15046 414 | 43685 415 | 47658 416 | 40008 417 | 9862 418 | 17563 419 | 560 420 | 15597 421 | 11317 422 | 47661 423 | 50185 424 | 26560 425 | 15814 426 | 39980 427 | 29415 428 | 29015 429 | 47632 430 | 44428 431 | 22692 432 | 29342 433 | 10175 434 | 28449 435 | 41283 436 | 47237 437 | 49305 438 | 22294 439 | 26304 440 | 44124 441 | 22585 442 | 16126 443 | 41189 444 | 51507 445 | 48572 446 | 47346 447 | 26977 448 | 29346 449 | 49578 450 | 30667 451 | 44423 452 | 29382 453 | 48780 454 | 13991 455 | 27489 456 | 27310 457 | 15145 458 | 32214 459 | 18315 460 | 44494 461 | 44859 462 | 50244 463 | 41895 464 | 13413 465 | 1717 466 | 23638 467 | 3953 468 | 44831 469 | 29246 470 | 14220 471 | 42149 472 | 44573 473 | 128 474 | 50566 475 | 4654 476 | 27305 477 | 26084 478 | 19767 479 | 106 480 | 50411 481 | 49867 482 | 45745 483 | 15900 484 | 39931 485 | 49306 486 | 42152 487 | 39935 488 | 12508 489 | 8281 490 | 45314 491 | 10392 492 | 11613 493 | 46262 494 | 45315 495 | 11168 496 | 13037 497 | 44296 498 | 44307 499 | 13454 500 | 11539 501 | 38492 502 | 43737 503 | 47995 504 | 49486 505 | 7076 506 | 39973 507 | 49208 508 | 5117 509 | 440 510 | 8350 511 | 49819 512 | 50074 513 | 46893 514 | 12705 515 | 22569 516 | 26911 517 | 13257 518 | 51181 519 | 18686 520 | 30697 521 | 44833 522 | 4933 523 | 22452 524 | 2418 525 | 49871 526 | 29339 527 | 50448 528 | 45238 529 | 45435 530 | 48380 531 | 49376 532 | 1657 533 | 23648 534 | 2454 535 | 42157 536 | 49835 537 | 2993 538 | 130 539 | 35339 540 | 41874 541 | 2507 542 | 49879 543 | 10895 544 | 26369 545 | 26915 546 | 49607 547 | 39504 548 | 30926 549 | 41326 550 | 44421 551 | 11356 552 | 47690 553 | 51760 554 | 47694 555 | 24580 556 | 48824 557 | 5181 558 | 48733 559 | 39399 560 | 420 561 | 31842 562 | 50184 563 | 49297 564 | 752 565 | 44489 566 | 39716 567 | 436 568 | 39873 569 | 1605 570 | 39997 571 | 44615 572 | 29245 573 | 48723 574 | 13191 575 | 39641 576 | 29243 577 | 50332 578 | 49803 579 | 4462 580 | 40013 581 | 42153 582 | 46893 583 | 29050 584 | 49503 585 | 13189 586 | 15505 587 | 48706 588 | 41183 589 | 46968 590 | 49901 591 | 48691 592 | 30872 593 | 49580 594 | 42077 595 | 49823 596 | 43685 597 | 10799 598 | 34 599 | 18812 600 | 43158 601 | 31842 602 | 31374 603 | 5647 604 | 22395 605 | 39973 606 | 43776 607 | 24433 608 | 26415 609 | 31825 610 | 24691 611 | 31157 612 | 27435 613 | 49457 614 | 44418 615 | 50129 616 | 39740 617 | 39629 618 | 49355 619 | 8352 620 | 20708 621 | 15884 622 | 20819 623 | 41892 624 | 49684 625 | 44215 626 | 28425 627 | 13332 628 | 11611 629 | 3953 630 | 25478 631 | 32174 632 | 52475 633 | 15249 634 | 49879 635 | 10897 636 | 32922 637 | 12267 638 | 128 639 | 25140 640 | 48738 641 | 50533 642 | 50307 643 | 49871 644 | 2027 645 | 16706 646 | 41322 647 | 44831 648 | 49367 649 | 6611 650 | 26533 651 | 45467 652 | 45227 653 | 19852 654 | 49818 655 | 39687 656 | 41874 657 | 43779 658 | 15781 659 | 44038 660 | 49827 661 | 48030 662 | 41184 663 | 8024 664 | 48855 665 | 47181 666 | 17518 667 | 51297 668 | 39714 669 | 48553 670 | 31120 671 | 22579 672 | 42155 673 | 27492 674 | 9063 675 | 43744 676 | 43099 677 | 32175 678 | 50278 679 | 12243 680 | 44340 681 | 39643 682 | 25885 683 | 50462 684 | 49354 685 | 48695 686 | 50349 687 | 43681 688 | 47717 689 | 29015 690 | 26868 691 | 43685 692 | 6045 693 | 23515 694 | 48718 695 | 42150 696 | 49879 697 | 13084 698 | 39484 699 | 39873 700 | 4867 701 | 49912 702 | 29082 703 | 15936 704 | 24721 705 | 25609 706 | 49503 707 | 43783 708 | 16526 709 | 49508 710 | 27488 711 | 11231 712 | 43743 713 | 43937 714 | 48467 715 | 3045 716 | 44400 717 | 22977 718 | 49834 719 | 43200 720 | 40012 721 | 42156 722 | 42113 723 | 50423 724 | 49849 725 | 44266 726 | 39654 727 | 27189 728 | 26852 729 | 50462 730 | 44836 731 | 50281 732 | 49859 733 | 27486 734 | 48447 735 | 42951 736 | 42148 737 | 38496 738 | 41974 739 | 30945 740 | 52735 741 | 45466 742 | 42926 743 | 47718 744 | 12335 745 | 51605 746 | 7928 747 | 44837 748 | 3186 749 | 22585 750 | 1918 751 | 44309 752 | 46228 753 | 45706 754 | 39968 755 | 11214 756 | 14011 757 | 44428 758 | 46938 759 | 25479 760 | 41202 761 | 50409 762 | 159 763 | 21547 764 | 45665 765 | 27160 766 | 50066 767 | 42004 768 | 41275 769 | 5708 770 | 10679 771 | 2316 772 | 45590 773 | 49032 774 | 11233 775 | 49600 776 | 3105 777 | 50455 778 | 48821 779 | 48379 780 | 39932 781 | 48571 782 | 50123 783 | 51500 784 | 24755 785 | 41311 786 | 50280 787 | 30697 788 | 50406 789 | 39475 790 | 597 791 | 4985 792 | 50362 793 | 8353 794 | 12981 795 | 15051 796 | 42001 797 | 49665 798 | 49882 799 | 49431 800 | 2710 801 | 48436 802 | 51320 803 | 27430 804 | 9844 805 | 39738 806 | 26029 807 | 42238 808 | 41206 809 | 18554 810 | 47390 811 | 48771 812 | 42122 813 | 43998 814 | 25660 815 | 46287 816 | 49228 817 | 30872 818 | 46677 819 | 23394 820 | 14867 821 | 42932 822 | 43781 823 | 49455 824 | 48796 825 | 42929 826 | 44404 827 | 27488 828 | 49306 829 | 31842 830 | 43931 831 | 42129 832 | 27716 833 | 16222 834 | 7772 835 | 22808 836 | 47559 837 | 26867 838 | 47618 839 | 6525 840 | 45761 841 | 30179 842 | 45306 843 | 49500 844 | 47655 845 | 12653 846 | 47375 847 | 48467 848 | 47689 849 | 32175 850 | 50149 851 | 22591 852 | 50359 853 | 39739 854 | 42139 855 | 49818 856 | 20879 857 | 48370 858 | 43733 859 | 43766 860 | 20884 861 | 47375 862 | 29015 863 | 11307 864 | 22569 865 | 28048 866 | 11214 867 | 23056 868 | 40013 869 | 6685 870 | 39972 871 | 29040 872 | 50062 873 | 41275 874 | 46871 875 | 29050 876 | 10984 877 | 12487 878 | 20828 879 | 43782 880 | 6409 881 | 42018 882 | 48822 883 | 3597 884 | 52664 885 | 29015 886 | 42156 887 | 50408 888 | 41214 889 | 18162 890 | 46891 891 | 36144 892 | 48823 893 | 18232 894 | 52700 895 | 47494 896 | 49580 897 | 27171 898 | 2558 899 | 49046 900 | 50227 901 | 26560 902 | 43738 903 | 41805 904 | 50281 905 | 12673 906 | 3962 907 | 46986 908 | 20379 909 | 29404 910 | 43198 911 | 26844 912 | 42160 913 | 23593 914 | 43928 915 | 45471 916 | 32187 917 | 44840 918 | 3893 919 | 45468 920 | 41319 921 | 49501 922 | 2640 923 | 25509 924 | 26846 925 | 11419 926 | 43690 927 | 49879 928 | 48379 929 | 49206 930 | 27492 931 | 1012 932 | 11469 933 | 42128 934 | 11420 935 | 48823 936 | 27081 937 | 44382 938 | 48572 939 | 42934 940 | 39983 941 | 45236 942 | 50119 943 | 15635 944 | 32913 945 | 130 946 | 15681 947 | 41324 948 | 43678 949 | 26706 950 | 33010 951 | 43227 952 | 41319 953 | 49046 954 | 49164 955 | 41226 956 | 19493 957 | 49833 958 | 45105 959 | 39442 960 | 43743 961 | 7773 962 | 26974 963 | 31403 964 | 48437 965 | 10882 966 | 44273 967 | 39422 968 | 6045 969 | 49819 970 | 29041 971 | 49503 972 | 49877 973 | 2732 974 | 23053 975 | 48814 976 | 44125 977 | 26087 978 | 41181 979 | 25381 980 | 28427 981 | 8528 982 | 42936 983 | 49431 984 | 16076 985 | 41293 986 | 29013 987 | 16367 988 | 21553 989 | 19499 990 | 50408 991 | 43775 992 | 41319 993 | 4884 994 | 49845 995 | 41843 996 | 29042 997 | 48728 998 | 42156 999 | 41324 1000 | 25478 1001 | -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "mlp-5000" 2 | all_model_checkpoint_paths: "mlp-0" 3 | all_model_checkpoint_paths: "mlp-500" 4 | all_model_checkpoint_paths: "mlp-1000" 5 | all_model_checkpoint_paths: "mlp-1500" 6 | all_model_checkpoint_paths: "mlp-2000" 7 | all_model_checkpoint_paths: "mlp-2500" 8 | all_model_checkpoint_paths: "mlp-3000" 9 | all_model_checkpoint_paths: "mlp-3500" 10 | all_model_checkpoint_paths: "mlp-4000" 11 | all_model_checkpoint_paths: "mlp-4500" 12 | all_model_checkpoint_paths: "mlp-5000" 13 | -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-0 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-0.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-0.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-0.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-0.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-0.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-0.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1000 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1000.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1000.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1000.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1500 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1500.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1500.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1500.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1500.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-1500.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-1500.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2000 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2000.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2000.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2000.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2500 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2500.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2500.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2500.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2500.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-2500.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-2500.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3000 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3000.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3000.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3000.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3500 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3500.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3500.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3500.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3500.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-3500.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-3500.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4000 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4000.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4000.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4000.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4500 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4500.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4500.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4500.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4500.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-4500.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-4500.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-500: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-500 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-500.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-500.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-500.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-500.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-500.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-500.meta -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-5000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-5000 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-5000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-5000.data-00000-of-00001 -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-5000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-5000.index -------------------------------------------------------------------------------- /Tensorflow_RL/Checkpoint/mlp-5000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/Checkpoint/mlp-5000.meta -------------------------------------------------------------------------------- /Tensorflow_RL/MNIST_data/t10k-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/MNIST_data/t10k-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /Tensorflow_RL/MNIST_data/t10k-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/MNIST_data/t10k-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /Tensorflow_RL/MNIST_data/train-images-idx3-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/MNIST_data/train-images-idx3-ubyte.gz -------------------------------------------------------------------------------- /Tensorflow_RL/MNIST_data/train-labels-idx1-ubyte.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/MNIST_data/train-labels-idx1-ubyte.gz -------------------------------------------------------------------------------- /Tensorflow_RL/RNN_Homework_Automated_trading.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 이더리움 시세 예측 with RNN\n", 8 | "이더리움은 비트코인과 같은 블록체인으로, 최근 엄청난 가격 폭등으로 이슈가 되고 있다. 이번 숙제에서는 이더리움의 가격을 예측하는 모델을 RNN으로 학습시켜본다." 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "\"hi\"" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "\n", 23 | "![](images/eth.png)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "\"hi\"\n", 31 | "\n", 32 | "- 제공되는 데이터는 미국의 가상화폐 거래소의 시세를 12/19/2016 @ 8:00pm (UTC) 를 기준으로 1800초(30분)마다 기록한 것이다.\n", 33 | "- 이를 학습하여, 다음 time step에서의 가격을 예측하는 sequential model을 RNN으로 학습해보자." 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 1, 39 | "metadata": { 40 | "collapsed": false 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "import numpy as np\n", 45 | "from numpy import genfromtxt\n", 46 | "raw_data = genfromtxt('1800_recent.csv', delimiter=',', skip_header=1)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 2, 52 | "metadata": { 53 | "collapsed": false 54 | }, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "(8437, 6)\n", 61 | "[[ 7.73181935e+00 7.64366106e+00 7.70923820e+00 7.64366106e+00\n", 62 | " 1.56108530e+02 2.03057873e+01]\n", 63 | " [ 7.68999818e+00 7.63542000e+00 7.64000000e+00 7.68999818e+00\n", 64 | " 5.26239564e+01 6.88413339e+00]\n", 65 | " [ 7.68999997e+00 7.63542011e+00 7.63643005e+00 7.68999985e+00\n", 66 | " 2.64346039e+00 3.43931810e-01]\n", 67 | " ..., \n", 68 | " [ 3.70000000e+02 3.63577192e+02 3.66546740e+02 3.65353691e+02\n", 69 | " 1.45889692e+05 3.97810160e+02]\n", 70 | " [ 3.71565000e+02 3.63913407e+02 3.65353691e+02 3.68400000e+02\n", 71 | " 1.61053736e+05 4.38938788e+02]\n", 72 | " [ 3.70899930e+02 3.66000000e+02 3.69825698e+02 3.68144535e+02\n", 73 | " 9.56366389e+04 2.59342470e+02]]\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "print raw_data.shape\n", 79 | "print raw_data" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": { 86 | "collapsed": true 87 | }, 88 | "outputs": [], 89 | "source": [] 90 | } 91 | ], 92 | "metadata": { 93 | "kernelspec": { 94 | "display_name": "Python 2", 95 | "language": "python", 96 | "name": "python2" 97 | }, 98 | "language_info": { 99 | "codemirror_mode": { 100 | "name": "ipython", 101 | "version": 2 102 | }, 103 | "file_extension": ".py", 104 | "mimetype": "text/x-python", 105 | "name": "python", 106 | "nbconvert_exporter": "python", 107 | "pygments_lexer": "ipython2", 108 | "version": "2.7.6" 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 0 113 | } 114 | -------------------------------------------------------------------------------- /Tensorflow_RL/Reinforcement_Learning_Tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Deep Reinforcement Learing\n", 8 | "\n", 9 | "Deep Reinforcement Learning이란, 기존의 강화학습에서의 Q function을 딥러닝으로 근사하는 모델을 의미한다. 대표적으로 구글 Deep Mind의 Atari와 AlphaGo 역시 이 Deep Reinforcement Learning 응용의 한가지이다.\n", 10 | "\n", 11 | "이번 파트에서는 Deep Reinforcement Learning을 이용해서 간단한 2차원 게임을 플레이하고, reward로 부터 스스로 학습하는 Kaparthy의 오픈소스 예제를 실습해 본다." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "\"Deep\n", 19 | "\n", 20 | "\n", 21 | "(출처: https://github.com/nivwusquorum/tensorflow-deepq)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "metadata": { 27 | "collapsed": true 28 | }, 29 | "source": [ 30 | "### Reinforcement Learning\n", 31 | "\n", 32 | "Reinforcement Learning, 이하 RL은 supervised learning과 달리 데이터에 대한 정확한 정답을 받지 않고, 내가 한 행동에 대한 reward feedback 만으로 학습을 수행하는 알고리즘이다. 이를 강화학습이라 부르며, 이것을 수행하는 가장 대표적인 알고리즘으로 Q-Learning 이 있다.\n", 33 | "\n", 34 | "RL을 이해하는 것은 매우 많은 공부를 필요로 하기 때문에, 우선 2D게임을 예로 들어 아주 기본적인 개념만 살펴본다.\n", 35 | "\n", 36 | "* **State**: 게임에서의 각 물체들의 위치, 속도, 벽과의 거리 등을 의미한다.\n", 37 | "* **Action**: 게임을 플레이하는 주인공의 행동을 의미한다. 여기서는 4가지 방향에 대한 움직임이 이에 해당한다.\n", 38 | "* **Reward**: 게임을 플레이하면서 받는 score. 여기서는 초록색을 먹으면 +1 , 빨간색을 먹으면 -1을 점수로 받는다.\n", 39 | "* **Value**: 해당 action 또는 state가 미래에 얼마나 큰 reward를 가져올 지에 대한 기댓값.\n", 40 | "* **Policy**: 주인공이 현재의 게임 State에서 Reward를 최대한 얻기 위해 Action을 선택하는 전략. 한마디로 [게임을 플레이하는 방법]을 의미한다.\n" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "### Environment Interaction(or Perception-Action Cycle) in Reinforcement Learning\n", 48 | "\n", 49 | "(출처 : Sutton, 1998, Reinforcement Learning: An Introduction)\n", 50 | "\n", 51 | "강화학습 학습이 다른 머신러닝 방법(supervised learning, unsupervised learning)과 다른 가장 큰 특징 중 하나는, 바로 환경과의 실시간 인터렉션을 가정한 학습 모델이라는 점이다. 강화학습은 생물체의 학습과정을 수학적으로 모델링하여 구현된 알고리즘이기 때문에 다른 방법들에 비해 보다 사람 혹은 동물의 학습과 유사하다." 52 | ] 53 | }, 54 | { 55 | "cell_type": "markdown", 56 | "metadata": {}, 57 | "source": [ 58 | "### Grid World\n", 59 | "![](images/gridworld2.png)\n", 60 | "
(출처 : Sutton, 1998, Reinforcement Learning: An Introduction)\n", 61 | "
State : 5x5\n", 62 | "
Action : 4방향이동\n", 63 | "
Reward : A에 도착하면 +10, B에 도착하면 +5, 벽에 부딪히면 -1, 그이외 0\n", 64 | "
Discounted Factor : 0.9" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "### Discounted Factor\n", 72 | "![](images/discounted_factor.png)\n" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "### Q table\n", 80 | "![](images/q_table.png)\n", 81 | "
(출처 : http://gruposagama.com/pages/q-learning.html)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "### Q Learning Algorithm\n", 89 | "![](images/qlearning.png)\n", 90 | "\n", 91 | "\n", 92 | "\n", 93 | "
(출처 : http://www.randomant.net/wp-content/uploads/2016/05/q_learning3.jpg)\n", 94 | "
(출처 : http://people.revoledu.com/kardi/tutorial/ReinforcementLearning/Q-Learning-Example.htm)\n" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "### Deep Reinforcement Learning : Q function -> Deep Learninig\n", 102 | "![](images/deeprl.png)" 103 | ] 104 | }, 105 | { 106 | "cell_type": "markdown", 107 | "metadata": {}, 108 | "source": [ 109 | "### Example of Deep Reinforcement Learning\n", 110 | "
\n", 111 | "아래의 예제 코드를 실행시키기 위해서는 리눅스 shell에서 다음의 명령어를 실행해 필요한 python package를 설치해야한다.\n", 112 | "####### pip install future euclid redis" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 6, 118 | "metadata": { 119 | "collapsed": false 120 | }, 121 | "outputs": [], 122 | "source": [ 123 | "%matplotlib inline\n", 124 | "import tensorflow as tf\n", 125 | "tf.reset_default_graph()\n", 126 | "\n", 127 | "from tf_rl.controller import DiscreteDeepQ, HumanController\n", 128 | "from tf_rl.simulation import KarpathyGame\n", 129 | "from tf_rl import simulate\n", 130 | "from tf_rl.models import MLP\n", 131 | "\n", 132 | "from __future__ import print_function" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "#### Environment Settings\n", 140 | "\n", 141 | "이제 우리가 원하는 게임 환경을 설정하고, 적절한 reward와 object의 개수 및 observation 을 조절한다" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 7, 147 | "metadata": { 148 | "collapsed": false 149 | }, 150 | "outputs": [], 151 | "source": [ 152 | "current_settings = {\n", 153 | " 'objects': [\n", 154 | " 'friend',\n", 155 | " 'enemy',\n", 156 | " ],\n", 157 | " 'colors': {\n", 158 | " 'hero': 'yellow',\n", 159 | " 'friend': 'green',\n", 160 | " 'enemy': 'red',\n", 161 | " },\n", 162 | " 'object_reward': {\n", 163 | " 'friend': 1,\n", 164 | " 'enemy': -1,\n", 165 | " },\n", 166 | " \"num_objects\": {\n", 167 | " 'friend' : 25,\n", 168 | " 'enemy' : 25,\n", 169 | " },\n", 170 | " \n", 171 | " 'hero_bounces_off_walls': False,\n", 172 | " 'world_size': (700,500),\n", 173 | " 'hero_initial_position': [400, 300],\n", 174 | " 'hero_initial_speed': [0, 0],\n", 175 | " \"maximum_speed\": [50, 50],\n", 176 | " \"object_radius\": 10.0,\n", 177 | " \"num_observation_lines\" : 32, # the number of antennas\n", 178 | " \"observation_line_length\": 240., # the length of antennas\n", 179 | " \"tolerable_distance_to_wall\": 50, \n", 180 | " \"wall_distance_penalty\": -0.0, # if the hero is close to wall, that receives penalty\n", 181 | " \"delta_v\": 50 # speed value\n", 182 | "}\n", 183 | "\n", 184 | "# create the game simulator\n", 185 | "g = KarpathyGame(current_settings)\n" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "#### Deep Learning Architecture\n", 193 | "\n", 194 | "이제 Q function을 근사하기 위한 딥러닝 모델을 만들어보자. 이번 예제에서는 위에서 보았던 4층짜리 MLP를 사용한다." 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 8, 200 | "metadata": { 201 | "collapsed": false 202 | }, 203 | "outputs": [], 204 | "source": [ 205 | "session = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth =True)))\n", 206 | "\n", 207 | " \n", 208 | "\n", 209 | "journalist = tf.summary.FileWriter(\"/tmp/drl\", session.graph)\n", 210 | "\n", 211 | "# Brain maps from observation to Q values for different actions.\n", 212 | "# Here it is a done using a multi layer perceptron with 2 hidden layers\n", 213 | "brain = MLP([g.observation_size,], [200, 200, g.num_actions], [tf.tanh, tf.tanh, tf.identity])" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "metadata": {}, 219 | "source": [ 220 | "#### Make an Agent\n", 221 | "\n", 222 | "이제 Discrete Deep Q learning 알고리즘이 이 게임을 플레이하면서 학습을 하도록 agent로 설정을 한다." 223 | ] 224 | }, 225 | { 226 | "cell_type": "code", 227 | "execution_count": 9, 228 | "metadata": { 229 | "collapsed": false 230 | }, 231 | "outputs": [ 232 | { 233 | "name": "stdout", 234 | "output_type": "stream", 235 | "text": [ 236 | "INFO:tensorflow:Summary name MLP/input_layer/W_0:0 is illegal; using MLP/input_layer/W_0_0 instead.\n" 237 | ] 238 | }, 239 | { 240 | "name": "stderr", 241 | "output_type": "stream", 242 | "text": [ 243 | "INFO:tensorflow:Summary name MLP/input_layer/W_0:0 is illegal; using MLP/input_layer/W_0_0 instead.\n" 244 | ] 245 | }, 246 | { 247 | "name": "stdout", 248 | "output_type": "stream", 249 | "text": [ 250 | "INFO:tensorflow:Summary name MLP/input_layer/W_0:0/gradients is illegal; using MLP/input_layer/W_0_0/gradients instead.\n" 251 | ] 252 | }, 253 | { 254 | "name": "stderr", 255 | "output_type": "stream", 256 | "text": [ 257 | "INFO:tensorflow:Summary name MLP/input_layer/W_0:0/gradients is illegal; using MLP/input_layer/W_0_0/gradients instead.\n" 258 | ] 259 | }, 260 | { 261 | "name": "stdout", 262 | "output_type": "stream", 263 | "text": [ 264 | "INFO:tensorflow:Summary name MLP/input_layer/b:0 is illegal; using MLP/input_layer/b_0 instead.\n" 265 | ] 266 | }, 267 | { 268 | "name": "stderr", 269 | "output_type": "stream", 270 | "text": [ 271 | "INFO:tensorflow:Summary name MLP/input_layer/b:0 is illegal; using MLP/input_layer/b_0 instead.\n" 272 | ] 273 | }, 274 | { 275 | "name": "stdout", 276 | "output_type": "stream", 277 | "text": [ 278 | "INFO:tensorflow:Summary name MLP/input_layer/b:0/gradients is illegal; using MLP/input_layer/b_0/gradients instead.\n" 279 | ] 280 | }, 281 | { 282 | "name": "stderr", 283 | "output_type": "stream", 284 | "text": [ 285 | "INFO:tensorflow:Summary name MLP/input_layer/b:0/gradients is illegal; using MLP/input_layer/b_0/gradients instead.\n" 286 | ] 287 | }, 288 | { 289 | "name": "stdout", 290 | "output_type": "stream", 291 | "text": [ 292 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/W_0:0 is illegal; using MLP/hidden_layer_0/W_0_0 instead.\n" 293 | ] 294 | }, 295 | { 296 | "name": "stderr", 297 | "output_type": "stream", 298 | "text": [ 299 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/W_0:0 is illegal; using MLP/hidden_layer_0/W_0_0 instead.\n" 300 | ] 301 | }, 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/W_0:0/gradients is illegal; using MLP/hidden_layer_0/W_0_0/gradients instead.\n" 307 | ] 308 | }, 309 | { 310 | "name": "stderr", 311 | "output_type": "stream", 312 | "text": [ 313 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/W_0:0/gradients is illegal; using MLP/hidden_layer_0/W_0_0/gradients instead.\n" 314 | ] 315 | }, 316 | { 317 | "name": "stdout", 318 | "output_type": "stream", 319 | "text": [ 320 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/b:0 is illegal; using MLP/hidden_layer_0/b_0 instead.\n" 321 | ] 322 | }, 323 | { 324 | "name": "stderr", 325 | "output_type": "stream", 326 | "text": [ 327 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/b:0 is illegal; using MLP/hidden_layer_0/b_0 instead.\n" 328 | ] 329 | }, 330 | { 331 | "name": "stdout", 332 | "output_type": "stream", 333 | "text": [ 334 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/b:0/gradients is illegal; using MLP/hidden_layer_0/b_0/gradients instead.\n" 335 | ] 336 | }, 337 | { 338 | "name": "stderr", 339 | "output_type": "stream", 340 | "text": [ 341 | "INFO:tensorflow:Summary name MLP/hidden_layer_0/b:0/gradients is illegal; using MLP/hidden_layer_0/b_0/gradients instead.\n" 342 | ] 343 | }, 344 | { 345 | "name": "stdout", 346 | "output_type": "stream", 347 | "text": [ 348 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/W_0:0 is illegal; using MLP/hidden_layer_1/W_0_0 instead.\n" 349 | ] 350 | }, 351 | { 352 | "name": "stderr", 353 | "output_type": "stream", 354 | "text": [ 355 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/W_0:0 is illegal; using MLP/hidden_layer_1/W_0_0 instead.\n" 356 | ] 357 | }, 358 | { 359 | "name": "stdout", 360 | "output_type": "stream", 361 | "text": [ 362 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/W_0:0/gradients is illegal; using MLP/hidden_layer_1/W_0_0/gradients instead.\n" 363 | ] 364 | }, 365 | { 366 | "name": "stderr", 367 | "output_type": "stream", 368 | "text": [ 369 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/W_0:0/gradients is illegal; using MLP/hidden_layer_1/W_0_0/gradients instead.\n" 370 | ] 371 | }, 372 | { 373 | "name": "stdout", 374 | "output_type": "stream", 375 | "text": [ 376 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/b:0 is illegal; using MLP/hidden_layer_1/b_0 instead.\n" 377 | ] 378 | }, 379 | { 380 | "name": "stderr", 381 | "output_type": "stream", 382 | "text": [ 383 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/b:0 is illegal; using MLP/hidden_layer_1/b_0 instead.\n" 384 | ] 385 | }, 386 | { 387 | "name": "stdout", 388 | "output_type": "stream", 389 | "text": [ 390 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/b:0/gradients is illegal; using MLP/hidden_layer_1/b_0/gradients instead.\n" 391 | ] 392 | }, 393 | { 394 | "name": "stderr", 395 | "output_type": "stream", 396 | "text": [ 397 | "INFO:tensorflow:Summary name MLP/hidden_layer_1/b:0/gradients is illegal; using MLP/hidden_layer_1/b_0/gradients instead.\n" 398 | ] 399 | }, 400 | { 401 | "name": "stdout", 402 | "output_type": "stream", 403 | "text": [ 404 | "INFO:tensorflow:Summary name input_layer_copy/b:0 is illegal; using input_layer_copy/b_0 instead.\n" 405 | ] 406 | }, 407 | { 408 | "name": "stderr", 409 | "output_type": "stream", 410 | "text": [ 411 | "INFO:tensorflow:Summary name input_layer_copy/b:0 is illegal; using input_layer_copy/b_0 instead.\n" 412 | ] 413 | }, 414 | { 415 | "name": "stdout", 416 | "output_type": "stream", 417 | "text": [ 418 | "INFO:tensorflow:Summary name input_layer_copy/W_0:0 is illegal; using input_layer_copy/W_0_0 instead.\n" 419 | ] 420 | }, 421 | { 422 | "name": "stderr", 423 | "output_type": "stream", 424 | "text": [ 425 | "INFO:tensorflow:Summary name input_layer_copy/W_0:0 is illegal; using input_layer_copy/W_0_0 instead.\n" 426 | ] 427 | }, 428 | { 429 | "name": "stdout", 430 | "output_type": "stream", 431 | "text": [ 432 | "INFO:tensorflow:Summary name hidden_layer_0_copy/b:0 is illegal; using hidden_layer_0_copy/b_0 instead.\n" 433 | ] 434 | }, 435 | { 436 | "name": "stderr", 437 | "output_type": "stream", 438 | "text": [ 439 | "INFO:tensorflow:Summary name hidden_layer_0_copy/b:0 is illegal; using hidden_layer_0_copy/b_0 instead.\n" 440 | ] 441 | }, 442 | { 443 | "name": "stdout", 444 | "output_type": "stream", 445 | "text": [ 446 | "INFO:tensorflow:Summary name hidden_layer_0_copy/W_0:0 is illegal; using hidden_layer_0_copy/W_0_0 instead.\n" 447 | ] 448 | }, 449 | { 450 | "name": "stderr", 451 | "output_type": "stream", 452 | "text": [ 453 | "INFO:tensorflow:Summary name hidden_layer_0_copy/W_0:0 is illegal; using hidden_layer_0_copy/W_0_0 instead.\n" 454 | ] 455 | }, 456 | { 457 | "name": "stdout", 458 | "output_type": "stream", 459 | "text": [ 460 | "INFO:tensorflow:Summary name hidden_layer_1_copy/b:0 is illegal; using hidden_layer_1_copy/b_0 instead.\n" 461 | ] 462 | }, 463 | { 464 | "name": "stderr", 465 | "output_type": "stream", 466 | "text": [ 467 | "INFO:tensorflow:Summary name hidden_layer_1_copy/b:0 is illegal; using hidden_layer_1_copy/b_0 instead.\n" 468 | ] 469 | }, 470 | { 471 | "name": "stdout", 472 | "output_type": "stream", 473 | "text": [ 474 | "INFO:tensorflow:Summary name hidden_layer_1_copy/W_0:0 is illegal; using hidden_layer_1_copy/W_0_0 instead.\n" 475 | ] 476 | }, 477 | { 478 | "name": "stderr", 479 | "output_type": "stream", 480 | "text": [ 481 | "INFO:tensorflow:Summary name hidden_layer_1_copy/W_0:0 is illegal; using hidden_layer_1_copy/W_0_0 instead.\n" 482 | ] 483 | } 484 | ], 485 | "source": [ 486 | "# The optimizer to use. Here we use RMSProp as recommended by the publication\n", 487 | "optimizer = tf.train.RMSPropOptimizer(learning_rate= 0.001, decay=0.9)\n", 488 | "\n", 489 | "# DiscreteDeepQ object\n", 490 | "current_controller = DiscreteDeepQ(g.observation_size, g.num_actions, brain, optimizer, session,\n", 491 | " discount_rate=0.99, exploration_period=5000, max_experience=10000, \n", 492 | " store_every_nth=4, train_every_nth=4,\n", 493 | " summary_writer=journalist)\n", 494 | "\n", 495 | "session.run(tf.global_variables_initializer())\n", 496 | "session.run(current_controller.target_network_update)\n", 497 | "#journalist.add_graph(session.graph_def)" 498 | ] 499 | }, 500 | { 501 | "cell_type": "markdown", 502 | "metadata": {}, 503 | "source": [ 504 | "#### Play the Game\n", 505 | "\n", 506 | "실제로 게임을 플레이하면서 강화학습이 일어나는 과정을 지켜보자." 507 | ] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "execution_count": 10, 512 | "metadata": { 513 | "collapsed": false 514 | }, 515 | "outputs": [ 516 | { 517 | "data": { 518 | "text/html": [ 519 | "\n", 520 | "\n", 521 | "\n", 522 | "\n", 523 | " \n", 526 | "\n", 527 | " \n", 530 | "\n", 531 | " \n", 532 | "\n", 533 | " \n", 534 | "\n", 535 | " \n", 536 | "\n", 537 | " \n", 538 | "\n", 539 | " \n", 540 | "\n", 541 | " \n", 542 | "\n", 543 | " \n", 544 | "\n", 545 | " \n", 546 | "\n", 547 | " \n", 548 | "\n", 549 | " \n", 550 | "\n", 551 | " \n", 552 | "\n", 553 | " \n", 554 | "\n", 555 | " \n", 556 | "\n", 557 | " \n", 558 | "\n", 559 | " \n", 560 | "\n", 561 | " \n", 562 | "\n", 563 | " \n", 564 | "\n", 565 | " \n", 566 | "\n", 567 | " \n", 568 | "\n", 569 | " \n", 570 | "\n", 571 | " \n", 572 | "\n", 573 | " \n", 574 | "\n", 575 | " \n", 576 | "\n", 577 | " \n", 578 | "\n", 579 | " \n", 580 | "\n", 581 | " \n", 582 | "\n", 583 | " \n", 584 | "\n", 585 | " \n", 586 | "\n", 587 | " \n", 588 | "\n", 589 | " \n", 590 | "\n", 591 | " \n", 592 | "\n", 593 | " \n", 594 | "\n", 595 | " \n", 598 | "\n", 599 | " \n", 602 | "\n", 603 | " \n", 606 | "\n", 607 | " \n", 610 | "\n", 611 | " \n", 614 | "\n", 615 | " \n", 618 | "\n", 619 | " \n", 622 | "\n", 623 | " \n", 626 | "\n", 627 | " \n", 630 | "\n", 631 | " \n", 634 | "\n", 635 | " \n", 638 | "\n", 639 | " \n", 642 | "\n", 643 | " \n", 646 | "\n", 647 | " \n", 650 | "\n", 651 | " \n", 654 | "\n", 655 | " \n", 658 | "\n", 659 | " \n", 662 | "\n", 663 | " \n", 666 | "\n", 667 | " \n", 670 | "\n", 671 | " \n", 674 | "\n", 675 | " \n", 678 | "\n", 679 | " \n", 682 | "\n", 683 | " \n", 686 | "\n", 687 | " \n", 690 | "\n", 691 | " \n", 694 | "\n", 695 | " \n", 698 | "\n", 699 | " \n", 702 | "\n", 703 | " \n", 706 | "\n", 707 | " \n", 710 | "\n", 711 | " \n", 714 | "\n", 715 | " \n", 718 | "\n", 719 | " \n", 722 | "\n", 723 | " \n", 726 | "\n", 727 | " \n", 730 | "\n", 731 | " \n", 734 | "\n", 735 | " \n", 738 | "\n", 739 | " \n", 742 | "\n", 743 | " \n", 746 | "\n", 747 | " \n", 750 | "\n", 751 | " \n", 754 | "\n", 755 | " \n", 758 | "\n", 759 | " \n", 762 | "\n", 763 | " \n", 766 | "\n", 767 | " \n", 770 | "\n", 771 | " \n", 774 | "\n", 775 | " \n", 778 | "\n", 779 | " \n", 782 | "\n", 783 | " \n", 786 | "\n", 787 | " \n", 790 | "\n", 791 | " \n", 794 | "\n", 795 | " \n", 798 | "\n", 799 | " \n", 800 | "\n", 801 | " fps = 113.5\n", 802 | "\n", 803 | " \n", 804 | "\n", 805 | " \n", 806 | "\n", 807 | " nearest wall = 99.1\n", 808 | "\n", 809 | " \n", 810 | "\n", 811 | " \n", 812 | "\n", 813 | " reward = 0.0\n", 814 | "\n", 815 | " \n", 816 | "\n", 817 | " \n", 818 | "\n", 819 | " objects eaten => enemy: 1, friend: 3\n", 820 | "\n", 821 | " \n", 822 | "\n", 823 | " \n", 824 | "\n" 825 | ], 826 | "text/plain": [ 827 | "" 828 | ] 829 | }, 830 | "metadata": {}, 831 | "output_type": "display_data" 832 | }, 833 | { 834 | "name": "stdout", 835 | "output_type": "stream", 836 | "text": [ 837 | "Interrupted\n" 838 | ] 839 | } 840 | ], 841 | "source": [ 842 | "FPS = 30\n", 843 | "ACTION_EVERY = 3\n", 844 | " \n", 845 | "fast_mode = True\n", 846 | "if fast_mode:\n", 847 | " WAIT, VISUALIZE_EVERY = False, 20\n", 848 | "else:\n", 849 | " WAIT, VISUALIZE_EVERY = True, 1\n", 850 | "\n", 851 | " \n", 852 | "try:\n", 853 | " with tf.device(\"/gpu:0\"):\n", 854 | " simulate(simulation=g,\n", 855 | " controller=current_controller,\n", 856 | " fps=FPS,\n", 857 | " visualize_every=VISUALIZE_EVERY,\n", 858 | " action_every=ACTION_EVERY,\n", 859 | " wait=WAIT,\n", 860 | " disable_training=False,\n", 861 | " simulation_resolution=0.001,\n", 862 | " save_path=None)\n", 863 | "except KeyboardInterrupt:\n", 864 | " print(\"Interrupted\")\n", 865 | " \n", 866 | "\n", 867 | "#session.close()" 868 | ] 869 | }, 870 | { 871 | "cell_type": "markdown", 872 | "metadata": {}, 873 | "source": [ 874 | "#### Applications\n", 875 | "* Parameter 들을 바꾸어 enemy와 friend의 개수 차이가 최대한 많이 나도록 agent를 학습시켜본다.\n", 876 | "* Boss object를 추가해본다.\n", 877 | "* Deep RL에서의 tensorboard를 열어서 visualize를 해본다." 878 | ] 879 | }, 880 | { 881 | "cell_type": "markdown", 882 | "metadata": {}, 883 | "source": [ 884 | "## * Materials\n", 885 | "\n", 886 | "
https://github.com/aikorea/awesome-rl\n", 887 | "
https://openai.com/blog/universe/\n", 888 | "
http://alpha.openai.com/miniwob/index.html\n" 889 | ] 890 | } 891 | ], 892 | "metadata": { 893 | "kernelspec": { 894 | "display_name": "Python 2", 895 | "language": "python", 896 | "name": "python2" 897 | }, 898 | "language_info": { 899 | "codemirror_mode": { 900 | "name": "ipython", 901 | "version": 2 902 | }, 903 | "file_extension": ".py", 904 | "mimetype": "text/x-python", 905 | "name": "python", 906 | "nbconvert_exporter": "python", 907 | "pygments_lexer": "ipython2", 908 | "version": "2.7.6" 909 | } 910 | }, 911 | "nbformat": 4, 912 | "nbformat_minor": 0 913 | } 914 | -------------------------------------------------------------------------------- /Tensorflow_RL/TensorFlow_Tutorial_RNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "#TensorFlow Tutorial with RNN\n", 8 | "\n", 9 | "본 문서는 TensorFlow 를 사용하여 기초적인 RNN을 구현하고 실험하기 위한 실습 자료이다.\n", 10 | "\n", 11 | "The code and comments are written by Dong-Hyun Kwak (imcomking@gmail.com)\n", 12 | "\n", 13 | "\"Creative
This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "### Recurrent Neural Networks\n", 21 | "Recurrent Neural Networks, 이하 RNN는 다음과 같은 구조를 가진 모델이다. RNN은 히든레이어에서 자기자신을 향하는 weight를 이용해 데이터간의 시간관계를 학습할 수 있다. 이러한 문제들을 시계열 학습이라고 부르며, 기존에 널리 쓰이던 Hidden Markov Model을 뉴럴넷을 이용해 구현했다고 볼 수 있다.\n", 22 | "\n", 23 | "\n", 24 | "![](images/rnn.png)\n" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "![](images/unroll_rnn.png)\n", 32 | "\n", 33 | "위의 구조는 1개의 Recurrent weight를 가진 hidden node이다. 이러한 hidden node들이 여러개를 모여 1개의 RNN layer를 형성하고, 이것이 다시 deep 하게 쌓이는 모델 또한 가능하다.(그러나 RNN은 deep 하게 쌓을 경우 학습이 쉽지 않다.)\n", 34 | "\n", 35 | "RNN의 경우 MLP나 CNN에 비해서 구현이 다소 복잡하다. 따라서 RNN은 TensorFlow에서 제공하는 추상화된 API 들을 이용해서 구현하는 것이 일반적이다." 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "metadata": {}, 41 | "source": [ 42 | "![](images/longterm.png)\n", 43 | "(출처: http://colah.github.io/posts/2015-08-Understanding-LSTMs/)\n", 44 | "\n", 45 | "- tf.contrib.rnn.BasicRNNCell : 기본 RNN cell이외에 매우 다양한 cell을 사용할 수 있다.\n", 46 | "https://www.tensorflow.org/api_guides/python/contrib.rnn#Core_RNN_Cells_for_use_with_TensorFlow_s_core_RNN_methods\n", 47 | "\n", 48 | "- tf.contrib.seq2seq.sequence_loss : N개의 sequence의 cross-entropy를 weighted sum 한 Loss. weight는 0or1의 masking용도로써 쓰인다.\n", 49 | "https://www.tensorflow.org/api_docs/python/tf/contrib/seq2seq/sequence_loss\n", 50 | "\n", 51 | "### Simple toy seq2seq with RNN / LSTM\n", 52 | "\n", 53 | "(이미지 출처: http://smile2x.tistory.com/archive/201605)\n", 54 | "(소스코드 출처: https://github.com/hunkim/DeepLearningZeroToAll)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 5, 60 | "metadata": { 61 | "collapsed": false, 62 | "scrolled": false 63 | }, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "outputs: Tensor(\"rnn/transpose:0\", shape=(1, 6, 10), dtype=float32)\n", 70 | "outputs_flat: Tensor(\"Reshape:0\", shape=(6, 10), dtype=float32)\n", 71 | "y_prediction: Tensor(\"Softmax:0\", shape=(6, 5), dtype=float32)\n", 72 | "y_prediction: Tensor(\"Reshape_1:0\", shape=(1, 6, 5), dtype=float32)\n", 73 | "Tensor(\"Placeholder_1:0\", shape=(?, 6, 5), dtype=float32)\n", 74 | "0 loss: 13.3657 prediction: [[1 0 1 1 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 75 | "Prediction str: ihiilo\n", 76 | "100 loss: 0.000930944 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 77 | "Prediction str: ihello\n", 78 | "200 loss: 0.000502015 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 79 | "Prediction str: ihello\n", 80 | "300 loss: 0.000329147 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 81 | "Prediction str: ihello\n", 82 | "400 loss: 0.000234132 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 83 | "Prediction str: ihello\n", 84 | "500 loss: 0.000175122 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 85 | "Prediction str: ihello\n", 86 | "600 loss: 0.00013602 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 87 | "Prediction str: ihello\n", 88 | "700 loss: 0.000108482 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 89 | "Prediction str: ihello\n", 90 | "800 loss: 8.88118e-05 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 91 | "Prediction str: ihello\n", 92 | "900 loss: 7.41488e-05 prediction: [[1 0 2 3 3 4]] true Y: [[1, 0, 2, 3, 3, 4]]\n", 93 | "Prediction str: ihello\n" 94 | ] 95 | } 96 | ], 97 | "source": [ 98 | "import tensorflow as tf\n", 99 | "import numpy as np\n", 100 | "tf.reset_default_graph()\n", 101 | "tf.set_random_seed(777) # reproducibility\n", 102 | "\n", 103 | "idx2char = ['h', 'i', 'e', 'l', 'o']\n", 104 | "# Teach hello: hihell -> ihello\n", 105 | "x_data = [[0, 1, 0, 2, 3, 3]] # hihell\n", 106 | "x_one_hot = [[[1, 0, 0, 0, 0], # h 0 \n", 107 | " [0, 1, 0, 0, 0], # i 1\n", 108 | " [1, 0, 0, 0, 0], # h 0\n", 109 | " [0, 0, 1, 0, 0], # e 2\n", 110 | " [0, 0, 0, 1, 0], # l 3\n", 111 | " [0, 0, 0, 1, 0]]] # l 3\n", 112 | "\n", 113 | "y_data = [[1, 0, 2, 3, 3, 4]] # ihello\n", 114 | "y_one_hot = [[[0, 1, 0, 0, 0], # i 1\n", 115 | " [1, 0, 0, 0, 0], # h 0\n", 116 | " [0, 0, 1, 0, 0], # e 2\n", 117 | " [0, 0, 0, 1, 0], # l 3\n", 118 | " [0, 0, 0, 1, 0], # l 3\n", 119 | " [0, 0, 0, 0, 1]]] # o 4\n", 120 | "\n", 121 | "input_dim = 5 # one-hot size\n", 122 | "hidden_size = 10 # output from the LSTM. 5 to directly predict one-hot\n", 123 | "output_dim = 5\n", 124 | "batch_size = 1 # one sentence\n", 125 | "sequence_length = 6 # |ihello| == 6\n", 126 | "\n", 127 | "X = tf.placeholder(tf.float32, [None, sequence_length, input_dim]) # X one-hot\n", 128 | "Y = tf.placeholder(tf.float32, [None, sequence_length, output_dim]) # Y one-hot\n", 129 | "\n", 130 | "cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n", 131 | "#cell = tf.contrib.rnn.GRUCell(num_units=hidden_size)\n", 132 | "#cell = tf.contrib.rnn.BasicLSTMCell(num_units=hidden_size)\n", 133 | "initial_state = cell.zero_state(batch_size, tf.float32)\n", 134 | "outputs, _states = tf.nn.dynamic_rnn(cell, X, initial_state=initial_state, dtype=tf.float32)\n", 135 | "print \"outputs: \",outputs\n", 136 | "\n", 137 | "outputs_flat = tf.reshape(outputs, [-1 , hidden_size])\n", 138 | "print \"outputs_flat: \",outputs_flat\n", 139 | "w = tf.Variable(tf.random_normal([hidden_size, output_dim]))\n", 140 | "b = tf.Variable(tf.random_normal([output_dim]))\n", 141 | "y_prediction = tf.nn.softmax(tf.matmul(outputs_flat, w) + b)\n", 142 | "print \"y_prediction:\", y_prediction\n", 143 | "y_prediction = tf.reshape(y_prediction, [-1 , sequence_length, output_dim])\n", 144 | "print \"y_prediction:\", y_prediction\n", 145 | "print Y\n", 146 | "\n", 147 | "y_label_pred = tf.argmax(y_prediction, 2)\n", 148 | " \n", 149 | "loss = -tf.reduce_sum(Y*tf.log(y_prediction), name = 'cross_entropy')\n", 150 | "\n", 151 | "train = tf.train.AdamOptimizer(learning_rate=0.1).minimize(loss)\n", 152 | "\n", 153 | "with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:\n", 154 | " sess.run(tf.global_variables_initializer())\n", 155 | " for i in range(1000):\n", 156 | " l, _ = sess.run([loss, train], feed_dict={X: x_one_hot, Y: y_one_hot})\n", 157 | " if i %100 ==0:\n", 158 | " _y_prediction, _y_label_pred = sess.run([y_prediction, y_label_pred], feed_dict={X: x_one_hot})\n", 159 | " print i, \"loss:\", l, \"prediction: \", _y_label_pred, \"true Y: \", y_data\n", 160 | "\n", 161 | " # print char using dic\n", 162 | " result_str = [idx2char[c] for c in np.squeeze(_y_label_pred)]\n", 163 | " print \"Prediction str: \", ''.join(result_str)" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "### Web Navigation Sequence Classification\n", 171 | "앞서 살펴본 단순한 toy 문제에서 한 층 나아가, 52739개의 웹페이지를 탐색한 데이터를 학습한다. 빠른 학습을 위해 전체 데이터 중에서 아주 일부(100만개중 1000개만 가져옴)만을 가져왔기 때문에 test accuracy는 극히 낮으므로, train accuracy이 높아지는 것만 확인하도록 하자.\n", 172 | "\n", 173 | "앞서 살펴본 예제는 seq2seq 이라는 형태의 RNN task이고, 본 예제는 전체 sequence를 보고서 어떤 class인지 예측하는 Sequence Classification 문제에 해당한다. 따라서 loss를 계산할 때 쓰이는 outputs가 어떻게 바뀌었는 지를 이해하는 것이 본 예제의 핵심이다.(마지막 step의 outputs만 가져옴)\n", 174 | "\n", 175 | "- tf.one_hot : 자동으로 one_hot encoding을 해주는 함수\n", 176 | "
https://www.tensorflow.org/api_docs/python/tf/one_hot\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 4, 182 | "metadata": { 183 | "collapsed": false, 184 | "scrolled": false 185 | }, 186 | "outputs": [ 187 | { 188 | "name": "stdout", 189 | "output_type": "stream", 190 | "text": [ 191 | "0 : 0.000217181 0.0\n", 192 | "Test accuracy: 0.000\n", 193 | "10 : 0.000216318 0.0\n", 194 | "20 : 0.000215133 0.0\n", 195 | "30 : 0.000213787 0.0\n", 196 | "40 : 0.000212321 0.0\n", 197 | "50 : 0.000210751 0.0\n", 198 | "Test accuracy: 0.000\n", 199 | "60 : 0.000209084 0.0\n", 200 | "70 : 0.000207321 0.0\n", 201 | "80 : 0.000205456 0.0\n", 202 | "90 : 0.000203478 0.0\n", 203 | "100 : 0.000201371 0.0\n", 204 | "Test accuracy: 0.000\n", 205 | "110 : 0.000199111 0.0\n", 206 | "120 : 0.000196676 0.00125\n", 207 | "130 : 0.000194038 0.00125\n", 208 | "140 : 0.000191169 0.0025\n", 209 | "150 : 0.000188032 0.0025\n", 210 | "Test accuracy: 0.000\n", 211 | "160 : 0.000184587 0.00625\n", 212 | "170 : 0.000180781 0.0175\n", 213 | "180 : 0.000176544 0.03125\n", 214 | "190 : 0.000171788 0.045\n", 215 | "200 : 0.000166384 0.0625\n", 216 | "Test accuracy: 0.000\n", 217 | "210 : 0.000160157 0.10875\n", 218 | "220 : 0.000152853 0.16\n", 219 | "230 : 0.000144084 0.21\n", 220 | "240 : 0.000133197 0.2875\n", 221 | "250 : 0.000119178 0.38375\n", 222 | "Test accuracy: 0.010\n", 223 | "260 : 0.000100746 0.50375\n", 224 | "270 : 7.69867e-05 0.6375\n", 225 | "280 : 4.94459e-05 0.7575\n", 226 | "290 : 2.52457e-05 0.86375\n", 227 | "300 : 1.06855e-05 0.95125\n", 228 | "Test accuracy: 0.025\n", 229 | "310 : 4.18085e-06 0.98375\n", 230 | "320 : 1.78268e-06 0.99875\n", 231 | "330 : 9.17186e-07 0.99875\n", 232 | "340 : 5.90027e-07 0.99875\n", 233 | "350 : 4.32424e-07 0.99875\n", 234 | "Test accuracy: 0.025\n", 235 | "360 : 3.37951e-07 1.0\n", 236 | "370 : 2.77822e-07 1.0\n", 237 | "380 : 2.38188e-07 1.0\n", 238 | "390 : 2.10426e-07 1.0\n", 239 | "400 : 1.89701e-07 1.0\n", 240 | "Test accuracy: 0.025\n", 241 | "410 : 1.7339e-07 1.0\n", 242 | "420 : 1.60043e-07 1.0\n", 243 | "430 : 1.48814e-07 1.0\n", 244 | "440 : 1.39175e-07 1.0\n", 245 | "450 : 1.30773e-07 1.0\n", 246 | "Test accuracy: 0.025\n", 247 | "460 : 1.23361e-07 1.0\n", 248 | "470 : 1.16762e-07 1.0\n", 249 | "480 : 1.10839e-07 1.0\n", 250 | "490 : 1.05487e-07 1.0\n", 251 | "500 : 1.00623e-07 1.0\n", 252 | "Test accuracy: 0.025\n", 253 | "510 : 9.61809e-08 1.0\n", 254 | "520 : 9.21056e-08 1.0\n", 255 | "530 : 8.83518e-08 1.0\n", 256 | "540 : 8.4882e-08 1.0\n", 257 | "550 : 8.16644e-08 1.0\n", 258 | "Test accuracy: 0.025\n", 259 | "560 : 7.86714e-08 1.0\n", 260 | "570 : 7.58805e-08 1.0\n", 261 | "580 : 7.32709e-08 1.0\n", 262 | "590 : 7.08254e-08 1.0\n", 263 | "600 : 6.8529e-08 1.0\n", 264 | "Test accuracy: 0.025\n", 265 | "610 : 6.63684e-08 1.0\n", 266 | "620 : 6.43314e-08 1.0\n", 267 | "630 : 6.24079e-08 1.0\n", 268 | "640 : 6.05886e-08 1.0\n", 269 | "650 : 5.88652e-08 1.0\n", 270 | "Test accuracy: 0.025\n", 271 | "660 : 5.72305e-08 1.0\n", 272 | "670 : 5.56775e-08 1.0\n", 273 | "680 : 5.42004e-08 1.0\n", 274 | "690 : 5.27937e-08 1.0\n", 275 | "700 : 5.14526e-08 1.0\n", 276 | "Test accuracy: 0.025\n", 277 | "710 : 5.01728e-08 1.0\n", 278 | "720 : 4.895e-08 1.0\n", 279 | "730 : 4.77805e-08 1.0\n", 280 | "740 : 4.66609e-08 1.0\n", 281 | "750 : 4.55882e-08 1.0\n", 282 | "Test accuracy: 0.025\n", 283 | "760 : 4.45596e-08 1.0\n", 284 | "770 : 4.35723e-08 1.0\n", 285 | "780 : 4.26238e-08 1.0\n", 286 | "790 : 4.17122e-08 1.0\n", 287 | "800 : 4.08354e-08 1.0\n", 288 | "Test accuracy: 0.025\n", 289 | "810 : 3.99914e-08 1.0\n", 290 | "820 : 3.91781e-08 1.0\n", 291 | "830 : 3.83944e-08 1.0\n", 292 | "840 : 3.76386e-08 1.0\n", 293 | "850 : 3.69089e-08 1.0\n", 294 | "Test accuracy: 0.025\n", 295 | "860 : 3.62044e-08 1.0\n", 296 | "870 : 3.55237e-08 1.0\n", 297 | "880 : 3.48656e-08 1.0\n", 298 | "890 : 3.42292e-08 1.0\n", 299 | "900 : 3.36133e-08 1.0\n", 300 | "Test accuracy: 0.025\n", 301 | "910 : 3.30171e-08 1.0\n", 302 | "920 : 3.24393e-08 1.0\n", 303 | "930 : 3.18794e-08 1.0\n", 304 | "940 : 3.13364e-08 1.0\n", 305 | "950 : 3.081e-08 1.0\n", 306 | "Test accuracy: 0.025\n", 307 | "960 : 3.02991e-08 1.0\n", 308 | "970 : 2.9803e-08 1.0\n", 309 | "980 : 2.93215e-08 1.0\n", 310 | "990 : 2.88535e-08 1.0\n" 311 | ] 312 | } 313 | ], 314 | "source": [ 315 | "import numpy as np\n", 316 | "import tensorflow as tf\n", 317 | "from sklearn.model_selection import train_test_split\n", 318 | "tf.reset_default_graph()\n", 319 | "\n", 320 | "train_x = np.loadtxt('1000_index_x.txt')\n", 321 | "train_y = np.loadtxt('1000_index_y.txt')\n", 322 | "\n", 323 | "nclasses = 52739 #len(items)\n", 324 | "nsteps = 4\n", 325 | "\n", 326 | "dimhidden = 100\n", 327 | "dimoutput = nclasses\n", 328 | "\n", 329 | "x = tf.placeholder(\"int32\", [None, nsteps])\n", 330 | "y = tf.placeholder(\"int32\", [None])\n", 331 | "x_one_hot = tf.one_hot(x, nclasses, on_value=1.0, off_value=0.0 , dtype='float')\n", 332 | "y_one_hot = tf.one_hot(y, nclasses, on_value=1.0, off_value=0.0, dtype='float')\n", 333 | "\n", 334 | "cell = tf.contrib.rnn.BasicLSTMCell(dimhidden)\n", 335 | "outputs, states = tf.nn.dynamic_rnn(cell, x_one_hot, dtype=tf.float32)\n", 336 | "last_output = outputs[:, -1, :]\n", 337 | "\n", 338 | "w = tf.Variable(tf.random_normal([dimhidden, dimoutput]))\n", 339 | "b = tf.Variable(tf.random_normal([dimoutput]))\n", 340 | "prediction = tf.nn.softmax(tf.matmul(last_output, w) + b)\n", 341 | "\n", 342 | "cost = -tf.reduce_mean( y_one_hot*tf.log(prediction+0.00000001))\n", 343 | "optm = tf.train.AdamOptimizer().minimize(cost) \n", 344 | "accr = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(prediction,1), tf.argmax(y_one_hot,1)), tf.float32))\n", 345 | "\n", 346 | "sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True)))\n", 347 | "sess.run(tf.global_variables_initializer())\n", 348 | "\n", 349 | "epochs = 1000\n", 350 | "x_train, x_test, y_train, y_test = train_test_split(train_x, train_y, test_size=0.2, random_state=0)\n", 351 | "\n", 352 | "for i in range(epochs):\n", 353 | " _, _cost, _accr = sess.run([optm, cost, accr], feed_dict={x: x_train, y: y_train})\n", 354 | " if i%10 == 0 :\n", 355 | " print i, \" : \", _cost , _accr\n", 356 | " \n", 357 | " if i%50 == 0 :\n", 358 | " _cost, test_acc = sess.run([cost, accr], feed_dict={x: x_test, y: y_test})\n", 359 | " print \"Test accuracy: %.3f\" % (test_acc)\n" 360 | ] 361 | }, 362 | { 363 | "cell_type": "markdown", 364 | "metadata": {}, 365 | "source": [ 366 | "## * Open-source TensorFlow Implementation\n", 367 | "\n", 368 | "아래 링크는 TensorFlow로 구현되어 공개된 여러 오픈소스 프로젝트들을 모아서 정리해 둔 페이지들이다. 이중 본인의 연구 분야와 관련있는 프로젝트를 clone, 수정하여 사용할 경우 개발시간을 크게 단축할 수 있다.\n", 369 | "\n", 370 | "https://github.com/tensorflow/models : Syntax Net, Magenta, Image2Txt\n", 371 | "
https://github.com/TensorFlowKR/awesome_tensorflow_implementations\n", 372 | "
https://github.com/aikorea/awesome-rl\n", 373 | "\n", 374 | "유명한 오픈소스 몇가지를 살펴보자." 375 | ] 376 | } 377 | ], 378 | "metadata": { 379 | "kernelspec": { 380 | "display_name": "Python 2", 381 | "language": "python", 382 | "name": "python2" 383 | }, 384 | "language_info": { 385 | "codemirror_mode": { 386 | "name": "ipython", 387 | "version": 2 388 | }, 389 | "file_extension": ".py", 390 | "mimetype": "text/x-python", 391 | "name": "python", 392 | "nbconvert_exporter": "python", 393 | "pygments_lexer": "ipython2", 394 | "version": "2.7.6" 395 | } 396 | }, 397 | "nbformat": 4, 398 | "nbformat_minor": 0 399 | } 400 | -------------------------------------------------------------------------------- /Tensorflow_RL/dbpedia_data/dbpedia_csv/classes.txt: -------------------------------------------------------------------------------- 1 | Company 2 | EducationalInstitution 3 | Artist 4 | Athlete 5 | OfficeHolder 6 | MeanOfTransportation 7 | Building 8 | NaturalPlace 9 | Village 10 | Animal 11 | Plant 12 | Album 13 | Film 14 | WrittenWork 15 | -------------------------------------------------------------------------------- /Tensorflow_RL/dbpedia_data/dbpedia_csv/readme.txt: -------------------------------------------------------------------------------- 1 | DBPedia Ontology Classification Dataset 2 | 3 | Version 2, Updated 09/09/2015 4 | 5 | 6 | LICENSE 7 | 8 | The DBpedia datasets are licensed under the terms of the Creative Commons Attribution-ShareAlike License and the GNU Free Documentation License. For more information, please refer to http://dbpedia.org. For a recent overview paper about DBpedia, please refer to: Jens Lehmann, Robert Isele, Max Jakob, Anja Jentzsch, Dimitris Kontokostas, Pablo N. Mendes, Sebastian Hellmann, Mohamed Morsey, Patrick van Kleef, Sören Auer, Christian Bizer: DBpedia – A Large-scale, Multilingual Knowledge Base Extracted from Wikipedia. Semantic Web Journal, Vol. 6 No. 2, pp 167–195, 2015. 9 | 10 | The DBPedia ontology classification dataset is constructed by Xiang Zhang (xiang.zhang@nyu.edu), licensed under the terms of the Creative Commons Attribution-ShareAlike License and the GNU Free Documentation License. It is used as a text classification benchmark in the following paper: Xiang Zhang, Junbo Zhao, Yann LeCun. Character-level Convolutional Networks for Text Classification. Advances in Neural Information Processing Systems 28 (NIPS 2015). 11 | 12 | 13 | DESCRIPTION 14 | 15 | The DBpedia ontology classification dataset is constructed by picking 14 non-overlapping classes from DBpedia 2014. They are listed in classes.txt. From each of thse 14 ontology classes, we randomly choose 40,000 training samples and 5,000 testing samples. Therefore, the total size of the training dataset is 560,000 and testing dataset 70,000. 16 | 17 | The files train.csv and test.csv contain all the training samples as comma-sparated values. There are 3 columns in them, corresponding to class index (1 to 14), title and content. The title and content are escaped using double quotes ("), and any internal double quote is escaped by 2 double quotes (""). There are no new lines in title or content. 18 | -------------------------------------------------------------------------------- /Tensorflow_RL/images/Cnn_layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/Cnn_layer.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/SimpleRNN01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/SimpleRNN01.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/apache2.0_license.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/apache2.0_license.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/big_cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/big_cnn.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/cnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/cnn.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/deeprl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/deeprl.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/discounted_factor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/discounted_factor.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/eth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/eth.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/example.gif -------------------------------------------------------------------------------- /Tensorflow_RL/images/excel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/excel.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/gridworld2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/gridworld2.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/longterm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/longterm.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/mlp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/mlp.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/mlp_total.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/mlp_total.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/mnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/mnist.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/news.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/news.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/q_learning3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/q_learning3.jpg -------------------------------------------------------------------------------- /Tensorflow_RL/images/q_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/q_table.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/qlearning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/qlearning.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/rl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/rl.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/rnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/rnn.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/rnn_seq2seq.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/rnn_seq2seq.jpg -------------------------------------------------------------------------------- /Tensorflow_RL/images/rnn_word.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/rnn_word.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/tensorboard_mlp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/tensorboard_mlp.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/tf_license.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/tf_license.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/unroll_rnn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/unroll_rnn.png -------------------------------------------------------------------------------- /Tensorflow_RL/images/word_embedding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Tensorflow_RL/images/word_embedding.png -------------------------------------------------------------------------------- /Tensorflow_RL/rnn_word_embedding_skflowipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "https://www.tensorflow.org/versions/r0.10/tutorials/word2vec/index.html#motivation-why-learn-word-embeddings\n", 8 | "\n", 9 | "\n", 10 | "\n", 11 | "\n", 12 | "\n", 13 | "### Recurrent Neural Networks\n", 14 | "Recurrent Neural Networks, 이하 RNN는 다음과 같은 구조를 가진 모델이다. RNN은 자기자신을 향하는 weight를 이용해 데이터간의 시간관계를 학습할 수 있다. 이러한 문제들을 시계열 학습이라고 부르며, 기존에 널리 쓰이던 Hidden Markov Model을 뉴럴넷을 이용해 구현했다고 볼 수 있다.\n", 15 | "\n", 16 | "\n" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "\n", 24 | "(출처: https://raw.githubusercontent.com/peterroelants/peterroelants.github.io/master/notebooks/RNN_implementation/img/SimpleRNN01.png)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "위의 구조는 1개의 Recurrent weight를 가진 hidden node이다. 이러한 hidden node들이 여러개를 모여 1개의 RNN layer를 형성하고, 이것이 다시 deep 하게 쌓이는 모델 또한 가능하다.(그러나 RNN은 deep 하게 쌓을 경우 학습이 쉽지 않다.)\n", 32 | "\n", 33 | "RNN의 경우 MLP나 CNN에 비해서 구현이 다소 복잡하다. 따라서 RNN은 skflow 라는 TensorFlow 공식 wrapping library를 활용해서 구현해보자.\n", 34 | "\n", 35 | "https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/learn/python/learn\n", 36 | "\n", 37 | "Skflow는 사용자가 최소한의 노력으로, 매우 추상화된 함수들을 사용해 deep learning 모델을 구축하고 학습할 수 있게 도와주는 역할을 한다.\n", 38 | "\n", 39 | "\n", 40 | "
아래의 예제 코드를 실행시키기 위해서는 리눅스 shell에서 다음의 명령어를 실행해 필요한 python package를 설치해야한다.\n", 41 | "####### pip install pandas" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "\n", 49 | "(출처: http://sebastianruder.com/word-embeddings-1/)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 1, 55 | "metadata": { 56 | "collapsed": false 57 | }, 58 | "outputs": [ 59 | { 60 | "name": "stderr", 61 | "output_type": "stream", 62 | "text": [ 63 | "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpnzKWsC\n", 64 | "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpnzKWsC\n", 65 | "WARNING:tensorflow:Using default config.\n", 66 | "WARNING:tensorflow:Using default config.\n", 67 | "WARNING:tensorflow:Setting feature info to TensorSignature(dtype=tf.int64, shape=TensorShape([Dimension(None), Dimension(10)]), is_sparse=False)\n", 68 | "WARNING:tensorflow:Setting feature info to TensorSignature(dtype=tf.int64, shape=TensorShape([Dimension(None), Dimension(10)]), is_sparse=False)\n", 69 | "WARNING:tensorflow:Setting targets info to TensorSignature(dtype=tf.float32, shape=TensorShape([Dimension(None), Dimension(15)]), is_sparse=False)\n", 70 | "WARNING:tensorflow:Setting targets info to TensorSignature(dtype=tf.float32, shape=TensorShape([Dimension(None), Dimension(15)]), is_sparse=False)\n", 71 | "WARNING:tensorflow:split_squeeze (from tensorflow.contrib.learn.python.learn.ops.array_ops) is deprecated and will be removed after 2016-08-01.\n", 72 | "Instructions for updating:\n", 73 | "Please use tf.unpack instead.\n", 74 | "WARNING:tensorflow:split_squeeze (from tensorflow.contrib.learn.python.learn.ops.array_ops) is deprecated and will be removed after 2016-08-01.\n", 75 | "Instructions for updating:\n", 76 | "Please use tf.unpack instead.\n", 77 | "WARNING:tensorflow:split_squeeze (from tensorflow.contrib.learn.python.learn.ops.array_ops) is deprecated and will be removed after 2016-08-01.\n", 78 | "Instructions for updating:\n", 79 | "Please use tf.unpack instead.\n", 80 | "WARNING:tensorflow:split_squeeze (from tensorflow.contrib.learn.python.learn.ops.array_ops) is deprecated and will be removed after 2016-08-01.\n", 81 | "Instructions for updating:\n", 82 | "Please use tf.unpack instead.\n" 83 | ] 84 | }, 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "Total words: 7664\n", 90 | "Accuracy: 0.485714\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "import numpy as np\n", 96 | "from sklearn import metrics\n", 97 | "import pandas\n", 98 | "\n", 99 | "import tensorflow as tf\n", 100 | "from tensorflow.contrib import learn\n", 101 | "\n", 102 | "### Downloads Training data, unpacks and reads DBpedia dataset.\n", 103 | "dbpedia = learn.datasets.load_dataset('dbpedia')\n", 104 | "X_train, y_train = pandas.DataFrame(dbpedia.train.data)[1], pandas.Series(dbpedia.train.target)\n", 105 | "X_test, y_test = pandas.DataFrame(dbpedia.test.data)[1], pandas.Series(dbpedia.test.target)\n", 106 | "\n", 107 | "### Process vocabulary\n", 108 | "MAX_DOCUMENT_LENGTH = 10\n", 109 | "\n", 110 | "vocab_processor = learn.preprocessing.VocabularyProcessor(MAX_DOCUMENT_LENGTH)\n", 111 | "X_train = np.array(list(vocab_processor.fit_transform(X_train)))\n", 112 | "X_test = np.array(list(vocab_processor.transform(X_test)))\n", 113 | "\n", 114 | "n_words = len(vocab_processor.vocabulary_)\n", 115 | "print('Total words: %d' % n_words)\n", 116 | "\n", 117 | "### Models\n", 118 | "EMBEDDING_SIZE = 50\n", 119 | "\n", 120 | "# Customized function to transform batched X into embeddings\n", 121 | "def input_op_fn(X):\n", 122 | " # Convert indexes of words into embeddings.\n", 123 | " # This creates embeddings matrix of [n_words, EMBEDDING_SIZE] and then\n", 124 | " # maps word indexes of the sequence into [batch_size, sequence_length,\n", 125 | " # EMBEDDING_SIZE].\n", 126 | " word_vectors = learn.ops.categorical_variable(X, n_classes=n_words,\n", 127 | " embedding_size=EMBEDDING_SIZE, name='words')\n", 128 | " # Split into list of embedding per word, while removing doc length dim.\n", 129 | " # word_list results to be a list of tensors [batch_size, EMBEDDING_SIZE].\n", 130 | " word_list = learn.ops.split_squeeze(1, MAX_DOCUMENT_LENGTH, word_vectors)\n", 131 | " return word_list\n", 132 | "\n", 133 | "# Single direction GRU with a single layer\n", 134 | "classifier = learn.TensorFlowRNNClassifier(rnn_size=EMBEDDING_SIZE, \n", 135 | " n_classes=15, cell_type='gru', input_op_fn=input_op_fn,\n", 136 | " num_layers=1, bidirectional=False, sequence_length=None,\n", 137 | " batch_size = 32, steps=1000, optimizer='Adam', learning_rate=0.01, continue_training=True)\n", 138 | "\n", 139 | "classifier.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth =True)))\n", 140 | "\n", 141 | "\n", 142 | "# train for 1000 steps & predict on test set.\n", 143 | "classifier.fit(X_train, y_train, logdir='/tmp/tf_examples/word_rnn')\n", 144 | "score = metrics.accuracy_score(y_test, classifier.predict(X_test))\n", 145 | "print('Accuracy: {0:f}'.format(score))\n", 146 | "\n", 147 | "classifier.sess.close()" 148 | ] 149 | } 150 | ], 151 | "metadata": { 152 | "kernelspec": { 153 | "display_name": "Python 2", 154 | "language": "python", 155 | "name": "python2" 156 | }, 157 | "language_info": { 158 | "codemirror_mode": { 159 | "name": "ipython", 160 | "version": 2 161 | }, 162 | "file_extension": ".py", 163 | "mimetype": "text/x-python", 164 | "name": "python", 165 | "nbconvert_exporter": "python", 166 | "pygments_lexer": "ipython2", 167 | "version": "2.7.6" 168 | } 169 | }, 170 | "nbformat": 4, 171 | "nbformat_minor": 0 172 | } 173 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/__init__.py: -------------------------------------------------------------------------------- 1 | from .simulate import simulate 2 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/controller/__init__.py: -------------------------------------------------------------------------------- 1 | from .discrete_deepq import DiscreteDeepQ 2 | from .human_controller import HumanController 3 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/controller/discrete_deepq.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import tensorflow as tf 4 | 5 | from collections import deque 6 | 7 | class DiscreteDeepQ(object): 8 | def __init__(self, observation_size, 9 | num_actions, 10 | observation_to_actions, 11 | optimizer, 12 | session, 13 | random_action_probability=0.05, 14 | exploration_period=1000, 15 | store_every_nth=5, 16 | train_every_nth=5, 17 | minibatch_size=32, 18 | discount_rate=0.95, 19 | max_experience=30000, 20 | target_network_update_rate=0.01, 21 | summary_writer=None): 22 | """Initialized the Deepq object. 23 | 24 | Based on: 25 | https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf 26 | 27 | Parameters 28 | ------- 29 | observation_size : int 30 | length of the vector passed as observation 31 | num_actions : int 32 | number of actions that the model can execute 33 | observation_to_actions: dali model 34 | model that implements activate function 35 | that can take in observation vector or a batch 36 | and returns scores (of unbounded values) for each 37 | action for each observation. 38 | input shape: [batch_size, observation_size] 39 | output shape: [batch_size, num_actions] 40 | optimizer: tf.solver.* 41 | optimizer for prediction error 42 | session: tf.Session 43 | session on which to execute the computation 44 | random_action_probability: float (0 to 1) 45 | exploration_period: int 46 | probability of choosing a random 47 | action (epsilon form paper) annealed linearly 48 | from 1 to random_action_probability over 49 | exploration_period 50 | store_every_nth: int 51 | to further decorrelate samples do not all 52 | transitions, but rather every nth transition. 53 | For example if store_every_nth is 5, then 54 | only 20% of all the transitions is stored. 55 | train_every_nth: int 56 | normally training_step is invoked every 57 | time action is executed. Depending on the 58 | setup that might be too often. When this 59 | variable is set set to n, then only every 60 | n-th time training_step is called will 61 | the training procedure actually be executed. 62 | minibatch_size: int 63 | number of state,action,reward,newstate 64 | tuples considered during experience reply 65 | dicount_rate: float (0 to 1) 66 | how much we care about future rewards. 67 | max_experience: int 68 | maximum size of the reply buffer 69 | target_network_update_rate: float 70 | how much to update target network after each 71 | iteration. Let's call target_network_update_rate 72 | alpha, target network T, and network N. Every 73 | time N gets updated we execute: 74 | T = (1-alpha)*T + alpha*N 75 | summary_writer: tf.train.SummaryWriter 76 | writer to log metrics 77 | """ 78 | # memorize arguments 79 | self.observation_size = observation_size 80 | self.num_actions = num_actions 81 | 82 | self.q_network = observation_to_actions 83 | self.optimizer = optimizer 84 | self.s = session 85 | 86 | self.random_action_probability = random_action_probability 87 | self.exploration_period = exploration_period 88 | self.store_every_nth = store_every_nth 89 | self.train_every_nth = train_every_nth 90 | self.minibatch_size = minibatch_size 91 | self.discount_rate = tf.constant(discount_rate) 92 | self.max_experience = max_experience 93 | self.target_network_update_rate = \ 94 | tf.constant(target_network_update_rate) 95 | 96 | # deepq state 97 | self.actions_executed_so_far = 0 98 | self.experience = deque() 99 | 100 | self.iteration = 0 101 | self.summary_writer = summary_writer 102 | 103 | self.number_of_times_store_called = 0 104 | self.number_of_times_train_called = 0 105 | 106 | self.create_variables() 107 | 108 | def linear_annealing(self, n, total, p_initial, p_final): 109 | """Linear annealing between p_initial and p_final 110 | over total steps - computes value at step n""" 111 | if n >= total: 112 | return p_final 113 | else: 114 | return p_initial - (n * (p_initial - p_final)) / (total) 115 | 116 | def create_variables(self): 117 | self.target_q_network = self.q_network.copy(scope="target_network") 118 | 119 | # FOR REGULAR ACTION SCORE COMPUTATION 120 | with tf.name_scope("taking_action"): 121 | self.observation = tf.placeholder(tf.float32, (None, self.observation_size), name="observation") 122 | self.action_scores = tf.identity(self.q_network(self.observation), name="action_scores") 123 | tf.summary.histogram("action_scores", self.action_scores) 124 | self.predicted_actions = tf.argmax(self.action_scores, dimension=1, name="predicted_actions") 125 | 126 | with tf.name_scope("estimating_future_rewards"): 127 | # FOR PREDICTING TARGET FUTURE REWARDS 128 | self.next_observation = tf.placeholder(tf.float32, (None, self.observation_size), name="next_observation") 129 | self.next_observation_mask = tf.placeholder(tf.float32, (None,), name="next_observation_mask") 130 | self.next_action_scores = tf.stop_gradient(self.target_q_network(self.next_observation)) 131 | tf.summary.histogram("target_action_scores", self.next_action_scores) 132 | self.rewards = tf.placeholder(tf.float32, (None,), name="rewards") 133 | target_values = tf.reduce_max(self.next_action_scores, reduction_indices=[1,]) * self.next_observation_mask 134 | self.future_rewards = self.rewards + self.discount_rate * target_values 135 | 136 | with tf.name_scope("q_value_precition"): 137 | # FOR PREDICTION ERROR 138 | self.action_mask = tf.placeholder(tf.float32, (None, self.num_actions), name="action_mask") 139 | self.masked_action_scores = tf.reduce_sum(self.action_scores * self.action_mask, reduction_indices=[1,]) 140 | temp_diff = self.masked_action_scores - self.future_rewards 141 | self.prediction_error = tf.reduce_mean(tf.square(temp_diff)) 142 | gradients = self.optimizer.compute_gradients(self.prediction_error) 143 | for i, (grad, var) in enumerate(gradients): 144 | if grad is not None: 145 | gradients[i] = (tf.clip_by_norm(grad, 5), var) 146 | # Add histograms for gradients. 147 | for grad, var in gradients: 148 | tf.summary.histogram(var.name, var) 149 | #if grad: 150 | if grad is not None: 151 | tf.summary.histogram(var.name + '/gradients', grad) 152 | self.train_op = self.optimizer.apply_gradients(gradients) 153 | 154 | # UPDATE TARGET NETWORK 155 | with tf.name_scope("target_network_update"): 156 | self.target_network_update = [] 157 | for v_source, v_target in zip(self.q_network.variables(), self.target_q_network.variables()): 158 | # this is equivalent to target = (1-alpha) * target + alpha * source 159 | update_op = v_target.assign_sub(self.target_network_update_rate * (v_target - v_source)) 160 | self.target_network_update.append(update_op) 161 | self.target_network_update = tf.group(*self.target_network_update) 162 | 163 | # summaries 164 | tf.summary.scalar("prediction_error", self.prediction_error) 165 | 166 | self.summarize = tf.summary.merge_all() 167 | self.no_op1 = tf.no_op() 168 | 169 | def action(self, observation): 170 | """Given observation returns the action that should be chosen using 171 | DeepQ learning strategy. Does not backprop.""" 172 | assert len(observation.shape) == 1, \ 173 | "Action is performed based on single observation." 174 | 175 | self.actions_executed_so_far += 1 176 | exploration_p = self.linear_annealing(self.actions_executed_so_far, 177 | self.exploration_period, 178 | 1.0, 179 | self.random_action_probability) 180 | 181 | if random.random() < exploration_p: 182 | return random.randint(0, self.num_actions - 1) 183 | else: 184 | return self.s.run(self.predicted_actions, {self.observation: observation[np.newaxis,:]})[0] 185 | 186 | def store(self, observation, action, reward, newobservation): 187 | """Store experience, where starting with observation and 188 | execution action, we arrived at the newobservation and got thetarget_network_update 189 | reward reward 190 | 191 | If newstate is None, the state/action pair is assumed to be terminal 192 | """ 193 | if self.number_of_times_store_called % self.store_every_nth == 0: 194 | self.experience.append((observation, action, reward, newobservation)) 195 | if len(self.experience) > self.max_experience: 196 | self.experience.popleft() 197 | self.number_of_times_store_called += 1 198 | 199 | def training_step(self): 200 | """Pick a self.minibatch_size exeperiences from reply buffer 201 | and backpropage the value function. 202 | """ 203 | if self.number_of_times_train_called % self.train_every_nth == 0: 204 | if len(self.experience) < self.minibatch_size: 205 | return 206 | 207 | # sample experience. 208 | samples = random.sample(range(len(self.experience)), self.minibatch_size) 209 | samples = [self.experience[i] for i in samples] 210 | 211 | # bach states 212 | states = np.empty((len(samples), self.observation_size)) 213 | newstates = np.empty((len(samples), self.observation_size)) 214 | action_mask = np.zeros((len(samples), self.num_actions)) 215 | 216 | newstates_mask = np.empty((len(samples),)) 217 | rewards = np.empty((len(samples),)) 218 | 219 | for i, (state, action, reward, newstate) in enumerate(samples): 220 | states[i] = state 221 | action_mask[i] = 0 222 | action_mask[i][action] = 1 223 | rewards[i] = reward 224 | if newstate is not None: 225 | newstates[i] = newstate 226 | newstates_mask[i] = 1 227 | else: 228 | newstates[i] = 0 229 | newstates_mask[i] = 0 230 | 231 | 232 | calculate_summaries = self.iteration % 100 == 0 and \ 233 | self.summary_writer is not None 234 | 235 | cost, _, summary_str = self.s.run([ 236 | self.prediction_error, 237 | self.train_op, 238 | self.summarize if calculate_summaries else self.no_op1, 239 | ], { 240 | self.observation: states, 241 | self.next_observation: newstates, 242 | self.next_observation_mask: newstates_mask, 243 | self.action_mask: action_mask, 244 | self.rewards: rewards, 245 | }) 246 | 247 | self.s.run(self.target_network_update) 248 | 249 | if calculate_summaries: 250 | self.summary_writer.add_summary(summary_str, self.iteration) 251 | 252 | self.iteration += 1 253 | 254 | self.number_of_times_train_called += 1 -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/controller/human_controller.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.append(os.path.abspath('../..')) 3 | 4 | from tf_rl.utils.getch import getch 5 | from redis import StrictRedis 6 | 7 | import random 8 | 9 | class HumanController(object): 10 | def __init__(self, mapping): 11 | self.mapping = mapping 12 | self.r = StrictRedis() 13 | self.experience = [] 14 | 15 | def action(self, o): 16 | return self.mapping[self.r.get("action")] 17 | #return random.randint(0,3) 18 | 19 | def store(self, observation, action, reward, newobservation): 20 | pass 21 | 22 | def training_step(self): 23 | pass 24 | 25 | 26 | 27 | def control_me(): 28 | r = StrictRedis() 29 | while True: 30 | c = getch() 31 | r.set("action", c) 32 | 33 | 34 | if __name__ == '__main__': 35 | control_me() 36 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/models.py: -------------------------------------------------------------------------------- 1 | import math 2 | import tensorflow as tf 3 | 4 | from .utils import base_name 5 | 6 | 7 | class Layer(object): 8 | def __init__(self, input_sizes, output_size, scope): 9 | """Cretes a neural network layer.""" 10 | if type(input_sizes) != list: 11 | input_sizes = [input_sizes] 12 | 13 | self.input_sizes = input_sizes 14 | self.output_size = output_size 15 | self.scope = scope or "Layer" 16 | 17 | with tf.variable_scope(self.scope): 18 | self.Ws = [] 19 | for input_idx, input_size in enumerate(input_sizes): 20 | W_name = "W_%d" % (input_idx,) 21 | W_initializer = tf.random_uniform_initializer( 22 | -1.0 / math.sqrt(input_size), 1.0 / math.sqrt(input_size)) 23 | W_var = tf.get_variable(W_name, (input_size, output_size), initializer=W_initializer) 24 | self.Ws.append(W_var) 25 | self.b = tf.get_variable("b", (output_size,), initializer=tf.constant_initializer(0)) 26 | 27 | def __call__(self, xs): 28 | if type(xs) != list: 29 | xs = [xs] 30 | assert len(xs) == len(self.Ws), \ 31 | "Expected %d input vectors, got %d" % (len(self.Ws), len(xs)) 32 | with tf.variable_scope(self.scope): 33 | return sum([tf.matmul(x, W) for x, W in zip(xs, self.Ws)]) + self.b 34 | 35 | def variables(self): 36 | return [self.b] + self.Ws 37 | 38 | def copy(self, scope=None): 39 | scope = scope or self.scope + "_copy" 40 | 41 | with tf.variable_scope(scope) as sc: 42 | for v in self.variables(): 43 | tf.get_variable(base_name(v), v.get_shape(), 44 | initializer=lambda x,dtype=tf.float32, partition_info=None: v.initialized_value()) 45 | sc.reuse_variables() 46 | return Layer(self.input_sizes, self.output_size, scope=sc) 47 | 48 | class MLP(object): 49 | def __init__(self, input_sizes, hiddens, nonlinearities, scope=None, given_layers=None): 50 | self.input_sizes = input_sizes 51 | # observation is 5 features(distance of each object and X,Y speed) of closest 32 object with hero(friend, enemy, wall) + 2 hero's own speed X,Y 52 | # ==> 5*32 + 2 = 162 features about the game 53 | self.hiddens = hiddens 54 | self.input_nonlinearity, self.layer_nonlinearities = nonlinearities[0], nonlinearities[1:] 55 | self.scope = scope or "MLP" 56 | 57 | assert len(hiddens) == len(nonlinearities), \ 58 | "Number of hiddens must be equal to number of nonlinearities" 59 | 60 | with tf.variable_scope(self.scope): 61 | if given_layers is not None: 62 | self.input_layer = given_layers[0] 63 | self.layers = given_layers[1:] 64 | else: 65 | self.input_layer = Layer(input_sizes, hiddens[0], scope="input_layer") # 135 -> 200 66 | self.layers = [] 67 | 68 | for l_idx, (h_from, h_to) in enumerate(zip(hiddens[:-1], hiddens[1:])): # hiddens == [200, 200, 4], so this mean, swifting the index by 1 69 | # (200, 200) , (200,4) 70 | self.layers.append(Layer(h_from, h_to, scope="hidden_layer_%d" % (l_idx,))) 71 | # this has 4 layers 72 | 73 | def __call__(self, xs): 74 | if type(xs) != list: 75 | xs = [xs] 76 | with tf.variable_scope(self.scope): 77 | hidden = self.input_nonlinearity(self.input_layer(xs)) 78 | for layer, nonlinearity in zip(self.layers, self.layer_nonlinearities): 79 | hidden = nonlinearity(layer(hidden)) 80 | return hidden 81 | 82 | def variables(self): 83 | res = self.input_layer.variables() 84 | for layer in self.layers: 85 | res.extend(layer.variables()) 86 | return res 87 | 88 | def copy(self, scope=None): 89 | scope = scope or self.scope + "_copy" 90 | nonlinearities = [self.input_nonlinearity] + self.layer_nonlinearities 91 | given_layers = [self.input_layer.copy()] + [layer.copy() for layer in self.layers] 92 | return MLP(self.input_sizes, self.hiddens, nonlinearities, scope=scope, 93 | given_layers=given_layers) -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/simulate.py: -------------------------------------------------------------------------------- 1 | import math 2 | import time 3 | 4 | from IPython.display import clear_output, display, HTML 5 | from itertools import count 6 | from os.path import join, exists 7 | from os import makedirs 8 | 9 | def simulate(simulation, 10 | controller= None, 11 | fps=60, 12 | visualize_every=1, 13 | action_every=1, 14 | simulation_resolution=None, 15 | wait=False, 16 | disable_training=False, 17 | save_path=None): 18 | """Start the simulation. Performs three tasks 19 | 20 | - visualizes simulation in iPython notebook 21 | - advances simulator state 22 | - reports state to controller and chooses actions 23 | to be performed. 24 | 25 | Parameters 26 | ------- 27 | simulation: tr_lr.simulation 28 | simulation that will be simulated ;-) 29 | controller: tr_lr.controller 30 | controller used 31 | fps: int 32 | frames per seconds 33 | visualize_every: int 34 | visualize every `visualize_every`-th frame. 35 | action_every: int 36 | take action every `action_every`-th frame 37 | simulation_resolution: float 38 | simulate at most 'simulation_resolution' seconds at a time. 39 | If None, the it is set to 1/FPS (default). 40 | wait: boolean 41 | whether to intentionally slow down the simulation 42 | to appear real time. 43 | disable_training: bool 44 | if true training_step is never called. 45 | save_path: str 46 | save svg visualization (only tl_rl.utils.svg 47 | supported for the moment) 48 | """ 49 | 50 | # prepare path to save simulation images 51 | if save_path is not None: 52 | if not exists(save_path): 53 | makedirs(save_path) 54 | last_image = 0 55 | 56 | # calculate simulation times 57 | chunks_per_frame = 1 58 | chunk_length_s = 1.0 / fps 59 | 60 | if simulation_resolution is not None: 61 | frame_length_s = 1.0 / fps 62 | chunks_per_frame = int(math.ceil(frame_length_s / simulation_resolution)) 63 | chunks_per_frame = max(chunks_per_frame, 1) 64 | chunk_length_s = frame_length_s / chunks_per_frame 65 | 66 | # state transition bookkeeping 67 | last_observation = None 68 | last_action = None 69 | 70 | simulation_started_time = time.time() 71 | 72 | for frame_no in count(): 73 | for _ in range(chunks_per_frame): 74 | simulation.step(chunk_length_s) 75 | 76 | if frame_no % action_every == 0: 77 | new_observation = simulation.observe() 78 | reward = simulation.collect_reward() 79 | # store last transition 80 | if last_observation is not None: 81 | controller.store(last_observation, last_action, reward, new_observation) 82 | 83 | # act 84 | new_action = controller.action(new_observation) # determine the action 85 | simulation.perform_action(new_action) # perform that action 86 | 87 | #train 88 | if not disable_training: 89 | controller.training_step() 90 | 91 | # update current state as last state. 92 | last_action = new_action 93 | last_observation = new_observation 94 | 95 | # adding 1 to make it less likely to happen at the same time as 96 | # action taking. 97 | if (frame_no + 1) % visualize_every == 0: 98 | fps_estimate = frame_no / (time.time() - simulation_started_time) 99 | clear_output(wait=True) 100 | svg_html = simulation.to_html(["fps = %.1f" % (fps_estimate,)]) 101 | display(svg_html) 102 | if save_path is not None: 103 | img_path = join(save_path, "%d.svg" % (last_image,)) 104 | with open(img_path, "w") as f: 105 | svg_html.write_svg(f) 106 | last_image += 1 107 | 108 | time_should_have_passed = frame_no / fps 109 | time_passed = (time.time() - simulation_started_time) 110 | if wait and (time_should_have_passed > time_passed): 111 | time.sleep(time_should_have_passed - time_passed) 112 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/simulation/__init__.py: -------------------------------------------------------------------------------- 1 | from .karpathy_game import KarpathyGame 2 | #from .double_pendulum import DoublePendulum 3 | #from .discrete_hill import DiscreteHill 4 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/simulation/karpathy_game.py: -------------------------------------------------------------------------------- 1 | import math 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import random 5 | import time 6 | 7 | from collections import defaultdict 8 | from euclid import Circle, Point2, Vector2, LineSegment2 9 | 10 | import tf_rl.utils.svg as svg 11 | 12 | class GameObject(object): 13 | # initialize the parameters of GameObject 14 | def __init__(self, position, speed, obj_type, settings): 15 | """Esentially represents circles of different kinds, which have 16 | position and speed.""" 17 | self.settings = settings 18 | self.radius = self.settings["object_radius"] 19 | 20 | self.obj_type = obj_type 21 | self.position = position 22 | self.speed = speed 23 | self.bounciness = 1.0 24 | 25 | # everytime this step is called 26 | def step(self, dt): 27 | """Move and bounce of walls.""" 28 | self.wall_collisions() 29 | self.move(dt) 30 | 31 | # wall collision check 32 | def wall_collisions(self): 33 | """Update speed upon collision with the wall.""" 34 | world_size = self.settings["world_size"] 35 | 36 | for dim in range(2): # check for X dim , Y dim 37 | if self.position[dim] - self.radius <= 0 and self.speed[dim] < 0: # left , top check 38 | self.speed[dim] = - self.speed[dim] * self.bounciness # then the speed is reversed 39 | elif self.position[dim] + self.radius + 1 >= world_size[dim] and self.speed[dim] > 0: # right , bottom check 40 | self.speed[dim] = - self.speed[dim] * self.bounciness # same 41 | 42 | # move this object 43 | def move(self, dt): # dt is the time. second 44 | """Move as if dt seconds passed""" 45 | self.position += dt * self.speed 46 | self.position = Point2(*self.position) # 2 dimension point 47 | 48 | # return itself as a circle type 49 | def as_circle(self): 50 | return Circle(self.position, float(self.radius)) 51 | 52 | # draw itself as a svg image 53 | def draw(self): 54 | """Return svg object for this item.""" 55 | color = self.settings["colors"][self.obj_type] 56 | return svg.Circle(self.position + Point2(10, 10), self.radius, color=color) 57 | 58 | 59 | class KarpathyGame(object): 60 | def __init__(self, settings): 61 | """Initiallize game simulator with settings""" 62 | self.settings = settings 63 | self.size = self.settings["world_size"] 64 | 65 | # make 4 walls 66 | self.walls = [LineSegment2(Point2(0,0), Point2(0,self.size[1])), 67 | LineSegment2(Point2(0,self.size[1]), Point2(self.size[0], self.size[1])), 68 | LineSegment2(Point2(self.size[0], self.size[1]), Point2(self.size[0], 0)), 69 | LineSegment2(Point2(self.size[0], 0), Point2(0,0))] 70 | 71 | # make hero object 72 | self.hero = GameObject(Point2(*self.settings["hero_initial_position"]), 73 | Vector2(*self.settings["hero_initial_speed"]), 74 | "hero", 75 | self.settings) 76 | if not self.settings["hero_bounces_off_walls"]: 77 | self.hero.bounciness = 0.0 # now hero has no bounciness 78 | 79 | self.objects = [] 80 | # spawn 25 friends, 25 enemy 81 | for obj_type, number in settings["num_objects"].items(): # 2 times run 82 | for _ in range(number): 83 | self.spawn_object(obj_type) 84 | 85 | # generate 32 number of antennas 86 | self.observation_lines = self.generate_observation_lines() 87 | 88 | self.object_reward = 0 89 | self.collected_rewards = [] 90 | 91 | # every observation_line sees one of objects or wall and 92 | # two numbers representing speed of the object (if applicable) 93 | self.eye_observation_size = len(self.settings["objects"]) + 3 # 2+3 --> maybe [friend, enemy, wall, speed X, Y] 94 | 95 | # additionally there are two numbers representing agents own speed. 96 | self.observation_size = self.eye_observation_size * len(self.observation_lines) + 2 # (5 * 32) + 2 ==> 2 is hero's own speed 97 | 98 | self.directions = [Vector2(*d) for d in [[1,0], [0,1], [-1,0],[0,-1]]] # there are 4 directions. up down left right 99 | self.num_actions = len(self.directions) # so num_actions is 4 100 | 101 | self.objects_eaten = defaultdict(lambda: 0) 102 | 103 | def perform_action(self, action_id): 104 | """Change speed to one of hero vectors""" 105 | assert 0 <= action_id < self.num_actions 106 | self.hero.speed *= 0.8 # remain 80% of speed 107 | self.hero.speed += self.directions[action_id] * self.settings["delta_v"] # delta_v == 50. so accel 50 speed 108 | 109 | def spawn_object(self, obj_type): 110 | """Spawn object of a given type and add it to the objects array""" 111 | radius = self.settings["object_radius"] # default == 10 112 | position = np.random.uniform([radius, radius], np.array(self.size) - radius) # randomly chooose X , Y position in the whole map 113 | position = Point2(float(position[0]), float(position[1])) 114 | max_speed = np.array(self.settings["maximum_speed"]) # max speed is [50, 50] 115 | speed = np.random.uniform(-max_speed, max_speed).astype(float) # randomly chooose X speed, Y speed from [-50,50] boundary 116 | speed = Vector2(float(speed[0]), float(speed[1])) 117 | 118 | self.objects.append(GameObject(position, speed, obj_type, self.settings)) # make GameObject with above setting and append in list 119 | 120 | # step function is called every frame 121 | def step(self, dt): 122 | """Simulate all the objects for a given ammount of time. 123 | 124 | Also resolve collisions with the hero""" 125 | 126 | # call step function of each object 127 | for obj in self.objects + [self.hero] : 128 | obj.step(dt) 129 | 130 | # collision check process 131 | self.resolve_collisions() 132 | 133 | def resolve_collisions(self): 134 | """If hero touches, hero eats. Also reward gets updated.""" 135 | collision_distance = 2 * self.settings["object_radius"] 136 | collision_distance2 = collision_distance ** 2 137 | to_remove = [] 138 | 139 | # for all object, if here touch that object, append that in the remove list 140 | for obj in self.objects: 141 | if self.squared_distance(self.hero.position, obj.position) < collision_distance2: 142 | to_remove.append(obj) 143 | 144 | # for all remove item 145 | for obj in to_remove: 146 | self.objects.remove(obj) # remove that item from the [self.objects] list 147 | self.objects_eaten[obj.obj_type] += 1 # and count the object_eaten number by obj_type 148 | self.object_reward += self.settings["object_reward"][obj.obj_type] # get a reward following the object_reward setting 149 | self.spawn_object(obj.obj_type) # and spawn that type object again ( randomly initialize) 150 | 151 | def squared_distance(self, p1, p2): 152 | return (p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2 153 | 154 | def inside_walls(self, point): 155 | """Check if the point is inside the walls""" 156 | EPS = 1e-4 157 | return (EPS <= point[0] < self.size[0] - EPS and 158 | EPS <= point[1] < self.size[1] - EPS) 159 | 160 | def observe(self): 161 | """Return observation vector. For all the observation directions it returns representation 162 | of the closest object to the hero - might be nothing, another object or a wall. 163 | Representation of observation for all the directions will be concatenated. 164 | """ 165 | num_obj_types = len(self.settings["objects"]) + 1 # 3 == (friend, enemy and wall) 166 | max_speed_x, max_speed_y = self.settings["maximum_speed"] 167 | 168 | observable_distance = self.settings["observation_line_length"] # length of antenna == 120 169 | 170 | relevant_objects = [obj for obj in self.objects 171 | if obj.position.distance(self.hero.position) < observable_distance] 172 | # recall the objects near from hero with given that length 173 | 174 | # objects sorted from closest to furthest 175 | relevant_objects.sort(key=lambda x: x.position.distance(self.hero.position)) 176 | 177 | observation = np.zeros(self.observation_size) # 32*5 (objects * features) + 2 ( hero's speed) 178 | observation_offset = 0 179 | for i, observation_line in enumerate(self.observation_lines): # for 32 lines 180 | # shift the antenna's center to hero position 181 | observation_line = LineSegment2(self.hero.position + Vector2(*observation_line.p1), self.hero.position + Vector2(*observation_line.p2)) 182 | 183 | observed_object = None 184 | # if end of observation line is outside of walls, we see the wall. 185 | if not self.inside_walls(observation_line.p2): # p1 is start, p2 is end of the line 186 | observed_object = "**wall**" 187 | # defaulyt see the wall, 188 | 189 | for obj in relevant_objects: # for near objects from hero, sorted from nearst to furtherst 190 | if observation_line.distance(obj.position) < self.settings["object_radius"]: # the distance between line and point < radius 191 | # this means, the line touch that object 192 | observed_object = obj # so that is observed_object 193 | break # this mean, one line can see only one object 194 | 195 | object_type_id = None 196 | speed_x, speed_y = 0, 0 197 | proximity = 0 # closetness 198 | 199 | if observed_object == "**wall**": # wall is seen 200 | object_type_id = num_obj_types - 1 # object type id of all == 2 201 | # a wall has fairly low speed... 202 | speed_x, speed_y = 0, 0 203 | # best candidate is intersection between observation_line and a wall, that's closest to the hero 204 | best_candidate = None 205 | 206 | for wall in self.walls: # for all wall about each lines (32 * 4 times) 207 | candidate = observation_line.intersect(wall) # LineSegment2 has intersect function 208 | # so this calculate the intersected point with wall 209 | if candidate is not None: 210 | if (best_candidate is None or best_candidate.distance(self.hero.position) > candidate.distance(self.hero.position)): 211 | best_candidate = candidate # change the best candidate 212 | 213 | if best_candidate is None: 214 | # assume it is due to rounding errors and wall is barely touching observation line 215 | proximity = observable_distance 216 | else: 217 | proximity = best_candidate.distance(self.hero.position) 218 | 219 | elif observed_object is not None: # agent seen 220 | object_type_id = self.settings["objects"].index(observed_object.obj_type) # index of that obj_type, -> 0:friend, 1:enemy 221 | speed_x, speed_y = tuple(observed_object.speed) # speed is 2dim vector 222 | intersection_segment = obj.as_circle().intersect(observation_line) # what is obj? relevant obj? 223 | assert intersection_segment is not None 224 | try: 225 | proximity = min(intersection_segment.p1.distance(self.hero.position), 226 | intersection_segment.p2.distance(self.hero.position)) 227 | except AttributeError: 228 | proximity = observable_distance 229 | 230 | 231 | for object_type_idx_loop in range(num_obj_types): # for (friend, enemy, wall) 232 | observation[observation_offset + object_type_idx_loop] = 1.0 # initialize each object's distance ratio feature with 1 ( 1 is maximum ) 233 | 234 | if object_type_id is not None: # this is always true 235 | observation[observation_offset + object_type_id] = proximity / observable_distance 236 | # the ratio of distance with hero on observed object 237 | 238 | # the 4th -> speed X ratio / 5th -> speed Y ratio of observed object 239 | observation[observation_offset + num_obj_types] = speed_x / max_speed_x 240 | observation[observation_offset + num_obj_types + 1] = speed_y / max_speed_y 241 | 242 | assert num_obj_types + 2 == self.eye_observation_size # maximum feature for each object is 5 243 | observation_offset += self.eye_observation_size # this means feature of observed object == 5 244 | 245 | 246 | # the last two observation is hero's own speed ratio 247 | observation[observation_offset] = self.hero.speed[0] / max_speed_x 248 | observation[observation_offset + 1] = self.hero.speed[1] / max_speed_y 249 | assert observation_offset + 2 == self.observation_size 250 | 251 | return observation 252 | 253 | 254 | def collect_reward(self): # collect reward for each steps 255 | """Return accumulated object eating score + current distance to walls score""" 256 | wall_reward = self.settings["wall_distance_penalty"] * np.exp(-self.distance_to_walls() / self.settings["tolerable_distance_to_wall"]) 257 | # currently, wall_reward == 0 258 | assert wall_reward < 1e-3, "You are rewarding hero for being close to the wall!" 259 | total_reward = wall_reward + self.object_reward # so total reward is just given from object collision 260 | self.object_reward = 0 261 | self.collected_rewards.append(total_reward) 262 | return total_reward 263 | 264 | 265 | def distance_to_walls(self): 266 | """Returns distance of a hero to walls""" 267 | res = float('inf') 268 | for wall in self.walls: 269 | res = min(res, self.hero.position.distance(wall)) 270 | return res - self.settings["object_radius"] 271 | 272 | 273 | def plot_reward(self, smoothing = 30): 274 | """Plot evolution of reward over time.""" 275 | plottable = self.collected_rewards[:] 276 | while len(plottable) > 1000: 277 | for i in range(0, len(plottable) - 1, 2): 278 | plottable[i//2] = (plottable[i] + plottable[i+1]) / 2 279 | plottable = plottable[:(len(plottable) // 2)] 280 | x = [] 281 | for i in range(smoothing, len(plottable)): 282 | chunk = plottable[i-smoothing:i] 283 | x.append(sum(chunk) / len(chunk)) 284 | plt.plot(list(range(len(x))), x) 285 | 286 | 287 | # make 32 number of antennas, with specific length 288 | def generate_observation_lines(self): 289 | """Generate observation segments in settings["num_observation_lines"] directions""" 290 | result = [] 291 | 292 | # make a line 293 | start = Point2(0.0, 0.0) 294 | end = Point2(self.settings["observation_line_length"], self.settings["observation_line_length"]) 295 | 296 | # the angle == 360 / 32 297 | for angle in np.linspace(0, 2*np.pi, self.settings["num_observation_lines"], endpoint=False): 298 | rotation = Point2(math.cos(angle), math.sin(angle)) # rotation each lines 299 | current_start = Point2(start[0] * rotation[0], start[1] * rotation[1]) 300 | current_end = Point2(end[0] * rotation[0], end[1] * rotation[1]) 301 | result.append( LineSegment2(current_start, current_end)) # list append the 32 lines 302 | return result 303 | 304 | 305 | def _repr_html_(self): 306 | return self.to_html() 307 | 308 | def to_html(self, stats=[]): 309 | """Return svg representation of the simulator""" 310 | 311 | stats = stats[:] 312 | recent_reward = self.collected_rewards[-100:] + [0] 313 | objects_eaten_str = ', '.join(["%s: %s" % (o,c) for o,c in self.objects_eaten.items()]) 314 | stats.extend([ 315 | "nearest wall = %.1f" % (self.distance_to_walls(),), 316 | "reward = %.1f" % (sum(recent_reward)/len(recent_reward),), 317 | "objects eaten => %s" % (objects_eaten_str,), 318 | ]) 319 | 320 | scene = svg.Scene((self.size[0] + 20, self.size[1] + 20 + 20 * len(stats))) 321 | scene.add(svg.Rectangle((10, 10), self.size)) 322 | 323 | 324 | for line in self.observation_lines: 325 | scene.add(svg.Line(line.p1 + self.hero.position + Point2(10,10), 326 | line.p2 + self.hero.position + Point2(10,10))) 327 | 328 | for obj in self.objects + [self.hero] : 329 | scene.add(obj.draw()) 330 | 331 | offset = self.size[1] + 15 332 | for txt in stats: 333 | scene.add(svg.Text((10, offset + 20), txt, 15)) 334 | offset += 20 335 | 336 | return scene 337 | 338 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/utils/__init__.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | def base_name(var): 4 | """Extracts value passed to name= when creating a variable""" 5 | return var.name.split('/')[-1].split(':')[0] 6 | 7 | def copy_variables(variables): 8 | res = {} 9 | for v in variables: 10 | name = base_name(v) 11 | copied_var = tf.Variable(v.initialized_value(), name=name) 12 | res[name] = copied_var 13 | return res 14 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/utils/event_queue.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from queue import PriorityQueue 4 | 5 | class EqItem(object): 6 | """Function and sechduled execution timestamp. 7 | 8 | This class is needed because if 9 | we use tuple instead, Python will ocassionally 10 | complaint that it does not know how to compare 11 | functions""" 12 | def __init__(self, ts, f): 13 | self.ts = ts 14 | self.f = f 15 | 16 | def __lt__(self, other): 17 | return self.ts < other.ts 18 | 19 | def __eq__(self, other): 20 | return self.ts == other.ts 21 | 22 | class EventQueue(object): 23 | def __init__(self): 24 | """Event queue for executing events at 25 | specific timepoints. 26 | 27 | In current form it is NOT thread safe.""" 28 | self.q = PriorityQueue() 29 | 30 | def schedule(self, f, ts): 31 | """Schedule f to be execute at time ts""" 32 | self.q.put(EqItem(ts, f)) 33 | 34 | def schedule_recurring(self, f, interval): 35 | """Schedule f to be run every interval seconds. 36 | 37 | It will be run for the first time interval seconds 38 | from now""" 39 | def recuring_f(): 40 | f() 41 | self.schedule(recuring_f, time.time() + interval) 42 | self.schedule(recuring_f, time.time() + interval) 43 | 44 | 45 | def run(self): 46 | """Execute events in the queue as timely as possible.""" 47 | while True: 48 | event = self.q.get() 49 | now = time.time() 50 | if now < event.ts: 51 | time.sleep(event.ts - now) 52 | event.f() 53 | 54 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/utils/geometry.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module assumes that all geometrical points are 3 | represented as 1D numpy arrays. 4 | 5 | It was designed and tested on 2D points, 6 | but if you try it on 3D points you may 7 | be pleasantly surprised ;-) 8 | """ 9 | import numpy as np 10 | 11 | 12 | def point_distance(x, y): 13 | """Returns euclidean distance between points x and y""" 14 | return np.linalg.norm(x-y) 15 | 16 | def point_projected_on_line(line_s, line_e, point): 17 | """Project point on line that goes through line_s and line_e 18 | 19 | assumes line_e is not equal or close to line_s 20 | """ 21 | line_along = line_e - line_s 22 | 23 | transformed_point = point - line_s 24 | 25 | point_dot_line = np.dot(transformed_point, line_along) 26 | line_along_norm = np.dot(line_along, line_along) 27 | 28 | transformed_projection = (point_dot_line / line_along_norm) * line_along 29 | 30 | return transformed_projection + line_s 31 | 32 | def point_segment_distance(segment_s, segment_e, point): 33 | """Returns distance from point to the closest point on segment 34 | connecting points segment_s and segment_e""" 35 | projected = point_projected_on_line(segment_s, segment_e, point) 36 | if np.isclose(point_distance(segment_s, projected) + point_distance(projected, segment_e), 37 | point_distance(segment_s, segment_e)): 38 | # projected on segment 39 | return point_distance(point, projected) 40 | else: 41 | return min(point_distance(point, segment_s), point_distance(point, segment_e)) 42 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/utils/getch.py: -------------------------------------------------------------------------------- 1 | class _Getch: 2 | """Gets a single character from standard input. Does not echo to the 3 | screen.""" 4 | def __init__(self): 5 | try: 6 | self.impl = _GetchWindows() 7 | except ImportError: 8 | self.impl = _GetchUnix() 9 | 10 | def __call__(self): return self.impl() 11 | 12 | 13 | class _GetchUnix: 14 | def __init__(self): 15 | import tty, sys 16 | 17 | def __call__(self): 18 | import sys, tty, termios 19 | fd = sys.stdin.fileno() 20 | old_settings = termios.tcgetattr(fd) 21 | try: 22 | tty.setraw(sys.stdin.fileno()) 23 | ch = sys.stdin.read(1) 24 | finally: 25 | termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) 26 | return ch 27 | 28 | 29 | class _GetchWindows: 30 | def __init__(self): 31 | import msvcrt 32 | 33 | def __call__(self): 34 | import msvcrt 35 | return msvcrt.getch() 36 | 37 | 38 | getch = _Getch() 39 | -------------------------------------------------------------------------------- /Tensorflow_RL/tf_rl/utils/svg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """\ 3 | SVG.py - Construct/display SVG scenes. 4 | 5 | The following code is a lightweight wrapper around SVG files. The metaphor 6 | is to construct a scene, add objects to it, and then write it to a file 7 | to display it. 8 | 9 | This program uses ImageMagick to display the SVG files. ImageMagick also 10 | does a remarkable job of converting SVG files into other formats. 11 | """ 12 | 13 | import os 14 | 15 | def colorstr(rgb): 16 | if type(rgb) == tuple: 17 | return "#%02x%02x%02x" % rgb 18 | else: 19 | return rgb 20 | 21 | def compute_style(style): 22 | color = style.get("color") 23 | style_str = [] 24 | if color is None: 25 | color="none" 26 | style_str.append('fill:%s;' % (colorstr(color),)) 27 | 28 | style_str = 'style="%s"' % (';'.join(style_str),) 29 | return style_str 30 | 31 | class Scene: 32 | def __init__(self, size=(400,400)): 33 | self.items = [] 34 | self.size = size 35 | 36 | def add(self,item): 37 | self.items.append(item) 38 | 39 | def strarray(self): 40 | var = [ 41 | "\n", 42 | "\n" % (self.size[1],self.size[0]), 43 | " \n" 45 | ] 46 | for item in self.items: var += item.strarray() 47 | var += [" \n\n"] 48 | return var 49 | 50 | def write_svg(self, file): 51 | file.writelines(self.strarray()) 52 | 53 | def _repr_html_(self): 54 | return '\n'.join(self.strarray()) 55 | 56 | class Line: 57 | def __init__(self,start,end): 58 | self.start = start #xy tuple 59 | self.end = end #xy tuple 60 | 61 | def strarray(self): 62 | return [" \n" %\ 63 | (self.start[0],self.start[1],self.end[0],self.end[1])] 64 | 65 | 66 | class Circle: 67 | def __init__(self,center,radius, **style_kwargs): 68 | self.center = center 69 | self.radius = radius 70 | self.style_kwargs = style_kwargs 71 | 72 | def strarray(self): 73 | style_str = compute_style(self.style_kwargs) 74 | 75 | return [ 76 | " \n" % (style_str,) 78 | ] 79 | 80 | class Rectangle: 81 | def __init__(self, origin, size, **style_kwargs): 82 | self.origin = origin 83 | self.size = size 84 | self.style_kwargs = style_kwargs 85 | 86 | def strarray(self): 87 | style_str = compute_style(self.style_kwargs) 88 | 89 | return [ 90 | " \n" % (self.size[0], style_str) 92 | ] 93 | 94 | class Text: 95 | def __init__(self,origin,text,size=24): 96 | self.origin = origin 97 | self.text = text 98 | self.size = size 99 | return 100 | 101 | def strarray(self): 102 | return [" \n" %\ 103 | (self.origin[0],self.origin[1],self.size), 104 | " %s\n" % self.text, 105 | " \n"] 106 | 107 | 108 | 109 | 110 | def test(): 111 | scene = Scene() 112 | scene.add(Rectangle((100,100),200,200,(0,255,255))) 113 | scene.add(Line((200,200),(200,300))) 114 | scene.add(Line((200,200),(300,200))) 115 | scene.add(Line((200,200),(100,200))) 116 | scene.add(Line((200,200),(200,100))) 117 | scene.add(Circle((200,200),30,(0,0,255))) 118 | scene.add(Circle((200,300),30,(0,255,0))) 119 | scene.add(Circle((300,200),30,(255,0,0))) 120 | scene.add(Circle((100,200),30,(255,255,0))) 121 | scene.add(Circle((200,100),30,(255,0,255))) 122 | scene.add(Text((50,50),"Testing SVG")) 123 | with open("test.svg", "w") as f: 124 | scene.write_svg(f) 125 | 126 | if __name__ == '__main__': 127 | test() 128 | -------------------------------------------------------------------------------- /Theano Tutorial Basic1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [ 10 | { 11 | "name": "stderr", 12 | "output_type": "stream", 13 | "text": [ 14 | "Using gpu device 0: GeForce GTX TITAN X (CNMeM is disabled)\n" 15 | ] 16 | } 17 | ], 18 | "source": [ 19 | "import theano\n", 20 | "import theano.tensor as T\n", 21 | "\n", 22 | "x = T.matrix('x')\n", 23 | "y = T.matrix('y')\n", 24 | "z = T.matrix('z')\n", 25 | "\n", 26 | "# (x+y) + (x+y) * z\n", 27 | "a = x + y\n", 28 | "b = a * z\n", 29 | "c = a + b\n", 30 | "\n", 31 | "f = theano.function(\n", 32 | " inputs = [x, y, z],\n", 33 | " outputs = c\n", 34 | ")\n" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 5, 40 | "metadata": { 41 | "collapsed": false 42 | }, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "[[ 1.05651045 -1.21079338 1.23110497 -0.04195251 2.67902255]\n", 49 | " [-0.13276182 1.76614249 -0.36022687 -0.59106445 -2.71669126]\n", 50 | " [-2.37245154 -1.12027502 4.23539543 -5.01190567 -0.88961536]\n", 51 | " [ 0.09296193 0.19824572 -0.89048773 1.89251935 2.93632889]]\n" 52 | ] 53 | } 54 | ], 55 | "source": [ 56 | "import numpy as np\n", 57 | "\n", 58 | "xx = np.random.randn(4, 5).astype('float32')\n", 59 | "yy = np.random.randn(4, 5).astype('float32')\n", 60 | "zz = np.random.randn(4, 5).astype('float32')\n", 61 | "\n", 62 | "print f(xx, yy, zz)" 63 | ] 64 | } 65 | ], 66 | "metadata": { 67 | "kernelspec": { 68 | "display_name": "Python 2", 69 | "language": "python", 70 | "name": "python2" 71 | }, 72 | "language_info": { 73 | "codemirror_mode": { 74 | "name": "ipython", 75 | "version": 2 76 | }, 77 | "file_extension": ".py", 78 | "mimetype": "text/x-python", 79 | "name": "python", 80 | "nbconvert_exporter": "python", 81 | "pygments_lexer": "ipython2", 82 | "version": "2.7.6" 83 | } 84 | }, 85 | "nbformat": 4, 86 | "nbformat_minor": 0 87 | } 88 | -------------------------------------------------------------------------------- /Theano_deeplearing_tutorial_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Theano_deeplearing_tutorial_1.pdf -------------------------------------------------------------------------------- /Theano_eskim.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Theano_eskim.pdf -------------------------------------------------------------------------------- /Theano_talk160630.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/Theano_talk160630.pdf -------------------------------------------------------------------------------- /example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "##간단한 신경망 구현하기" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "두 개의 층으로 이루어진 신경망 구현\n", 15 | "\n", 16 | "입력층: 1000차원 벡터를 입력으로 받음\n", 17 | "hidden layer 1: 100차원\n", 18 | "hidden layer 2: 10차원\n", 19 | "\n", 20 | "입력 데이터 갯수: 64개\n", 21 | "\n", 22 | "weight matrix 1 (입력층과 hidden layer 1사이) : 1000X100 행렬\n", 23 | "\n", 24 | "weight matrix 2 (hidden layer1과 hidden layer 2사이) : 100X10 행렬" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 4, 30 | "metadata": { 31 | "collapsed": false 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import tensorflow as tf\n", 36 | "import numpy as np\n", 37 | "\n", 38 | "#ㅜㅜN: 데이터 갯수, D: 데이터 차원, H: 첫 번째 hidden layer 의 노드 수, C: 두 번째 hidden layer의 노드 수\n", 39 | "N, D, H, C = 64, 1000, 100, 10\n", 40 | "\n", 41 | "x = tf.placeholder(tf.float32, shape = [None, D], name='x')\n", 42 | "y = tf.placeholder(tf.float32, shape = [None, C], name ='y')\n", 43 | "\n", 44 | "#w1, w2 initialize\n", 45 | "w1 = tf.Variable(1e-3 * np.random.randn(D, H).astype(np.float32), name='w1')\n", 46 | "w2 = tf.Variable(1e-3 * np.random.randn(H, C).astype(np.float32), name='w2')\n", 47 | "\n", 48 | "#output 계산\n", 49 | "with tf.name_scope('scores') as scope:\n", 50 | " a = tf.matmul(x, w1)\n", 51 | " a_relu = tf.nn.relu(a)\n", 52 | " scores = tf.matmul(a_relu, w2)\n", 53 | "\n", 54 | "#loss 계산, 엔트로피로 정의\n", 55 | "with tf.name_scope('loss') as scope:\n", 56 | " probs = tf.nn.softmax(scores)\n", 57 | " loss = -tf.reduce_sum(y * tf.log(probs))\n", 58 | " \n", 59 | "loss_summary = tf.scalar_summary('loss', loss)\n", 60 | "w1_hist = tf.histogram_summary('w1', w1)\n", 61 | "w2_hist = tf.histogram_summary('w2', w2)\n", 62 | "\n", 63 | "#learning_rate 초기값 정의\n", 64 | "learning_rate = 1e-2\n", 65 | "#Gradient Decent 방법으로 모델 파라미터 학습\n", 66 | "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n", 67 | "\n", 68 | "xx = np.random.randn(N, D).astype(np.float32)\n", 69 | "yy = np.zeros([N,C]).astype(np.float32)\n", 70 | "yy[np.arange(N), np.random.randint(C, size=N)] = 1" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": { 77 | "collapsed": true 78 | }, 79 | "outputs": [], 80 | "source": [] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": { 86 | "collapsed": true 87 | }, 88 | "outputs": [], 89 | "source": [] 90 | } 91 | ], 92 | "metadata": { 93 | "kernelspec": { 94 | "display_name": "Python 2", 95 | "language": "python", 96 | "name": "python2" 97 | }, 98 | "language_info": { 99 | "codemirror_mode": { 100 | "name": "ipython", 101 | "version": 2 102 | }, 103 | "file_extension": ".py", 104 | "mimetype": "text/x-python", 105 | "name": "python", 106 | "nbconvert_exporter": "python", 107 | "pygments_lexer": "ipython2", 108 | "version": "2.7.6" 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 0 113 | } 114 | -------------------------------------------------------------------------------- /examples/lady.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/examples/lady.jpg -------------------------------------------------------------------------------- /examples/moheo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/examples/moheo.jpg -------------------------------------------------------------------------------- /examples/mom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/examples/mom.jpg -------------------------------------------------------------------------------- /images/ae_single.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/images/ae_single.png -------------------------------------------------------------------------------- /images/ae_stacked.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/images/ae_stacked.png -------------------------------------------------------------------------------- /images/higher_layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/images/higher_layer.png -------------------------------------------------------------------------------- /images/lower_layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/images/lower_layer.png -------------------------------------------------------------------------------- /images/sae.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/images/sae.jpg -------------------------------------------------------------------------------- /mnist.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/mnist.hdf5 -------------------------------------------------------------------------------- /setup_nbserver.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | home_dir = os.environ['HOME'] 4 | 5 | if os.path.isdir("$HOME/.ipython/profile_nbserver")is False: 6 | os.system("ipython profile create nbserver") 7 | else: 8 | os.system("echo profile_nbserver is already exist.") 9 | 10 | from IPython.lib import passwd 11 | pwsha = passwd() 12 | 13 | config_str = """ 14 | # Server config 15 | c = get_config() 16 | c.NotebookApp.ip = '*' 17 | c.NotebookApp.open_browser = False 18 | c.NotebookApp.password = u'{}' 19 | # It is a good idea to put it on a known, fixed port 20 | c.NotebookApp.port = 8888 21 | c.NotebookApp.notebook_dir = u'/' 22 | """.format(pwsha) 23 | 24 | with open(home_dir+"/.ipython/profile_nbserver/ipython_notebook_config.py", "w") as cf: 25 | cf.write(config_str) 26 | 27 | 28 | #os.system("sudo printf \"\nexport PATH=/usr/local/cuda/bin:$PATH\nexport LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH\n\" >> /root/.bashrc") 29 | #os.system("sudo printf \"\nshell -/bin/bash\n\" >> /root/.screenrc") 30 | 31 | # this is usually make some error... do not use this! 32 | #os.system("screen -dRR -dmS ipython_notebook ipython notebook --profile=nbserver;") 33 | 34 | # https://www.gnu.org/software/screen/manual/screen.html 35 | # screen install check 36 | # 37 | -------------------------------------------------------------------------------- /sshd_config: -------------------------------------------------------------------------------- 1 | UseLogin yes 2 | 3 | # What ports, IPs and protocols we listen for 4 | Port 22 5 | # Use these options to restrict which interfaces/protocols sshd will bind to 6 | #ListenAddress :: 7 | #ListenAddress 0.0.0.0 8 | Protocol 2 9 | # HostKeys for protocol version 2 10 | HostKey /etc/ssh/ssh_host_rsa_key 11 | HostKey /etc/ssh/ssh_host_dsa_key 12 | HostKey /etc/ssh/ssh_host_ecdsa_key 13 | HostKey /etc/ssh/ssh_host_ed25519_key 14 | #Privilege Separation is turned on for security 15 | UsePrivilegeSeparation yes 16 | 17 | # Lifetime and size of ephemeral version 1 server key 18 | KeyRegenerationInterval 3600 19 | ServerKeyBits 1024 20 | 21 | # Logging 22 | SyslogFacility AUTH 23 | LogLevel INFO 24 | 25 | # Authentication: 26 | LoginGraceTime 120 27 | PermitRootLogin yes 28 | # root login is possible 29 | StrictModes yes 30 | 31 | RSAAuthentication no 32 | PubkeyAuthentication yes 33 | #AuthorizedKeysFile %h/.ssh/authorized_keys 34 | 35 | # Don't read the user's ~/.rhosts and ~/.shosts files 36 | IgnoreRhosts yes 37 | # For this to work you will also need host keys in /etc/ssh_known_hosts 38 | RhostsRSAAuthentication no 39 | # similar for protocol version 2 40 | HostbasedAuthentication no 41 | # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication 42 | #IgnoreUserKnownHosts yes 43 | 44 | # To enable empty passwords, change to yes (NOT RECOMMENDED) 45 | PermitEmptyPasswords no 46 | 47 | # Change to yes to enable challenge-response passwords (beware issues with 48 | # some PAM modules and threads) 49 | ChallengeResponseAuthentication no 50 | 51 | # Change to no to disable tunnelled clear text passwords 52 | PasswordAuthentication yes 53 | 54 | X11Forwarding no 55 | X11DisplayOffset 10 56 | PrintMotd no 57 | PrintLastLog no 58 | TCPKeepAlive yes 59 | #UseLogin no 60 | 61 | #MaxStartups 10:30:60 62 | #Banner /etc/issue.net 63 | 64 | # Allow client to pass locale environment variables 65 | AcceptEnv LANG LC_* 66 | 67 | Subsystem sftp internal-sftp 68 | 69 | # Set this to 'yes' to enable PAM authentication, account processing, 70 | # and session processing. If this is enabled, PAM authentication will 71 | # be allowed through the ChallengeResponseAuthentication and 72 | # PasswordAuthentication. Depending on your PAM configuration, 73 | # PAM authentication via ChallengeResponseAuthentication may bypass 74 | # the setting of "PermitRootLogin without-password". 75 | # If you just want the PAM account and session checks to run without 76 | # PAM authentication, then enable this but set PasswordAuthentication 77 | # and ChallengeResponseAuthentication to 'no'. 78 | UsePAM no 79 | #UsePAM yes makes a connection error. exit code 254 80 | -------------------------------------------------------------------------------- /stories.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/stories.txt -------------------------------------------------------------------------------- /trained_ae.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/trained_ae.hdf5 -------------------------------------------------------------------------------- /trained_ae.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bi-lab/deeplearning_tutorial/de4b86bd26a05d289a636c0c61d518200f1f9ded/trained_ae.pkl --------------------------------------------------------------------------------