├── day_1 ├── data ├── modelzoo ├── vocabulary.py ├── 0_Using_Pretrained_Embeddings.ipynb └── figures │ └── intro_to_pytorch │ ├── computational_graph_forward.svg │ ├── pytorch_variable.svg │ └── computational_graph_backward.svg ├── day_2 ├── data ├── modelzoo └── vocabulary.py ├── docs ├── recipes │ ├── packed_sequences.rst │ ├── index.rst │ ├── compute_conv_size.rst │ └── load_pretrained_vectors.rst ├── day2 │ ├── adventures │ │ ├── conv_then_rnn.rst │ │ ├── index.rst │ │ ├── lvg.rst │ │ ├── interpolation.rst │ │ └── lookups.rst │ ├── patterns │ │ ├── stacking.rst │ │ ├── index.rst │ │ ├── gating.rst │ │ └── attention.rst │ ├── warmup.rst │ ├── tensorfu1.rst │ ├── tensorfu2.rst │ ├── failfastprototypemode.rst │ └── sampling.rst ├── .gitignore ├── errata.rst ├── _build │ ├── html │ │ ├── objects.inv │ │ ├── _static │ │ │ ├── up.png │ │ │ ├── down.png │ │ │ ├── file.png │ │ │ ├── minus.png │ │ │ ├── plus.png │ │ │ ├── comment.png │ │ │ ├── up-pressed.png │ │ │ ├── ajax-loader.gif │ │ │ ├── comment-close.png │ │ │ ├── down-pressed.png │ │ │ ├── comment-bright.png │ │ │ ├── fonts │ │ │ │ ├── Lato-Bold.ttf │ │ │ │ ├── Lato-Regular.ttf │ │ │ │ ├── Inconsolata-Bold.ttf │ │ │ │ ├── RobotoSlab-Bold.ttf │ │ │ │ ├── Inconsolata-Regular.ttf │ │ │ │ ├── RobotoSlab-Regular.ttf │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ └── fontawesome-webfont.woff │ │ │ ├── css │ │ │ │ └── badge_only.css │ │ │ ├── js │ │ │ │ └── theme.js │ │ │ ├── pygments.css │ │ │ ├── doctools.js │ │ │ ├── underscore.js │ │ │ └── basic.css │ │ ├── .buildinfo │ │ ├── _sources │ │ │ ├── faq.rst.txt │ │ │ ├── download_data.rst.txt │ │ │ ├── index.rst.txt │ │ │ └── environment_setup.rst.txt │ │ ├── genindex.html │ │ ├── search.html │ │ ├── faq.html │ │ ├── download_data.html │ │ ├── searchindex.js │ │ └── index.html │ └── doctrees │ │ ├── faq.doctree │ │ ├── index.doctree │ │ ├── environment.pickle │ │ ├── download_data.doctree │ │ └── environment_setup.doctree ├── day1 │ ├── index.rst │ ├── takehome.rst │ └── solutions.rst ├── migration.rst ├── Makefile ├── faq.rst ├── download_data.rst ├── index.rst ├── conf.py └── environment_setup.rst ├── requirements.txt ├── data └── README.md ├── README.md ├── modelzoo └── README.md ├── LICENSE ├── LOCAL_RUN_README.md └── .gitignore /day_1/data: -------------------------------------------------------------------------------- 1 | ../data -------------------------------------------------------------------------------- /day_2/data: -------------------------------------------------------------------------------- 1 | ../data -------------------------------------------------------------------------------- /day_1/modelzoo: -------------------------------------------------------------------------------- 1 | ../modelzoo -------------------------------------------------------------------------------- /day_2/modelzoo: -------------------------------------------------------------------------------- 1 | ../modelzoo -------------------------------------------------------------------------------- /docs/recipes/packed_sequences.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/day2/adventures/conv_then_rnn.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build/ 2 | _static/ 3 | _templates/ 4 | -------------------------------------------------------------------------------- /docs/day2/patterns/stacking.rst: -------------------------------------------------------------------------------- 1 | Stacking 2 | ======== 3 | -------------------------------------------------------------------------------- /docs/errata.rst: -------------------------------------------------------------------------------- 1 | Errata 2 | ====== 3 | 4 | Please check back. 5 | 6 | (Last updated, April 29, 2018) -------------------------------------------------------------------------------- /docs/day2/patterns/index.rst: -------------------------------------------------------------------------------- 1 | NN Patterns 2 | =========== 3 | 4 | .. toctree:: 5 | 6 | attention 7 | -------------------------------------------------------------------------------- /docs/_build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/objects.inv -------------------------------------------------------------------------------- /docs/_build/doctrees/faq.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/doctrees/faq.doctree -------------------------------------------------------------------------------- /docs/_build/html/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/up.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | nltk 3 | annoy 4 | seaborn 5 | numpy 6 | matplotlib 7 | scikit-learn 8 | tqdm 9 | ipywidgets 10 | -------------------------------------------------------------------------------- /docs/_build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/doctrees/index.doctree -------------------------------------------------------------------------------- /docs/_build/html/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/down.png -------------------------------------------------------------------------------- /docs/_build/html/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/file.png -------------------------------------------------------------------------------- /docs/_build/html/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/minus.png -------------------------------------------------------------------------------- /docs/_build/html/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/plus.png -------------------------------------------------------------------------------- /docs/_build/html/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/comment.png -------------------------------------------------------------------------------- /docs/_build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/_build/html/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/up-pressed.png -------------------------------------------------------------------------------- /docs/_build/doctrees/download_data.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/doctrees/download_data.doctree -------------------------------------------------------------------------------- /docs/_build/html/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/_build/html/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/comment-close.png -------------------------------------------------------------------------------- /docs/_build/html/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/_build/html/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /docs/day1/index.rst: -------------------------------------------------------------------------------- 1 | Day 1 2 | ===== 3 | 4 | Here you will find things from Day 1! 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | takehome 10 | -------------------------------------------------------------------------------- /docs/_build/doctrees/environment_setup.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/doctrees/environment_setup.doctree -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/day2/adventures/index.rst: -------------------------------------------------------------------------------- 1 | Choose Your Own Adventures 2 | ========================== 3 | 4 | .. toctree:: 5 | 6 | interpolation 7 | lookups 8 | lvg 9 | 10 | -------------------------------------------------------------------------------- /docs/_build/html/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joosthub/pytorch-nlp-tutorial-ny2018/master/docs/_build/html/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/_build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 35c45cfadc00e89350681f38ef462d90 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/recipes/index.rst: -------------------------------------------------------------------------------- 1 | Recipes and PyTorch patterns 2 | ============================ 3 | 4 | 5 | In this section, you will find a set of recipes for doing various things with PyTorch. 6 | 7 | .. toctree:: 8 | 9 | load_pretrained_vectors 10 | compute_conv_size 11 | -------------------------------------------------------------------------------- /docs/day2/patterns/gating.rst: -------------------------------------------------------------------------------- 1 | Gating 2 | ====== 3 | 4 | 5 | 6 | .. code-block:: python 7 | 8 | 9 | resetgate = F.sigmoid(i_r + h_r) 10 | inputgate = F.sigmoid(i_i + h_i) 11 | newgate = F.tanh(i_n + resetgate * h_n) 12 | hy = newgate + inputgate * (hidden - newgate) 13 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # Data Folder 2 | 3 | There are several files you will need. you can download them in zip format 4 | from [here](https://drive.google.com/file/d/0B2hg7DTHpfLsdHhEUVhHWU5hUXc/view?usp=sharing). 5 | 6 | 7 | The files you need are: 8 | 9 | - surnames.csv 10 | - trump.csv 11 | - glove.6B.100d.txt 12 | - zhnews.csv 13 | - firstnames.csv 14 | - amazon_train_small.csv 15 | 16 | -------------------------------------------------------------------------------- /docs/migration.rst: -------------------------------------------------------------------------------- 1 | Migrating to PyTorch 0.4.0 2 | ========================== 3 | 4 | The training session at O'Reilly AI in NYC, 2018 will be conducted using PyTorch 0.3.1. However, as of the end of April, PyTorch 0.4.0 has been released! 5 | 6 | To help you understand how to migrate, the PyTorch folks have a wonderful migration guide found `here `_. 7 | 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Natural Language Processing (NLP) with PyTorch 2 | 3 | Welcome to the repository for the training tutorial as the 2018 O'Reilly AI Conference 4 | in NYC on April 29 and 30, 2018. 5 | 6 | You can visit the Read-The-Docs rendering of this repository at [dl4nlp.info](http://dl4nlp.info) or at the longer urls of [http://pytorch-nlp-tutorial-ny2018.readthedocs.io](http://pytorch-nlp-tutorial-ny2018.readthedocs.io/en/latest/) 7 | 8 | -------------------------------------------------------------------------------- /docs/recipes/compute_conv_size.rst: -------------------------------------------------------------------------------- 1 | Compute Convolution Sizes 2 | ========================= 3 | 4 | 5 | .. code-block:: python 6 | 7 | import math 8 | 9 | def conv_shape_helper_1d(input_seq_len, kernel_size, stride=1, padding=0, dilation=1): 10 | kernel_width = dilation * (kernel_size - 1) + 1 11 | tensor_size = input_seq_len + 2 * padding 12 | return math.floor((tensor_size - kernel_width) / stride + 1) 13 | -------------------------------------------------------------------------------- /docs/day1/takehome.rst: -------------------------------------------------------------------------------- 1 | Take-Home Exercises 2 | =================== 3 | 4 | Exercise 1 5 | ---------- 6 | 7 | Implement Deep Continuous Bag-of-Words (CBOW). 8 | `Here is a link to the paper! `_ 9 | 10 | 11 | Exercise 2 12 | ---------- 13 | 14 | Complete ConvNet example to do evaluation (in the same manner as the MLP example). 15 | 16 | Exercise 3 17 | ---------- 18 | 19 | Implement a convnet classifier to classify names 20 | 21 | Things to try: with and without padding and changing strides. 22 | 23 | -------------------------------------------------------------------------------- /modelzoo/README.md: -------------------------------------------------------------------------------- 1 | # Model Zoo files 2 | 3 | In this folder, you should place the saved states that we have pre-trained. 4 | 5 | You can download them from [here](https://drive.google.com/file/d/0B2hg7DTHpfLsZW44aTRVd2FrbEE/view?usp=sharing) 6 | 7 | 8 | You should have the following files: 9 | 10 | - trump_twitter.vocab 11 | - surnames_classify.vocab 12 | - charnn_emb16_hid64_surnames_classify.state 13 | - charnn_emb16_hid64_surnames_predict.state 14 | - charnn_emb16_hid64_surnames_conditionally_predict.state 15 | - wordrnn_emb100_hid64_trump_tweets_predict_fresh_train_8_min.state 16 | - wordrnn_emb100_hid64_trump_tweets_predict.state 17 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = pytorch-nlp-tutorial-sf2017 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | Frequency Asked Questions 2 | ========================= 3 | 4 | On this page, you will find a list of questions that we either anticipate 5 | people will ask or that we have been asked previously. They are intended to 6 | be the first stop for any confusion or trouble that might occur. 7 | 8 | 9 | Do I Need to have a NVIDIA GPU enabled laptop? 10 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 11 | 12 | Nope! While having a NVIDIA GPU enabled laptop will make the training run 13 | faster, we provide instructions for people who do not have one. 14 | 15 | If you are plan on working on Natural Language Processing/Deep Learning in the future, 16 | a GPU enabled laptop might be a good investment. 17 | -------------------------------------------------------------------------------- /docs/_build/html/_sources/faq.rst.txt: -------------------------------------------------------------------------------- 1 | Frequency Asked Questions 2 | ========================= 3 | 4 | On this page, you will find a list of questions that we either anticipate 5 | people will ask or that we have been asked previously. They are intended to 6 | be the first stop for any confusion or trouble that might occur. 7 | 8 | 9 | Do I Need to have a NVIDIA GPU enabled laptop? 10 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 11 | 12 | Nope! While having a NVIDIA GPU enabled laptop will make the training run 13 | faster, we provide instructions for people who do not have one. 14 | 15 | If you are plan on working on Natural Language Processing/Deep Learning in the future, 16 | a GPU enabled laptop might be a good investment. 17 | -------------------------------------------------------------------------------- /docs/day2/warmup.rst: -------------------------------------------------------------------------------- 1 | Warm Up Exercise 2 | ================ 3 | 4 | To get you back into the PyTorch groove, let's do some easy exercises. You will have 10 minutes. See how far you can get. 5 | 6 | 1. Use :code:`torch.randn` to create two tensors of size (29, 30, 32) and and (32, 100). 7 | 2. Use :code:`torch.matmul` to matrix multiply the two tensors. 8 | 3. Use :code:`torch.sum` on the resulting tensor, passing the optional argument of :code:`dim=1` to sum across the 1st dimension. Before you run this, can you predict the size? 9 | 4. Create a new long tensor of size (3, 10) from the :code:`np.random.randint` method. 10 | 5. Use this new long tensor to index into the tensor from step 3. 11 | 6. Use :code:`torch.mean` to average across the last dimension in the tensor from step 5. 12 | -------------------------------------------------------------------------------- /docs/day2/tensorfu1.rst: -------------------------------------------------------------------------------- 1 | Tensor-Fu-1 2 | =========== 3 | 4 | 5 | Exercise 1 6 | ---------- 7 | 8 | .. code-block:: python 9 | 10 | import torch 11 | from torch import nn 12 | x = torch.randn(9, 10) 13 | 14 | 15 | Exercise 2 16 | ---------- 17 | 18 | .. code-block:: python 19 | 20 | import torch 21 | from torch import nn 22 | 23 | x2dim = torch.randn(9, 10) 24 | 25 | # required and default parameters: 26 | # fc = nn.Linear(in_features, out_features) 27 | 28 | Task: Create a linear layer which works wih x2dim 29 | 30 | 31 | Exercise 3 32 | ---------- 33 | 34 | 35 | .. code-block:: python 36 | 37 | import torch 38 | from torch import nn 39 | 40 | x3dim = torch.randn(9, 10, 11) 41 | 42 | # required and default parameters: 43 | # conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, stride=1, padding=0) 44 | 45 | Task: Create a convolution which works on x3dim 46 | 47 | 48 | -------------------------------------------------------------------------------- /docs/day2/tensorfu2.rst: -------------------------------------------------------------------------------- 1 | Tensor-Fu-2 2 | =========== 3 | 4 | Exercise 1 5 | ---------- 6 | 7 | .. code-block:: python 8 | 9 | indices = torch.arange(10).long() 10 | indices = torch.from_numpy(np.random.randint(0, 10, size=(10,))) 11 | 12 | emb = nn.Embedding(num_embeddings=100, embedding_dim=16) 13 | emb(indices) 14 | 15 | Task: Get the above code to work. 16 | Use the second indices method and change the size to a matrix (such as (10,11)). 17 | 18 | Exercise 2 19 | ---------- 20 | 21 | Task: Create a MultiEmbedding class which can input two sets of indices, embed them, and concat the results! 22 | 23 | .. code-block:: python 24 | 25 | class MultiEmbedding(nn.Module): 26 | def __init__(self, num_embeddings1, num_embeddings2, embedding_dim1, embedding_dim2): 27 | pass 28 | 29 | def forward(self, indices1, indices2): 30 | # use something like 31 | # z = torch.concat([x, y], dim=1) 32 | 33 | pass -------------------------------------------------------------------------------- /docs/download_data.rst: -------------------------------------------------------------------------------- 1 | Getting the Data 2 | ================ 3 | 4 | In this training, there are two options of participating. 5 | 6 | Option 1: Download and Setup things on your laptop 7 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 8 | 9 | The first option is to download the data below, setup the environment, and download the notebooks when we make them available. 10 | If you choose this options but do not download the data before the first day, we will have several flash drives with the data on it. 11 | 12 | Please visit `this link `_ to download the data. 13 | 14 | 15 | Option 2: Use O'Reilly's online resource through your browser 16 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 17 | 18 | The second option is to use an online resource provided by O'Reilly. On the first day of this training, you will be provided with a link to a JupyterHub instance where the environment will be pre-made and ready to go! If you choose this option, you do not have to do anything until you arrive on Sunday. 19 | You are still required to bring your laptop. 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Joostware AI Research Corp. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/_build/html/_sources/download_data.rst.txt: -------------------------------------------------------------------------------- 1 | Getting the Data 2 | ================ 3 | 4 | In this training, there are two options of participating. 5 | 6 | Option 1: Download and Setup things on your laptop 7 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 8 | 9 | The first option is to download the data below, setup the environment, and download the notebooks when we make them available. 10 | If you choose this options but do not download the data before the first day, we will have several flash drives with the data on it. 11 | 12 | Please visit `this link `_ to download the data. 13 | 14 | 15 | Option 2: Use O'Reilly's online resource through your browser 16 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 17 | 18 | The second option is to use an online resource provided by O'Reilly. On the first day of this training, you will be provided with a link to a JupyterHub instance where the environment will be pre-made and ready to go! If you choose this option, you do not have to do anything until you arrive on Sunday. 19 | You are still required to bring your laptop. 20 | 21 | -------------------------------------------------------------------------------- /docs/recipes/load_pretrained_vectors.rst: -------------------------------------------------------------------------------- 1 | Loading Pretrained Vectors 2 | ========================== 3 | 4 | It can be extremely useful to make a model which had as advantageous starting point. 5 | 6 | To do this, we can set the values of the embedding matrix. 7 | 8 | 9 | .. code-block:: python 10 | 11 | # we give an example of this function in the day 1, word vector notebook 12 | word_to_index, word_vectors, word_vector_size = load_word_vectors() 13 | 14 | 15 | # now, we want to iterate over our vocabulary items 16 | for word, emb_index in vectorizer.word_vocab.items(): 17 | # if the word is in the loaded glove vectors 18 | if word.lower() in word_to_index: 19 | # get the index into the glove vectors 20 | glove_index = word_to_index[word.lower()] 21 | # get the glove vector itself and convert to pytorch structure 22 | glove_vec = torch.FloatTensor(word_vectors[glove_index]) 23 | 24 | # this only matters if using cuda :) 25 | if settings.CUDA: 26 | glove_vec = glove_vec.cuda() 27 | 28 | # finally, if net is our network, and emb is the embedding layer: 29 | net.emb.weight.data[emb_index, :].set_(glove_vec) 30 | -------------------------------------------------------------------------------- /LOCAL_RUN_README.md: -------------------------------------------------------------------------------- 1 | # Running this repository locally 2 | 3 | During the training session, O'Reilly will be providing a JupyterHub for participants. 4 | However, during that time and after the training ends, if you would like to run the contents of this repository, you can do so with the following steps: 5 | 6 | 1. Follow the instructions on [environment setup](http://dl4nlp.info/en/latest/environment_setup.html). 7 | 2. Download the files in the data/ and modelzoo/ folders. The download link is in the `README.md` file in each respective folder. The downloaded files will be zip files. They should be unzipped so that the files are inside these folders. 8 | - in other word, it should be `data/surnames.csv` etc. 9 | - Please make sure this is correct! It may unzip with an extra directory and be `data/data/surnames.csv` and this would be incorrect. 10 | 3. In a terminal with the `dl4nlp` (or equivalent) environment activated (see step 1 if you don't have this), navigate back to the top level (so you can see day_1, etc) and run the following command: 11 | 12 | ```bash 13 | DL4NLPROOT=$(pwd) jupyter notebook --notebook-dir=$(pwd) 14 | ``` 15 | 16 | If you would like to run with CUDA enabled, please run: 17 | 18 | ```bash 19 | ENABLE_CUDA_DL4NLP=1 DL4NLPROOT=$(pwd) jupyter notebook --notebook-dir=$(pwd) 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/day2/adventures/lvg.rst: -------------------------------------------------------------------------------- 1 | A New Load-Vectorize-Generate 2 | ============================= 3 | 4 | In this exercise, you should look into the two datasets that are not included in the exercises. There are two datasets to work with. The first is the Amazon Review dataset. 5 | 6 | .. code-block:: python 7 | 8 | from local_settings import settings 9 | import pandas as pd 10 | 11 | data = pd.read_csv(settings.AMAZON_FILENAME, names=['rating', 'title', 'review']) 12 | print(data.head()) 13 | 14 | The Amazon Reviews Dataset does not come with a precompute train-test split. One thing that would be important is to select a subset to do that. 15 | 16 | The other is the first names dataset. You can load with: 17 | 18 | .. code-block:: python 19 | 20 | from local_settings import settings 21 | import pandas as pd 22 | 23 | data = pd.read_csv(settings.FIRSTNAMES_CSV) 24 | print(data.head()) 25 | 26 | 27 | For these two datasets, you should write a Raw dataset which loads the data. Then, you should write a Vectorizer which creates the relevant vocabularies from the 'fit' method and transforms a raw dataset into a vectorized dataset using the 'transform' method. Finally, you should write a Vectorized datset which implements the required :code:`__len__` and :code:`__getitem__` methods. 28 | 29 | The make_generator can be reused. 30 | -------------------------------------------------------------------------------- /docs/day2/failfastprototypemode.rst: -------------------------------------------------------------------------------- 1 | Fail Fast Prototype Mode 2 | ======================== 3 | 4 | When building neural networks, you want things to either work or fail fast. Long iteration loops are the truest enemy of the machine learning practitioner. 5 | 6 | 7 | To that end, the following techniques will help you out. 8 | 9 | .. code-block:: python 10 | 11 | import torch 12 | from torch import nn 13 | from torch.autograd import Variable 14 | # note, Variable deprecates in 0.4.0 15 | 16 | # 2dim tensor.. aka a matrix 17 | x = Variable(torch.randn(4, 5)) 18 | 19 | # this is the same as: 20 | batch_size = 4 21 | feature_size = 5 22 | x = Variable(torch.randn(batch_size, feature_size)) 23 | 24 | 25 | You can construct whatever prototype variables you want doing this. 26 | 27 | Prototyping an embedding 28 | ^^^^^^^^^^^^^^^^^^^^^^^^ 29 | 30 | 31 | .. code-block:: python 32 | 33 | import torch 34 | from torch import nn 35 | from torch.autograd import Variable 36 | # note, Variable deprecates in 0.4.0 37 | 38 | batch_size = 4 39 | sequence_size = 5 40 | integer_range = 100 41 | embedding_size = 25 42 | # notice rand vs randn. rand is uniform (0,1), and randn is normal (-1,1) 43 | random_numbers = torch.rand(batch_size, sequence_size) * integer_range 44 | x = Variable(random_numbers.long()) 45 | 46 | embedder = nn.Embedding(num_embeddings=integer_range, 47 | embedding_dim=embedding_size) 48 | 49 | print(embedder(x).shape) 50 | 51 | 52 | -------------------------------------------------------------------------------- /docs/day1/solutions.rst: -------------------------------------------------------------------------------- 1 | Solutions 2 | ========= 3 | 4 | Problem 1 5 | --------- 6 | 7 | .. code-block:: python 8 | 9 | def f(x): 10 | if x.data[0] > 0: 11 | return torch.sin(x) 12 | else: 13 | return torch.cos(x) 14 | 15 | x = torch.autograd.Variable(torch.FloatTensor([1]), 16 | requires_grad=True) 17 | 18 | y = f(x) 19 | print(y) 20 | 21 | y.backward() 22 | 23 | x.grad 24 | 25 | y.grad_fn 26 | 27 | Problem 2 28 | --------- 29 | 30 | .. code-block:: python 31 | 32 | def cbow(phrase): 33 | words = phrase.split(" ") 34 | embeddings = [] 35 | for word in words: 36 | if word in glove.word_to_index: 37 | embeddings.append(glove.get_embedding(word)) 38 | embeddings = np.stack(embeddings) 39 | return np.mean(embeddings, axis=0) 40 | 41 | cbow("the dog flew over the moon").shape 42 | 43 | # >> (100,) 44 | 45 | def cbow_sim(phrase1, phrase2): 46 | vec1 = cbow(phrase1) 47 | vec2 = cbow(phrase2) 48 | return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2)) 49 | 50 | cbow_sim("green apple", "green apple") 51 | # >> 1.0 52 | 53 | cbow_sim("green apple", "apple green") 54 | # >> 1.0 55 | 56 | cbow_sim("green apple", "red potato") 57 | # >> 0.749 58 | 59 | cbow_sim("green apple", "green alien") 60 | # >> 0.683 61 | 62 | cbow_sim("green apple", "blue alien") 63 | # >> 0.5799815958114477 64 | 65 | cbow_sim("eat an apple", "ingest an apple") 66 | # >> 0.9304712574359718 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | day_1/data/* 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | docs/_build 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | .DS_Store 105 | amazon_train_small.csv 106 | glove.6B.zip 107 | names_test.csv 108 | names_test_delip_version.csv 109 | names_train.csv 110 | snli_1.0.zip 111 | trump.csv 112 | data/ 113 | -------------------------------------------------------------------------------- /docs/day2/adventures/interpolation.rst: -------------------------------------------------------------------------------- 1 | Exercise: Interpolating Between Vectors 2 | ======================================= 3 | 4 | One fun option for the conditional generation code is to interpolate 5 | between the learned hidden vectors. 6 | 7 | To do this, first look at the code for sampling given a specific nationality: 8 | 9 | .. code-block:: python 10 | :linenos: 11 | 12 | def sample_n_for_nationality(nationality, n=10, temp=0.8): 13 | assert nationality in vectorizer.nationality_vocab.keys(), 'not a nationality we trained on' 14 | keys = [nationality] * n 15 | init_vector = long_variable([vectorizer.nationality_vocab[key] for key in keys]) 16 | init_vector = net.conditional_emb(init_vector) 17 | samples = decode_matrix(vectorizer, 18 | sample(net.emb, net.rnn, net.fc, 19 | init_vector, 20 | make_initial_x(n, vectorizer), 21 | temp=temp)) 22 | return list(zip(keys, samples)) 23 | 24 | As you can see, we create a list of keys that is the length of the number of samples we want (n). 25 | And we use that list to retrieve the correct index from the vocabulary. 26 | Finally, we use that index in the conditional embedding inside the network to get the 27 | initial hidden state for the sampler. 28 | 29 | To do this exercise, write a function that has the following signature: 30 | 31 | .. code-block:: python 32 | 33 | def interpolate_n_samples_from_two_nationalities(nationality1, nationality2, weight, n=10, temp=0.8): 34 | print('awesome stuff here') 35 | 36 | 37 | This should retrieve the :code:`init_vectors` for two different nationalities. Then, using the weight, combine the init vectors as :code:`weight * init_vector1 + (1 - weight) * init_vector2`. 38 | 39 | For fun, after you finish this function, write a for loop which loops over the weight from 0.1 to 0.9 to see how it affects the generation. 40 | -------------------------------------------------------------------------------- /docs/day2/patterns/attention.rst: -------------------------------------------------------------------------------- 1 | Design Pattern: Attention 2 | ========================= 3 | 4 | Attention is a useful pattern for when you want to take a collection of vectors---whether it be a sequence of vectors representing a sequence of words, or an unordered collections of vectors representing a collection of attributes---and summarize them into a single vector. This has similar analogs to the CBOW examples we saw on Day 1, but instead of just averaging or using max pooling, we are learning a function which learns to compute the weights for each of the vectors before summing them together. 5 | 6 | Importantly, the weights that the attention module is learning is a valid probability distribution. This means that weighting the vectors by the value the attention module learns can additionally be seen as computing the Expection. Or, it could as interpolating. In any case, attention's main use is to select 'softly' amongst a set of vectors. 7 | 8 | The attention vector has several different published forms. The one below is very simple and just learns a single vector as the attention mechanism. 9 | 10 | Using the :code:`new_parameter` function we have been using for the RNN notebooks: 11 | 12 | .. code-block:: python 13 | 14 | def new_parameter(*size): 15 | out = Parameter(FloatTensor(*size)) 16 | torch.nn.init.xavier_normal(out) 17 | return out 18 | 19 | We can then do: 20 | 21 | .. code-block:: python 22 | 23 | class Attention(nn.Module): 24 | def __init__(self, attention_size): 25 | super(Attention, self).__init__() 26 | self.attention = new_parameter(attention_size, 1) 27 | 28 | def forward(self, x_in): 29 | # after this, we have (batch, dim1) with a diff weight per each cell 30 | attention_score = torch.matmul(x_in, self.attention).squeeze() 31 | attention_score = F.softmax(attention_score).view(x_in.size(0), x_in.size(1), 1) 32 | scored_x = x_in * attention_score 33 | 34 | # now, sum across dim 1 to get the expected feature vector 35 | condensed_x = torch.sum(scored_x, dim=1) 36 | 37 | return condensed_x 38 | 39 | 40 | 41 | attn = Attention(100) 42 | x = Variable(torch.randn(16,30,100)) 43 | attn(x).size() == (16,100) 44 | 45 | 46 | -------------------------------------------------------------------------------- /docs/day2/adventures/lookups.rst: -------------------------------------------------------------------------------- 1 | Exercise: Fast Lookups for Encoded Sequences 2 | ========================================== 3 | 4 | Let's suppose that you want to embed or encode something that you want to look up at a later date. 5 | For example, you could be embedded things that need to be identified (such as a song). Or maybe you want to just find the neighbors of a new data point. 6 | 7 | In any case, using the approximate nearest neighbors libraries are wonderful for this. 8 | For this exercise, we will use Spotify's annoy library (we saw this on day 1, in the pretrained word vector notebook). You should aim to complete the following steps: 9 | 10 | 1. Load the network from the Day 2, 01 notebook using the pre-trained weights. 11 | - You could use the 02 notebook, but we want to get a single vector per each sequence. 12 | - So, to use 02, you would need to port the :code:`column_gather` function. 13 | - One reason why you might be interested in doing this is because the 02 objective function learned a better final vector representation. 14 | 2. Given a loaded network with pre-trained weights, write a function which does nearly exactly what the forward function does, but doesn't apply the fully connected layer. 15 | - This is because we want the feature vector just before the fully connected. 16 | - it is common to assume that the penultimate layer has learned more generalizable features than the final layer (which is used in softmax computations and is this used to being normalize inducing a probability distribution). 17 | - The code for this shoud look something like: 18 | 19 | .. code-block:: python 20 | 21 | def get_penultimate(net, x_in, x_lengths=None): 22 | x_in = net.emb(x_in) 23 | x_mid = net.conv(x_in.permute(0, 2, 1)).permute(0, 2, 1) 24 | y_out = net.rnn(x_in) 25 | 26 | if x_lengths is not None: 27 | y_out = column_gather(y_out, x_lengths) 28 | else: 29 | y_out = y_out[:, -1, :] 30 | 31 | return y_out 32 | 33 | 3. As you get penultimate vectors for each datapoint, store them in spotify's annoy. This requires specifying some label for the vector. Using :code:`vectorizer.surname_vocab.lookup` is how you can retrieve the character for each index value in the network inputs. There are some 'decode' functions in the day 2 02 and 03 notebooks. 34 | 4. Once everything is added to spotify's annoy, you can then look up any surname and find the set of nearest neighbors! Kind of cool! this is one way to do the `k nearest neighbor classification rule `_. 35 | -------------------------------------------------------------------------------- /docs/day2/sampling.rst: -------------------------------------------------------------------------------- 1 | Exercise: Sampling from an RNN 2 | ============================== 3 | 4 | The goal of sampling from an RNN is to initialize the sequence in some way, feed it into the recurrent computation, and retrieve the next prediction. 5 | 6 | To start, we create the initial vectors: 7 | 8 | .. code-block:: python 9 | 10 | start_index = vectorizer.surname_vocab.start_index 11 | batch_size = 2 12 | # hidden_size = whatever hidden size the model is set to 13 | 14 | initial_h = Variable(torch.ones(batch_size, hidden_size)) 15 | initial_x_index = Variable(torch.ones(batch_size).long()) * start_index 16 | 17 | Then, we need to use these vectors to retrieve the next prediction: 18 | 19 | .. code-block:: python 20 | 21 | # model is stored in variable called `net` 22 | 23 | x_t = net.emb(initial_x_index) 24 | print(x_t.shape) 25 | h_t = net.rnn._compute_next_hidden(x_t, initial_h) 26 | 27 | y_t = net.fc(h_t) 28 | 29 | 30 | Now that we have a prediction vector, we can create a probability distribution and sample from it. Note we include a temperature hyper parameter for controlling how strongly we sample from the distribution (at high temperatures, everything is uniform, at low temperatures below 1, small differences are magnified). The temperature is always greater than 0. 31 | 32 | .. code-block:: python 33 | 34 | temperature = 1.0 35 | prediction_vector = F.softmax(y_t / temperature, dim=1) 36 | x_index_t = torch.multinomial(y_t, 1)[:, 0] 37 | 38 | 39 | Now we can start the cycle over again: 40 | 41 | .. code-block:: python 42 | 43 | x_t = net.emb(x_index_t) 44 | h_t = net.rnn._compute_next_hidden(x_t, h_t) 45 | 46 | y_t = net.fc(h_t) 47 | 48 | Write a for loop which repeats this sequence and appends the x_t variable to a list. 49 | 50 | Then, we can do the following: 51 | 52 | .. code-block:: python 53 | 54 | final_x_indices = torch.stack(x_indices).squeeze().permute(1, 0) 55 | 56 | # stop here if you don't know what cpu, data, and numpy do. Ask away! 57 | final_x_indices = final_x_indices.cpu().data.numpy() 58 | 59 | # loop over the items in the batch 60 | results = [] 61 | for i in range(len(final_x_indices)): 62 | tokens = [] 63 | index_vector = final_x_indices[i] 64 | for x_index in index_vector: 65 | if vectorizer.surname_vocab.start_index == x_index: 66 | continue 67 | elif vectorizer.surname_vocab.end_index == x_index: 68 | break 69 | else: 70 | token = vectorizer.surname_vocab.lookup(x_index) 71 | tokens.append(token) 72 | 73 | sampled_surname = "".join(tokens) 74 | results.append(sampled_surname) 75 | tokens = [] -------------------------------------------------------------------------------- /docs/_build/html/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. pytorch-nlp-tutorial-sf2017 documentation master file, created by 2 | sphinx-quickstart on Sun Sep 3 13:31:44 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Natural Language Processing (NLP) with PyTorch 7 | ============================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :hidden: 12 | :caption: Extra Resources 13 | 14 | download_data 15 | environment_setup 16 | faq 17 | 18 | .. toctree:: 19 | :hidden: 20 | :caption: Day 1 Materials 21 | 22 | 23 | .. toctree:: 24 | :hidden: 25 | :maxdepth: 3 26 | :caption: Day 2 Materials 27 | 28 | 29 | 30 | Hello! This is a directory of resources for a training tutorial to be 31 | given at the O'Reilly AI Conference in New York City on Sunday, April 29, and Monday, April 30, 2018. 32 | 33 | Please read below for general information. You can find the github repository at `this link `_. Please note that there are two ways to engage in this training (desribed below). 34 | 35 | More information will be added to this site as the training progresses. 36 | Specifically, we will be adding a 'recipes' section, 'errata' section, and a 'bonus exercise' section as the training progresses! 37 | 38 | General Information 39 | ------------------- 40 | 41 | Prerequisites: 42 | ^^^^^^^^^^^^^ 43 | 44 | - A working knowledge of Python and the command line 45 | - Familiarity with precalc math (multiply matrices, dot products of vectors, etc.) and derivatives of simple functions (If you are new to linear algebra, this video course is handy.) 46 | - A general understanding of machine learning (setting up experiments, evaluation, etc.) (useful but not required) 47 | 48 | Hardware and/or installation requirements: 49 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 50 | 51 | - There are two options: 52 | 1. **Using O'Reilly's online resources**. For this, you only needs a laptop; on the first day, we will provide you with credentials and a URL to use an online computing resource (a JupyterHub instance) provided by O'Reilly. You will be able to access Jupyter notebooks through this and they will persist until the end of the second day of training (September 18th). This option is not limited by what operating system you have. You will need to have a browser installed. 53 | 2. **Setting everything up locally**. For this, you need a laptop with the PyTorch environment set up. This is only recommended if you want to have the environment locally or have a laptop with a GPU. (If you have trouble following the provided instructions or if you find any mistakes, please file an issue `here `_.) This option is limited to Macs and Linux users only (sorry Windows users!). Be sure you check the `LOCAL_RUN_README.md `_. 54 | -------------------------------------------------------------------------------- /docs/_build/html/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;overflow-y:scroll;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{max-height:100%}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pytorch-nlp-tutorial-sf2017 documentation master file, created by 2 | sphinx-quickstart on Sun Sep 3 13:31:44 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Natural Language Processing (NLP) with PyTorch 7 | ============================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :hidden: 12 | :caption: Extra Resources 13 | 14 | download_data 15 | environment_setup 16 | faq 17 | migration 18 | 19 | .. toctree:: 20 | :hidden: 21 | :caption: Day 1 Materials 22 | 23 | day1/solutions 24 | 25 | .. toctree:: 26 | :hidden: 27 | :maxdepth: 3 28 | :caption: Day 2 Materials 29 | 30 | day2/warmup 31 | day2/failfastprototypemode 32 | day2/tensorfu1 33 | day2/tensorfu2 34 | day2/adventures/interpolation 35 | day2/sampling 36 | day2/patterns/attention 37 | 38 | 39 | 40 | For participants of the Training Tutorial in NY, please fill out this form! 41 | https://goo.gl/forms/iLRlpoutWBy3As8Q2 42 | 43 | Hello! This is a directory of resources for a training tutorial to be 44 | given at the O'Reilly AI Conference in New York City on Sunday, April 29, and Monday, April 30, 2018. 45 | 46 | Please read below for general information. You can find the github repository at `this link `_. Please note that there are two ways to engage in this training (desribed below). 47 | 48 | More information will be added to this site as the training progresses. 49 | Specifically, we will be adding a 'recipes' section, 'errata' section, and a 'bonus exercise' section as the training progresses! 50 | 51 | General Information 52 | ------------------- 53 | 54 | Prerequisites: 55 | ^^^^^^^^^^^^^ 56 | 57 | - A working knowledge of Python and the command line 58 | - Familiarity with precalc math (multiply matrices, dot products of vectors, etc.) and derivatives of simple functions (If you are new to linear algebra, this video course is handy.) 59 | - A general understanding of machine learning (setting up experiments, evaluation, etc.) (useful but not required) 60 | 61 | Hardware and/or installation requirements: 62 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 63 | 64 | - There are two options: 65 | 1. **Using O'Reilly's online resources**. For this, you only needs a laptop; on the first day, we will provide you with credentials and a URL to use an online computing resource (a JupyterHub instance) provided by O'Reilly. You will be able to access Jupyter notebooks through this and they will persist until the end of the second day of training (April 30th). This option is not limited by what operating system you have. You will need to have a browser installed. 66 | 2. **Setting everything up locally**. For this, you need a laptop with the PyTorch environment set up. This is only recommended if you want to have the environment locally or have a laptop with a GPU. (If you have trouble following the provided instructions or if you find any mistakes, please file an issue `here `_.) This option is limited to Macs and Linux users only (sorry Windows users!). Be sure you check the `LOCAL_RUN_README.md `_. 67 | -------------------------------------------------------------------------------- /docs/_build/html/_static/js/theme.js: -------------------------------------------------------------------------------- 1 | require=function r(s,a,l){function c(i,n){if(!a[i]){if(!s[i]){var e="function"==typeof require&&require;if(!n&&e)return e(i,!0);if(u)return u(i,!0);var t=new Error("Cannot find module '"+i+"'");throw t.code="MODULE_NOT_FOUND",t}var o=a[i]={exports:{}};s[i][0].call(o.exports,function(n){var e=s[i][1][n];return c(e||n)},o,o.exports,r,s,a,l)}return a[i].exports}for(var u="function"==typeof require&&require,n=0;n"),i("table.docutils.footnote").wrap("
"),i("table.docutils.citation").wrap("
"),i(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var e=i(this);expand=i(''),expand.on("click",function(n){return t.toggleCurrent(e),n.stopPropagation(),!1}),e.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}0this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav}),function(){for(var r=0,n=["ms","moz","webkit","o"],e=0;e 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Index — NLP with PyTorch documentation 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
43 | 44 | 45 | 93 | 94 |
95 | 96 | 97 | 103 | 104 | 105 |
106 | 107 |
108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 |
126 | 127 |
    128 | 129 |
  • Docs »
    • 130 | 131 |
  • Index
    • 132 | 133 | 134 |
  • 135 | 136 | 137 | 138 |
    • 139 | 140 |
141 | 142 | 143 |
144 |
145 |
146 |
147 | 148 | 149 |

Index

150 | 151 |
152 | 153 |
154 | 155 | 156 |
157 | 158 |
159 |
160 | 161 | 162 |
163 | 164 |
165 |

166 | © Copyright 2017, Brian McMahan and Delip Rao. 167 | 168 |

169 |
170 | Built with Sphinx using a theme provided by Read the Docs. 171 | 172 |
173 | 174 |
175 |
176 | 177 |
178 | 179 |
180 | 181 | 182 | 183 | 184 | 185 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 215 | 216 | 217 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # pytorch-nlp-tutorial-sf2017 documentation build configuration file, created by 5 | # sphinx-quickstart on Sun Sep 3 13:31:44 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.mathjax'] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = 'NLP with PyTorch' 50 | copyright = '2017, Brian McMahan and Delip Rao' 51 | author = 'Brian McMahan and Delip Rao' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This patterns also effect to html_static_path and html_extra_path 72 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = 'sphinx' 76 | 77 | # If true, `todo` and `todoList` produce output, else they produce nothing. 78 | todo_include_todos = False 79 | 80 | 81 | # -- Options for HTML output ---------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = 'sphinx_rtd_theme' 87 | 88 | # Theme options are theme-specific and customize the look and feel of a theme 89 | # further. For a list of options available for each theme, see the 90 | # documentation. 91 | # 92 | # html_theme_options = {} 93 | 94 | # Add any paths that contain custom static files (such as style sheets) here, 95 | # relative to this directory. They are copied after the builtin static files, 96 | # so a file named "default.css" will overwrite the builtin "default.css". 97 | html_static_path = ['_static'] 98 | 99 | 100 | # -- Options for HTMLHelp output ------------------------------------------ 101 | 102 | # Output file base name for HTML help builder. 103 | htmlhelp_basename = 'pytorch-nlp-tutorial-sf2017doc' 104 | 105 | 106 | # -- Options for LaTeX output --------------------------------------------- 107 | 108 | latex_elements = { 109 | # The paper size ('letterpaper' or 'a4paper'). 110 | # 111 | # 'papersize': 'letterpaper', 112 | 113 | # The font size ('10pt', '11pt' or '12pt'). 114 | # 115 | # 'pointsize': '10pt', 116 | 117 | # Additional stuff for the LaTeX preamble. 118 | # 119 | # 'preamble': '', 120 | 121 | # Latex figure (float) alignment 122 | # 123 | # 'figure_align': 'htbp', 124 | } 125 | 126 | # Grouping the document tree into LaTeX files. List of tuples 127 | # (source start file, target name, title, 128 | # author, documentclass [howto, manual, or own class]). 129 | latex_documents = [ 130 | (master_doc, 'pytorch-nlp-tutorial-sf2017.tex', 'pytorch-nlp-tutorial-sf2017 Documentation', 131 | 'Brian McMahan and Delip Rao', 'manual'), 132 | ] 133 | 134 | 135 | # -- Options for manual page output --------------------------------------- 136 | 137 | # One entry per manual page. List of tuples 138 | # (source start file, name, description, authors, manual section). 139 | man_pages = [ 140 | (master_doc, 'pytorch-nlp-tutorial-sf2017', 'pytorch-nlp-tutorial-sf2017 Documentation', 141 | [author], 1) 142 | ] 143 | 144 | 145 | # -- Options for Texinfo output ------------------------------------------- 146 | 147 | # Grouping the document tree into Texinfo files. List of tuples 148 | # (source start file, target name, title, author, 149 | # dir menu entry, description, category) 150 | texinfo_documents = [ 151 | (master_doc, 'pytorch-nlp-tutorial-sf2017', 'pytorch-nlp-tutorial-sf2017 Documentation', 152 | author, 'pytorch-nlp-tutorial-sf2017', 'One line description of project.', 153 | 'Miscellaneous'), 154 | ] 155 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /docs/environment_setup.rst: -------------------------------------------------------------------------------- 1 | Environment Setup 2 | ================= 3 | 4 | On this page, you will find not only the list of dependencies to install 5 | for the tutorial, but a description of how to install them. This tutorial assumes 6 | you have a laptop with OSX or Linux. If you use Windows, you might have to install 7 | a virtual machine to get a UNIX-like environment to continue with the rest of this 8 | instruction. A lot of this instruction is more verbose than needed to accomodate 9 | participants of different skill levels. 10 | 11 | **Please note that these are only optional. On the first day of this training, you will be provided with a link to a JupyterHub instance where the environment will be pre-made and ready to go!** 12 | 13 | 0. Get Anaconda 14 | --------------- 15 | 16 | Anaconda is a Python (and R) distribution that aims to provide everything 17 | needed for common scientific and machine learning situations out-of-the-box. 18 | We chose Anaconda for this tutorial as it significantly simplifies Python 19 | dependency management. 20 | 21 | In practice, Anaconda can be used to manage different environment and packages. 22 | This setup document will assume that you have Anaconda installed as your default 23 | Python distribution. 24 | 25 | You can download Anaconda here: https://www.continuum.io/downloads 26 | 27 | After installing Anaconda, you can access its command-line interface 28 | with the :code:`conda` command. 29 | 30 | 31 | 1. Create a new environment 32 | --------------------------- 33 | 34 | Environments are a tool for sanitary software development. By this, we mean that 35 | you can install specific versions of packages without worrying that it breaks 36 | a dependency elsewhere. 37 | 38 | Here is how you can create an environment with Anaconda 39 | 40 | .. code-block:: bash 41 | 42 | conda create -n dl4nlp python=3.6 43 | 44 | 45 | 2. Install Dependencies 46 | ----------------------- 47 | 48 | 2a. Activate the environment 49 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 50 | 51 | After creating the environment, you need to **activate** the environment: 52 | 53 | .. code-block:: bash 54 | 55 | source activate dl4nlp 56 | 57 | After an environment is activated, it might prepend/append itself to your 58 | console prompt to let you know it is active. 59 | 60 | With the environment activated, any installation commands 61 | (whether it is :code:`pip install X`, :code:`python setup.py install` or using 62 | Anaconda's install command :code:`conda install X`) will only install inside 63 | the environment. 64 | 65 | 2b. Install IPython and Jupyter 66 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 67 | 68 | Two core dependencies are IPython and Jupyter. Let's install them first: 69 | 70 | .. code-block:: bash 71 | 72 | conda install ipython 73 | conda install jupyter 74 | 75 | To allow a jupyter notebooks to use this environment as their kernel, it 76 | needs to be linked: 77 | 78 | .. code-block:: bash 79 | 80 | python -m ipykernel install --user --name dl4nlp 81 | 82 | 2c. Installing CUDA (optional) 83 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 84 | 85 | NOTE: CUDA is currently not supported out of the conda package control manager. 86 | Please refer to pytorch's github repository for compilation instructions. 87 | 88 | If you have a CUDA compatible GPU, it is worthwhile to take advantage of it as 89 | it can significantly speedup training and make your PyTorch experimentation more 90 | enjoyable. 91 | 92 | To install CUDA: 93 | 94 | 1. Download CUDA appropriate to your OS/Arch from `here `_. 95 | 2. Follow installation steps for your architecture/OS. For Ubuntu/x86_64, see `here `_. 96 | 3. Download and install CUDNN from `here `_. 97 | 98 | Make sure you have the latest CUDA (8.0) and CUDNN (7.0). 99 | 100 | 2d. Install PyTorch 101 | ^^^^^^^^^^^^^^^^^^^ 102 | 103 | There are instructions on http://pytorch.org which detail how to install it. 104 | If you have been following along so far and have Anaconda installed with CUDA enabled, you can simply do: 105 | 106 | 107 | .. code-block:: bash 108 | 109 | conda install pytorch torchvision cuda80 -c soumith 110 | 111 | The widget on PyTorch.org will let you select the right command line for your specific OS/Arch. 112 | 113 | PLEASE NOTE. Make sure you have PyTorch 0.3.0. PyTorch has recently released version 0.4.0, but it has many code changes that we will not be incorporating at this time. The Anaconda installation method for this is: 114 | 115 | .. code-block:: bash 116 | 117 | conda install pytorch=0.3.1 torchvision -c pytorch 118 | 119 | If you would like to install using pips and wheels: 120 | 121 | .. code-block:: bash 122 | 123 | pip install http://download.pytorch.org/whl/cpu/torch-0.3.1-cp36-cp36m-linux_x86_64.whl 124 | pip install torchvision 125 | 126 | 127 | 128 | 2e. Clone (or Download) Repository 129 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 130 | 131 | At this point, you may have already cloned the tutorial repository. But if 132 | you have not, you will need it for the next step. 133 | 134 | .. code-block:: bash 135 | 136 | git clone https://github.com/joosthub/pytorch-nlp-tutorial-ny2018.git 137 | 138 | If you do not have git or do not want to use it, you can also 139 | `download the repository as a zip file `_ 140 | 141 | 2f. Install Dependencies from Repository 142 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 143 | 144 | Assuming the you have cloned (or downloaded and unzipped) the repository, 145 | please navigate to the directory in your terminal. Then, you can do the following: 146 | 147 | .. code-block:: bash 148 | 149 | pip install -r requirements.txt 150 | -------------------------------------------------------------------------------- /docs/_build/html/_sources/environment_setup.rst.txt: -------------------------------------------------------------------------------- 1 | Environment Setup 2 | ================= 3 | 4 | On this page, you will find not only the list of dependencies to install 5 | for the tutorial, but a description of how to install them. This tutorial assumes 6 | you have a laptop with OSX or Linux. If you use Windows, you might have to install 7 | a virtual machine to get a UNIX-like environment to continue with the rest of this 8 | instruction. A lot of this instruction is more verbose than needed to accomodate 9 | participants of different skill levels. 10 | 11 | **Please note that these are only optional. On the first day of this training, you will be provided with a link to a JupyterHub instance where the environment will be pre-made and ready to go!** 12 | 13 | 0. Get Anaconda 14 | --------------- 15 | 16 | Anaconda is a Python (and R) distribution that aims to provide everything 17 | needed for common scientific and machine learning situations out-of-the-box. 18 | We chose Anaconda for this tutorial as it significantly simplifies Python 19 | dependency management. 20 | 21 | In practice, Anaconda can be used to manage different environment and packages. 22 | This setup document will assume that you have Anaconda installed as your default 23 | Python distribution. 24 | 25 | You can download Anaconda here: https://www.continuum.io/downloads 26 | 27 | After installing Anaconda, you can access its command-line interface 28 | with the :code:`conda` command. 29 | 30 | 31 | 1. Create a new environment 32 | --------------------------- 33 | 34 | Environments are a tool for sanitary software development. By this, we mean that 35 | you can install specific versions of packages without worrying that it breaks 36 | a dependency elsewhere. 37 | 38 | Here is how you can create an environment with Anaconda 39 | 40 | .. code-block:: bash 41 | 42 | conda create -n dl4nlp python=3.6 43 | 44 | 45 | 2. Install Dependencies 46 | ----------------------- 47 | 48 | 2a. Activate the environment 49 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 50 | 51 | After creating the environment, you need to **activate** the environment: 52 | 53 | .. code-block:: bash 54 | 55 | source activate dl4nlp 56 | 57 | After an environment is activated, it might prepend/append itself to your 58 | console prompt to let you know it is active. 59 | 60 | With the environment activated, any installation commands 61 | (whether it is :code:`pip install X`, :code:`python setup.py install` or using 62 | Anaconda's install command :code:`conda install X`) will only install inside 63 | the environment. 64 | 65 | 2b. Install IPython and Jupyter 66 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 67 | 68 | Two core dependencies are IPython and Jupyter. Let's install them first: 69 | 70 | .. code-block:: bash 71 | 72 | conda install ipython 73 | conda install jupyter 74 | 75 | To allow a jupyter notebooks to use this environment as their kernel, it 76 | needs to be linked: 77 | 78 | .. code-block:: bash 79 | 80 | python -m ipykernel install --user --name dl4nlp 81 | 82 | 2c. Installing CUDA (optional) 83 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 84 | 85 | NOTE: CUDA is currently not supported out of the conda package control manager. 86 | Please refer to pytorch's github repository for compilation instructions. 87 | 88 | If you have a CUDA compatible GPU, it is worthwhile to take advantage of it as 89 | it can significantly speedup training and make your PyTorch experimentation more 90 | enjoyable. 91 | 92 | To install CUDA: 93 | 94 | 1. Download CUDA appropriate to your OS/Arch from `here `_. 95 | 2. Follow installation steps for your architecture/OS. For Ubuntu/x86_64, see `here `_. 96 | 3. Download and install CUDNN from `here `_. 97 | 98 | Make sure you have the latest CUDA (8.0) and CUDNN (7.0). 99 | 100 | 2d. Install PyTorch 101 | ^^^^^^^^^^^^^^^^^^^ 102 | 103 | There are instructions on http://pytorch.org which detail how to install it. 104 | If you have been following along so far and have Anaconda installed with CUDA enabled, you can simply do: 105 | 106 | 107 | .. code-block:: bash 108 | 109 | conda install pytorch torchvision cuda80 -c soumith 110 | 111 | The widget on PyTorch.org will let you select the right command line for your specific OS/Arch. 112 | 113 | PLEASE NOTE. Make sure you have PyTorch 0.3.0. PyTorch has recently released version 0.4.0, but it has many code changes that we will not be incorporating at this time. The Anaconda installation method for this is: 114 | 115 | .. code-block:: bash 116 | 117 | conda install pytorch=0.3.1 torchvision -c pytorch 118 | 119 | If you would like to install using pips and wheels: 120 | 121 | .. code-block:: bash 122 | 123 | pip install http://download.pytorch.org/whl/cpu/torch-0.3.1-cp36-cp36m-linux_x86_64.whl 124 | pip install torchvision 125 | 126 | 127 | 128 | 2e. Clone (or Download) Repository 129 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 130 | 131 | At this point, you may have already cloned the tutorial repository. But if 132 | you have not, you will need it for the next step. 133 | 134 | .. code-block:: bash 135 | 136 | git clone https://github.com/joosthub/pytorch-nlp-tutorial-ny2018.git 137 | 138 | If you do not have git or do not want to use it, you can also 139 | `download the repository as a zip file `_ 140 | 141 | 2f. Install Dependencies from Repository 142 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 143 | 144 | Assuming the you have cloned (or downloaded and unzipped) the repository, 145 | please navigate to the directory in your terminal. Then, you can do the following: 146 | 147 | .. code-block:: bash 148 | 149 | pip install -r requirements.txt 150 | -------------------------------------------------------------------------------- /docs/_build/html/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Search — NLP with PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 |
42 | 43 | 44 | 92 | 93 |
94 | 95 | 96 | 102 | 103 | 104 |
105 | 106 |
107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 |
125 | 126 |
    127 | 128 |
  • Docs »
    • 129 | 130 |
  • Search
    • 131 | 132 | 133 |
  • 134 | 135 | 136 | 137 |
    • 138 | 139 |
140 | 141 | 142 |
143 |
144 |
145 |
146 | 147 | 155 | 156 | 157 |
158 | 159 |
160 | 161 |
162 | 163 |
164 |
165 | 166 | 167 |
168 | 169 |
170 |

171 | © Copyright 2017, Brian McMahan and Delip Rao. 172 | 173 |

174 |
175 | Built with Sphinx using a theme provided by Read the Docs. 176 | 177 |
178 | 179 |
180 |
181 | 182 |
183 | 184 |
185 | 186 | 187 | 188 | 189 | 190 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 221 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | -------------------------------------------------------------------------------- /docs/_build/html/faq.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Frequency Asked Questions — NLP with PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
43 | 44 | 45 | 96 | 97 |
98 | 99 | 100 | 106 | 107 | 108 |
109 | 110 |
111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 |
129 | 130 |
    131 | 132 |
  • Docs »
    • 133 | 134 |
  • Frequency Asked Questions
    • 135 | 136 | 137 |
  • 138 | 139 | 140 | View page source 141 | 142 | 143 |
    • 144 | 145 |
146 | 147 | 148 |
149 |
150 |
151 |
152 | 153 |
154 |

Frequency Asked Questions

155 |

On this page, you will find a list of questions that we either anticipate 156 | people will ask or that we have been asked previously. They are intended to 157 | be the first stop for any confusion or trouble that might occur.

158 |
159 |

Do I Need to have a NVIDIA GPU enabled laptop?

160 |

Nope! While having a NVIDIA GPU enabled laptop will make the training run 161 | faster, we provide instructions for people who do not have one.

162 |

If you are plan on working on Natural Language Processing/Deep Learning in the future, 163 | a GPU enabled laptop might be a good investment.

164 |
165 |
166 | 167 | 168 |
169 | 170 |
171 |
172 | 173 | 179 | 180 | 181 |
182 | 183 |
184 |

185 | © Copyright 2017, Brian McMahan and Delip Rao. 186 | 187 |

188 |
189 | Built with Sphinx using a theme provided by Read the Docs. 190 | 191 |
192 | 193 |
194 |
195 | 196 |
197 | 198 |
199 | 200 | 201 | 202 | 203 | 204 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 234 | 235 | 236 | -------------------------------------------------------------------------------- /docs/_build/html/download_data.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Getting the Data — NLP with PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 |
44 | 45 | 46 | 98 | 99 |
100 | 101 | 102 | 108 | 109 | 110 |
111 | 112 |
113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 |
131 | 132 |
    133 | 134 |
  • Docs »
    • 135 | 136 |
  • Getting the Data
    • 137 | 138 | 139 |
  • 140 | 141 | 142 | View page source 143 | 144 | 145 |
    • 146 | 147 |
148 | 149 | 150 |
151 |
152 |
153 |
154 | 155 |
156 |

Getting the Data

157 |

In this training, there are two options of participating.

158 |
159 |

Option 1: Download and Setup things on your laptop

160 |

The first option is to download the data below, setup the environment, and download the notebooks when we make them available. 161 | If you choose this options but do not download the data before the first day, we will have several flash drives with the data on it.

162 |

Please visit this link to download the data.

163 |
164 |
165 |

Option 2: Use O’Reilly’s online resource through your browser

166 |

The second option is to use an online resource provided by O’Reilly. On the first day of this training, you will be provided with a link to a JupyterHub instance where the environment will be pre-made and ready to go! If you choose this option, you do not have to do anything until you arrive on Sunday. 167 | You are still required to bring your laptop.

168 |
169 |
170 | 171 | 172 |
173 | 174 |
175 |
176 | 177 | 185 | 186 | 187 |
188 | 189 |
190 |

191 | © Copyright 2017, Brian McMahan and Delip Rao. 192 | 193 |

194 |
195 | Built with Sphinx using a theme provided by Read the Docs. 196 | 197 |
198 | 199 |
200 |
201 | 202 |
203 | 204 |
205 | 206 | 207 | 208 | 209 | 210 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 240 | 241 | 242 | -------------------------------------------------------------------------------- /docs/_build/html/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({docnames:["day1/index","day1/takehome","day2/adventures/conv_then_rnn","day2/adventures/index","day2/adventures/interpolation","day2/adventures/lookups","day2/adventures/lvg","day2/patterns/attention","day2/patterns/gating","day2/patterns/index","day2/patterns/stacking","day2/prefpractice","day2/warmup","download_data","environment_setup","errata","faq","index","recipes/compute_conv_size","recipes/index","recipes/load_pretrained_vectors","recipes/packed_sequences"],envversion:53,filenames:["day1/index.rst","day1/takehome.rst","day2/adventures/conv_then_rnn.rst","day2/adventures/index.rst","day2/adventures/interpolation.rst","day2/adventures/lookups.rst","day2/adventures/lvg.rst","day2/patterns/attention.rst","day2/patterns/gating.rst","day2/patterns/index.rst","day2/patterns/stacking.rst","day2/prefpractice.rst","day2/warmup.rst","download_data.rst","environment_setup.rst","errata.rst","faq.rst","index.rst","recipes/compute_conv_size.rst","recipes/index.rst","recipes/load_pretrained_vectors.rst","recipes/packed_sequences.rst"],objects:{},objnames:{},objtypes:{},terms:{"18th":17,"1st":12,"break":14,"case":[5,7],"class":7,"default":14,"final":[4,5,6,20],"function":[4,5,7,17,20],"import":[6,18],"long":12,"new":[3,5,12,17],"return":[4,5,7,18],"super":7,"try":1,"while":16,And:4,But:14,For:[4,5,6,14,17],One:[4,5,6],The:[5,6,7,13,14],Then:[4,6,14],There:[5,6,14,17],Use:12,Using:[5,7,17],With:14,__getitem__:6,__init__:7,__len__:6,abl:17,access:[14,17],accomod:14,across:[7,12],added:[5,17],adding:17,addition:7,advantag:[14,20],affect:4,after:[4,7,14],aim:[5,14],algebra:17,allow:14,along:14,alreadi:14,also:14,amazon:6,amazon_filenam:6,amongst:7,analog:7,ani:[5,7,14,16,17],annoi:5,anticip:16,anyth:13,append:14,appli:5,appropri:14,approxim:5,april:[15,17],arch:14,architectur:14,argument:12,arriv:13,assert:4,assum:[5,14],attent:9,attention_s:7,attention_scor:7,attn:7,attribut:7,avail:13,averag:[7,12],awesom:4,back:[12,15],bag:1,batch:7,becaus:5,been:[7,14,16],befor:[5,7,12,13],being:5,below:[7,13,17],better:5,between:3,bonu:17,box:14,bring:13,browser:17,can:[4,5,6,7,12,14,17,20],cbow:[1,7],cell:7,chang:[1,14],charact:5,check:[15,17],choos:13,chose:14,citi:17,classif:5,classifi:1,code:[4,5,14],collect:7,column_gath:5,com:14,combin:4,come:6,command:[14,17],common:[5,14],compat:14,compil:14,complet:[1,5],comput:[5,7,17,19],conda:14,condensed_x:7,condit:4,conditional_emb:4,confer:17,confus:16,connect:5,consol:14,continu:[1,14],continuum:14,control:14,conv:5,conv_shape_helper_1d:18,convert:20,convnet:1,convolut:19,cool:5,core:14,correct:4,could:[5,7],cours:17,cp36:14,cp36m:14,cpu:14,creat:[4,6,12],credenti:17,cuda80:14,cuda:20,cudnn:14,current:14,dai:[5,7,13,14,17,20],data:[5,6,20],datapoint:5,dataset:6,date:5,datset:6,decod:5,decode_matrix:4,deep:[1,16],def:[4,5,7,18],deriv:17,descript:14,desrib:17,detail:14,develop:14,diff:7,differ:[4,7,14],dilat:18,dim1:7,dim:[7,12],dimens:12,directori:[14,17],distribut:[5,7,14],dl4nlp:14,document:14,doe:[5,6],doesn:5,doing:[5,19],dot:17,drive:13,each:[5,7],easi:12,either:16,els:5,elsewher:14,emb:[4,5,20],emb_index:20,embed:[4,5,20],enabl:14,encod:3,end:17,engag:17,enjoy:14,environ:[13,17],errata:17,etc:17,evalu:[1,17],everyth:[5,14,17],exactli:5,exampl:[1,5,7,20],exercis:[0,3,6,17],expect:7,experi:17,experiment:14,extrem:20,familiar:17,far:[12,14],fast:3,faster:16,featur:[5,7],file:[14,17],find:[0,5,14,16,17,19],finish:4,first:[4,6,13,14,16,17],firstnames_csv:6,fit:6,flash:13,floattensor:[7,20],floor:18,follow:[4,5,14,17],form:7,forward:[5,7],from:[0,4,5,6,12],fulli:5,fun:4,futur:16,gener:[3,4],generaliz:5,get:[4,5,7,12,20],get_penultim:5,git:14,github:[14,17],give:20,given:[4,5,17],glove:20,glove_index:20,glove_vec:20,good:16,gpu:[14,17],groov:12,h_i:8,h_n:8,h_r:8,had:20,handi:17,has:[4,5,7,14],have:[7,12,13,14,17],head:6,hello:17,here:[0,1,4,14,17],hidden:[4,8],home:0,how:[4,5,12,14],http:14,i_i:8,i_n:8,i_r:8,identifi:5,implement:[1,6],importantli:7,includ:6,incorpor:14,index:[4,5,12,20],induc:5,init:[4,7],init_vector1:4,init_vector2:4,init_vector:4,initi:4,input:5,input_seq_len:18,inputg:8,insid:[4,14],instanc:[13,14,17],instead:7,instruct:[14,16,17],intend:16,interest:5,interfac:14,interpol:[3,7],interpolate_n_samples_from_two_nation:4,invest:16,ipykernel:14,issu:17,item:20,iter:20,its:14,itself:[14,20],joosthub:14,jupyt:17,jupyterhub:[13,14,17],just:[5,7],kei:4,kernel:14,kernel_s:18,kernel_width:18,kind:5,know:14,knowledg:17,label:5,languag:16,laptop:[14,17],last:[12,15],later:5,latest:14,layer:[5,20],learn:[4,5,7,14,16,17],length:4,let:[5,12,14],level:14,librari:5,like:[5,14],limit:17,line:[14,17],linear:17,link:[1,13,14,17],linux:[14,17],linux_x86_64:14,list:[4,14,16],load:[3,5,19],load_word_vector:20,local:17,local_run_readm:17,local_set:6,long_vari:4,look:[4,5,6],lookup:3,loop:4,lot:14,lower:20,mac:17,machin:[14,17],made:[13,14],mai:14,main:7,make:[13,14,16,20],make_gener:6,make_initial_x:4,manag:14,mani:14,manner:1,math:[17,18],matmul:[7,12],matric:17,matrix:[12,20],matter:20,max:7,mayb:5,mean:[7,12,14],mechan:7,method:[6,12,14],might:[5,14,16],minut:12,mistak:17,mlp:1,model:20,modul:7,mondai:17,more:[5,14,17],multipli:[12,17],name:[1,6,14],nation:4,nationality1:4,nationality2:4,nationality_vocab:4,natur:16,navig:14,nearest:5,nearli:5,need:[5,14,17],neighbor:5,net:[4,5,20],network:[4,5,20],new_paramet:7,newgat:8,next:14,nlp:14,none:5,nope:16,normal:5,note:[14,17],notebook:[5,7,13,14,17,20],now:[7,20],number:4,ny2018:14,object:5,occur:16,onc:5,one:[5,7,16],onli:[14,17,20],onlin:17,oper:17,option:[4,12,17],org:14,osx:14,other:6,our:20,out:[7,14],over:[4,20],packag:14,pad:[1,18],page:[14,16],panda:6,paper:1,paramet:7,particip:[13,14],pass:12,pattern:7,penultim:5,peopl:16,per:[5,7],permut:5,persist:17,pip:14,plan:16,pleas:[13,14,15,17],point:[5,14,20],pool:7,port:5,practic:14,pre:[5,13,14],precalc:17,precomput:6,predict:12,prepend:14,pretrain:[5,19],previous:16,print:[4,6],probabl:[5,7],process:16,product:17,progress:17,prompt:14,provid:[13,14,16,17],publish:7,python:[14,17],pytorch:[12,20],randint:12,randn:[7,12],random:12,rate:6,raw:6,read:17,read_csv:6,readi:[13,14],reason:5,recent:14,recip:17,recommend:17,refer:14,reilli:17,releas:14,relev:6,repositori:17,repres:7,represent:5,requir:[5,6,13,14],resetg:8,resourc:17,rest:14,result:12,retriev:[4,5],reus:6,review:6,right:14,rnn:[4,5,7],rule:5,run:[12,16],same:1,sampl:4,sample_n_for_nation:4,sampler:4,sanitari:14,saw:[5,7],scientif:14,scored_x:7,second:[13,17],section:[17,19],see:[4,12,14],seen:7,select:[6,7,14],self:7,septemb:17,sequenc:[3,7],set:[5,6,7,17,19,20],set_:20,sever:[7,13],shoud:5,should:[4,5,6],sigmoid:8,signatur:4,significantli:14,similar:7,simpl:[7,17],simpli:14,simplifi:14,singl:[5,7],site:17,situat:14,size:[7,12,19],skill:14,softli:7,softmax:[5,7],softwar:14,some:[5,12],someth:5,song:5,sorri:17,soumith:14,sourc:14,specif:[4,14,17],specifi:5,speedup:14,split:6,spotifi:5,squeez:7,start:20,state:4,step:[5,12,14],still:13,stop:16,store:5,stride:[1,18],structur:20,stuff:4,subset:6,sum:[7,12],summar:7,sundai:[13,17],support:14,suppos:5,sure:[14,17],surnam:5,surname_vocab:5,system:17,take:[0,7,14],tanh:8,temp:4,tensor:12,tensor_s:18,termin:14,test:6,than:[5,14],thei:[16,17],them:[5,7,13,14],thi:[4,5,6,7,12,13,14,16,17,19,20],thing:[0,1,5,6,19],through:17,time:14,titl:6,togeth:7,tool:14,torch:[7,12,14,20],torchvis:14,train:[4,5,6,13,14,16,17],transform:6,troubl:[16,17],tutori:[14,17],two:[4,6,12,13,14,17],txt:14,ubuntu:14,understand:17,unix:14,unord:7,until:[13,17],unzip:14,updat:15,url:17,use:[4,5,7,13,14,17],used:[5,14],useful:[7,17,20],user:[14,17],using:[4,5,6,7,14,20],valid:7,valu:[5,7,20],variabl:7,variou:19,vector:[3,5,7,17,19],verbos:14,veri:7,version:14,video:17,view:7,virtual:14,visit:13,vocabulari:[4,6,20],wai:[5,17],want:[4,5,7,14,17,20],weight:[4,5,7,20],what:[5,17],wheel:14,when:[7,13],where:[13,14],whether:[7,14],which:[4,5,6,7,14,20],whl:14,who:16,why:5,widget:14,window:[14,17],without:[1,14],wonder:5,word:[1,5,7,20],word_to_index:20,word_vector:20,word_vector_s:20,word_vocab:20,work:[6,16,17],worri:14,worthwhil:14,would:[5,6,14],write:[4,5,6],www:14,x86_64:14,x_in:[5,7],x_length:5,x_mid:5,xavier_norm:7,y_out:5,york:17,you:[0,4,5,6,7,12,13,14,16,17,19],your:14,zip:[4,14]},titles:["Day 1","Take-Home Exercises","<no title>","Choose Your Own Adventures","Exercise: Interpolating Between Vectors","Exercise: Fast Lookups for Encoded Sequences","A New Load-Vectorize-Generate","Attention","Gating","NN Patterns","Stacking","<no title>","Warm Up Exercise","Getting the Data","Environment Setup","Errata","Frequency Asked Questions","Natural Language Processing (NLP) with PyTorch","Compute Convolution Sizes","Recipes and PyTorch patterns","Loading Pretrained Vectors","<no title>"],titleterms:{"new":[6,14],Use:13,activ:14,adventur:3,anaconda:14,ask:16,attent:7,between:4,browser:13,choos:3,clone:14,comput:18,convolut:18,creat:14,cuda:14,dai:0,data:13,depend:14,download:[13,14],enabl:16,encod:5,environ:14,errata:15,exercis:[1,4,5,12],fast:5,frequenc:16,from:14,gate:8,gener:[6,17],get:[13,14],gpu:16,hardwar:17,have:16,home:1,inform:17,instal:[14,17],interpol:4,ipython:14,jupyt:14,languag:17,laptop:[13,16],load:[6,20],lookup:5,natur:17,need:16,nlp:17,nvidia:16,onlin:13,option:[13,14],own:3,pattern:[9,19],prerequisit:17,pretrain:20,process:17,pytorch:[14,17,19],question:16,recip:19,reilli:13,repositori:14,requir:17,resourc:13,sequenc:5,setup:[13,14],size:18,stack:10,take:1,thing:13,through:13,vector:[4,6,20],warm:12,your:[3,13]}}) -------------------------------------------------------------------------------- /docs/_build/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Natural Language Processing (NLP) with PyTorch — NLP with PyTorch documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
43 | 44 | 45 | 93 | 94 |
95 | 96 | 97 | 103 | 104 | 105 |
106 | 107 |
108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 |
126 | 127 |
    128 | 129 |
  • Docs »
    • 130 | 131 |
  • Natural Language Processing (NLP) with PyTorch
    • 132 | 133 | 134 |
  • 135 | 136 | 137 | View page source 138 | 139 | 140 |
    • 141 | 142 |
143 | 144 | 145 |
146 |
147 |
148 |
149 | 150 |
151 |

Natural Language Processing (NLP) with PyTorch

152 |
153 |
154 |
155 |
156 |
157 |
158 |

Hello! This is a directory of resources for a training tutorial to be 159 | given at the O’Reilly AI Conference in New York City on Sunday, April 29, and Monday, April 30, 2018.

160 |

Please read below for general information. You can find the github repository at this link. Please note that there are two ways to engage in this training (desribed below).

161 |

More information will be added to this site as the training progresses. 162 | Specifically, we will be adding a ‘recipes’ section, ‘errata’ section, and a ‘bonus exercise’ section as the training progresses!

163 |
164 |

General Information

165 |
166 |

Prerequisites:

167 |
    168 |
  • A working knowledge of Python and the command line
    • 169 |
  • Familiarity with precalc math (multiply matrices, dot products of vectors, etc.) and derivatives of simple functions (If you are new to linear algebra, this video course is handy.)
    • 170 |
  • A general understanding of machine learning (setting up experiments, evaluation, etc.) (useful but not required)
    • 171 |
172 |
173 |
174 |

Hardware and/or installation requirements:

175 |
    176 |
  • 177 |
    There are two options:
    178 |
      179 |
    1. Using O’Reilly’s online resources. For this, you only needs a laptop; on the first day, we will provide you with credentials and a URL to use an online computing resource (a JupyterHub instance) provided by O’Reilly. You will be able to access Jupyter notebooks through this and they will persist until the end of the second day of training (September 18th). This option is not limited by what operating system you have. You will need to have a browser installed.
      2. 180 |
    2. Setting everything up locally. For this, you need a laptop with the PyTorch environment set up. This is only recommended if you want to have the environment locally or have a laptop with a GPU. (If you have trouble following the provided instructions or if you find any mistakes, please file an issue here.) This option is limited to Macs and Linux users only (sorry Windows users!). Be sure you check the LOCAL_RUN_README.md.
      3. 181 |
    182 |
    183 |
    184 |
    • 185 |
186 |
187 |
188 |
189 | 190 | 191 |
192 | 193 |
194 |
195 | 196 | 202 | 203 | 204 |
205 | 206 |
207 |

208 | © Copyright 2017, Brian McMahan and Delip Rao. 209 | 210 |

211 |
212 | Built with Sphinx using a theme provided by Read the Docs. 213 | 214 |
215 | 216 |
217 |
218 | 219 |
220 | 221 |
222 | 223 | 224 | 225 | 226 | 227 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 257 | 258 | 259 | -------------------------------------------------------------------------------- /day_1/vocabulary.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | 3 | import numpy as np 4 | from torch.utils.data import Dataset 5 | import six 6 | 7 | import json 8 | 9 | 10 | class Vocabulary(object): 11 | """ 12 | An implementation that manages the interface between a token dataset and the 13 | machine learning algorithm. 14 | """ 15 | 16 | def __init__(self, use_unks=False, unk_token="", 17 | use_mask=False, mask_token="", use_start_end=False, 18 | start_token="", end_token=""): 19 | """ 20 | Args: 21 | use_unks (bool): The vocabulary will output UNK tokens for out of 22 | vocabulary items. 23 | [default=False] 24 | unk_token (str): The token used for unknown tokens. 25 | If `use_unks` is True, this will be added to the vocabulary. 26 | [default=''] 27 | use_mask (bool): The vocabulary will reserve the 0th index for a mask token. 28 | This is used to handle variable lengths in sequence models. 29 | [default=False] 30 | mask_token (str): The token used for the mask. 31 | Note: mostly a placeholder; it's unlikely the token will be seen. 32 | [default=''] 33 | use_start_end (bool): The vocabulary will reserve indices for two tokens 34 | that represent the start and end of a sequence. 35 | [default=False] 36 | start_token: The token used to indicate the start of a sequence. 37 | If `use_start_end` is True, this will be added to the vocabulary. 38 | [default=''] 39 | end_token: The token used to indicate the end of a sequence 40 | If `use_start_end` is True, this will be added to the vocabulary. 41 | [default=''] 42 | """ 43 | 44 | self._mapping = {} # str -> int 45 | self._flip = {} # int -> str; 46 | self._counts = Counter() # int -> int; count occurrences 47 | self._forced_unks = set() # force tokens to unk (e.g. if < 5 occurrences) 48 | self._i = 0 49 | self._frozen = False 50 | self._frequency_threshold = -1 51 | 52 | # mask token for use in masked recurrent networks 53 | # usually need to be the 0th index 54 | self.use_mask = use_mask 55 | self.mask_token = mask_token 56 | if self.use_mask: 57 | self.add(self.mask_token) 58 | 59 | # unk token for out of vocabulary tokens 60 | self.use_unks = use_unks 61 | self.unk_token = unk_token 62 | if self.use_unks: 63 | self.add(self.unk_token) 64 | 65 | # start token for sequence models 66 | self.use_start_end = use_start_end 67 | self.start_token = start_token 68 | self.end_token = end_token 69 | if self.use_start_end: 70 | self.add(self.start_token) 71 | self.add(self.end_token) 72 | 73 | def iterkeys(self): 74 | for k in self._mapping.keys(): 75 | if k == self.unk_token or k == self.mask_token: 76 | continue 77 | else: 78 | yield k 79 | 80 | def keys(self): 81 | return list(self.iterkeys()) 82 | 83 | def iteritems(self): 84 | for key, value in self._mapping.items(): 85 | if key == self.unk_token or key == self.mask_token: 86 | continue 87 | yield key, value 88 | 89 | def items(self): 90 | return list(self.iteritems()) 91 | 92 | def values(self): 93 | return [value for _, value in self.iteritems()] 94 | 95 | def __getitem__(self, k): 96 | if self._frozen: 97 | if k in self._mapping: 98 | out_index = self._mapping[k] 99 | elif self.use_unks: 100 | out_index = self.unk_index 101 | else: # case: frozen, don't want unks, raise exception 102 | raise VocabularyException("Vocabulary is frozen. " + 103 | "Key '{}' not found.".format(k)) 104 | if out_index in self._forced_unks: 105 | out_index = self.unk_index 106 | elif k in self._mapping: # case: normal 107 | out_index = self._mapping[k] 108 | self._counts[out_index] += 1 109 | else: 110 | out_index = self._mapping[k] = self._i 111 | self._i += 1 112 | self._flip[out_index] = k 113 | self._counts[out_index] = 1 114 | 115 | return out_index 116 | 117 | def add(self, k): 118 | return self.__getitem__(k) 119 | 120 | def add_many(self, x): 121 | return [self.add(k) for k in x] 122 | 123 | def lookup(self, i): 124 | try: 125 | return self._flip[i] 126 | except KeyError: 127 | raise VocabularyException("Key {} not in Vocabulary".format(i)) 128 | 129 | def lookup_many(self, x): 130 | for k in x: 131 | yield self.lookup(k) 132 | 133 | def map(self, sequence, include_start_end=False): 134 | if include_start_end: 135 | yield self.start_index 136 | 137 | for item in sequence: 138 | yield self[item] 139 | 140 | if include_start_end: 141 | yield self.end_index 142 | 143 | def freeze(self, use_unks=False, frequency_cutoff=-1): 144 | self.use_unks = use_unks 145 | self._frequency_cutoff = frequency_cutoff 146 | 147 | if use_unks and self.unk_token not in self: 148 | self.add(self.unk_token) 149 | 150 | if self._frequency_cutoff > 0: 151 | for token, count in self._counts.items(): 152 | if count < self._frequency_cutoff: 153 | self._forced_unks.add(token) 154 | 155 | self._frozen = True 156 | 157 | def unfreeze(self): 158 | self._frozen = False 159 | 160 | def get_counts(self): 161 | return {self._flip[i]: count for i, count in self._counts.items()} 162 | 163 | def get_count(self, token=None, index=None): 164 | if token is None and index is None: 165 | return None 166 | elif token is not None and index is not None: 167 | print("Cannot do two things at once; choose one") 168 | elif token is not None: 169 | return self._counts[self[token]] 170 | elif index is not None: 171 | return self._counts[index] 172 | else: 173 | raise Exception("impossible condition") 174 | 175 | @property 176 | def unk_index(self): 177 | if self.unk_token not in self: 178 | return None 179 | return self._mapping[self.unk_token] 180 | 181 | @property 182 | def mask_index(self): 183 | if self.mask_token not in self: 184 | return None 185 | return self._mapping[self.mask_token] 186 | 187 | @property 188 | def start_index(self): 189 | if self.start_token not in self: 190 | return None 191 | return self._mapping[self.start_token] 192 | 193 | @property 194 | def end_index(self): 195 | if self.end_token not in self: 196 | return None 197 | return self._mapping[self.end_token] 198 | 199 | def __contains__(self, k): 200 | return k in self._mapping 201 | 202 | def __len__(self): 203 | return len(self._mapping) 204 | 205 | def __repr__(self): 206 | return "".format(len(self), self._frozen) 207 | 208 | 209 | def get_serializable_contents(self): 210 | """ 211 | Creats a dict containing the necessary information to recreate this instance 212 | """ 213 | config = {"_mapping": self._mapping, 214 | "_flip": self._flip, 215 | "_frozen": self._frozen, 216 | "_i": self._i, 217 | "_counts": list(self._counts.items()), 218 | "_frequency_threshold": self._frequency_threshold, 219 | "use_unks": self.use_unks, 220 | "unk_token": self.unk_token, 221 | "use_mask": self.use_mask, 222 | "mask_token": self.mask_token, 223 | "use_start_end": self.use_start_end, 224 | "start_token": self.start_token, 225 | "end_token": self.end_token} 226 | return config 227 | 228 | @classmethod 229 | def deserialize_from_contents(cls, content): 230 | """ 231 | Recreate a Vocabulary instance; expect same dict as output in `serialize` 232 | """ 233 | try: 234 | _mapping = content.pop("_mapping") 235 | _flip = content.pop("_flip") 236 | _i = content.pop("_i") 237 | _frozen = content.pop("_frozen") 238 | _counts = content.pop("_counts") 239 | _frequency_threshold = content.pop("_frequency_threshold") 240 | except KeyError: 241 | raise Exception("unable to deserialize vocabulary") 242 | if isinstance(list(_flip.keys())[0], six.string_types): 243 | _flip = {int(k): v for k, v in _flip.items()} 244 | out = cls(**content) 245 | out._mapping = _mapping 246 | out._flip = _flip 247 | out._i = _i 248 | out._counts = Counter(dict(_counts)) 249 | out._frequency_threshold = _frequency_threshold 250 | 251 | if _frozen: 252 | out.freeze(out.use_unks) 253 | 254 | return out 255 | 256 | -------------------------------------------------------------------------------- /day_2/vocabulary.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | 3 | import numpy as np 4 | from torch.utils.data import Dataset 5 | import six 6 | 7 | import json 8 | 9 | 10 | class Vocabulary(object): 11 | """ 12 | An implementation that manages the interface between a token dataset and the 13 | machine learning algorithm. 14 | """ 15 | 16 | def __init__(self, use_unks=False, unk_token="", 17 | use_mask=False, mask_token="", use_start_end=False, 18 | start_token="", end_token=""): 19 | """ 20 | Args: 21 | use_unks (bool): The vocabulary will output UNK tokens for out of 22 | vocabulary items. 23 | [default=False] 24 | unk_token (str): The token used for unknown tokens. 25 | If `use_unks` is True, this will be added to the vocabulary. 26 | [default=''] 27 | use_mask (bool): The vocabulary will reserve the 0th index for a mask token. 28 | This is used to handle variable lengths in sequence models. 29 | [default=False] 30 | mask_token (str): The token used for the mask. 31 | Note: mostly a placeholder; it's unlikely the token will be seen. 32 | [default=''] 33 | use_start_end (bool): The vocabulary will reserve indices for two tokens 34 | that represent the start and end of a sequence. 35 | [default=False] 36 | start_token: The token used to indicate the start of a sequence. 37 | If `use_start_end` is True, this will be added to the vocabulary. 38 | [default=''] 39 | end_token: The token used to indicate the end of a sequence 40 | If `use_start_end` is True, this will be added to the vocabulary. 41 | [default=''] 42 | """ 43 | 44 | self._mapping = {} # str -> int 45 | self._flip = {} # int -> str; 46 | self._counts = Counter() # int -> int; count occurrences 47 | self._forced_unks = set() # force tokens to unk (e.g. if < 5 occurrences) 48 | self._i = 0 49 | self._frozen = False 50 | self._frequency_threshold = -1 51 | 52 | # mask token for use in masked recurrent networks 53 | # usually need to be the 0th index 54 | self.use_mask = use_mask 55 | self.mask_token = mask_token 56 | if self.use_mask: 57 | self.add(self.mask_token) 58 | 59 | # unk token for out of vocabulary tokens 60 | self.use_unks = use_unks 61 | self.unk_token = unk_token 62 | if self.use_unks: 63 | self.add(self.unk_token) 64 | 65 | # start token for sequence models 66 | self.use_start_end = use_start_end 67 | self.start_token = start_token 68 | self.end_token = end_token 69 | if self.use_start_end: 70 | self.add(self.start_token) 71 | self.add(self.end_token) 72 | 73 | def iterkeys(self): 74 | for k in self._mapping.keys(): 75 | if k == self.unk_token or k == self.mask_token: 76 | continue 77 | else: 78 | yield k 79 | 80 | def keys(self): 81 | return list(self.iterkeys()) 82 | 83 | def iteritems(self): 84 | for key, value in self._mapping.items(): 85 | if key == self.unk_token or key == self.mask_token: 86 | continue 87 | yield key, value 88 | 89 | def items(self): 90 | return list(self.iteritems()) 91 | 92 | def values(self): 93 | return [value for _, value in self.iteritems()] 94 | 95 | def __getitem__(self, k): 96 | if self._frozen: 97 | if k in self._mapping: 98 | out_index = self._mapping[k] 99 | elif self.use_unks: 100 | out_index = self.unk_index 101 | else: # case: frozen, don't want unks, raise exception 102 | raise VocabularyException("Vocabulary is frozen. " + 103 | "Key '{}' not found.".format(k)) 104 | if out_index in self._forced_unks: 105 | out_index = self.unk_index 106 | elif k in self._mapping: # case: normal 107 | out_index = self._mapping[k] 108 | self._counts[out_index] += 1 109 | else: 110 | out_index = self._mapping[k] = self._i 111 | self._i += 1 112 | self._flip[out_index] = k 113 | self._counts[out_index] = 1 114 | 115 | return out_index 116 | 117 | def add(self, k): 118 | return self.__getitem__(k) 119 | 120 | def add_many(self, x): 121 | return [self.add(k) for k in x] 122 | 123 | def lookup(self, i): 124 | try: 125 | return self._flip[i] 126 | except KeyError: 127 | raise VocabularyException("Key {} not in Vocabulary".format(i)) 128 | 129 | def lookup_many(self, x): 130 | for k in x: 131 | yield self.lookup(k) 132 | 133 | def map(self, sequence, include_start_end=False): 134 | if include_start_end: 135 | yield self.start_index 136 | 137 | for item in sequence: 138 | yield self[item] 139 | 140 | if include_start_end: 141 | yield self.end_index 142 | 143 | def freeze(self, use_unks=False, frequency_cutoff=-1): 144 | self.use_unks = use_unks 145 | self._frequency_cutoff = frequency_cutoff 146 | 147 | if use_unks and self.unk_token not in self: 148 | self.add(self.unk_token) 149 | 150 | if self._frequency_cutoff > 0: 151 | for token, count in self._counts.items(): 152 | if count < self._frequency_cutoff: 153 | self._forced_unks.add(token) 154 | 155 | self._frozen = True 156 | 157 | def unfreeze(self): 158 | self._frozen = False 159 | 160 | def get_counts(self): 161 | return {self._flip[i]: count for i, count in self._counts.items()} 162 | 163 | def get_count(self, token=None, index=None): 164 | if token is None and index is None: 165 | return None 166 | elif token is not None and index is not None: 167 | print("Cannot do two things at once; choose one") 168 | elif token is not None: 169 | return self._counts[self[token]] 170 | elif index is not None: 171 | return self._counts[index] 172 | else: 173 | raise Exception("impossible condition") 174 | 175 | @property 176 | def unk_index(self): 177 | if self.unk_token not in self: 178 | return None 179 | return self._mapping[self.unk_token] 180 | 181 | @property 182 | def mask_index(self): 183 | if self.mask_token not in self: 184 | return None 185 | return self._mapping[self.mask_token] 186 | 187 | @property 188 | def start_index(self): 189 | if self.start_token not in self: 190 | return None 191 | return self._mapping[self.start_token] 192 | 193 | @property 194 | def end_index(self): 195 | if self.end_token not in self: 196 | return None 197 | return self._mapping[self.end_token] 198 | 199 | def __contains__(self, k): 200 | return k in self._mapping 201 | 202 | def __len__(self): 203 | return len(self._mapping) 204 | 205 | def __repr__(self): 206 | return "".format(len(self), self._frozen) 207 | 208 | 209 | def get_serializable_contents(self): 210 | """ 211 | Creats a dict containing the necessary information to recreate this instance 212 | """ 213 | config = {"_mapping": self._mapping, 214 | "_flip": self._flip, 215 | "_frozen": self._frozen, 216 | "_i": self._i, 217 | "_counts": list(self._counts.items()), 218 | "_frequency_threshold": self._frequency_threshold, 219 | "use_unks": self.use_unks, 220 | "unk_token": self.unk_token, 221 | "use_mask": self.use_mask, 222 | "mask_token": self.mask_token, 223 | "use_start_end": self.use_start_end, 224 | "start_token": self.start_token, 225 | "end_token": self.end_token} 226 | return config 227 | 228 | @classmethod 229 | def deserialize_from_contents(cls, content): 230 | """ 231 | Recreate a Vocabulary instance; expect same dict as output in `serialize` 232 | """ 233 | try: 234 | _mapping = content.pop("_mapping") 235 | _flip = content.pop("_flip") 236 | _i = content.pop("_i") 237 | _frozen = content.pop("_frozen") 238 | _counts = content.pop("_counts") 239 | _frequency_threshold = content.pop("_frequency_threshold") 240 | except KeyError: 241 | raise Exception("unable to deserialize vocabulary") 242 | if isinstance(list(_flip.keys())[0], six.string_types): 243 | _flip = {int(k): v for k, v in _flip.items()} 244 | out = cls(**content) 245 | out._mapping = _mapping 246 | out._flip = _flip 247 | out._i = _i 248 | out._counts = Counter(dict(_counts)) 249 | out._frequency_threshold = _frequency_threshold 250 | 251 | if _frozen: 252 | out.freeze(out.use_unks) 253 | 254 | return out 255 | 256 | -------------------------------------------------------------------------------- /docs/_build/html/_static/doctools.js: -------------------------------------------------------------------------------- 1 | /* 2 | * doctools.js 3 | * ~~~~~~~~~~~ 4 | * 5 | * Sphinx JavaScript utilities for all documentation. 6 | * 7 | * :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /** 13 | * select a different prefix for underscore 14 | */ 15 | $u = _.noConflict(); 16 | 17 | /** 18 | * make the code below compatible with browsers without 19 | * an installed firebug like debugger 20 | if (!window.console || !console.firebug) { 21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir", 22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", 23 | "profile", "profileEnd"]; 24 | window.console = {}; 25 | for (var i = 0; i < names.length; ++i) 26 | window.console[names[i]] = function() {}; 27 | } 28 | */ 29 | 30 | /** 31 | * small helper function to urldecode strings 32 | */ 33 | jQuery.urldecode = function(x) { 34 | return decodeURIComponent(x).replace(/\+/g, ' '); 35 | }; 36 | 37 | /** 38 | * small helper function to urlencode strings 39 | */ 40 | jQuery.urlencode = encodeURIComponent; 41 | 42 | /** 43 | * This function returns the parsed url parameters of the 44 | * current request. Multiple values per key are supported, 45 | * it will always return arrays of strings for the value parts. 46 | */ 47 | jQuery.getQueryParameters = function(s) { 48 | if (typeof s === 'undefined') 49 | s = document.location.search; 50 | var parts = s.substr(s.indexOf('?') + 1).split('&'); 51 | var result = {}; 52 | for (var i = 0; i < parts.length; i++) { 53 | var tmp = parts[i].split('=', 2); 54 | var key = jQuery.urldecode(tmp[0]); 55 | var value = jQuery.urldecode(tmp[1]); 56 | if (key in result) 57 | result[key].push(value); 58 | else 59 | result[key] = [value]; 60 | } 61 | return result; 62 | }; 63 | 64 | /** 65 | * highlight a given string on a jquery object by wrapping it in 66 | * span elements with the given class name. 67 | */ 68 | jQuery.fn.highlightText = function(text, className) { 69 | function highlight(node, addItems) { 70 | if (node.nodeType === 3) { 71 | var val = node.nodeValue; 72 | var pos = val.toLowerCase().indexOf(text); 73 | if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { 74 | var span; 75 | var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg"); 76 | if (isInSVG) { 77 | span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); 78 | } else { 79 | span = document.createElement("span"); 80 | span.className = className; 81 | } 82 | span.appendChild(document.createTextNode(val.substr(pos, text.length))); 83 | node.parentNode.insertBefore(span, node.parentNode.insertBefore( 84 | document.createTextNode(val.substr(pos + text.length)), 85 | node.nextSibling)); 86 | node.nodeValue = val.substr(0, pos); 87 | if (isInSVG) { 88 | var bbox = span.getBBox(); 89 | var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); 90 | rect.x.baseVal.value = bbox.x; 91 | rect.y.baseVal.value = bbox.y; 92 | rect.width.baseVal.value = bbox.width; 93 | rect.height.baseVal.value = bbox.height; 94 | rect.setAttribute('class', className); 95 | var parentOfText = node.parentNode.parentNode; 96 | addItems.push({ 97 | "parent": node.parentNode, 98 | "target": rect}); 99 | } 100 | } 101 | } 102 | else if (!jQuery(node).is("button, select, textarea")) { 103 | jQuery.each(node.childNodes, function() { 104 | highlight(this, addItems); 105 | }); 106 | } 107 | } 108 | var addItems = []; 109 | var result = this.each(function() { 110 | highlight(this, addItems); 111 | }); 112 | for (var i = 0; i < addItems.length; ++i) { 113 | jQuery(addItems[i].parent).before(addItems[i].target); 114 | } 115 | return result; 116 | }; 117 | 118 | /* 119 | * backward compatibility for jQuery.browser 120 | * This will be supported until firefox bug is fixed. 121 | */ 122 | if (!jQuery.browser) { 123 | jQuery.uaMatch = function(ua) { 124 | ua = ua.toLowerCase(); 125 | 126 | var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || 127 | /(webkit)[ \/]([\w.]+)/.exec(ua) || 128 | /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || 129 | /(msie) ([\w.]+)/.exec(ua) || 130 | ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || 131 | []; 132 | 133 | return { 134 | browser: match[ 1 ] || "", 135 | version: match[ 2 ] || "0" 136 | }; 137 | }; 138 | jQuery.browser = {}; 139 | jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; 140 | } 141 | 142 | /** 143 | * Small JavaScript module for the documentation. 144 | */ 145 | var Documentation = { 146 | 147 | init : function() { 148 | this.fixFirefoxAnchorBug(); 149 | this.highlightSearchWords(); 150 | this.initIndexTable(); 151 | 152 | }, 153 | 154 | /** 155 | * i18n support 156 | */ 157 | TRANSLATIONS : {}, 158 | PLURAL_EXPR : function(n) { return n === 1 ? 0 : 1; }, 159 | LOCALE : 'unknown', 160 | 161 | // gettext and ngettext don't access this so that the functions 162 | // can safely bound to a different name (_ = Documentation.gettext) 163 | gettext : function(string) { 164 | var translated = Documentation.TRANSLATIONS[string]; 165 | if (typeof translated === 'undefined') 166 | return string; 167 | return (typeof translated === 'string') ? translated : translated[0]; 168 | }, 169 | 170 | ngettext : function(singular, plural, n) { 171 | var translated = Documentation.TRANSLATIONS[singular]; 172 | if (typeof translated === 'undefined') 173 | return (n == 1) ? singular : plural; 174 | return translated[Documentation.PLURALEXPR(n)]; 175 | }, 176 | 177 | addTranslations : function(catalog) { 178 | for (var key in catalog.messages) 179 | this.TRANSLATIONS[key] = catalog.messages[key]; 180 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); 181 | this.LOCALE = catalog.locale; 182 | }, 183 | 184 | /** 185 | * add context elements like header anchor links 186 | */ 187 | addContextElements : function() { 188 | $('div[id] > :header:first').each(function() { 189 | $('\u00B6'). 190 | attr('href', '#' + this.id). 191 | attr('title', _('Permalink to this headline')). 192 | appendTo(this); 193 | }); 194 | $('dt[id]').each(function() { 195 | $('\u00B6'). 196 | attr('href', '#' + this.id). 197 | attr('title', _('Permalink to this definition')). 198 | appendTo(this); 199 | }); 200 | }, 201 | 202 | /** 203 | * workaround a firefox stupidity 204 | * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 205 | */ 206 | fixFirefoxAnchorBug : function() { 207 | if (document.location.hash && $.browser.mozilla) 208 | window.setTimeout(function() { 209 | document.location.href += ''; 210 | }, 10); 211 | }, 212 | 213 | /** 214 | * highlight the search words provided in the url in the text 215 | */ 216 | highlightSearchWords : function() { 217 | var params = $.getQueryParameters(); 218 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; 219 | if (terms.length) { 220 | var body = $('div.body'); 221 | if (!body.length) { 222 | body = $('body'); 223 | } 224 | window.setTimeout(function() { 225 | $.each(terms, function() { 226 | body.highlightText(this.toLowerCase(), 'highlighted'); 227 | }); 228 | }, 10); 229 | $('

' + _('Hide Search Matches') + '

') 231 | .appendTo($('#searchbox')); 232 | } 233 | }, 234 | 235 | /** 236 | * init the domain index toggle buttons 237 | */ 238 | initIndexTable : function() { 239 | var togglers = $('img.toggler').click(function() { 240 | var src = $(this).attr('src'); 241 | var idnum = $(this).attr('id').substr(7); 242 | $('tr.cg-' + idnum).toggle(); 243 | if (src.substr(-9) === 'minus.png') 244 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 245 | else 246 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 247 | }).css('display', ''); 248 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 249 | togglers.click(); 250 | } 251 | }, 252 | 253 | /** 254 | * helper function to hide the search marks again 255 | */ 256 | hideSearchWords : function() { 257 | $('#searchbox .highlight-link').fadeOut(300); 258 | $('span.highlighted').removeClass('highlighted'); 259 | }, 260 | 261 | /** 262 | * make the url absolute 263 | */ 264 | makeURL : function(relativeURL) { 265 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 266 | }, 267 | 268 | /** 269 | * get the current relative url 270 | */ 271 | getCurrentURL : function() { 272 | var path = document.location.pathname; 273 | var parts = path.split(/\//); 274 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 275 | if (this === '..') 276 | parts.pop(); 277 | }); 278 | var url = parts.join('/'); 279 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 280 | }, 281 | 282 | initOnKeyListeners: function() { 283 | $(document).keyup(function(event) { 284 | var activeElementType = document.activeElement.tagName; 285 | // don't navigate when in search box or textarea 286 | if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { 287 | switch (event.keyCode) { 288 | case 37: // left 289 | var prevHref = $('link[rel="prev"]').prop('href'); 290 | if (prevHref) { 291 | window.location.href = prevHref; 292 | return false; 293 | } 294 | case 39: // right 295 | var nextHref = $('link[rel="next"]').prop('href'); 296 | if (nextHref) { 297 | window.location.href = nextHref; 298 | return false; 299 | } 300 | } 301 | } 302 | }); 303 | } 304 | }; 305 | 306 | // quick alias for translations 307 | _ = Documentation.gettext; 308 | 309 | $(document).ready(function() { 310 | Documentation.init(); 311 | }); -------------------------------------------------------------------------------- /docs/_build/html/_static/underscore.js: -------------------------------------------------------------------------------- 1 | // Underscore.js 1.3.1 2 | // (c) 2009-2012 Jeremy Ashkenas, DocumentCloud Inc. 3 | // Underscore is freely distributable under the MIT license. 4 | // Portions of Underscore are inspired or borrowed from Prototype, 5 | // Oliver Steele's Functional, and John Resig's Micro-Templating. 6 | // For all details and documentation: 7 | // http://documentcloud.github.com/underscore 8 | (function(){function q(a,c,d){if(a===c)return a!==0||1/a==1/c;if(a==null||c==null)return a===c;if(a._chain)a=a._wrapped;if(c._chain)c=c._wrapped;if(a.isEqual&&b.isFunction(a.isEqual))return a.isEqual(c);if(c.isEqual&&b.isFunction(c.isEqual))return c.isEqual(a);var e=l.call(a);if(e!=l.call(c))return false;switch(e){case "[object String]":return a==String(c);case "[object Number]":return a!=+a?c!=+c:a==0?1/a==1/c:a==+c;case "[object Date]":case "[object Boolean]":return+a==+c;case "[object RegExp]":return a.source== 9 | c.source&&a.global==c.global&&a.multiline==c.multiline&&a.ignoreCase==c.ignoreCase}if(typeof a!="object"||typeof c!="object")return false;for(var f=d.length;f--;)if(d[f]==a)return true;d.push(a);var f=0,g=true;if(e=="[object Array]"){if(f=a.length,g=f==c.length)for(;f--;)if(!(g=f in a==f in c&&q(a[f],c[f],d)))break}else{if("constructor"in a!="constructor"in c||a.constructor!=c.constructor)return false;for(var h in a)if(b.has(a,h)&&(f++,!(g=b.has(c,h)&&q(a[h],c[h],d))))break;if(g){for(h in c)if(b.has(c, 10 | h)&&!f--)break;g=!f}}d.pop();return g}var r=this,G=r._,n={},k=Array.prototype,o=Object.prototype,i=k.slice,H=k.unshift,l=o.toString,I=o.hasOwnProperty,w=k.forEach,x=k.map,y=k.reduce,z=k.reduceRight,A=k.filter,B=k.every,C=k.some,p=k.indexOf,D=k.lastIndexOf,o=Array.isArray,J=Object.keys,s=Function.prototype.bind,b=function(a){return new m(a)};if(typeof exports!=="undefined"){if(typeof module!=="undefined"&&module.exports)exports=module.exports=b;exports._=b}else r._=b;b.VERSION="1.3.1";var j=b.each= 11 | b.forEach=function(a,c,d){if(a!=null)if(w&&a.forEach===w)a.forEach(c,d);else if(a.length===+a.length)for(var e=0,f=a.length;e2;a== 12 | null&&(a=[]);if(y&&a.reduce===y)return e&&(c=b.bind(c,e)),f?a.reduce(c,d):a.reduce(c);j(a,function(a,b,i){f?d=c.call(e,d,a,b,i):(d=a,f=true)});if(!f)throw new TypeError("Reduce of empty array with no initial value");return d};b.reduceRight=b.foldr=function(a,c,d,e){var f=arguments.length>2;a==null&&(a=[]);if(z&&a.reduceRight===z)return e&&(c=b.bind(c,e)),f?a.reduceRight(c,d):a.reduceRight(c);var g=b.toArray(a).reverse();e&&!f&&(c=b.bind(c,e));return f?b.reduce(g,c,d,e):b.reduce(g,c)};b.find=b.detect= 13 | function(a,c,b){var e;E(a,function(a,g,h){if(c.call(b,a,g,h))return e=a,true});return e};b.filter=b.select=function(a,c,b){var e=[];if(a==null)return e;if(A&&a.filter===A)return a.filter(c,b);j(a,function(a,g,h){c.call(b,a,g,h)&&(e[e.length]=a)});return e};b.reject=function(a,c,b){var e=[];if(a==null)return e;j(a,function(a,g,h){c.call(b,a,g,h)||(e[e.length]=a)});return e};b.every=b.all=function(a,c,b){var e=true;if(a==null)return e;if(B&&a.every===B)return a.every(c,b);j(a,function(a,g,h){if(!(e= 14 | e&&c.call(b,a,g,h)))return n});return e};var E=b.some=b.any=function(a,c,d){c||(c=b.identity);var e=false;if(a==null)return e;if(C&&a.some===C)return a.some(c,d);j(a,function(a,b,h){if(e||(e=c.call(d,a,b,h)))return n});return!!e};b.include=b.contains=function(a,c){var b=false;if(a==null)return b;return p&&a.indexOf===p?a.indexOf(c)!=-1:b=E(a,function(a){return a===c})};b.invoke=function(a,c){var d=i.call(arguments,2);return b.map(a,function(a){return(b.isFunction(c)?c||a:a[c]).apply(a,d)})};b.pluck= 15 | function(a,c){return b.map(a,function(a){return a[c]})};b.max=function(a,c,d){if(!c&&b.isArray(a))return Math.max.apply(Math,a);if(!c&&b.isEmpty(a))return-Infinity;var e={computed:-Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;b>=e.computed&&(e={value:a,computed:b})});return e.value};b.min=function(a,c,d){if(!c&&b.isArray(a))return Math.min.apply(Math,a);if(!c&&b.isEmpty(a))return Infinity;var e={computed:Infinity};j(a,function(a,b,h){b=c?c.call(d,a,b,h):a;bd?1:0}),"value")};b.groupBy=function(a,c){var d={},e=b.isFunction(c)?c:function(a){return a[c]};j(a,function(a,b){var c=e(a,b);(d[c]||(d[c]=[])).push(a)});return d};b.sortedIndex=function(a, 17 | c,d){d||(d=b.identity);for(var e=0,f=a.length;e>1;d(a[g])=0})})};b.difference=function(a){var c=b.flatten(i.call(arguments,1));return b.filter(a,function(a){return!b.include(c,a)})};b.zip=function(){for(var a=i.call(arguments),c=b.max(b.pluck(a,"length")),d=Array(c),e=0;e=0;d--)b=[a[d].apply(this,b)];return b[0]}}; 24 | b.after=function(a,b){return a<=0?b():function(){if(--a<1)return b.apply(this,arguments)}};b.keys=J||function(a){if(a!==Object(a))throw new TypeError("Invalid object");var c=[],d;for(d in a)b.has(a,d)&&(c[c.length]=d);return c};b.values=function(a){return b.map(a,b.identity)};b.functions=b.methods=function(a){var c=[],d;for(d in a)b.isFunction(a[d])&&c.push(d);return c.sort()};b.extend=function(a){j(i.call(arguments,1),function(b){for(var d in b)a[d]=b[d]});return a};b.defaults=function(a){j(i.call(arguments, 25 | 1),function(b){for(var d in b)a[d]==null&&(a[d]=b[d])});return a};b.clone=function(a){return!b.isObject(a)?a:b.isArray(a)?a.slice():b.extend({},a)};b.tap=function(a,b){b(a);return a};b.isEqual=function(a,b){return q(a,b,[])};b.isEmpty=function(a){if(b.isArray(a)||b.isString(a))return a.length===0;for(var c in a)if(b.has(a,c))return false;return true};b.isElement=function(a){return!!(a&&a.nodeType==1)};b.isArray=o||function(a){return l.call(a)=="[object Array]"};b.isObject=function(a){return a===Object(a)}; 26 | b.isArguments=function(a){return l.call(a)=="[object Arguments]"};if(!b.isArguments(arguments))b.isArguments=function(a){return!(!a||!b.has(a,"callee"))};b.isFunction=function(a){return l.call(a)=="[object Function]"};b.isString=function(a){return l.call(a)=="[object String]"};b.isNumber=function(a){return l.call(a)=="[object Number]"};b.isNaN=function(a){return a!==a};b.isBoolean=function(a){return a===true||a===false||l.call(a)=="[object Boolean]"};b.isDate=function(a){return l.call(a)=="[object Date]"}; 27 | b.isRegExp=function(a){return l.call(a)=="[object RegExp]"};b.isNull=function(a){return a===null};b.isUndefined=function(a){return a===void 0};b.has=function(a,b){return I.call(a,b)};b.noConflict=function(){r._=G;return this};b.identity=function(a){return a};b.times=function(a,b,d){for(var e=0;e/g,">").replace(/"/g,""").replace(/'/g,"'").replace(/\//g,"/")};b.mixin=function(a){j(b.functions(a), 28 | function(c){K(c,b[c]=a[c])})};var L=0;b.uniqueId=function(a){var b=L++;return a?a+b:b};b.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var t=/.^/,u=function(a){return a.replace(/\\\\/g,"\\").replace(/\\'/g,"'")};b.template=function(a,c){var d=b.templateSettings,d="var __p=[],print=function(){__p.push.apply(__p,arguments);};with(obj||{}){__p.push('"+a.replace(/\\/g,"\\\\").replace(/'/g,"\\'").replace(d.escape||t,function(a,b){return"',_.escape("+ 29 | u(b)+"),'"}).replace(d.interpolate||t,function(a,b){return"',"+u(b)+",'"}).replace(d.evaluate||t,function(a,b){return"');"+u(b).replace(/[\r\n\t]/g," ")+";__p.push('"}).replace(/\r/g,"\\r").replace(/\n/g,"\\n").replace(/\t/g,"\\t")+"');}return __p.join('');",e=new Function("obj","_",d);return c?e(c,b):function(a){return e.call(this,a,b)}};b.chain=function(a){return b(a).chain()};var m=function(a){this._wrapped=a};b.prototype=m.prototype;var v=function(a,c){return c?b(a).chain():a},K=function(a,c){m.prototype[a]= 30 | function(){var a=i.call(arguments);H.call(a,this._wrapped);return v(c.apply(b,a),this._chain)}};b.mixin(b);j("pop,push,reverse,shift,sort,splice,unshift".split(","),function(a){var b=k[a];m.prototype[a]=function(){var d=this._wrapped;b.apply(d,arguments);var e=d.length;(a=="shift"||a=="splice")&&e===0&&delete d[0];return v(d,this._chain)}});j(["concat","join","slice"],function(a){var b=k[a];m.prototype[a]=function(){return v(b.apply(this._wrapped,arguments),this._chain)}});m.prototype.chain=function(){this._chain= 31 | true;return this};m.prototype.value=function(){return this._wrapped}}).call(this); 32 | -------------------------------------------------------------------------------- /day_1/0_Using_Pretrained_Embeddings.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from annoy import AnnoyIndex\n", 10 | "import numpy as np\n", 11 | "import torch\n", 12 | "from tqdm import tqdm_notebook\n", 13 | "from argparse import Namespace" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 5, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "args = Namespace(\n", 23 | " glove_filename='data/glove.6B.100d.txt'\n", 24 | ")" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 15, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "def load_word_vectors(filename):\n", 34 | " word_to_index = {}\n", 35 | " word_vectors = []\n", 36 | " \n", 37 | " with open(filename) as fp:\n", 38 | " for line in tqdm_notebook(fp.readlines(), leave=False):\n", 39 | " line = line.split(\" \")\n", 40 | " \n", 41 | " word = line[0]\n", 42 | " word_to_index[word] = len(word_to_index)\n", 43 | " \n", 44 | " vec = np.array([float(x) for x in line[1:]])\n", 45 | " word_vectors.append(vec)\n", 46 | " \n", 47 | " return word_to_index, word_vectors" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 16, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "class PreTrainedEmbeddings(object):\n", 57 | " def __init__(self, glove_filename):\n", 58 | " self.word_to_index, self.word_vectors = load_word_vectors(glove_filename)\n", 59 | " self.word_vector_size = len(self.word_vectors[0])\n", 60 | " \n", 61 | " self.index_to_word = {v: k for k, v in self.word_to_index.items()}\n", 62 | " self.index = AnnoyIndex(self.word_vector_size, metric='euclidean')\n", 63 | " print('Building Index')\n", 64 | " for _, i in tqdm_notebook(self.word_to_index.items(), leave=False):\n", 65 | " self.index.add_item(i, self.word_vectors[i])\n", 66 | " self.index.build(50)\n", 67 | " print('Finished!')\n", 68 | " \n", 69 | " def get_embedding(self, word):\n", 70 | " return self.word_vectors[self.word_to_index[word]]\n", 71 | " \n", 72 | " def closest(self, word, n=1):\n", 73 | " vector = self.get_embedding(word)\n", 74 | " nn_indices = self.index.get_nns_by_vector(vector, n)\n", 75 | " return [self.index_to_word[neighbor] for neighbor in nn_indices]\n", 76 | " \n", 77 | " def closest_v(self, vector, n=1):\n", 78 | " nn_indices = self.index.get_nns_by_vector(vector, n)\n", 79 | " return [self.index_to_word[neighbor] for neighbor in nn_indices]\n", 80 | " \n", 81 | " def sim(self, w1, w2):\n", 82 | " return np.dot(self.get_embedding(w1), self.get_embedding(w2))" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 17, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "data": { 92 | "application/vnd.jupyter.widget-view+json": { 93 | "model_id": "", 94 | "version_major": 2, 95 | "version_minor": 0 96 | }, 97 | "text/plain": [ 98 | "HBox(children=(IntProgress(value=0, max=400000), HTML(value='')))" 99 | ] 100 | }, 101 | "metadata": {}, 102 | "output_type": "display_data" 103 | }, 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "\r", 109 | "Building Index\n" 110 | ] 111 | }, 112 | { 113 | "data": { 114 | "application/vnd.jupyter.widget-view+json": { 115 | "model_id": "", 116 | "version_major": 2, 117 | "version_minor": 0 118 | }, 119 | "text/plain": [ 120 | "HBox(children=(IntProgress(value=0, max=400000), HTML(value='')))" 121 | ] 122 | }, 123 | "metadata": {}, 124 | "output_type": "display_data" 125 | }, 126 | { 127 | "name": "stdout", 128 | "output_type": "stream", 129 | "text": [ 130 | "Finished!\n" 131 | ] 132 | } 133 | ], 134 | "source": [ 135 | "glove = PreTrainedEmbeddings(args.glove_filename)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 18, 141 | "metadata": { 142 | "scrolled": true 143 | }, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "['apple', 'microsoft', 'dell', 'pc', 'compaq']" 149 | ] 150 | }, 151 | "execution_count": 18, 152 | "metadata": {}, 153 | "output_type": "execute_result" 154 | } 155 | ], 156 | "source": [ 157 | "glove.closest('apple', n=5)" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 19, 163 | "metadata": { 164 | "scrolled": true 165 | }, 166 | "outputs": [ 167 | { 168 | "data": { 169 | "text/plain": [ 170 | "['plane', 'airplane', 'jet', 'flight', 'crashed']" 171 | ] 172 | }, 173 | "execution_count": 19, 174 | "metadata": {}, 175 | "output_type": "execute_result" 176 | } 177 | ], 178 | "source": [ 179 | "glove.closest('plane', n=5)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": 34, 185 | "metadata": {}, 186 | "outputs": [ 187 | { 188 | "data": { 189 | "text/plain": [ 190 | "(26.873448266652, 16.501491855324)" 191 | ] 192 | }, 193 | "execution_count": 34, 194 | "metadata": {}, 195 | "output_type": "execute_result" 196 | } 197 | ], 198 | "source": [ 199 | "glove.sim('beer', 'wine'), glove.sim('beer', 'gasoline')" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": {}, 205 | "source": [ 206 | "** Lexical relationships uncovered by word embeddings **" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 35, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "def SAT_analogy(w1, w2, w3):\n", 216 | " '''\n", 217 | " Solves problems of the type:\n", 218 | " w1 : w2 :: w3 : __\n", 219 | " '''\n", 220 | " closest_words = []\n", 221 | " try:\n", 222 | " w1v = glove.get_embedding(w1)\n", 223 | " w2v = glove.get_embedding(w2)\n", 224 | " w3v = glove.get_embedding(w3)\n", 225 | " w4v = w3v + (w2v - w1v)\n", 226 | " closest_words = glove.closest_v(w4v, n=5)\n", 227 | " closest_words = [w for w in closest_words if w not in [w1, w2, w3]]\n", 228 | " except:\n", 229 | " pass\n", 230 | " if len(closest_words) == 0:\n", 231 | " print(':-(')\n", 232 | " else:\n", 233 | " print('{} : {} :: {} : {}'.format(w1, w2, w3, closest_words[0]))" 234 | ] 235 | }, 236 | { 237 | "cell_type": "markdown", 238 | "metadata": {}, 239 | "source": [ 240 | "**Pronouns**" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": 36, 246 | "metadata": {}, 247 | "outputs": [ 248 | { 249 | "name": "stdout", 250 | "output_type": "stream", 251 | "text": [ 252 | "man : he :: woman : she\n" 253 | ] 254 | } 255 | ], 256 | "source": [ 257 | "SAT_analogy('man', 'he', 'woman')" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": {}, 263 | "source": [ 264 | "** Verb-Noun relationships **" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 37, 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "name": "stdout", 274 | "output_type": "stream", 275 | "text": [ 276 | "fly : plane :: sail : ship\n" 277 | ] 278 | } 279 | ], 280 | "source": [ 281 | "SAT_analogy('fly', 'plane', 'sail')" 282 | ] 283 | }, 284 | { 285 | "cell_type": "markdown", 286 | "metadata": {}, 287 | "source": [ 288 | "**Noun-Noun relationships**" 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 38, 294 | "metadata": {}, 295 | "outputs": [ 296 | { 297 | "name": "stdout", 298 | "output_type": "stream", 299 | "text": [ 300 | "cat : kitten :: dog : pug\n" 301 | ] 302 | } 303 | ], 304 | "source": [ 305 | "SAT_analogy('cat', 'kitten', 'dog')" 306 | ] 307 | }, 308 | { 309 | "cell_type": "markdown", 310 | "metadata": {}, 311 | "source": [ 312 | "**Hypernymy**" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 39, 318 | "metadata": {}, 319 | "outputs": [ 320 | { 321 | "name": "stdout", 322 | "output_type": "stream", 323 | "text": [ 324 | "blue : color :: dog : animal\n" 325 | ] 326 | } 327 | ], 328 | "source": [ 329 | "SAT_analogy('blue', 'color', 'dog')" 330 | ] 331 | }, 332 | { 333 | "cell_type": "markdown", 334 | "metadata": { 335 | "collapsed": true 336 | }, 337 | "source": [ 338 | "**Meronymy**" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 40, 344 | "metadata": {}, 345 | "outputs": [ 346 | { 347 | "name": "stdout", 348 | "output_type": "stream", 349 | "text": [ 350 | "leg : legs :: hand : hands\n" 351 | ] 352 | } 353 | ], 354 | "source": [ 355 | "SAT_analogy('leg', 'legs', 'hand')" 356 | ] 357 | }, 358 | { 359 | "cell_type": "markdown", 360 | "metadata": {}, 361 | "source": [ 362 | "**Troponymy**" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 41, 368 | "metadata": {}, 369 | "outputs": [ 370 | { 371 | "name": "stdout", 372 | "output_type": "stream", 373 | "text": [ 374 | "talk : communicate :: read : correctly\n" 375 | ] 376 | } 377 | ], 378 | "source": [ 379 | "SAT_analogy('talk', 'communicate', 'read')" 380 | ] 381 | }, 382 | { 383 | "cell_type": "markdown", 384 | "metadata": {}, 385 | "source": [ 386 | "**Metonymy**" 387 | ] 388 | }, 389 | { 390 | "cell_type": "code", 391 | "execution_count": 42, 392 | "metadata": {}, 393 | "outputs": [ 394 | { 395 | "name": "stdout", 396 | "output_type": "stream", 397 | "text": [ 398 | "blue : democrat :: red : republican\n" 399 | ] 400 | } 401 | ], 402 | "source": [ 403 | "SAT_analogy('blue', 'democrat', 'red')" 404 | ] 405 | }, 406 | { 407 | "cell_type": "markdown", 408 | "metadata": {}, 409 | "source": [ 410 | "**Misc**" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 43, 416 | "metadata": {}, 417 | "outputs": [ 418 | { 419 | "name": "stdout", 420 | "output_type": "stream", 421 | "text": [ 422 | "man : doctor :: woman : nurse\n" 423 | ] 424 | } 425 | ], 426 | "source": [ 427 | "SAT_analogy('man', 'doctor', 'woman')" 428 | ] 429 | }, 430 | { 431 | "cell_type": "code", 432 | "execution_count": 44, 433 | "metadata": {}, 434 | "outputs": [ 435 | { 436 | "name": "stdout", 437 | "output_type": "stream", 438 | "text": [ 439 | "man : leader :: woman : opposition\n" 440 | ] 441 | } 442 | ], 443 | "source": [ 444 | "SAT_analogy('man', 'leader', 'woman')" 445 | ] 446 | } 447 | ], 448 | "metadata": { 449 | "kernelspec": { 450 | "display_name": "Python 3", 451 | "language": "python", 452 | "name": "python3" 453 | }, 454 | "language_info": { 455 | "codemirror_mode": { 456 | "name": "ipython", 457 | "version": 3 458 | }, 459 | "file_extension": ".py", 460 | "mimetype": "text/x-python", 461 | "name": "python", 462 | "nbconvert_exporter": "python", 463 | "pygments_lexer": "ipython3", 464 | "version": "3.6.5" 465 | } 466 | }, 467 | "nbformat": 4, 468 | "nbformat_minor": 2 469 | } 470 | -------------------------------------------------------------------------------- /day_1/figures/intro_to_pytorch/computational_graph_forward.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /day_1/figures/intro_to_pytorch/pytorch_variable.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/_build/html/_static/basic.css: -------------------------------------------------------------------------------- 1 | /* 2 | * basic.css 3 | * ~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- basic theme. 6 | * 7 | * :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /* -- main layout ----------------------------------------------------------- */ 13 | 14 | div.clearer { 15 | clear: both; 16 | } 17 | 18 | /* -- relbar ---------------------------------------------------------------- */ 19 | 20 | div.related { 21 | width: 100%; 22 | font-size: 90%; 23 | } 24 | 25 | div.related h3 { 26 | display: none; 27 | } 28 | 29 | div.related ul { 30 | margin: 0; 31 | padding: 0 0 0 10px; 32 | list-style: none; 33 | } 34 | 35 | div.related li { 36 | display: inline; 37 | } 38 | 39 | div.related li.right { 40 | float: right; 41 | margin-right: 5px; 42 | } 43 | 44 | /* -- sidebar --------------------------------------------------------------- */ 45 | 46 | div.sphinxsidebarwrapper { 47 | padding: 10px 5px 0 10px; 48 | } 49 | 50 | div.sphinxsidebar { 51 | float: left; 52 | width: 230px; 53 | margin-left: -100%; 54 | font-size: 90%; 55 | word-wrap: break-word; 56 | overflow-wrap : break-word; 57 | } 58 | 59 | div.sphinxsidebar ul { 60 | list-style: none; 61 | } 62 | 63 | div.sphinxsidebar ul ul, 64 | div.sphinxsidebar ul.want-points { 65 | margin-left: 20px; 66 | list-style: square; 67 | } 68 | 69 | div.sphinxsidebar ul ul { 70 | margin-top: 0; 71 | margin-bottom: 0; 72 | } 73 | 74 | div.sphinxsidebar form { 75 | margin-top: 10px; 76 | } 77 | 78 | div.sphinxsidebar input { 79 | border: 1px solid #98dbcc; 80 | font-family: sans-serif; 81 | font-size: 1em; 82 | } 83 | 84 | div.sphinxsidebar #searchbox input[type="text"] { 85 | width: 170px; 86 | } 87 | 88 | img { 89 | border: 0; 90 | max-width: 100%; 91 | } 92 | 93 | /* -- search page ----------------------------------------------------------- */ 94 | 95 | ul.search { 96 | margin: 10px 0 0 20px; 97 | padding: 0; 98 | } 99 | 100 | ul.search li { 101 | padding: 5px 0 5px 20px; 102 | background-image: url(file.png); 103 | background-repeat: no-repeat; 104 | background-position: 0 7px; 105 | } 106 | 107 | ul.search li a { 108 | font-weight: bold; 109 | } 110 | 111 | ul.search li div.context { 112 | color: #888; 113 | margin: 2px 0 0 30px; 114 | text-align: left; 115 | } 116 | 117 | ul.keywordmatches li.goodmatch a { 118 | font-weight: bold; 119 | } 120 | 121 | /* -- index page ------------------------------------------------------------ */ 122 | 123 | table.contentstable { 124 | width: 90%; 125 | margin-left: auto; 126 | margin-right: auto; 127 | } 128 | 129 | table.contentstable p.biglink { 130 | line-height: 150%; 131 | } 132 | 133 | a.biglink { 134 | font-size: 1.3em; 135 | } 136 | 137 | span.linkdescr { 138 | font-style: italic; 139 | padding-top: 5px; 140 | font-size: 90%; 141 | } 142 | 143 | /* -- general index --------------------------------------------------------- */ 144 | 145 | table.indextable { 146 | width: 100%; 147 | } 148 | 149 | table.indextable td { 150 | text-align: left; 151 | vertical-align: top; 152 | } 153 | 154 | table.indextable ul { 155 | margin-top: 0; 156 | margin-bottom: 0; 157 | list-style-type: none; 158 | } 159 | 160 | table.indextable > tbody > tr > td > ul { 161 | padding-left: 0em; 162 | } 163 | 164 | table.indextable tr.pcap { 165 | height: 10px; 166 | } 167 | 168 | table.indextable tr.cap { 169 | margin-top: 10px; 170 | background-color: #f2f2f2; 171 | } 172 | 173 | img.toggler { 174 | margin-right: 3px; 175 | margin-top: 3px; 176 | cursor: pointer; 177 | } 178 | 179 | div.modindex-jumpbox { 180 | border-top: 1px solid #ddd; 181 | border-bottom: 1px solid #ddd; 182 | margin: 1em 0 1em 0; 183 | padding: 0.4em; 184 | } 185 | 186 | div.genindex-jumpbox { 187 | border-top: 1px solid #ddd; 188 | border-bottom: 1px solid #ddd; 189 | margin: 1em 0 1em 0; 190 | padding: 0.4em; 191 | } 192 | 193 | /* -- domain module index --------------------------------------------------- */ 194 | 195 | table.modindextable td { 196 | padding: 2px; 197 | border-collapse: collapse; 198 | } 199 | 200 | /* -- general body styles --------------------------------------------------- */ 201 | 202 | div.body p, div.body dd, div.body li, div.body blockquote { 203 | -moz-hyphens: auto; 204 | -ms-hyphens: auto; 205 | -webkit-hyphens: auto; 206 | hyphens: auto; 207 | } 208 | 209 | a.headerlink { 210 | visibility: hidden; 211 | } 212 | 213 | h1:hover > a.headerlink, 214 | h2:hover > a.headerlink, 215 | h3:hover > a.headerlink, 216 | h4:hover > a.headerlink, 217 | h5:hover > a.headerlink, 218 | h6:hover > a.headerlink, 219 | dt:hover > a.headerlink, 220 | caption:hover > a.headerlink, 221 | p.caption:hover > a.headerlink, 222 | div.code-block-caption:hover > a.headerlink { 223 | visibility: visible; 224 | } 225 | 226 | div.body p.caption { 227 | text-align: inherit; 228 | } 229 | 230 | div.body td { 231 | text-align: left; 232 | } 233 | 234 | .first { 235 | margin-top: 0 !important; 236 | } 237 | 238 | p.rubric { 239 | margin-top: 30px; 240 | font-weight: bold; 241 | } 242 | 243 | img.align-left, .figure.align-left, object.align-left { 244 | clear: left; 245 | float: left; 246 | margin-right: 1em; 247 | } 248 | 249 | img.align-right, .figure.align-right, object.align-right { 250 | clear: right; 251 | float: right; 252 | margin-left: 1em; 253 | } 254 | 255 | img.align-center, .figure.align-center, object.align-center { 256 | display: block; 257 | margin-left: auto; 258 | margin-right: auto; 259 | } 260 | 261 | .align-left { 262 | text-align: left; 263 | } 264 | 265 | .align-center { 266 | text-align: center; 267 | } 268 | 269 | .align-right { 270 | text-align: right; 271 | } 272 | 273 | /* -- sidebars -------------------------------------------------------------- */ 274 | 275 | div.sidebar { 276 | margin: 0 0 0.5em 1em; 277 | border: 1px solid #ddb; 278 | padding: 7px 7px 0 7px; 279 | background-color: #ffe; 280 | width: 40%; 281 | float: right; 282 | } 283 | 284 | p.sidebar-title { 285 | font-weight: bold; 286 | } 287 | 288 | /* -- topics ---------------------------------------------------------------- */ 289 | 290 | div.topic { 291 | border: 1px solid #ccc; 292 | padding: 7px 7px 0 7px; 293 | margin: 10px 0 10px 0; 294 | } 295 | 296 | p.topic-title { 297 | font-size: 1.1em; 298 | font-weight: bold; 299 | margin-top: 10px; 300 | } 301 | 302 | /* -- admonitions ----------------------------------------------------------- */ 303 | 304 | div.admonition { 305 | margin-top: 10px; 306 | margin-bottom: 10px; 307 | padding: 7px; 308 | } 309 | 310 | div.admonition dt { 311 | font-weight: bold; 312 | } 313 | 314 | div.admonition dl { 315 | margin-bottom: 0; 316 | } 317 | 318 | p.admonition-title { 319 | margin: 0px 10px 5px 0px; 320 | font-weight: bold; 321 | } 322 | 323 | div.body p.centered { 324 | text-align: center; 325 | margin-top: 25px; 326 | } 327 | 328 | /* -- tables ---------------------------------------------------------------- */ 329 | 330 | table.docutils { 331 | border: 0; 332 | border-collapse: collapse; 333 | } 334 | 335 | table caption span.caption-number { 336 | font-style: italic; 337 | } 338 | 339 | table caption span.caption-text { 340 | } 341 | 342 | table.docutils td, table.docutils th { 343 | padding: 1px 8px 1px 5px; 344 | border-top: 0; 345 | border-left: 0; 346 | border-right: 0; 347 | border-bottom: 1px solid #aaa; 348 | } 349 | 350 | table.footnote td, table.footnote th { 351 | border: 0 !important; 352 | } 353 | 354 | th { 355 | text-align: left; 356 | padding-right: 5px; 357 | } 358 | 359 | table.citation { 360 | border-left: solid 1px gray; 361 | margin-left: 1px; 362 | } 363 | 364 | table.citation td { 365 | border-bottom: none; 366 | } 367 | 368 | /* -- figures --------------------------------------------------------------- */ 369 | 370 | div.figure { 371 | margin: 0.5em; 372 | padding: 0.5em; 373 | } 374 | 375 | div.figure p.caption { 376 | padding: 0.3em; 377 | } 378 | 379 | div.figure p.caption span.caption-number { 380 | font-style: italic; 381 | } 382 | 383 | div.figure p.caption span.caption-text { 384 | } 385 | 386 | /* -- field list styles ----------------------------------------------------- */ 387 | 388 | table.field-list td, table.field-list th { 389 | border: 0 !important; 390 | } 391 | 392 | .field-list ul { 393 | margin: 0; 394 | padding-left: 1em; 395 | } 396 | 397 | .field-list p { 398 | margin: 0; 399 | } 400 | 401 | .field-name { 402 | -moz-hyphens: manual; 403 | -ms-hyphens: manual; 404 | -webkit-hyphens: manual; 405 | hyphens: manual; 406 | } 407 | 408 | /* -- other body styles ----------------------------------------------------- */ 409 | 410 | ol.arabic { 411 | list-style: decimal; 412 | } 413 | 414 | ol.loweralpha { 415 | list-style: lower-alpha; 416 | } 417 | 418 | ol.upperalpha { 419 | list-style: upper-alpha; 420 | } 421 | 422 | ol.lowerroman { 423 | list-style: lower-roman; 424 | } 425 | 426 | ol.upperroman { 427 | list-style: upper-roman; 428 | } 429 | 430 | dl { 431 | margin-bottom: 15px; 432 | } 433 | 434 | dd p { 435 | margin-top: 0px; 436 | } 437 | 438 | dd ul, dd table { 439 | margin-bottom: 10px; 440 | } 441 | 442 | dd { 443 | margin-top: 3px; 444 | margin-bottom: 10px; 445 | margin-left: 30px; 446 | } 447 | 448 | dt:target, span.highlighted { 449 | background-color: #fbe54e; 450 | } 451 | 452 | rect.highlighted { 453 | fill: #fbe54e; 454 | } 455 | 456 | dl.glossary dt { 457 | font-weight: bold; 458 | font-size: 1.1em; 459 | } 460 | 461 | .optional { 462 | font-size: 1.3em; 463 | } 464 | 465 | .sig-paren { 466 | font-size: larger; 467 | } 468 | 469 | .versionmodified { 470 | font-style: italic; 471 | } 472 | 473 | .system-message { 474 | background-color: #fda; 475 | padding: 5px; 476 | border: 3px solid red; 477 | } 478 | 479 | .footnote:target { 480 | background-color: #ffa; 481 | } 482 | 483 | .line-block { 484 | display: block; 485 | margin-top: 1em; 486 | margin-bottom: 1em; 487 | } 488 | 489 | .line-block .line-block { 490 | margin-top: 0; 491 | margin-bottom: 0; 492 | margin-left: 1.5em; 493 | } 494 | 495 | .guilabel, .menuselection { 496 | font-family: sans-serif; 497 | } 498 | 499 | .accelerator { 500 | text-decoration: underline; 501 | } 502 | 503 | .classifier { 504 | font-style: oblique; 505 | } 506 | 507 | abbr, acronym { 508 | border-bottom: dotted 1px; 509 | cursor: help; 510 | } 511 | 512 | /* -- code displays --------------------------------------------------------- */ 513 | 514 | pre { 515 | overflow: auto; 516 | overflow-y: hidden; /* fixes display issues on Chrome browsers */ 517 | } 518 | 519 | span.pre { 520 | -moz-hyphens: none; 521 | -ms-hyphens: none; 522 | -webkit-hyphens: none; 523 | hyphens: none; 524 | } 525 | 526 | td.linenos pre { 527 | padding: 5px 0px; 528 | border: 0; 529 | background-color: transparent; 530 | color: #aaa; 531 | } 532 | 533 | table.highlighttable { 534 | margin-left: 0.5em; 535 | } 536 | 537 | table.highlighttable td { 538 | padding: 0 0.5em 0 0.5em; 539 | } 540 | 541 | div.code-block-caption { 542 | padding: 2px 5px; 543 | font-size: small; 544 | } 545 | 546 | div.code-block-caption code { 547 | background-color: transparent; 548 | } 549 | 550 | div.code-block-caption + div > div.highlight > pre { 551 | margin-top: 0; 552 | } 553 | 554 | div.code-block-caption span.caption-number { 555 | padding: 0.1em 0.3em; 556 | font-style: italic; 557 | } 558 | 559 | div.code-block-caption span.caption-text { 560 | } 561 | 562 | div.literal-block-wrapper { 563 | padding: 1em 1em 0; 564 | } 565 | 566 | div.literal-block-wrapper div.highlight { 567 | margin: 0; 568 | } 569 | 570 | code.descname { 571 | background-color: transparent; 572 | font-weight: bold; 573 | font-size: 1.2em; 574 | } 575 | 576 | code.descclassname { 577 | background-color: transparent; 578 | } 579 | 580 | code.xref, a code { 581 | background-color: transparent; 582 | font-weight: bold; 583 | } 584 | 585 | h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { 586 | background-color: transparent; 587 | } 588 | 589 | .viewcode-link { 590 | float: right; 591 | } 592 | 593 | .viewcode-back { 594 | float: right; 595 | font-family: sans-serif; 596 | } 597 | 598 | div.viewcode-block:target { 599 | margin: -1px -10px; 600 | padding: 0 10px; 601 | } 602 | 603 | /* -- math display ---------------------------------------------------------- */ 604 | 605 | img.math { 606 | vertical-align: middle; 607 | } 608 | 609 | div.body div.math p { 610 | text-align: center; 611 | } 612 | 613 | span.eqno { 614 | float: right; 615 | } 616 | 617 | span.eqno a.headerlink { 618 | position: relative; 619 | left: 0px; 620 | z-index: 1; 621 | } 622 | 623 | div.math:hover a.headerlink { 624 | visibility: visible; 625 | } 626 | 627 | /* -- printout stylesheet --------------------------------------------------- */ 628 | 629 | @media print { 630 | div.document, 631 | div.documentwrapper, 632 | div.bodywrapper { 633 | margin: 0 !important; 634 | width: 100%; 635 | } 636 | 637 | div.sphinxsidebar, 638 | div.related, 639 | div.footer, 640 | #top-link { 641 | display: none; 642 | } 643 | } -------------------------------------------------------------------------------- /day_1/figures/intro_to_pytorch/computational_graph_backward.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | --------------------------------------------------------------------------------