├── Readme.md ├── .gitignore ├── jupyter_hello.py ├── jupyter_hello.ipynb ├── linux ├── run-nvidia-docker ├── install-docker └── install-nvidia-docker ├── docker-compose.yml ├── mac └── install-docker ├── 1-Assignment.py ├── 1-5-torchvision.py ├── Dockerfile ├── 1-4-gpu.py ├── 2-1-perceptron.py ├── 2-2.py ├── 2-4.py ├── 3-1.py ├── 1-3.py ├── 2-3.py ├── 2-Assignment.py ├── 1-4-gradient.py ├── 4-3.py ├── 1-5-datasets.py ├── 4-Assignment.py ├── 4-5.py ├── 2-5.py ├── 4-4.py ├── readme.md ├── Iris.csv ├── 3-Assignment.py ├── 3-2_3.py ├── 3-4_5.py └── Admission_Predict.csv /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | .vscode/ 3 | -------------------------------------------------------------------------------- /jupyter_hello.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import torch 3 | torch.__version__ -------------------------------------------------------------------------------- /jupyter_hello.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 2 6 | } 7 | -------------------------------------------------------------------------------- /linux/run-nvidia-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | docker build --tag pytorch-gpu . 4 | docker run --runtime nvidia -p 8888:8888 --volume $(pwd):/src pytorch-gpu 5 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | notebook: 4 | build: ./ 5 | image: anaconda-pytorch-notebook 6 | volumes: 7 | - ./:/src 8 | ports: 9 | - "8888:8888" 10 | -------------------------------------------------------------------------------- /mac/install-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # homebrew if you don't already have it 4 | which brew || ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 5 | 6 | # docker goes on as a cask, it's a 'mac app bundle' 7 | brew cask install docker 8 | brew install docker-compose -------------------------------------------------------------------------------- /linux/install-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # plain docker 5 | sudo apt install apt-transport-https ca-certificates curl software-properties-common 6 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 7 | # Add the package repositories 8 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 9 | sudo apt update 10 | # You may need to pin a specific version with docker-ce=... if nvidia docker is lagging the current docker release 11 | sudo apt install docker-ce docker-compose 12 | sudo docker run hello-world 13 | 14 | 15 | -------------------------------------------------------------------------------- /linux/install-nvidia-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # make sure to have installed docker first 5 | # and note, nvidia docker is tightly paired to docker versions 6 | # so you may need to remove and reinstall a specific version of docker 7 | 8 | # nvidia docker 9 | # Add the package repositories 10 | curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | \ 11 | sudo apt-key add - 12 | distribution=$(. /etc/os-release;echo $ID$VERSION_ID) 13 | curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ 14 | sudo tee /etc/apt/sources.list.d/nvidia-docker.list 15 | sudo apt update 16 | 17 | # Install nvidia-docker2 and reload the Docker daemon configuration 18 | sudo apt install -y nvidia-docker2 19 | sudo pkill -SIGHUP dockerd 20 | 21 | # Test nvidia-smi with the latest official CUDA image 22 | sudo docker run --runtime=nvidia --rm nvidia/cuda nvidia-smi 23 | 24 | 25 | -------------------------------------------------------------------------------- /1-Assignment.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import torchvision 3 | import torch 4 | 5 | # get some data -- don't forget to download it 6 | mnist = torchvision.datasets.MNIST('./var', 7 | download=True, 8 | transform=torchvision.transforms.ToTensor()) 9 | 10 | mnist[0] 11 | 12 | #%% 13 | 14 | # each batch, let's make a tensor of batch averages 15 | batches = torch.utils.data.DataLoader(mnist, 16 | batch_size=32) 17 | 18 | batch_averages = torch.Tensor([ 19 | batch[0].mean() for batch in batches 20 | ]) 21 | 22 | #%% 23 | # and there we have it 24 | batch_averages.mean() 25 | 26 | #%% now just for kicks -- let's compute the average a bit by hand 27 | # notice that the overall average is different than the batch-wise 28 | # average -- this is normal something to think about when 29 | # maching learning with batch training 30 | all_images = torch.cat([ 31 | image for image, label in mnist 32 | ]) 33 | 34 | all_images.shape, all_images.mean() -------------------------------------------------------------------------------- /1-5-torchvision.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # there are also a lot of predefined datasets in torchvision 3 | import torch 4 | import torchvision 5 | import matplotlib.pyplot as plt 6 | dir(torchvision.datasets) 7 | 8 | #%% 9 | # let's take a look at some of this data 10 | # really handy built in download! 11 | cifar = torchvision.datasets.CIFAR10('./var', download=True) 12 | cifar[0] 13 | 14 | #%% 15 | # looks like this is an image 16 | fig = plt.figure(figsize=(1,1)) 17 | sub = fig.add_subplot(111) 18 | sub.imshow(cifar[0][0]) 19 | 20 | #%% 21 | # how, that's a frog -- but -- we need a tensor of a 22 | # frog -- so that's where transforms come in 23 | # transforms are built in to torchvision and are 24 | # objects that implement __call__ can change the data 25 | from torchvision import transforms 26 | pipeline = transforms.Compose([ 27 | transforms.ToTensor() 28 | ]) 29 | cifar_tr = torchvision.datasets.CIFAR10('./var', transform=pipeline) 30 | 31 | #%% 32 | cifar_tr[0] -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 2 | 3 | 4 | ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 5 | ENV PATH /opt/conda/bin:$PATH 6 | 7 | RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ 8 | libglib2.0-0 libxext6 libsm6 libxrender1 \ 9 | git mercurial subversion 10 | 11 | RUN wget --quiet https://repo.anaconda.com/archive/Anaconda3-5.3.0-Linux-x86_64.sh -O ~/anaconda.sh && \ 12 | /bin/bash ~/anaconda.sh -b -p /opt/conda && \ 13 | rm ~/anaconda.sh && \ 14 | ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ 15 | echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ 16 | conda install pytorch torchvision cuda100 -c pytorch && \ 17 | echo "conda activate base" >> ~/.bashrc 18 | 19 | #all the code samples for the video series 20 | VOLUME ["/src"] 21 | 22 | #serve up a jupyter notebook 23 | WORKDIR /src 24 | EXPOSE 8888 25 | 26 | #this has security disabled which is less fuss for learning purposes 27 | CMD jupyter notebook --port=8888 --ip=0.0.0.0 --allow-root --NotebookApp.token='' --NotebookApp.disable_check_xsrf=True 28 | -------------------------------------------------------------------------------- /1-4-gpu.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # You'll need a GPU to get this to run! 3 | # `./linux/install-docker` 4 | # `./linux/install-nvidia-docker` 5 | # `./linux/run-nvidia-docker` 6 | # get your notebook server started 7 | 8 | # now let's talk about devices 9 | import torch 10 | cpu = torch.device('cpu') 11 | gpu = torch.device('cuda') 12 | cpu, gpu 13 | 14 | #%% 15 | # when you allocate a tensor, it's on a device, in the contexgt 16 | # of that device, if you don't specify, it's on the CPU 17 | x = torch.tensor([1.5]) 18 | x, x.device 19 | 20 | #%% 21 | # you can explicitly place, which is how I do it in general 22 | y = torch.tensor([2.5], device=cpu) 23 | y, y.device 24 | 25 | #%% 26 | # and now -- GPU 27 | z = torch.tensor([3.5], device=gpu) 28 | z, z.device 29 | 30 | #%% 31 | # you cannot mix devices, this is the important thing to remember 32 | # particularly when loading up data -- make sure you put things 33 | # together on a device! 34 | x + y + z 35 | 36 | #%% 37 | # but you can move things around to work on the gpu 38 | a = x.to(gpu) + y.to(gpu) + z 39 | a 40 | 41 | #%% 42 | # and you can move things back to the CPU 43 | b = a.to(cpu) 44 | b, b.device -------------------------------------------------------------------------------- /2-1-perceptron.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Here we can use network x to create a -- network. This isn't 3 | # exactly how a neural network works in practice, but it is a 4 | # great way to create a visualization you can modify in code 5 | 6 | #%% 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | import networkx as nx 10 | import math 11 | 12 | #%% 13 | # Building a graph with network X, a neural network 14 | # consists of three basic kinds of nodes 15 | # - inputs 16 | # - activations, these are the connections between all nodes 17 | # - outputs, this is how you tell what your network did 18 | 19 | #%% 20 | dense = nx.Graph() 21 | inputs = {i: (0, i) for i in range(0, 5)} 22 | activations = {i+100: (1, i) for i in range(0, 5)} 23 | outputs= {i+1000: (2, i) for i in range(0, 2)} 24 | all = {**inputs, **activations, **outputs} 25 | # and now -- fully connected, every input talks to every 26 | # activation -- this is the classic neural network 27 | for input in inputs: 28 | for activation in activations: 29 | dense.add_edge(input, activation) 30 | for activation in activations: 31 | for output in outputs: 32 | dense.add_edge(activation, output) 33 | nx.draw_networkx_nodes(dense, all, 34 | nodelist=all.keys(), node_color='b') 35 | nx.draw_networkx_edges(dense, all, edge_color='w') 36 | axes = plt.axis('off') 37 | 38 | #%% 39 | # in practice, these graphs are represented as tensors at each 40 | # layer and are connected via operations, such as a tensor 41 | # product which mathematically connects nodes via 42 | # multiplication and addition -------------------------------------------------------------------------------- /2-2.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Starting off, we need to import torch 3 | import torch 4 | 5 | #%% 6 | # this neural network will focus just on creating the network 7 | # and not on data or data loading, so we'll keep this simple and 8 | # build up the network step by step, in this first set of code 9 | # we'll work with dummy random inputs and outputs, and connect 10 | # them in a network 11 | 12 | #%% 13 | # inputs - -this is a 'batch' of size 1, with 1 color channel -- 14 | # imagine this is greyscale, and 64x and 64y pixels 15 | 16 | #%% 17 | inputs = torch.rand(1, 1, 64, 64) 18 | inputs 19 | 20 | #%% 21 | # outputs -- pretend we are building a binary classifier, 22 | # so we'll have to output possibilites, with a batch size of 1 23 | # we'll use rand again, so each thing can be a little bit 24 | # category 0 and a little bit category 1 25 | 26 | #%% 27 | outputs = torch.rand(1, 2) 28 | outputs 29 | 30 | #%% 31 | # OK in a real model, those inputs and outputs would be your actual 32 | # data, loaded up, in datasets, converted into batches 33 | # for this simple model, it's just tensors, no data wrangling 34 | # now for a sequential network, let's do a simple multi 35 | # layer perceptron 36 | 37 | 38 | #%% 39 | # now we start up a model with layers of linear -- these will 40 | # themselves have tensors inside filled with random numbers 41 | # these random numbers are called parameters, and these 42 | # parameters are the things that machine learning learns 43 | # basically -- the parameters -- sometimes called weights 44 | # are updated by learning algorithms, searching for the best 45 | # available answer 46 | 47 | #%% 48 | model = torch.nn.Sequential( 49 | # input features are the size of one image 50 | # outputs are how many we have when done 51 | # the 64 has to 'match' the final dimnension of the input 52 | # try changing it to another number to see errors! 53 | torch.nn.Linear(64, 256), 54 | torch.nn.Linear(256, 256), 55 | torch.nn.Linear(256, 2), 56 | ) 57 | 58 | #%% 59 | # and -- this isn't learning, we're just running our random 60 | # initialized linear network over our input 61 | 62 | #%% 63 | result = model(inputs) 64 | result, result.shape 65 | 66 | #%% 67 | # hmm -- that's not two convenient output labels, we have some 68 | # more work to do in the next videos -- but we have a model! 69 | -------------------------------------------------------------------------------- /2-4.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Starting off, we need to import torch 3 | import torch 4 | 5 | #%% 6 | # here are the inputs and outputs from the last video 7 | # as well as the model with activations 8 | 9 | #%% 10 | inputs = torch.rand(1, 1, 64, 64) 11 | outputs = torch.rand(1, 2) 12 | model = torch.nn.Sequential( 13 | torch.nn.Linear(64, 256), 14 | torch.nn.ReLU(), 15 | torch.nn.Linear(256, 256), 16 | torch.nn.ReLU(), 17 | torch.nn.Linear(256, 2), 18 | ) 19 | 20 | #%% 21 | # now we have our inputs as sample data and our outputs 22 | # we are trying to generate with our network -- the 23 | # application of a network -- it's computation to turn input 24 | # into output is done with a forward pass simply by calling 25 | # the model as a function over the inputs 26 | 27 | #%% 28 | test_results = model(inputs) 29 | test_results 30 | 31 | #%% 32 | # don't forget the loss function! the loss function is the 33 | # key driver to compute gradients, which are needed to drive 34 | # learning 35 | 36 | #%% 37 | loss = torch.nn.MSELoss()(test_results, outputs) 38 | loss 39 | 40 | 41 | #%% 42 | # now we compute the gradients, this is done for each forward 43 | # pass after you have compute the loss -- basically you are 44 | # zeroing out as the gradients will differ on each pass 45 | # once the gradients are zeroed, then you use the loss to 46 | # drive the backward propagation of gradients through the model 47 | # this is pretty much the heart of what pytorch does for you -- 48 | # automatically computing gradients and propagating them back 49 | 50 | #%% 51 | model.zero_grad() 52 | loss.backward() 53 | 54 | #%% 55 | # and now -- we'll do some very simple learning -- remember 56 | # that the gradients tell you how far away you are from the 57 | # right answer -- so you move in the opposite direction to 58 | # get to the answer, meaning -- we just subtract! 59 | # one additional concept here -- learning rate, which we'll 60 | # experiment with more in later videos, but for now we'll use 61 | # a very simple constant learning rate 62 | 63 | #%% 64 | learning_rate = 0.001 65 | for parameter in model.parameters(): 66 | parameter.data -= parameter.grad.data * learning_rate 67 | 68 | 69 | #%% 70 | # now -- we've learned -- and should be closer to the answer 71 | # let's run the model with our new updated parameters 72 | # and see... 73 | 74 | #%% 75 | after_learning = model(inputs) 76 | loss_after_learning = torch.nn.MSELoss()(after_learning, outputs) 77 | loss_after_learning 78 | 79 | #%% 80 | # yep -- that's a smaller loss, we are closer to the answer -------------------------------------------------------------------------------- /3-1.py: -------------------------------------------------------------------------------- 1 | 2 | #%% 3 | # First, we'll need to load up a dataset. Pandas is a great 4 | # tool to use to load csv data you may find, which we 5 | # will later turn into tensors. 6 | # Let's start with the Dataset 7 | 8 | #%% 9 | 10 | import torch 11 | import pandas 12 | from torch.utils.data import Dataset 13 | 14 | class MushroomDataset(Dataset): 15 | 16 | def __init__(self): 17 | '''Load up the data. 18 | ''' 19 | self.data = pandas.read_csv('./mushrooms.csv') 20 | 21 | def __len__(self): 22 | '''How much data do we have? 23 | ''' 24 | return len(self.data) 25 | 26 | def __getitem__(self, idx): 27 | '''Grab one data sample 28 | 29 | Arguments: 30 | idx {int} -- data at this position. 31 | ''' 32 | return self.data.iloc[idx][0:1] 33 | # pretty simple when we start from pandas 34 | # here is a dataset loaded, with a single sample 35 | shrooms = MushroomDataset() 36 | len(shrooms), shrooms[0] 37 | 38 | #%% 39 | # Well -- we have some clearly identifiable properties, but we 40 | # have this all in one dataset, we're going to need to separate 41 | # out the inputs from the outputs 42 | 43 | #%% 44 | class MushroomDataset(Dataset): 45 | 46 | def __init__(self): 47 | '''Load up the data. 48 | ''' 49 | self.data = pandas.read_csv('./mushrooms.csv') 50 | 51 | def __len__(self): 52 | '''How much data do we have? 53 | ''' 54 | return len(self.data) 55 | 56 | def __getitem__(self, idx): 57 | '''Grab one data sample 58 | 59 | Arguments: 60 | idx {int, tensor} -- data at this position. 61 | ''' 62 | # handle being passed a tensor as an index 63 | if type(idx) is torch.Tensor: 64 | idx = idx.item() 65 | return self.data.iloc[idx][1:], self.data.iloc[idx][0:1] 66 | 67 | shrooms = MushroomDataset() 68 | shrooms[0] 69 | 70 | #%% 71 | # One more thing to think about -- testing and training data 72 | # we need some set of data samples we don't use in training to 73 | # verify that our model can generalize -- 74 | # that it can make a classification 75 | # for an unseen sample and hasn't merely 76 | # memorized the input data 77 | 78 | #%% 79 | number_for_testing = int(len(shrooms) * 0.05) 80 | number_for_training = len(shrooms) - number_for_testing 81 | train, test = torch.utils.data.random_split(shrooms, 82 | [number_for_training, number_for_testing]) 83 | len(test), len(train) 84 | 85 | #%% 86 | test[0] 87 | -------------------------------------------------------------------------------- /1-3.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import torch 3 | torch.__version__ 4 | 5 | #%% 6 | # lots of ways to create a tensor 7 | # https://pytorch.org/docs/stable/torch.html#creation-ops 8 | # a tensor is really just a multidimensional array, starting 9 | # with a simple empty array 10 | e = torch.empty(2, 2) 11 | e 12 | 13 | #%% 14 | # ok - that's strange -- there are values in that array! this 15 | # isn't a true random, it's just whatever was in memory -- if 16 | # you really want random, which is pretty often 17 | r = torch.rand(2, 2) 18 | r 19 | 20 | #%% 21 | # that's more like it and sometimes you just want specific values 22 | # like a good old zero 23 | z = torch.zeros(2, 2) 24 | z 25 | 26 | #%% 27 | # or specifc constants, let's make some threes! 28 | c = torch.full((2, 2), 3) 29 | c 30 | 31 | #%% 32 | # the most flexible is the `torch.tensor` creation method, you can 33 | # pass it data in a lot of formats -- starting with lists 34 | l = torch.tensor([[1, 2], [3, 4]]) 35 | l 36 | 37 | #%% 38 | # as well as interoperate with numpy arrays, which is very 39 | # handy to work with data you may have already processed 40 | # with other machine learning tools liek sklearn 41 | import numpy 42 | n = numpy.linspace(0, 5, 5) 43 | n 44 | 45 | #%% 46 | # turning this into pytorch is as easy as you would wish 47 | nn = torch.tensor(n) 48 | nn 49 | 50 | #%% 51 | # and back again is easy too! 52 | nn.numpy() 53 | 54 | #%% 55 | # arrays support conventional operations -- size and slice 56 | nn.shape 57 | 58 | #%% 59 | nn[1:], nn[0] 60 | 61 | #%% 62 | # in any creation method, you can also specify the data type 63 | # like using a full precision floating point 64 | s = torch.ones(3, 3, dtype=torch.float) 65 | s 66 | 67 | #%% 68 | # all kinds of math operations are available 69 | # https://pytorch.org/docs/stable/torch.html#math-operations 70 | # math is straightforward operatos for common operations 71 | # like addition 72 | eye = torch.eye(3, 3) 73 | eye + torch.zeros(3, 3) 74 | 75 | #%% 76 | # subtraction 77 | eye - torch.ones(3, 3) 78 | 79 | #%% 80 | # broadcast multiplication of a constant 81 | eye * 3 82 | 83 | #%% 84 | # or division... 85 | eye / 3 86 | 87 | #%% 88 | # element wise tensor multiplication 89 | eye * torch.full((3,3), 4) 90 | 91 | #%% 92 | # and you might not have seen this before, but a dot product 93 | # operator in python 94 | x = torch.rand(3, 4) 95 | y = torch.rand(4, 3) 96 | x @ y 97 | 98 | #%% 99 | # and handy machine learning component operations 100 | # like getting the index of the maximum value 101 | torch.tensor([1 , 2, 5, 3, 0]).argmax() -------------------------------------------------------------------------------- /2-3.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Starting off, we need to import torch 3 | 4 | #%% 5 | import torch 6 | import matplotlib.pyplot as plt 7 | plt.style.use('ggplot') 8 | 9 | #%% 10 | # here are the inputs and outputs from the last videoA 11 | 12 | #%% 13 | inputs = torch.rand(1, 1, 64, 64) 14 | outputs = torch.rand(1, 2) 15 | 16 | #%% 17 | # here is our model from the last video -- notice everything is 18 | # linear -- this limits what out model can learn, so we need 19 | # something else -- an activation function 20 | 21 | #%% 22 | model = torch.nn.Sequential( 23 | torch.nn.Linear(64, 256), 24 | torch.nn.Linear(256, 256), 25 | torch.nn.Linear(256, 2), 26 | ) 27 | 28 | #%% 29 | # this is just about the simplest activation functional possible 30 | # the RELU 31 | # it is nonlinear in a straightforward way -- it starts out flat 32 | # and then it inflects at 0 -- two linear parts making 33 | # non linear part -- the advantage is -- it is very fast 34 | 35 | #%% 36 | x = torch.range(-1, 1, 0.1) 37 | y = torch.nn.functional.relu(x) 38 | plt.plot(x.numpy(), y.numpy()) 39 | 40 | #%% 41 | # now we can update our model with relu 42 | model = torch.nn.Sequential( 43 | torch.nn.Linear(64, 256), 44 | torch.nn.ReLU(), 45 | torch.nn.Linear(256, 256), 46 | torch.nn.ReLU(), 47 | torch.nn.Linear(256, 2), 48 | ) 49 | 50 | #%% 51 | # and the model needs feedback in order to learn, and this is the 52 | # role of the loss function -- it simply tells you how far away 53 | # from the right answer we are 54 | # you can think of -- and it's not to far off -- of machine 55 | # learning as taking a random guess, and saying 'how far wrong' 56 | # and then updating that guess -- this would be a silly strategy 57 | # as a person, but computers can guess fast 58 | # 59 | # there are a lot of choices for loss functions, a classic 60 | # one is the Mean Squared Error, this is related to the 61 | # classic distance you may have learned in school -- 62 | # A^2 + B^2 = C^2 when looking at right triangles, but 63 | # -- this is generalized into high dimension tensors 64 | # you'll hear it referred to as the L2 (because of the square) 65 | # or Euclidean distance as well 66 | 67 | #%% 68 | results = model(inputs) 69 | loss = torch.nn.MSELoss()(results, outputs) 70 | loss 71 | 72 | #%% 73 | # and finally -- the gradient -- when I said machine learning 74 | # makes a lot of guesses, there is a bit more to it - it 75 | # makes educated guesses -- that education is in the gradient 76 | # 77 | # the gradient tells the machine learning model, based on 78 | # the loss -- which direction it is away from the right 79 | # answer 80 | # and that will be the subject of our next video -------------------------------------------------------------------------------- /2-Assignment.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # starting with imports 3 | import torch 4 | 5 | #%% 6 | # here is some iteration -- lowering the number 7 | # of hidden parameters until we no longer can get the 8 | # gradiens to vanish 9 | # this is a bit of dynamic model generation, which is 10 | # a kind of meta-learning 11 | 12 | 13 | #%% 14 | # inputs and outputs, just random values -- we'll work with real 15 | # data in subsequent videos 16 | inputs = torch.rand(1, 1, 64, 64) 17 | outputs = torch.rand(1, 2) 18 | 19 | #%% 20 | # keep track of how many learning steps it took 21 | # at each number of parameters 22 | learning_steps = [] 23 | 24 | #%% 25 | for number_of_parameters in range(256, 1, -1): 26 | class Model(torch.nn.Module): 27 | def __init__(self): 28 | super().__init__() 29 | self.layer_one = torch.nn.Linear(64, number_of_parameters) 30 | self.activation_one = torch.nn.ReLU() 31 | self.layer_two = torch.nn.Linear(number_of_parameters, number_of_parameters) 32 | self.activation_two = torch.nn.ReLU() 33 | # this is a pretty big number -- because we are flattening 34 | # which turned 64 * 256 into a flat array like tensor 35 | self.shape_outputs = torch.nn.Linear(number_of_parameters * 64, 2) 36 | 37 | def forward(self, inputs): 38 | buffer = self.layer_one(inputs) 39 | buffer = self.activation_one(buffer) 40 | buffer = self.layer_two(buffer) 41 | buffer = self.activation_two(buffer) 42 | buffer = buffer.flatten(start_dim=1) 43 | return self.shape_outputs(buffer) 44 | 45 | model = Model() 46 | loss_function = torch.nn.MSELoss() 47 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 48 | # a limit on how much learning we do 49 | for i in range(10000): 50 | # the optimizer reaches into the model and will zero out 51 | optimizer.zero_grad() 52 | results = model(inputs) 53 | loss = loss_function(results, outputs) 54 | loss.backward() 55 | optimizer.step() 56 | # now -- look for vanishing gradients 57 | gradients = 0.0 58 | for parameter in model.parameters(): 59 | gradients += parameter.grad.data.sum() 60 | if abs(gradients) <= 0.0001: 61 | learning_steps.append((number_of_parameters, i, results)) 62 | break 63 | 64 | #%% 65 | learning_steps 66 | 67 | 68 | #%% 69 | # looks like it still learns -- but it gets a lot harder when 70 | # the number of parameters converges to the number of outputs 71 | import matplotlib.pyplot as plt 72 | plt.style.use('ggplot') 73 | learning_steps = [step[1] for step in learning_steps] 74 | plt.plot(learning_steps) 75 | 76 | -------------------------------------------------------------------------------- /1-4-gradient.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # understanding gradients is a bit of math, but we'll try to keep this 3 | # simple -- bascially a numerical gradient tells you which direction 4 | # you need to move when you are machine learning -- positive or 5 | # negative -- and well as having an actual numerical value 6 | # that tells you 'how much' you should move 7 | 8 | # a machine learning loop takes the gradeients for a group 9 | # of tensor operations, and then updates the value of the tensors 10 | # that are being 'learned' using the product of the gradients 11 | # and the learning rate 12 | 13 | # so if you know a bit of math, you know a gradient is a numerical 14 | # representation of a derivative -- which means you know to ask 15 | # 'a gradient with respect to what?' -- and the answer there 16 | # is a loss function, you use gradients to figure the direction 17 | # to go to make your loss smaller 18 | 19 | # here is the simplest example from grade school algebra I could 20 | # cook up -- before you knew algebra -- this was a bit of a 21 | # mystery how to solve it, and I know I personally tried -- 22 | # just plain guessing the numbers to 'solve' equations -- 23 | # machine learning is a bit like that, but instead of just plain 24 | # guessing, we use the gradient to figure how far off, and what 25 | # our next guess should be --= OK 26 | 27 | #%% 28 | import torch 29 | 30 | # X + 1 = 3 -- that's our little bit of algebra 31 | 32 | # here is our random initial guess 33 | X = torch.rand(1, requires_grad=True) 34 | # and our formula 35 | Y = X + 1.0 36 | Y 37 | 38 | #%% 39 | # now, our loss is -- how far are we off from 3? 40 | def mse(Y): 41 | diff = 3.0 - Y 42 | return (diff * diff).sum() / 2 43 | 44 | #%% 45 | # the gradient on our X -- that tells us which direction 46 | # we are 'off' from the right answer -- let's look when we are too high 47 | loss = mse(Y) 48 | loss.backward() 49 | X.grad 50 | 51 | #%% 52 | # now -- let's use that gradient to solve some grade school 53 | # algebra with simple machine learning 54 | learning_rate = 1e-3 55 | # here is our learning loop 56 | for i in range(0, 10000): 57 | Y = X + 1.0 58 | loss = mse(Y) 59 | # here is the 'backpropagation' of the gradient 60 | loss.backward() 61 | # and here is the 'learning', so we turn off the graidents 62 | # from being updated temporarily 63 | with torch.no_grad(): 64 | # the gradient tells you which direction you are off 65 | # so you go in the opposite direction to correct the problem 66 | X -= learning_rate * X.grad 67 | # and we zero out the gradients to get fresh values on 68 | # each learning loop iteration 69 | X.grad.zero_() 70 | # and -- here is our answer 71 | X 72 | # OK -- you can see that this is approximate -- and that's an 73 | # important point -- machine learning is going to approximate 74 | # and you can control how close you get to the target answer 75 | # by altering your learning rate or your number of iterations 76 | # experiment with this by altering the `learning_rate` 77 | # and the number of loops in `range` -------------------------------------------------------------------------------- /4-3.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Images are quite convenient in pytorch, there are a few built 3 | # in datasets -- let's take a look at the classic -- MNIST 4 | 5 | 6 | #%% 7 | import torch 8 | import matplotlib.pyplot as plt 9 | import numpy as np 10 | 11 | 12 | #%% 13 | import torchvision 14 | mnist = torchvision.datasets.MNIST('./var', download=True) 15 | mnist[0][0] 16 | 17 | #%% 18 | # looks like a squiggly 5 -- let's check the label 19 | mnist[0][1] 20 | 21 | #%% 22 | # now the data is actually images, so we're going to need to 23 | # turn it into tensors, which is conveniently built in 24 | 25 | #%% 26 | import torchvision.transforms as transforms 27 | 28 | transform = transforms.Compose( 29 | [transforms.ToTensor(), 30 | transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) 31 | 32 | train = torchvision.datasets.MNIST('./var', train=True, transform=transform) 33 | trainloader = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True) 34 | test = torchvision.datasets.MNIST('./var', train=False, transform=transform) 35 | testloader = torch.utils.data.DataLoader(test, batch_size=len(test), shuffle=True) 36 | 37 | 38 | #%% 39 | # and now to define a very simple convolutional network 40 | 41 | import torch.nn as nn 42 | import torch.nn.functional as F 43 | 44 | 45 | class Net(nn.Module): 46 | def __init__(self): 47 | super(Net, self).__init__() 48 | # in channels, out channels (filters!), kernel size (square) 49 | # one channel -- this is greyscale 50 | self.conv1 = nn.Conv2d(1, 3, 3) 51 | # pooling divides in half with 52 | # kernel size, stride the same as 2 53 | self.pool = nn.MaxPool2d(2, 2) 54 | # now here is where you start to need to think about 55 | # the size of the image 56 | self.conv2 = nn.Conv2d(3, 6, 3) 57 | self.fc1 = nn.Linear(150, 128) 58 | self.fc2 = nn.Linear(128, 128) 59 | # ten digits -- ten outputs 60 | self.fc3 = nn.Linear(128, 10) 61 | 62 | def forward(self, x): 63 | x = self.pool(F.relu(self.conv1(x))) 64 | x = self.pool(F.relu(self.conv2(x))) 65 | x = x.flatten(start_dim=1) 66 | # this is a good place to see the size for debugging 67 | # print(x.shape) 68 | x = F.relu(self.fc1(x)) 69 | x = F.relu(self.fc2(x)) 70 | x = self.fc3(x) 71 | return x 72 | 73 | 74 | net = Net() 75 | #%% 76 | # loss functions, here we are using cross entropy loss, which 77 | # actuall does the softmax for us 78 | 79 | #%% 80 | import torch.optim as optim 81 | 82 | loss_function = nn.CrossEntropyLoss() 83 | optimizer = optim.Adam(net.parameters()) 84 | 85 | 86 | #%% 87 | # and the training loop 88 | 89 | #%% 90 | for epoch in range(16): 91 | for inputs, outputs in trainloader: 92 | optimizer.zero_grad() 93 | results = net(inputs) 94 | loss = loss_function(results, outputs) 95 | loss.backward() 96 | optimizer.step() 97 | print("Loss: {0}".format(loss)) 98 | 99 | #%% 100 | # now let's use that classification report to see how well we are doing 101 | 102 | 103 | #%% 104 | import sklearn.metrics 105 | for inputs, actual in testloader: 106 | results = net(inputs).argmax(dim=1).numpy() 107 | accuracy = sklearn.metrics.accuracy_score(actual, results) 108 | print(accuracy) 109 | 110 | print(sklearn.metrics.classification_report(actual, results)) -------------------------------------------------------------------------------- /1-5-datasets.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # To do machine learning you need data, and there are three concepts 3 | # to master here, Dataset, Dataloader, and transforms 4 | 5 | #%% 6 | # Let's make use of pandas and CSV data to create a dataset. 7 | 8 | import torch 9 | import pandas 10 | from torch.utils.data import Dataset 11 | 12 | class IrisDataset(Dataset): 13 | 14 | def __init__(self): 15 | '''Load up the data. 16 | ''' 17 | self.data = pandas.read_csv('./Iris.csv') 18 | 19 | def __len__(self): 20 | '''How much data do we have? 21 | ''' 22 | return len(self.data) 23 | 24 | def __getitem__(self, idx): 25 | '''Grab one data sample 26 | 27 | Arguments: 28 | idx {int} -- data at this position. 29 | ''' 30 | return self.data.iloc[idx] 31 | # pretty simple when we start from pandas 32 | # here is a dataset loaded, with a single sample 33 | iris = IrisDataset() 34 | len(iris), iris[0] 35 | #%% 36 | # To do machine learning you need data, and there are three concepts 37 | # to master here, Dataset, Dataloader, and transforms 38 | 39 | #%% 40 | # Let's make use of pandas and CSV data to create a dataset. 41 | 42 | import torch 43 | import pandas 44 | from torch.utils.data import Dataset 45 | 46 | class IrisDataset(Dataset): 47 | 48 | def __init__(self): 49 | '''Load up the data. 50 | ''' 51 | self.data = pandas.read_csv('./Iris.csv') 52 | 53 | def __len__(self): 54 | '''How much data do we have? 55 | ''' 56 | return len(self.data) 57 | 58 | def __getitem__(self, idx): 59 | '''Grab one data sample 60 | 61 | Arguments: 62 | idx {int} -- data at this position. 63 | ''' 64 | return self.data.iloc[idx] 65 | # pretty simple when we start from pandas 66 | # here is a dataset loaded, with a single sample 67 | iris = IrisDataset() 68 | len(iris), iris[0] 69 | 70 | #%% 71 | # Now, the small problem is -- we have a named tuple, 72 | # and we're going to need a tensor for inputs and 73 | # the target label -- so we need to transform 74 | 75 | class TensorIrisDataset(IrisDataset): 76 | def __getitem__(self, idx): 77 | '''Get a single sample that is 78 | {values:, label:} 79 | ''' 80 | sample = super().__getitem__(idx) 81 | return { 82 | 'tensor': torch.Tensor( 83 | [sample.SepalLengthCm, 84 | sample.SepalWidthCm, 85 | sample.PetalLengthCm, 86 | sample.PetalWidthCm] 87 | ), 88 | 'label': sample.Species 89 | } 90 | 91 | # and output... 92 | tensors = TensorIrisDataset() 93 | len(tensors), tensors[0] 94 | 95 | #%% 96 | # Training almost always takes place in batches 97 | # so pytorch has a very convenient loader that can take 98 | # a dataset and turn it into batches so you can iterate 99 | from torch.utils.data import DataLoader 100 | 101 | loader = DataLoader(tensors, batch_size=16, shuffle=True) 102 | for batch in loader: 103 | print(batch) 104 | 105 | # see how the data comes out in batches, and the last batch 106 | # tries to be as large as it can 107 | 108 | #%% 109 | # And -- there is even a parallel possibility 110 | # this is a pretty small dataset so it's not really 111 | # essential, but here is how you use it 112 | 113 | parallel_loader = DataLoader(tensors, 114 | batch_size=16, shuffle=True, num_workers=4) 115 | for batch in parallel_loader: 116 | print(batch) 117 | -------------------------------------------------------------------------------- /4-Assignment.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Here I'm swapping out MNIST for CIFAR, which is object recognition 3 | # -- and it is 3 channel color image 4 | 5 | 6 | #%%% 7 | import torch 8 | import torchvision 9 | import torchvision.transforms as transforms 10 | import torch.nn as nn 11 | 12 | 13 | 14 | #%% 15 | cifar = torchvision.datasets.CIFAR10('./var', download=True) 16 | transform = transforms.Compose([ 17 | transforms.ToTensor(), 18 | ]) 19 | cifar[0][0] 20 | 21 | #%% 22 | train = torchvision.datasets.CIFAR10('./var', train=True, transform=transform) 23 | trainloader = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True) 24 | test = torchvision.datasets.CIFAR10('./var', train=False, transform=transform) 25 | testloader = torch.utils.data.DataLoader(test, batch_size=len(test), shuffle=True) 26 | 27 | #%% 28 | class SlimAlexNet(nn.Module): 29 | 30 | def __init__(self, num_classes=10): 31 | super().__init__() 32 | self.features = nn.Sequential( 33 | # three color input channels 34 | nn.Conv2d(3, 32, kernel_size=3, stride=1), 35 | nn.ReLU(inplace=True), 36 | nn.MaxPool2d(kernel_size=3, stride=2), 37 | nn.Conv2d(32, 64, kernel_size=3), 38 | nn.ReLU(inplace=True), 39 | nn.MaxPool2d(kernel_size=3, stride=2), 40 | nn.Conv2d(64, 128, kernel_size=3, padding=1), 41 | nn.ReLU(inplace=True), 42 | nn.Conv2d(128, 256, kernel_size=3, padding=1), 43 | nn.ReLU(inplace=True), 44 | nn.Conv2d(256, 128, kernel_size=3, padding=1), 45 | nn.ReLU(inplace=True), 46 | nn.MaxPool2d(kernel_size=3, stride=2), 47 | ) 48 | self.classifier = nn.Sequential( 49 | nn.Dropout(), 50 | # this is the shape after flattening 51 | nn.Linear(512, 1024), 52 | nn.ReLU(inplace=True), 53 | nn.Dropout(), 54 | nn.Linear(1024, 1024), 55 | nn.ReLU(inplace=True), 56 | nn.Linear(1024, num_classes), 57 | ) 58 | 59 | def forward(self, x): 60 | x = self.features(x) 61 | x = x.flatten(start_dim=1) 62 | # here is where I figure out the shape to get 63 | # the right number of parameters in the next layer 64 | # print(x.shape) 65 | x = self.classifier(x) 66 | return x 67 | 68 | 69 | #%% 70 | net = SlimAlexNet(num_classes=10) 71 | loss_function = torch.nn.CrossEntropyLoss() 72 | optimizer = torch.optim.Adam(net.parameters()) 73 | if torch.cuda.is_available(): 74 | device = torch.device('cuda') 75 | else: 76 | device = torch.device('cpu') 77 | 78 | net.to(device) 79 | 80 | # train this longer 81 | for epoch in range(64): 82 | total_loss = 0 83 | for inputs, outputs in trainloader: 84 | inputs = inputs.to(device) 85 | outputs = outputs.to(device) 86 | optimizer.zero_grad() 87 | results = net(inputs) 88 | loss = loss_function(results, outputs) 89 | total_loss += loss.item() 90 | loss.backward() 91 | optimizer.step() 92 | print("Loss: {0}".format(total_loss / len(trainloader))) 93 | 94 | #%% 95 | # let's see how much better this is! 96 | 97 | #%% 98 | import sklearn.metrics 99 | for inputs, actual in testloader: 100 | inputs = inputs.to(device) 101 | results = net(inputs).argmax(dim=1).to('cpu').numpy() 102 | accuracy = sklearn.metrics.accuracy_score(actual, results) 103 | print(accuracy) 104 | 105 | print(sklearn.metrics.classification_report(actual, results)) 106 | -------------------------------------------------------------------------------- /4-5.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Now -- let's take a look at a full featured convolutional network 3 | # by investigating AlexNet -- this was one of the early 'truly deep' 4 | # networks and is actually a great basis for study. 5 | 6 | 7 | #%%% 8 | import torch 9 | import torchvision 10 | import torchvision.transforms as transforms 11 | import torch.nn as nn 12 | 13 | 14 | #%% 15 | # and now our data 16 | 17 | #%% 18 | mnist = torchvision.datasets.MNIST('./var', download=True) 19 | transform = transforms.Compose([ 20 | transforms.ToTensor(), 21 | ]) 22 | 23 | train = torchvision.datasets.MNIST('./var', train=True, transform=transform) 24 | trainloader = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True) 25 | test = torchvision.datasets.MNIST('./var', train=False, transform=transform) 26 | testloader = torch.utils.data.DataLoader(test, batch_size=len(test), shuffle=True) 27 | 28 | #%% 29 | # we can use use AlexNet -- it's built in to torchvisionm but we'll 30 | # modify it a bit for our smaller images and grayscale 31 | 32 | #%% 33 | class SlimAlexNet(nn.Module): 34 | 35 | def __init__(self, num_classes=10): 36 | super().__init__() 37 | self.features = nn.Sequential( 38 | nn.Conv2d(1, 32, kernel_size=3, stride=1), 39 | nn.ReLU(inplace=True), 40 | nn.MaxPool2d(kernel_size=3, stride=2), 41 | nn.Conv2d(32, 64, kernel_size=3), 42 | nn.ReLU(inplace=True), 43 | nn.MaxPool2d(kernel_size=3, stride=2), 44 | nn.Conv2d(64, 128, kernel_size=3, padding=1), 45 | nn.ReLU(inplace=True), 46 | nn.Conv2d(128, 256, kernel_size=3, padding=1), 47 | nn.ReLU(inplace=True), 48 | nn.Conv2d(256, 128, kernel_size=3, padding=1), 49 | nn.ReLU(inplace=True), 50 | nn.MaxPool2d(kernel_size=3, stride=2), 51 | ) 52 | self.classifier = nn.Sequential( 53 | nn.Dropout(), 54 | nn.Linear(128, 1024), 55 | nn.ReLU(inplace=True), 56 | nn.Dropout(), 57 | nn.Linear(1024, 1024), 58 | nn.ReLU(inplace=True), 59 | nn.Linear(1024, num_classes), 60 | ) 61 | 62 | def forward(self, x): 63 | x = self.features(x) 64 | x = x.flatten(start_dim=1) 65 | x = self.classifier(x) 66 | return x 67 | 68 | 69 | #%% 70 | net = SlimAlexNet(num_classes=10) 71 | loss_function = torch.nn.CrossEntropyLoss() 72 | optimizer = torch.optim.Adam(net.parameters()) 73 | 74 | #%% 75 | # and the training loop, with CUDA support 76 | 77 | #%% 78 | if torch.cuda.is_available(): 79 | device = torch.device('cuda') 80 | else: 81 | device = torch.device('cpu') 82 | 83 | net.to(device) 84 | 85 | for epoch in range(16): 86 | total_loss = 0 87 | for inputs, outputs in trainloader: 88 | inputs = inputs.to(device) 89 | outputs = outputs.to(device) 90 | optimizer.zero_grad() 91 | results = net(inputs) 92 | loss = loss_function(results, outputs) 93 | total_loss += loss.item() 94 | loss.backward() 95 | optimizer.step() 96 | print("Loss: {0}".format(total_loss / len(trainloader))) 97 | 98 | #%% 99 | # let's see how much better this is! 100 | 101 | #%% 102 | import sklearn.metrics 103 | for inputs, actual in testloader: 104 | inputs = inputs.to(device) 105 | results = net(inputs).argmax(dim=1).to('cpu').numpy() 106 | accuracy = sklearn.metrics.accuracy_score(actual, results) 107 | print(accuracy) 108 | 109 | print(sklearn.metrics.classification_report(actual, results)) 110 | -------------------------------------------------------------------------------- /2-5.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # starting with imports 3 | import torch 4 | 5 | #%% 6 | # now we'll create a model in the way that you are most likely 7 | # going to use pytorch in practice, by creating a reusable model 8 | # module -- this is simply a class with layer member variables 9 | # and a forward method that does the actual computation 10 | # we're going to build the same model we did before with linear 11 | # and relu, but we'll also fix up our model and get the output 12 | # shape we really want -- which is a tensor with two elements 13 | 14 | 15 | #%% 16 | # inputs and outputs, just random values -- we'll work with real 17 | # data in subsequent videos 18 | inputs = torch.rand(1, 1, 64, 64) 19 | outputs = torch.rand(1, 2) 20 | 21 | #%% 22 | # and now on to our module 23 | 24 | class Model(torch.nn.Module): 25 | 26 | def __init__(self): 27 | ''' 28 | The constructor is the place to set up each of the layers 29 | and activations. 30 | ''' 31 | 32 | super().__init__() 33 | self.layer_one = torch.nn.Linear(64, 256) 34 | self.activation_one = torch.nn.ReLU() 35 | self.layer_two = torch.nn.Linear(256, 256) 36 | self.activation_two = torch.nn.ReLU() 37 | # this is a pretty big number -- because we are flattening 38 | # which turned 64 * 256 into a flat array like tensor 39 | self.shape_outputs = torch.nn.Linear(16384, 2) 40 | 41 | def forward(self, inputs): 42 | buffer = self.layer_one(inputs) 43 | buffer = self.activation_one(buffer) 44 | buffer = self.layer_two(buffer) 45 | buffer = self.activation_two(buffer) 46 | # and here -- we correct the model to give us the output 47 | # shape we want -- starting with dimension one to 48 | # preserve the batch dimension -- we only have a bactch 49 | # of one item, but dealing with batches will becomre more 50 | # important as we process real data in later videos 51 | buffer = buffer.flatten(start_dim=1) 52 | return self.shape_outputs(buffer) 53 | 54 | #%% 55 | # now let's run our model over our inputs 56 | 57 | #%% 58 | model = Model() 59 | test_results = model(inputs) 60 | test_results 61 | 62 | #%% 63 | # now -- let's learn, this time creating a learning loop 64 | # with a built in optimizer -- we'll let it cycle keeping track 65 | # of our gradients, and when our gradients 'vanish' -- we'll 66 | # stop learning and see how close we are to our model 67 | # being able to generate the correct outputs 68 | 69 | #%% 70 | loss_function = torch.nn.MSELoss() 71 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01) 72 | # a limit on how much learning we do 73 | for i in range(10000): 74 | # the optimizer reaches into the model and will zero out 75 | optimizer.zero_grad() 76 | results = model(inputs) 77 | loss = loss_function(results, outputs) 78 | loss.backward() 79 | optimizer.step() 80 | # now -- look for vanishing gradients 81 | gradients = 0.0 82 | for parameter in model.parameters(): 83 | gradients += parameter.grad.data.sum() 84 | if abs(gradients) <= 0.0001: 85 | print(gradients) 86 | print('gradient vanished at iteration {0}'.format(i)) 87 | break 88 | 89 | 90 | #%% 91 | # relatively quick to get to no gradients, let's look at the answer 92 | model(inputs), outputs 93 | 94 | #%% 95 | # spot on! 96 | # this illustrates how networks can learn arbitrary functions, 97 | # in this case -- extremely arbitrary, we learned random data! 98 | # keep this in mind as you are doing machine learning on real data 99 | # -- networks are powerful enough to fix nearly any data, including 100 | # random, which means in effect the algorithm memorized the 101 | # inputs in a kind of sophisticated mathematical hashtable 102 | # -- when this happens, we call it overfitting -- meaning 103 | # the model knows only the inputs it is trained on and cannot 104 | # deal with previously unseen inputs -- think about this 105 | # when you make your models! -------------------------------------------------------------------------------- /4-4.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Let's take a look at what is going on inside these convolutions 3 | # by viewing the layer output channels as images. It's an interesting technique 4 | # to get more of a feel for what the machine learner 'sees' 5 | 6 | 7 | #%%% 8 | import torch 9 | import matplotlib.pyplot as plt 10 | import numpy as np 11 | import torchvision 12 | import torchvision.transforms as transforms 13 | 14 | #%% 15 | mnist = torchvision.datasets.MNIST('./var', download=True) 16 | transform = transforms.Compose([transforms.ToTensor()]) 17 | 18 | train = torchvision.datasets.MNIST('./var', train=True, transform=transform) 19 | trainloader = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True) 20 | test = torchvision.datasets.MNIST('./var', train=False, transform=transform) 21 | testloader = torch.utils.data.DataLoader(test, batch_size=len(test), shuffle=True) 22 | 23 | #%% 24 | # let's plot a tensor as an image -- this hasn't had any machine learning 25 | # just yet -- it is only the source image data 26 | 27 | #%% 28 | for inputs, outputs in trainloader: 29 | #slice out one channel 30 | image = inputs[0][0] 31 | plt.imshow(image.numpy(), cmap=plt.get_cmap('binary')) 32 | break 33 | 34 | 35 | 36 | #%% 37 | # OK -- that's an image - now let's train up a simple convolutional network 38 | # and then augment it by saving intermediate tensors, the thing to know here 39 | # is the convolutional tensors have multiple filters, we go from 40 | # one color channel to three -- so we'll have some interesting choices when 41 | # we visualize! 42 | 43 | 44 | 45 | #%% 46 | import torch.nn as nn 47 | import torch.nn.functional as F 48 | 49 | class Net(nn.Module): 50 | def __init__(self): 51 | super(Net, self).__init__() 52 | # in channels, out channels (filters!), kernel size (square) 53 | # one channel -- this is greyscale 54 | self.conv1 = nn.Conv2d(1, 3, 3) 55 | # pooling divides in half with 56 | # kernel size, stride the same as 2 57 | self.pool = nn.MaxPool2d(2, 2) 58 | # now here is where you start to need to think about 59 | # the size of the image 60 | self.conv2 = nn.Conv2d(3, 6, 3) 61 | self.fc1 = nn.Linear(150, 128) 62 | self.fc2 = nn.Linear(128, 128) 63 | # ten digits -- ten outputs 64 | self.fc3 = nn.Linear(128, 10) 65 | 66 | def forward(self, x): 67 | x = self.pool(F.relu(self.conv1(x))) 68 | self.after_conv1 = x 69 | x = self.pool(F.relu(self.conv2(x))) 70 | self.after_conv2 = x 71 | x = x.flatten(start_dim=1) 72 | # this is a good place to see the size for debugging 73 | # print(x.shape) 74 | x = F.relu(self.fc1(x)) 75 | x = F.relu(self.fc2(x)) 76 | x = self.fc3(x) 77 | return x 78 | 79 | 80 | net = Net() 81 | #%% 82 | # loss functions, here we are using cross entropy loss, which 83 | # actuall does the softmax for us -- convience feature in pytorch 84 | 85 | #%% 86 | import torch.optim as optim 87 | 88 | loss_function = nn.CrossEntropyLoss() 89 | optimizer = optim.Adam(net.parameters()) 90 | 91 | 92 | #%% 93 | # and the training loop 94 | 95 | #%% 96 | for epoch in range(16): 97 | for inputs, outputs in trainloader: 98 | optimizer.zero_grad() 99 | results = net(inputs) 100 | loss = loss_function(results, outputs) 101 | loss.backward() 102 | optimizer.step() 103 | print("Loss: {0}".format(loss)) 104 | 105 | #%% 106 | # ok -- now we have a trained model -- now we can visualize! 107 | # pyplot is a bit odd when you make multiple images -- the 108 | # trick is to remember it is a bit modal - you create a figure 109 | # which means the plots you call are 'to' that figure implicitly 110 | # and then you add subplots which are (rows, columns, index) 111 | # and it is one based from left to right, top to bottom 112 | # 113 | # we'll make a figure with 3 rows, 6 columns to show the source 114 | # image, then the first filter of three channels 115 | # followed by the second filter of six channels 116 | 117 | 118 | #%% 119 | for inputs, outputs in trainloader: 120 | # multi image figure 121 | figure = plt.figure() 122 | # the original image 123 | image = inputs[0][0] 124 | 125 | figure.add_subplot(3, 6, 1) 126 | plt.imshow(image.numpy(), cmap=plt.get_cmap('binary')) 127 | output = net(inputs) 128 | # remember we have a batch in the model -- and this 129 | # has a gradient, so we'll need it detached to get numpy format 130 | filter_one = net.after_conv1[0].detach() 131 | for i in range(3): 132 | figure.add_subplot(3, 6, 6 + 1 + i) 133 | plt.imshow(filter_one[i].numpy(), cmap=plt.get_cmap('binary')) 134 | 135 | filter_two = net.after_conv2[0].detach() 136 | for i in range(6): 137 | figure.add_subplot(3, 6, 12 + 1 + i) 138 | plt.imshow(filter_two[i].numpy(), cmap=plt.get_cmap('binary')) 139 | plt.show() 140 | 141 | break 142 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # PyTorch Deep Learning in 7 Days[Video] 2 | This is the code repository for [PyTorch Deep Learning in 7 Days[Video]](https://prod.packtpub.com/in/big-data-and-business-intelligence/pytorch-deep-learning-7-days-video), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the video course from start to finish. 3 | ## About the Video Course 4 | PyTorch is Facebook’s latest Python-based framework for Deep Learning. It has the ability to create dynamic Neural Networks on CPUs and GPUs, both with a significantly less code compared to other competing frameworks. PyTorch has a unique interface that makes it as easy to learn as NumPy. 5 | 6 | This 7-day course is for those who are in a hurry to get started with PyTorch. You will be introduced to the most commonly used Deep Learning models, techniques, and algorithms through PyTorch code. This course is an attempt to break the myth that Deep Learning is complicated and show you that with the right choice of tools combined with a simple and intuitive explanation of core concepts, Deep Learning is as accessible as any other application development technologies out there. It’s a journey from diving deep into the fundamentals to getting acquainted with the advance concepts such as Transfer Learning, Natural Language Processing and implementation of Generative Adversarial Networks. 7 | 8 | By the end of the course, you will be able to build Deep Learning applications with PyTorch. 9 | 10 |

What You Will Learn

11 |
12 |
19 | 20 | ## Instructions and Navigation 21 | ### Assumed Knowledge 22 | This course is for software development professionals and machine learning enthusiasts, who have heard the hype of Deep Learning and want to learn it to stay relevant in their field. Basic knowledge of machine learning concepts and Python programming is required. 23 | 24 | # Getting Started 25 | 26 | ## Linux 27 | This is set up with scripts to run on Linus to install `docker` 28 | and `nvidia-docker` to allow GPU support. 29 | 30 | `sudo ./linux/install-docker` 31 | 32 | and if you want GPU follow up with 33 | 34 | `sudo ./linux/install-nvidia` 35 | 36 | If all is well, you will see a listing of your video cards: 37 | 38 | ``` 39 | +-----------------------------------------------------------------------------+ 40 | | NVIDIA-SMI 415.25 Driver Version: 415.25 CUDA Version: 10.0 | 41 | |-------------------------------+----------------------+----------------------+ 42 | | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | 43 | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | 44 | |===============================+======================+======================| 45 | | 0 TITAN RTX Off | 00000000:03:00.0 On | N/A | 46 | | 41% 37C P8 9W / 280W | 1036MiB / 24165MiB | 13% Default | 47 | +-------------------------------+----------------------+----------------------+ 48 | 49 | +-----------------------------------------------------------------------------+ 50 | | Processes: GPU Memory | 51 | | GPU PID Type Process name Usage | 52 | |=============================================================================| 53 | +-----------------------------------------------------------------------------+ 54 | ``` 55 | 56 | ## Mac 57 | 58 | Run this to use `brew` to get docker installed. 59 | 60 | `./mac/install-docker` 61 | 62 | 63 | # Running Notebooks 64 | 65 | This is configured to run with `docker-compose`, so just start things up with 66 | 67 | `docker-compose up` 68 | 69 | And then you can just go to http://localhost:8888 to get started. All the notbook security is 70 | switched off as this is packed up for learning purposes, one less thing to worry about. 71 | 72 | 73 | ### Technical Requirements 74 | This course has the following software requirements:
75 | OS: Any compatible flavor of Linux from the link in minimum requirements
76 | 77 | Browser: Google Chrome, Firefox latest version
78 | 79 | Others: Pytorch (Pytorch.org), Anaconda distribution for Python 3.6 and above from https://repo.continuum.io/archive
80 | 81 | ## Related Products 82 | * [Deep Learning with PyTorch [Video]](https://prod.packtpub.com/in/big-data-and-business-intelligence/deep-learning-pytorch-video) 83 | 84 | * [Deep Learning Projects with PyTorch [Video]](https://prod.packtpub.com/in/application-development/deep-learning-projects-pytorch-video) 85 | 86 | * [Deep Learning with PyTorch Quick Start Guide](https://prod.packtpub.com/in/big-data-and-business-intelligence/deep-learning-pytorch-quick-start-guide) 87 | 88 | 89 | -------------------------------------------------------------------------------- /Iris.csv: -------------------------------------------------------------------------------- 1 | Id,SepalLengthCm,SepalWidthCm,PetalLengthCm,PetalWidthCm,Species 2 | 1,5.1,3.5,1.4,0.2,Iris-setosa 3 | 2,4.9,3.0,1.4,0.2,Iris-setosa 4 | 3,4.7,3.2,1.3,0.2,Iris-setosa 5 | 4,4.6,3.1,1.5,0.2,Iris-setosa 6 | 5,5.0,3.6,1.4,0.2,Iris-setosa 7 | 6,5.4,3.9,1.7,0.4,Iris-setosa 8 | 7,4.6,3.4,1.4,0.3,Iris-setosa 9 | 8,5.0,3.4,1.5,0.2,Iris-setosa 10 | 9,4.4,2.9,1.4,0.2,Iris-setosa 11 | 10,4.9,3.1,1.5,0.1,Iris-setosa 12 | 11,5.4,3.7,1.5,0.2,Iris-setosa 13 | 12,4.8,3.4,1.6,0.2,Iris-setosa 14 | 13,4.8,3.0,1.4,0.1,Iris-setosa 15 | 14,4.3,3.0,1.1,0.1,Iris-setosa 16 | 15,5.8,4.0,1.2,0.2,Iris-setosa 17 | 16,5.7,4.4,1.5,0.4,Iris-setosa 18 | 17,5.4,3.9,1.3,0.4,Iris-setosa 19 | 18,5.1,3.5,1.4,0.3,Iris-setosa 20 | 19,5.7,3.8,1.7,0.3,Iris-setosa 21 | 20,5.1,3.8,1.5,0.3,Iris-setosa 22 | 21,5.4,3.4,1.7,0.2,Iris-setosa 23 | 22,5.1,3.7,1.5,0.4,Iris-setosa 24 | 23,4.6,3.6,1.0,0.2,Iris-setosa 25 | 24,5.1,3.3,1.7,0.5,Iris-setosa 26 | 25,4.8,3.4,1.9,0.2,Iris-setosa 27 | 26,5.0,3.0,1.6,0.2,Iris-setosa 28 | 27,5.0,3.4,1.6,0.4,Iris-setosa 29 | 28,5.2,3.5,1.5,0.2,Iris-setosa 30 | 29,5.2,3.4,1.4,0.2,Iris-setosa 31 | 30,4.7,3.2,1.6,0.2,Iris-setosa 32 | 31,4.8,3.1,1.6,0.2,Iris-setosa 33 | 32,5.4,3.4,1.5,0.4,Iris-setosa 34 | 33,5.2,4.1,1.5,0.1,Iris-setosa 35 | 34,5.5,4.2,1.4,0.2,Iris-setosa 36 | 35,4.9,3.1,1.5,0.1,Iris-setosa 37 | 36,5.0,3.2,1.2,0.2,Iris-setosa 38 | 37,5.5,3.5,1.3,0.2,Iris-setosa 39 | 38,4.9,3.1,1.5,0.1,Iris-setosa 40 | 39,4.4,3.0,1.3,0.2,Iris-setosa 41 | 40,5.1,3.4,1.5,0.2,Iris-setosa 42 | 41,5.0,3.5,1.3,0.3,Iris-setosa 43 | 42,4.5,2.3,1.3,0.3,Iris-setosa 44 | 43,4.4,3.2,1.3,0.2,Iris-setosa 45 | 44,5.0,3.5,1.6,0.6,Iris-setosa 46 | 45,5.1,3.8,1.9,0.4,Iris-setosa 47 | 46,4.8,3.0,1.4,0.3,Iris-setosa 48 | 47,5.1,3.8,1.6,0.2,Iris-setosa 49 | 48,4.6,3.2,1.4,0.2,Iris-setosa 50 | 49,5.3,3.7,1.5,0.2,Iris-setosa 51 | 50,5.0,3.3,1.4,0.2,Iris-setosa 52 | 51,7.0,3.2,4.7,1.4,Iris-versicolor 53 | 52,6.4,3.2,4.5,1.5,Iris-versicolor 54 | 53,6.9,3.1,4.9,1.5,Iris-versicolor 55 | 54,5.5,2.3,4.0,1.3,Iris-versicolor 56 | 55,6.5,2.8,4.6,1.5,Iris-versicolor 57 | 56,5.7,2.8,4.5,1.3,Iris-versicolor 58 | 57,6.3,3.3,4.7,1.6,Iris-versicolor 59 | 58,4.9,2.4,3.3,1.0,Iris-versicolor 60 | 59,6.6,2.9,4.6,1.3,Iris-versicolor 61 | 60,5.2,2.7,3.9,1.4,Iris-versicolor 62 | 61,5.0,2.0,3.5,1.0,Iris-versicolor 63 | 62,5.9,3.0,4.2,1.5,Iris-versicolor 64 | 63,6.0,2.2,4.0,1.0,Iris-versicolor 65 | 64,6.1,2.9,4.7,1.4,Iris-versicolor 66 | 65,5.6,2.9,3.6,1.3,Iris-versicolor 67 | 66,6.7,3.1,4.4,1.4,Iris-versicolor 68 | 67,5.6,3.0,4.5,1.5,Iris-versicolor 69 | 68,5.8,2.7,4.1,1.0,Iris-versicolor 70 | 69,6.2,2.2,4.5,1.5,Iris-versicolor 71 | 70,5.6,2.5,3.9,1.1,Iris-versicolor 72 | 71,5.9,3.2,4.8,1.8,Iris-versicolor 73 | 72,6.1,2.8,4.0,1.3,Iris-versicolor 74 | 73,6.3,2.5,4.9,1.5,Iris-versicolor 75 | 74,6.1,2.8,4.7,1.2,Iris-versicolor 76 | 75,6.4,2.9,4.3,1.3,Iris-versicolor 77 | 76,6.6,3.0,4.4,1.4,Iris-versicolor 78 | 77,6.8,2.8,4.8,1.4,Iris-versicolor 79 | 78,6.7,3.0,5.0,1.7,Iris-versicolor 80 | 79,6.0,2.9,4.5,1.5,Iris-versicolor 81 | 80,5.7,2.6,3.5,1.0,Iris-versicolor 82 | 81,5.5,2.4,3.8,1.1,Iris-versicolor 83 | 82,5.5,2.4,3.7,1.0,Iris-versicolor 84 | 83,5.8,2.7,3.9,1.2,Iris-versicolor 85 | 84,6.0,2.7,5.1,1.6,Iris-versicolor 86 | 85,5.4,3.0,4.5,1.5,Iris-versicolor 87 | 86,6.0,3.4,4.5,1.6,Iris-versicolor 88 | 87,6.7,3.1,4.7,1.5,Iris-versicolor 89 | 88,6.3,2.3,4.4,1.3,Iris-versicolor 90 | 89,5.6,3.0,4.1,1.3,Iris-versicolor 91 | 90,5.5,2.5,4.0,1.3,Iris-versicolor 92 | 91,5.5,2.6,4.4,1.2,Iris-versicolor 93 | 92,6.1,3.0,4.6,1.4,Iris-versicolor 94 | 93,5.8,2.6,4.0,1.2,Iris-versicolor 95 | 94,5.0,2.3,3.3,1.0,Iris-versicolor 96 | 95,5.6,2.7,4.2,1.3,Iris-versicolor 97 | 96,5.7,3.0,4.2,1.2,Iris-versicolor 98 | 97,5.7,2.9,4.2,1.3,Iris-versicolor 99 | 98,6.2,2.9,4.3,1.3,Iris-versicolor 100 | 99,5.1,2.5,3.0,1.1,Iris-versicolor 101 | 100,5.7,2.8,4.1,1.3,Iris-versicolor 102 | 101,6.3,3.3,6.0,2.5,Iris-virginica 103 | 102,5.8,2.7,5.1,1.9,Iris-virginica 104 | 103,7.1,3.0,5.9,2.1,Iris-virginica 105 | 104,6.3,2.9,5.6,1.8,Iris-virginica 106 | 105,6.5,3.0,5.8,2.2,Iris-virginica 107 | 106,7.6,3.0,6.6,2.1,Iris-virginica 108 | 107,4.9,2.5,4.5,1.7,Iris-virginica 109 | 108,7.3,2.9,6.3,1.8,Iris-virginica 110 | 109,6.7,2.5,5.8,1.8,Iris-virginica 111 | 110,7.2,3.6,6.1,2.5,Iris-virginica 112 | 111,6.5,3.2,5.1,2.0,Iris-virginica 113 | 112,6.4,2.7,5.3,1.9,Iris-virginica 114 | 113,6.8,3.0,5.5,2.1,Iris-virginica 115 | 114,5.7,2.5,5.0,2.0,Iris-virginica 116 | 115,5.8,2.8,5.1,2.4,Iris-virginica 117 | 116,6.4,3.2,5.3,2.3,Iris-virginica 118 | 117,6.5,3.0,5.5,1.8,Iris-virginica 119 | 118,7.7,3.8,6.7,2.2,Iris-virginica 120 | 119,7.7,2.6,6.9,2.3,Iris-virginica 121 | 120,6.0,2.2,5.0,1.5,Iris-virginica 122 | 121,6.9,3.2,5.7,2.3,Iris-virginica 123 | 122,5.6,2.8,4.9,2.0,Iris-virginica 124 | 123,7.7,2.8,6.7,2.0,Iris-virginica 125 | 124,6.3,2.7,4.9,1.8,Iris-virginica 126 | 125,6.7,3.3,5.7,2.1,Iris-virginica 127 | 126,7.2,3.2,6.0,1.8,Iris-virginica 128 | 127,6.2,2.8,4.8,1.8,Iris-virginica 129 | 128,6.1,3.0,4.9,1.8,Iris-virginica 130 | 129,6.4,2.8,5.6,2.1,Iris-virginica 131 | 130,7.2,3.0,5.8,1.6,Iris-virginica 132 | 131,7.4,2.8,6.1,1.9,Iris-virginica 133 | 132,7.9,3.8,6.4,2.0,Iris-virginica 134 | 133,6.4,2.8,5.6,2.2,Iris-virginica 135 | 134,6.3,2.8,5.1,1.5,Iris-virginica 136 | 135,6.1,2.6,5.6,1.4,Iris-virginica 137 | 136,7.7,3.0,6.1,2.3,Iris-virginica 138 | 137,6.3,3.4,5.6,2.4,Iris-virginica 139 | 138,6.4,3.1,5.5,1.8,Iris-virginica 140 | 139,6.0,3.0,4.8,1.8,Iris-virginica 141 | 140,6.9,3.1,5.4,2.1,Iris-virginica 142 | 141,6.7,3.1,5.6,2.4,Iris-virginica 143 | 142,6.9,3.1,5.1,2.3,Iris-virginica 144 | 143,5.8,2.7,5.1,1.9,Iris-virginica 145 | 144,6.8,3.2,5.9,2.3,Iris-virginica 146 | 145,6.7,3.3,5.7,2.5,Iris-virginica 147 | 146,6.7,3.0,5.2,2.3,Iris-virginica 148 | 147,6.3,2.5,5.0,1.9,Iris-virginica 149 | 148,6.5,3.0,5.2,2.0,Iris-virginica 150 | 149,6.2,3.4,5.4,2.3,Iris-virginica 151 | 150,5.9,3.0,5.1,1.8,Iris-virginica 152 | -------------------------------------------------------------------------------- /3-Assignment.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Here is my version of the assignment working on graduate 3 | # school admission dataset as a regression problem 4 | 5 | #%% 6 | import torch 7 | import pandas 8 | import dateutil 9 | from torch.utils.data import Dataset 10 | 11 | class OneHotSeriesEncoder(): 12 | def __init__(self, series): 13 | '''Given a single pandas series, creaet an encoder 14 | that can turn values from that series into a one hot 15 | pytorch tensor. 16 | 17 | Arguments: 18 | series {pandas.Series} -- encode this 19 | ''' 20 | unique_values = series.unique() 21 | self.ordinals = {val: i for i, val in enumerate(unique_values)} 22 | self.encoder = torch.eye(len(unique_values), len(unique_values)) 23 | 24 | def __getitem__(self, value): 25 | '''Turn a value into a tensor 26 | 27 | Arguments: 28 | value {} -- Value to encode, anything that can be hashed 29 | but most likely a string 30 | 31 | Returns: 32 | [torch.Tensor] -- a one dimensional tensor with encoded values. 33 | ''' 34 | 35 | return self.encoder[self.ordinals[value]] 36 | 37 | 38 | class DateEncoder(): 39 | def __getitem__(self, datestring): 40 | '''Encode into a tensor [year, month, date] 41 | given an input date string. 42 | 43 | Arguments: 44 | datestring {string} -- date string, best bet is ISO format 45 | ''' 46 | parsed = dateutil.parser.parse(datestring) 47 | return torch.Tensor([parsed.year, parsed.month, parsed.day]) 48 | 49 | class MixedCSV(Dataset): 50 | def __init__(self, datafile, output_series_name, 51 | date_series_names, categorical_series_names, 52 | ignore_series_names): 53 | '''Load the dataset and create needed encoders for 54 | each series. 55 | 56 | Arguments: 57 | datafile {string} -- path to data file 58 | output_series_name {string} -- use this series/column as output 59 | date_series_names {list} -- column names of dates 60 | categorical_series_names {list} -- column names of categories 61 | ignore_series_names {list} -- column names to skip 62 | ''' 63 | self.dataset = pandas.read_csv(datafile) 64 | self.output_series_name = output_series_name 65 | self.encoders = {} 66 | for series_name in date_series_names: 67 | self.encoders[series_name] = DateEncoder() 68 | for series_name in categorical_series_names: 69 | self.encoders[series_name] = OneHotSeriesEncoder( 70 | self.dataset[series_name] 71 | ) 72 | self.ignore = ignore_series_names 73 | 74 | def __len__(self): 75 | return len(self.dataset) 76 | 77 | def __getitem__(self, index): 78 | '''Return an (input, output) tensor tuple 79 | with all categories one hot encoded. 80 | 81 | Arguments: 82 | index {[type]} -- [description] 83 | ''' 84 | if type(index) is torch.Tensor: 85 | index = index.item() 86 | sample = self.dataset.iloc[index] 87 | 88 | output = torch.Tensor([sample[self.output_series_name]]) 89 | 90 | input_components = [] 91 | for name, value in sample.items(): 92 | if name in self.ignore: 93 | continue 94 | elif name in self.encoders: 95 | input_components.append( 96 | self.encoders[name][value] 97 | ) 98 | else: 99 | input_components.append(torch.Tensor([value])) 100 | input = torch.cat(input_components) 101 | return input, output 102 | #%% 103 | categorical = [ 104 | 'Research', 105 | ] 106 | 107 | dates = [ 108 | 109 | ] 110 | 111 | discard = [ 112 | 'Serial No.' 113 | ] 114 | #%% 115 | 116 | data = MixedCSV('./Admission_Predict.csv', 117 | 'Chance of Admit ', #tricky follow space in the key name here! 118 | dates, 119 | categorical, 120 | discard 121 | ) 122 | #%% 123 | # And here is a regression model 124 | class Model(torch.nn.Module): 125 | 126 | def __init__(self, input_dimensions, size=256): 127 | ''' 128 | The constructor is the place to set up each of the layers 129 | and activations. 130 | ''' 131 | super().__init__() 132 | self.layer_one = torch.nn.Linear(input_dimensions, size) 133 | self.activation_one = torch.nn.ReLU() 134 | self.layer_two = torch.nn.Linear(size, size) 135 | self.activation_two = torch.nn.ReLU() 136 | self.shape_outputs = torch.nn.Linear(size, 1) 137 | 138 | def forward(self, inputs): 139 | 140 | buffer = self.layer_one(inputs) 141 | buffer = self.activation_one(buffer) 142 | buffer = self.layer_two(buffer) 143 | buffer = self.activation_two(buffer) 144 | buffer = self.shape_outputs(buffer) 145 | return buffer 146 | 147 | 148 | # I ended up making a much smaller model, given the small number 149 | # of features and small number of samples 150 | model = Model(data[0][0].shape[0], size=32) 151 | optimizer = torch.optim.Adam(model.parameters()) 152 | loss_function = torch.nn.MSELoss() 153 | 154 | 155 | #%% 156 | # and now our training loop 157 | # I ended up with a much larger batch size 158 | 159 | number_for_testing = int(len(data) * 0.05) 160 | number_for_training = len(data) - number_for_testing 161 | train, test = torch.utils.data.random_split(data, 162 | [number_for_training, number_for_testing]) 163 | training = torch.utils.data.DataLoader(train, batch_size=number_for_training, shuffle=True) 164 | for epoch in range(256): 165 | for inputs, outputs in training: 166 | optimizer.zero_grad() 167 | results = model(inputs) 168 | loss = loss_function(results, outputs) 169 | loss.backward() 170 | optimizer.step() 171 | print("Loss: {0}".format(loss)) 172 | 173 | #%% 174 | # quick check 175 | actual = test[0][1] 176 | predicted = model(test[0][0]) 177 | actual, predicted 178 | 179 | 180 | #%% 181 | import sklearn.metrics 182 | 183 | testing = torch.utils.data.DataLoader(test, batch_size=len(test), shuffle=False) 184 | for inputs, outputs in testing: 185 | predicted = model(inputs).detach().numpy() 186 | actual = outputs.numpy() 187 | print(sklearn.metrics.r2_score(actual, predicted)) -------------------------------------------------------------------------------- /3-2_3.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | #%% 4 | # When we have structured data, it is often cateogorical, meaning 5 | # a series of discrete values, often strings or ID numbers, that 6 | # are not intendted to be interpreted mathematically -- things 7 | # as simple as category labels, states, country names -- or 8 | # a numerical example -- postal codes are actually categorical 9 | 10 | # In order to get this kind of data ready for machine learning, 11 | # we need to encode it properly -- which leads us to 12 | # one hot encoding 13 | # this is a method where each category is represented 14 | # in a dataset as a single dimension, 15 | # encoded 0 or 1 indicating if that category is set 16 | 17 | # There are naturally a lot of ways to write the code to turn a 18 | # categorical variable into a one hot encoded tensor -- 19 | # one interesting idea is to use an identity matrix 20 | # assuming you have three discreet values A, B, C, 21 | # represent those 22 | # as a list ['A', 'B', 'C'], and then A is 0, B is 1, C is 2 as 23 | # ordinal index values -- we can use an identity matrix to turn 24 | # the ordinal into a one hot encoded representation 25 | 26 | #%% 27 | import torch 28 | from torch.utils.data import Dataset 29 | import pandas 30 | 31 | #%% 32 | one_hots = torch.eye(3, 3) 33 | one_hots 34 | 35 | #%% 36 | ordinals = {c: i for i, c in enumerate(['A', 'B', 'C'])} 37 | ordinals 38 | 39 | #%% 40 | one_hots[ordinals['A']] 41 | 42 | #%% 43 | # so this is an encoder - turning the letters into a bitmap 44 | # now we need to just generalize this to work on a whole dataset! 45 | 46 | #%% 47 | 48 | class OneHotEncoder(): 49 | def __init__(self, series): 50 | '''Given a single pandas series, creaet an encoder 51 | that can turn values from that series into a one hot 52 | pytorch tensor. 53 | 54 | Arguments: 55 | series {pandas.Series} -- encode this 56 | ''' 57 | unique_values = series.unique() 58 | self.ordinals = { 59 | val: i for i, val in enumerate(unique_values) 60 | } 61 | self.encoder = torch.eye( 62 | len(unique_values), len(unique_values) 63 | ) 64 | 65 | def __getitem__(self, value): 66 | '''Turn a value into a tensor 67 | 68 | Arguments: 69 | value {} -- Value to encode, 70 | anything that can be hashed but most likely a string 71 | 72 | Returns: 73 | [torch.Tensor] -- a one dimensional tensor 74 | ''' 75 | 76 | return self.encoder[self.ordinals[value]] 77 | 78 | class CategoricalCSV(Dataset): 79 | def __init__(self, datafile, output_series_name): 80 | '''Load the dataset and create needed encoders for 81 | each series. 82 | 83 | Arguments: 84 | datafile {string} -- path to data file 85 | output_series_name {string} -- series/column name 86 | ''' 87 | self.dataset = pandas.read_csv(datafile) 88 | self.output_series_name = output_series_name 89 | self.encoders = {} 90 | for series_name, series in self.dataset.items(): 91 | # create a per series encoder 92 | self.encoders[series_name] = OneHotEncoder(series) 93 | 94 | def __len__(self): 95 | return len(self.dataset) 96 | 97 | def __getitem__(self, index): 98 | '''Return an (input, output) tensor tuple 99 | with all categories one hot encoded. 100 | 101 | Arguments: 102 | index {[type]} -- [description] 103 | ''' 104 | if type(index) is torch.Tensor: 105 | index = index.item() 106 | sample = self.dataset.iloc[index] 107 | output = self.encoders[self.output_series_name][ 108 | sample[self.output_series_name] 109 | ] 110 | input_components = [] 111 | for name, value in sample.items(): 112 | if name != self.output_series_name: 113 | input_components.append( 114 | self.encoders[name][value] 115 | ) 116 | input = torch.cat(input_components) 117 | return input, output 118 | 119 | 120 | shrooms = CategoricalCSV('./mushrooms.csv', 'class') 121 | shrooms[0] 122 | 123 | #%% 124 | # now that is a lot of ones and zeros, but if you look closer, you can see 125 | # that out of each range of numbers, there is a clearly a 1 signal 126 | # that category value flag is on 127 | 128 | # at this point -- we have tensors from pure CSC text data -- and 129 | # are ready to start learning! 130 | 131 | #%% 3-3 132 | # let's start off creating a simple network, we'll make the 133 | # input and output dimensions variable, and introduce a new 134 | # activation -- Softmax -- this is a function that smooths out 135 | # values to get the total of all possibilities to sum to 1 136 | # you may recognize this as a probability -- that's the effect 137 | # you can take classifier results and nudge them into probability 138 | # classes, which is very useful with a binary classifier trying 139 | # to distinguish one thing from another 140 | 141 | # a new loss function -- cross entropy -- makes a debut, 142 | # which is # very useful for classifiers, 143 | # it differs from mean squared error 144 | # in that it is less concerned about the actual value 145 | # in a prediction 146 | # than it is in which prediction has the largest value -- 147 | # the idea here 148 | # is that for a binary classifier, the answer with the highest 149 | # probability is 'the' prediction, even though it may not be 100% 150 | 151 | #%% 152 | class Model(torch.nn.Module): 153 | 154 | def __init__(self, input_dimensions, 155 | output_dimensions, size=128): 156 | ''' 157 | The constructor is the place to set up each of the layers 158 | and activations. 159 | ''' 160 | super().__init__() 161 | self.layer_one = torch.nn.Linear(input_dimensions, size) 162 | self.activation_one = torch.nn.ReLU() 163 | self.layer_two = torch.nn.Linear(size, size) 164 | self.activation_two = torch.nn.ReLU() 165 | self.shape_outputs = torch.nn.Linear(size, 166 | output_dimensions) 167 | 168 | def forward(self, inputs): 169 | 170 | buffer = self.layer_one(inputs) 171 | buffer = self.activation_one(buffer) 172 | buffer = self.layer_two(buffer) 173 | buffer = self.activation_two(buffer) 174 | buffer = self.shape_outputs(buffer) 175 | return torch.nn.functional.softmax(buffer, dim=-1) 176 | 177 | model = Model(shrooms[0][0].shape[0], shrooms[0][1].shape[0]) 178 | optimizer = torch.optim.Adam(model.parameters()) 179 | loss_function = torch.nn.BCELoss() 180 | 181 | #%% 182 | # now let's run a training loop, we'll go through the dataset 183 | # multiple times -- a loop through the dataset is conventionally 184 | # called an epoch, inside of each epoch, 185 | # we'll go through each batch 186 | 187 | #%% 188 | number_for_testing = int(len(shrooms) * 0.05) 189 | number_for_training = len(shrooms) - number_for_testing 190 | train, test = torch.utils.data.random_split(shrooms, 191 | [number_for_training, number_for_testing]) 192 | training = torch.utils.data.DataLoader(train, 193 | batch_size=16, shuffle=True) 194 | for epoch in range(4): 195 | for inputs, outputs in training: 196 | optimizer.zero_grad() 197 | results = model(inputs) 198 | loss = loss_function(results, outputs) 199 | loss.backward() 200 | optimizer.step() 201 | print("Loss: {0}".format(loss)) 202 | 203 | 204 | #%% 205 | # now let's take a look at accuracy, this is a place where 206 | # we can reach to sklearn - 207 | # which has multiple evaluation functions and metrics 208 | 209 | #%% 210 | import sklearn.metrics 211 | 212 | #%% 213 | # accuracy is somewhat what you'd guess, 214 | # the percentage of the time 215 | # that your model is getting the right answer -- 216 | # let's look with 217 | # our test data, 218 | # comparing the actual test output with our model 219 | # output on test 220 | 221 | # here we are taking the argmax -- this turns one-hots back into 222 | # integers -- say you have a one hot [1, 0] -- the arg max is 0 223 | # since the maximum value is in slot zero 224 | 225 | # we compute the argmax along dimension 1 -- 226 | # dim=1, remember dim 0 227 | # in this case is the batch, so each batch entry is indexed in 228 | # 0 dimension, each one hot encoding is in the 1 dimension 229 | 230 | #%% 231 | testing = torch.utils.data.DataLoader(test, 232 | batch_size=len(test), shuffle=False) 233 | for inputs, outputs in testing: 234 | results = model(inputs).argmax(dim=1).numpy() 235 | actual = outputs.argmax(dim=1).numpy() 236 | accuracy = sklearn.metrics.accuracy_score(actual, results) 237 | print(accuracy) 238 | 239 | #%% 240 | # and, you can see how accurate you are -- per class this is 241 | # a way to tell if you model is better or worse at making i 242 | # certain 243 | # kinds of predictions 244 | 245 | #%% 246 | sklearn.metrics.confusion_matrix(actual, results) 247 | 248 | #%% 249 | # you read this left to right 250 | # true positive, false positive 251 | # false negative, true negative 252 | 253 | #%% 254 | # even better, you can get a handy classification report, 255 | # which is easy to read 256 | 257 | #%% 258 | print(sklearn.metrics.classification_report(actual, results)) 259 | -------------------------------------------------------------------------------- /3-4_5.py: -------------------------------------------------------------------------------- 1 | #%% 2 | # Regression has a different output format than classification -- 3 | # it is in real values, good old floating point numbers. 4 | 5 | # This particular dataset also has real valued data points, 6 | # so we're 7 | # going to update our data loader to be more configurable column 8 | # wise how to encode values. We'll start be brining back our one 9 | # hot column encoder. 10 | 11 | #%% 12 | import torch 13 | import pandas 14 | 15 | class OneHotEncoder(): 16 | def __init__(self, series): 17 | '''Given a single pandas series, create an encoder 18 | that can turn values from that series into a one hot 19 | pytorch tensor. 20 | 21 | Arguments: 22 | series {pandas.Series} -- encode this 23 | ''' 24 | unique_values = series.unique() 25 | self.ordinals = { 26 | val: i for i, val in enumerate(unique_values)} 27 | self.encoder = torch.eye( 28 | len(unique_values), len(unique_values)) 29 | 30 | def __getitem__(self, value): 31 | '''Turn a value into a tensor 32 | 33 | Arguments: 34 | value {} -- Value to encode 35 | but most likely a string 36 | 37 | Returns: 38 | [torch.Tensor] -- a one dimensional tensor 39 | ''' 40 | 41 | return self.encoder[self.ordinals[value]] 42 | 43 | #%% 44 | # now we could make an encoder for numerical values, 45 | # but the values 46 | # already are numbers, so this is in effect 'no encoder' -- i 47 | # so we'll implement it that way. 48 | # time to load up the dataset and learn about our columns 49 | 50 | #%% 51 | look = pandas.read_csv('./kc_house_data.csv') 52 | look.iloc[0] 53 | 54 | 55 | #%% 56 | # looking at that data, let's make a configuration of the 57 | # categorical columns, some of these are judgement -- for example 58 | # it's easy to think of a 1-5 scale as categorical or 59 | # as real valued, that 60 | # will be something to experiment with in the assignment 61 | 62 | #%% 63 | categorical = [ 64 | 'waterfront', 65 | 'view', 66 | 'condition', 67 | 'grade', 68 | ] 69 | 70 | #%% 71 | # One interesting column in there -- id -- that one doesn't look like real 72 | # data, just a key from a database. Values that are 'unique' like this 73 | # you need to throw out, otherwise you can end up making a 74 | # machine learning hash-table! 75 | # This isn't a hard and fast rule, but a good one to think 76 | # about in practice 77 | # any unique value in a sample isn't likely to generalize well, 78 | # there isn't 79 | # any data for the network to comapre. 80 | # Whether it is a unique keyword 81 | # or a unique number value, be on the lookout for these. 82 | 83 | #%% 84 | discard = [ 85 | 'id' 86 | ] 87 | 88 | #%% 89 | # And the really tricky bit -- look at that date column. 90 | # Time is a tricky 91 | # one to think about, as there are seasonal effects, 92 | # and in some sense 93 | # we recognize this in how we write out time -- 94 | # years, months, and days 95 | # let's break this feature into three numerical features. 96 | 97 | #%% 98 | import dateutil 99 | 100 | 101 | class DateEncoder(): 102 | def __getitem__(self, datestring): 103 | '''Encode into a tensor [year, month, date] 104 | given an input date string. 105 | 106 | Arguments: 107 | datestring {string} -- date string, ISO format 108 | ''' 109 | parsed = dateutil.parser.parse(datestring) 110 | return torch.Tensor( 111 | [parsed.year, parsed.month, parsed.day]) 112 | 113 | dates = ['date'] 114 | DateEncoder()['20141013T000000'] 115 | 116 | 117 | #%% 118 | from torch.utils.data import Dataset 119 | 120 | class MixedCSV(Dataset): 121 | def __init__(self, datafile, output_series_name, 122 | date_series_names, categorical_series_names, 123 | ignore_series_names): 124 | '''Load the dataset and create needed encoders for 125 | each series. 126 | 127 | Arguments: 128 | datafile {string} -- path to data file 129 | output_series_name {string} -- use this series/column as output 130 | date_series_names {list} -- column names of dates 131 | categorical_series_names {list} -- column names 132 | ignore_series_names {list} -- column names to skip 133 | ''' 134 | self.dataset = pandas.read_csv(datafile) 135 | self.output_series_name = output_series_name 136 | self.encoders = {} 137 | for series_name in date_series_names: 138 | self.encoders[series_name] = DateEncoder() 139 | for series_name in categorical_series_names: 140 | self.encoders[series_name] = OneHotEncoder( 141 | self.dataset[series_name] 142 | ) 143 | self.ignore = ignore_series_names 144 | 145 | def __len__(self): 146 | return len(self.dataset) 147 | 148 | def __getitem__(self, index): 149 | '''Return an (input, output) tensor tuple 150 | with all categories one hot encoded. 151 | 152 | Arguments: 153 | index {[type]} -- [description] 154 | ''' 155 | if type(index) is torch.Tensor: 156 | index = index.item() 157 | sample = self.dataset.iloc[index] 158 | 159 | output = torch.Tensor([sample[self.output_series_name]]) 160 | 161 | input_components = [] 162 | for name, value in sample.items(): 163 | if name in self.ignore: 164 | continue 165 | elif name in self.encoders: 166 | input_components.append( 167 | self.encoders[name][value] 168 | ) 169 | else: 170 | input_components.append(torch.Tensor([value])) 171 | input = torch.cat(input_components) 172 | return input, output 173 | 174 | houses = MixedCSV('./kc_house_data.csv', 175 | 'price', 176 | dates, 177 | categorical, 178 | discard 179 | ) 180 | houses[0] 181 | 182 | 183 | 184 | 185 | 186 | #%% 187 | # 3-5 188 | # The big differences for a regression network are in the output 189 | # rather than a softmax or a probability, 190 | # we can simply emit a real valued # number 191 | # Depending on the model - 192 | # this number isn't simply a 0-1, and in our case 193 | # we're looking to emit a price in dollars, 194 | # so it'll be 6 figures. 195 | 196 | #%% 197 | class Model(torch.nn.Module): 198 | 199 | def __init__(self, input_dimensions, size=128): 200 | ''' 201 | The constructor is the place to set up each layer 202 | and activations. 203 | ''' 204 | super().__init__() 205 | self.layer_one = torch.nn.Linear(input_dimensions, size) 206 | self.activation_one = torch.nn.ReLU() 207 | self.layer_two = torch.nn.Linear(size, size) 208 | self.activation_two = torch.nn.ReLU() 209 | self.shape_outputs = torch.nn.Linear(size, 1) 210 | 211 | def forward(self, inputs): 212 | 213 | buffer = self.layer_one(inputs) 214 | buffer = self.activation_one(buffer) 215 | buffer = self.layer_two(buffer) 216 | buffer = self.activation_two(buffer) 217 | buffer = self.shape_outputs(buffer) 218 | return buffer 219 | 220 | model = Model(houses[0][0].shape[0]) 221 | optimizer = torch.optim.Adam(model.parameters()) 222 | loss_function = torch.nn.MSELoss() 223 | 224 | 225 | #%% 226 | # and now our training loop 227 | 228 | number_for_testing = int(len(houses) * 0.05) 229 | number_for_training = len(houses) - number_for_testing 230 | train, test = torch.utils.data.random_split(houses, 231 | [number_for_training, number_for_testing]) 232 | training = torch.utils.data.DataLoader( 233 | train, batch_size=64, shuffle=True) 234 | for epoch in range(16): 235 | for inputs, outputs in training: 236 | optimizer.zero_grad() 237 | results = model(inputs) 238 | loss = loss_function(results, outputs) 239 | loss.backward() 240 | optimizer.step() 241 | print("Loss: {0}".format(loss)) 242 | 243 | 244 | #%% 245 | # notice those loss numbers are large, since we are computing 246 | # loss in terms of dollars, and the error involves a square, 247 | # you always 248 | # need to get a sense of error relative to your target numbers 249 | # -- another 250 | # way to think of this would be to create a model 251 | # that gives an output 252 | # on the range 0-1 and multiply that by the 253 | # range of values you 254 | # see in the output say 0-1000000 for houses, 255 | # but as you can see from these 256 | # outputs, our model seems plenty well able to 257 | # learn with large number output 258 | 259 | # let's use our test data and see what we get 260 | 261 | #%% 262 | # here is our actual , and w 263 | actual = test[0][1] 264 | predicted = model(test[0][0]) 265 | actual, predicted 266 | 267 | #%% 268 | # wow - that's pretty good for an quick eyeball check, let's 269 | # take a look at the overall error for all our test data 270 | # for this we'll reach back to sklearn and use R^2, 271 | # which gives a score 272 | # of 0-1, one being best, and is a standard method 273 | # to judge the quality 274 | # of a regression model 275 | 276 | #%% 277 | import sklearn.metrics 278 | import torch.utils.data 279 | 280 | testing = torch.utils.data.DataLoader( 281 | test, batch_size=len(test), shuffle=False) 282 | for inputs, outputs in testing: 283 | predicted = model(inputs).detach().numpy() 284 | actual = outputs.numpy() 285 | print(sklearn.metrics.r2_score(actual, predicted)) 286 | 287 | 288 | #%% 289 | # pretty good, that's actually better than I expected 290 | # when I started 291 | # up this model -- turns out house prices are quite predictable 292 | # in this dataset -- I've actually seen this done 293 | # for local housing prices 294 | # by some of my co workers when they were moving 295 | # to maximize their 296 | # return and minimze their risk -- 297 | # a pretty useful application if you 298 | # can get some local MLS data and are planning a move! 299 | -------------------------------------------------------------------------------- /Admission_Predict.csv: -------------------------------------------------------------------------------- 1 | Serial No.,GRE Score,TOEFL Score,University Rating,SOP,LOR ,CGPA,Research,Chance of Admit 2 | 1,337,118,4,4.5,4.5,9.65,1,0.92 3 | 2,324,107,4,4,4.5,8.87,1,0.76 4 | 3,316,104,3,3,3.5,8,1,0.72 5 | 4,322,110,3,3.5,2.5,8.67,1,0.8 6 | 5,314,103,2,2,3,8.21,0,0.65 7 | 6,330,115,5,4.5,3,9.34,1,0.9 8 | 7,321,109,3,3,4,8.2,1,0.75 9 | 8,308,101,2,3,4,7.9,0,0.68 10 | 9,302,102,1,2,1.5,8,0,0.5 11 | 10,323,108,3,3.5,3,8.6,0,0.45 12 | 11,325,106,3,3.5,4,8.4,1,0.52 13 | 12,327,111,4,4,4.5,9,1,0.84 14 | 13,328,112,4,4,4.5,9.1,1,0.78 15 | 14,307,109,3,4,3,8,1,0.62 16 | 15,311,104,3,3.5,2,8.2,1,0.61 17 | 16,314,105,3,3.5,2.5,8.3,0,0.54 18 | 17,317,107,3,4,3,8.7,0,0.66 19 | 18,319,106,3,4,3,8,1,0.65 20 | 19,318,110,3,4,3,8.8,0,0.63 21 | 20,303,102,3,3.5,3,8.5,0,0.62 22 | 21,312,107,3,3,2,7.9,1,0.64 23 | 22,325,114,4,3,2,8.4,0,0.7 24 | 23,328,116,5,5,5,9.5,1,0.94 25 | 24,334,119,5,5,4.5,9.7,1,0.95 26 | 25,336,119,5,4,3.5,9.8,1,0.97 27 | 26,340,120,5,4.5,4.5,9.6,1,0.94 28 | 27,322,109,5,4.5,3.5,8.8,0,0.76 29 | 28,298,98,2,1.5,2.5,7.5,1,0.44 30 | 29,295,93,1,2,2,7.2,0,0.46 31 | 30,310,99,2,1.5,2,7.3,0,0.54 32 | 31,300,97,2,3,3,8.1,1,0.65 33 | 32,327,103,3,4,4,8.3,1,0.74 34 | 33,338,118,4,3,4.5,9.4,1,0.91 35 | 34,340,114,5,4,4,9.6,1,0.9 36 | 35,331,112,5,4,5,9.8,1,0.94 37 | 36,320,110,5,5,5,9.2,1,0.88 38 | 37,299,106,2,4,4,8.4,0,0.64 39 | 38,300,105,1,1,2,7.8,0,0.58 40 | 39,304,105,1,3,1.5,7.5,0,0.52 41 | 40,307,108,2,4,3.5,7.7,0,0.48 42 | 41,308,110,3,3.5,3,8,1,0.46 43 | 42,316,105,2,2.5,2.5,8.2,1,0.49 44 | 43,313,107,2,2.5,2,8.5,1,0.53 45 | 44,332,117,4,4.5,4,9.1,0,0.87 46 | 45,326,113,5,4.5,4,9.4,1,0.91 47 | 46,322,110,5,5,4,9.1,1,0.88 48 | 47,329,114,5,4,5,9.3,1,0.86 49 | 48,339,119,5,4.5,4,9.7,0,0.89 50 | 49,321,110,3,3.5,5,8.85,1,0.82 51 | 50,327,111,4,3,4,8.4,1,0.78 52 | 51,313,98,3,2.5,4.5,8.3,1,0.76 53 | 52,312,100,2,1.5,3.5,7.9,1,0.56 54 | 53,334,116,4,4,3,8,1,0.78 55 | 54,324,112,4,4,2.5,8.1,1,0.72 56 | 55,322,110,3,3,3.5,8,0,0.7 57 | 56,320,103,3,3,3,7.7,0,0.64 58 | 57,316,102,3,2,3,7.4,0,0.64 59 | 58,298,99,2,4,2,7.6,0,0.46 60 | 59,300,99,1,3,2,6.8,1,0.36 61 | 60,311,104,2,2,2,8.3,0,0.42 62 | 61,309,100,2,3,3,8.1,0,0.48 63 | 62,307,101,3,4,3,8.2,0,0.47 64 | 63,304,105,2,3,3,8.2,1,0.54 65 | 64,315,107,2,4,3,8.5,1,0.56 66 | 65,325,111,3,3,3.5,8.7,0,0.52 67 | 66,325,112,4,3.5,3.5,8.92,0,0.55 68 | 67,327,114,3,3,3,9.02,0,0.61 69 | 68,316,107,2,3.5,3.5,8.64,1,0.57 70 | 69,318,109,3,3.5,4,9.22,1,0.68 71 | 70,328,115,4,4.5,4,9.16,1,0.78 72 | 71,332,118,5,5,5,9.64,1,0.94 73 | 72,336,112,5,5,5,9.76,1,0.96 74 | 73,321,111,5,5,5,9.45,1,0.93 75 | 74,314,108,4,4.5,4,9.04,1,0.84 76 | 75,314,106,3,3,5,8.9,0,0.74 77 | 76,329,114,2,2,4,8.56,1,0.72 78 | 77,327,112,3,3,3,8.72,1,0.74 79 | 78,301,99,2,3,2,8.22,0,0.64 80 | 79,296,95,2,3,2,7.54,1,0.44 81 | 80,294,93,1,1.5,2,7.36,0,0.46 82 | 81,312,105,3,2,3,8.02,1,0.5 83 | 82,340,120,4,5,5,9.5,1,0.96 84 | 83,320,110,5,5,4.5,9.22,1,0.92 85 | 84,322,115,5,4,4.5,9.36,1,0.92 86 | 85,340,115,5,4.5,4.5,9.45,1,0.94 87 | 86,319,103,4,4.5,3.5,8.66,0,0.76 88 | 87,315,106,3,4.5,3.5,8.42,0,0.72 89 | 88,317,107,2,3.5,3,8.28,0,0.66 90 | 89,314,108,3,4.5,3.5,8.14,0,0.64 91 | 90,316,109,4,4.5,3.5,8.76,1,0.74 92 | 91,318,106,2,4,4,7.92,1,0.64 93 | 92,299,97,3,5,3.5,7.66,0,0.38 94 | 93,298,98,2,4,3,8.03,0,0.34 95 | 94,301,97,2,3,3,7.88,1,0.44 96 | 95,303,99,3,2,2.5,7.66,0,0.36 97 | 96,304,100,4,1.5,2.5,7.84,0,0.42 98 | 97,306,100,2,3,3,8,0,0.48 99 | 98,331,120,3,4,4,8.96,1,0.86 100 | 99,332,119,4,5,4.5,9.24,1,0.9 101 | 100,323,113,3,4,4,8.88,1,0.79 102 | 101,322,107,3,3.5,3.5,8.46,1,0.71 103 | 102,312,105,2,2.5,3,8.12,0,0.64 104 | 103,314,106,2,4,3.5,8.25,0,0.62 105 | 104,317,104,2,4.5,4,8.47,0,0.57 106 | 105,326,112,3,3.5,3,9.05,1,0.74 107 | 106,316,110,3,4,4.5,8.78,1,0.69 108 | 107,329,111,4,4.5,4.5,9.18,1,0.87 109 | 108,338,117,4,3.5,4.5,9.46,1,0.91 110 | 109,331,116,5,5,5,9.38,1,0.93 111 | 110,304,103,5,5,4,8.64,0,0.68 112 | 111,305,108,5,3,3,8.48,0,0.61 113 | 112,321,109,4,4,4,8.68,1,0.69 114 | 113,301,107,3,3.5,3.5,8.34,1,0.62 115 | 114,320,110,2,4,3.5,8.56,0,0.72 116 | 115,311,105,3,3.5,3,8.45,1,0.59 117 | 116,310,106,4,4.5,4.5,9.04,1,0.66 118 | 117,299,102,3,4,3.5,8.62,0,0.56 119 | 118,290,104,4,2,2.5,7.46,0,0.45 120 | 119,296,99,2,3,3.5,7.28,0,0.47 121 | 120,327,104,5,3,3.5,8.84,1,0.71 122 | 121,335,117,5,5,5,9.56,1,0.94 123 | 122,334,119,5,4.5,4.5,9.48,1,0.94 124 | 123,310,106,4,1.5,2.5,8.36,0,0.57 125 | 124,308,108,3,3.5,3.5,8.22,0,0.61 126 | 125,301,106,4,2.5,3,8.47,0,0.57 127 | 126,300,100,3,2,3,8.66,1,0.64 128 | 127,323,113,3,4,3,9.32,1,0.85 129 | 128,319,112,3,2.5,2,8.71,1,0.78 130 | 129,326,112,3,3.5,3,9.1,1,0.84 131 | 130,333,118,5,5,5,9.35,1,0.92 132 | 131,339,114,5,4,4.5,9.76,1,0.96 133 | 132,303,105,5,5,4.5,8.65,0,0.77 134 | 133,309,105,5,3.5,3.5,8.56,0,0.71 135 | 134,323,112,5,4,4.5,8.78,0,0.79 136 | 135,333,113,5,4,4,9.28,1,0.89 137 | 136,314,109,4,3.5,4,8.77,1,0.82 138 | 137,312,103,3,5,4,8.45,0,0.76 139 | 138,316,100,2,1.5,3,8.16,1,0.71 140 | 139,326,116,2,4.5,3,9.08,1,0.8 141 | 140,318,109,1,3.5,3.5,9.12,0,0.78 142 | 141,329,110,2,4,3,9.15,1,0.84 143 | 142,332,118,2,4.5,3.5,9.36,1,0.9 144 | 143,331,115,5,4,3.5,9.44,1,0.92 145 | 144,340,120,4,4.5,4,9.92,1,0.97 146 | 145,325,112,2,3,3.5,8.96,1,0.8 147 | 146,320,113,2,2,2.5,8.64,1,0.81 148 | 147,315,105,3,2,2.5,8.48,0,0.75 149 | 148,326,114,3,3,3,9.11,1,0.83 150 | 149,339,116,4,4,3.5,9.8,1,0.96 151 | 150,311,106,2,3.5,3,8.26,1,0.79 152 | 151,334,114,4,4,4,9.43,1,0.93 153 | 152,332,116,5,5,5,9.28,1,0.94 154 | 153,321,112,5,5,5,9.06,1,0.86 155 | 154,324,105,3,3,4,8.75,0,0.79 156 | 155,326,108,3,3,3.5,8.89,0,0.8 157 | 156,312,109,3,3,3,8.69,0,0.77 158 | 157,315,105,3,2,2.5,8.34,0,0.7 159 | 158,309,104,2,2,2.5,8.26,0,0.65 160 | 159,306,106,2,2,2.5,8.14,0,0.61 161 | 160,297,100,1,1.5,2,7.9,0,0.52 162 | 161,315,103,1,1.5,2,7.86,0,0.57 163 | 162,298,99,1,1.5,3,7.46,0,0.53 164 | 163,318,109,3,3,3,8.5,0,0.67 165 | 164,317,105,3,3.5,3,8.56,0,0.68 166 | 165,329,111,4,4.5,4,9.01,1,0.81 167 | 166,322,110,5,4.5,4,8.97,0,0.78 168 | 167,302,102,3,3.5,5,8.33,0,0.65 169 | 168,313,102,3,2,3,8.27,0,0.64 170 | 169,293,97,2,2,4,7.8,1,0.64 171 | 170,311,99,2,2.5,3,7.98,0,0.65 172 | 171,312,101,2,2.5,3.5,8.04,1,0.68 173 | 172,334,117,5,4,4.5,9.07,1,0.89 174 | 173,322,110,4,4,5,9.13,1,0.86 175 | 174,323,113,4,4,4.5,9.23,1,0.89 176 | 175,321,111,4,4,4,8.97,1,0.87 177 | 176,320,111,4,4.5,3.5,8.87,1,0.85 178 | 177,329,119,4,4.5,4.5,9.16,1,0.9 179 | 178,319,110,3,3.5,3.5,9.04,0,0.82 180 | 179,309,108,3,2.5,3,8.12,0,0.72 181 | 180,307,102,3,3,3,8.27,0,0.73 182 | 181,300,104,3,3.5,3,8.16,0,0.71 183 | 182,305,107,2,2.5,2.5,8.42,0,0.71 184 | 183,299,100,2,3,3.5,7.88,0,0.68 185 | 184,314,110,3,4,4,8.8,0,0.75 186 | 185,316,106,2,2.5,4,8.32,0,0.72 187 | 186,327,113,4,4.5,4.5,9.11,1,0.89 188 | 187,317,107,3,3.5,3,8.68,1,0.84 189 | 188,335,118,5,4.5,3.5,9.44,1,0.93 190 | 189,331,115,5,4.5,3.5,9.36,1,0.93 191 | 190,324,112,5,5,5,9.08,1,0.88 192 | 191,324,111,5,4.5,4,9.16,1,0.9 193 | 192,323,110,5,4,5,8.98,1,0.87 194 | 193,322,114,5,4.5,4,8.94,1,0.86 195 | 194,336,118,5,4.5,5,9.53,1,0.94 196 | 195,316,109,3,3.5,3,8.76,0,0.77 197 | 196,307,107,2,3,3.5,8.52,1,0.78 198 | 197,306,105,2,3,2.5,8.26,0,0.73 199 | 198,310,106,2,3.5,2.5,8.33,0,0.73 200 | 199,311,104,3,4.5,4.5,8.43,0,0.7 201 | 200,313,107,3,4,4.5,8.69,0,0.72 202 | 201,317,103,3,2.5,3,8.54,1,0.73 203 | 202,315,110,2,3.5,3,8.46,1,0.72 204 | 203,340,120,5,4.5,4.5,9.91,1,0.97 205 | 204,334,120,5,4,5,9.87,1,0.97 206 | 205,298,105,3,3.5,4,8.54,0,0.69 207 | 206,295,99,2,2.5,3,7.65,0,0.57 208 | 207,315,99,2,3.5,3,7.89,0,0.63 209 | 208,310,102,3,3.5,4,8.02,1,0.66 210 | 209,305,106,2,3,3,8.16,0,0.64 211 | 210,301,104,3,3.5,4,8.12,1,0.68 212 | 211,325,108,4,4.5,4,9.06,1,0.79 213 | 212,328,110,4,5,4,9.14,1,0.82 214 | 213,338,120,4,5,5,9.66,1,0.95 215 | 214,333,119,5,5,4.5,9.78,1,0.96 216 | 215,331,117,4,4.5,5,9.42,1,0.94 217 | 216,330,116,5,5,4.5,9.36,1,0.93 218 | 217,322,112,4,4.5,4.5,9.26,1,0.91 219 | 218,321,109,4,4,4,9.13,1,0.85 220 | 219,324,110,4,3,3.5,8.97,1,0.84 221 | 220,312,104,3,3.5,3.5,8.42,0,0.74 222 | 221,313,103,3,4,4,8.75,0,0.76 223 | 222,316,110,3,3.5,4,8.56,0,0.75 224 | 223,324,113,4,4.5,4,8.79,0,0.76 225 | 224,308,109,2,3,4,8.45,0,0.71 226 | 225,305,105,2,3,2,8.23,0,0.67 227 | 226,296,99,2,2.5,2.5,8.03,0,0.61 228 | 227,306,110,2,3.5,4,8.45,0,0.63 229 | 228,312,110,2,3.5,3,8.53,0,0.64 230 | 229,318,112,3,4,3.5,8.67,0,0.71 231 | 230,324,111,4,3,3,9.01,1,0.82 232 | 231,313,104,3,4,4.5,8.65,0,0.73 233 | 232,319,106,3,3.5,2.5,8.33,1,0.74 234 | 233,312,107,2,2.5,3.5,8.27,0,0.69 235 | 234,304,100,2,2.5,3.5,8.07,0,0.64 236 | 235,330,113,5,5,4,9.31,1,0.91 237 | 236,326,111,5,4.5,4,9.23,1,0.88 238 | 237,325,112,4,4,4.5,9.17,1,0.85 239 | 238,329,114,5,4.5,5,9.19,1,0.86 240 | 239,310,104,3,2,3.5,8.37,0,0.7 241 | 240,299,100,1,1.5,2,7.89,0,0.59 242 | 241,296,101,1,2.5,3,7.68,0,0.6 243 | 242,317,103,2,2.5,2,8.15,0,0.65 244 | 243,324,115,3,3.5,3,8.76,1,0.7 245 | 244,325,114,3,3.5,3,9.04,1,0.76 246 | 245,314,107,2,2.5,4,8.56,0,0.63 247 | 246,328,110,4,4,2.5,9.02,1,0.81 248 | 247,316,105,3,3,3.5,8.73,0,0.72 249 | 248,311,104,2,2.5,3.5,8.48,0,0.71 250 | 249,324,110,3,3.5,4,8.87,1,0.8 251 | 250,321,111,3,3.5,4,8.83,1,0.77 252 | 251,320,104,3,3,2.5,8.57,1,0.74 253 | 252,316,99,2,2.5,3,9,0,0.7 254 | 253,318,100,2,2.5,3.5,8.54,1,0.71 255 | 254,335,115,4,4.5,4.5,9.68,1,0.93 256 | 255,321,114,4,4,5,9.12,0,0.85 257 | 256,307,110,4,4,4.5,8.37,0,0.79 258 | 257,309,99,3,4,4,8.56,0,0.76 259 | 258,324,100,3,4,5,8.64,1,0.78 260 | 259,326,102,4,5,5,8.76,1,0.77 261 | 260,331,119,4,5,4.5,9.34,1,0.9 262 | 261,327,108,5,5,3.5,9.13,1,0.87 263 | 262,312,104,3,3.5,4,8.09,0,0.71 264 | 263,308,103,2,2.5,4,8.36,1,0.7 265 | 264,324,111,3,2.5,1.5,8.79,1,0.7 266 | 265,325,110,2,3,2.5,8.76,1,0.75 267 | 266,313,102,3,2.5,2.5,8.68,0,0.71 268 | 267,312,105,2,2,2.5,8.45,0,0.72 269 | 268,314,107,3,3,3.5,8.17,1,0.73 270 | 269,327,113,4,4.5,5,9.14,0,0.83 271 | 270,308,108,4,4.5,5,8.34,0,0.77 272 | 271,306,105,2,2.5,3,8.22,1,0.72 273 | 272,299,96,2,1.5,2,7.86,0,0.54 274 | 273,294,95,1,1.5,1.5,7.64,0,0.49 275 | 274,312,99,1,1,1.5,8.01,1,0.52 276 | 275,315,100,1,2,2.5,7.95,0,0.58 277 | 276,322,110,3,3.5,3,8.96,1,0.78 278 | 277,329,113,5,5,4.5,9.45,1,0.89 279 | 278,320,101,2,2.5,3,8.62,0,0.7 280 | 279,308,103,2,3,3.5,8.49,0,0.66 281 | 280,304,102,2,3,4,8.73,0,0.67 282 | 281,311,102,3,4.5,4,8.64,1,0.68 283 | 282,317,110,3,4,4.5,9.11,1,0.8 284 | 283,312,106,3,4,3.5,8.79,1,0.81 285 | 284,321,111,3,2.5,3,8.9,1,0.8 286 | 285,340,112,4,5,4.5,9.66,1,0.94 287 | 286,331,116,5,4,4,9.26,1,0.93 288 | 287,336,118,5,4.5,4,9.19,1,0.92 289 | 288,324,114,5,5,4.5,9.08,1,0.89 290 | 289,314,104,4,5,5,9.02,0,0.82 291 | 290,313,109,3,4,3.5,9,0,0.79 292 | 291,307,105,2,2.5,3,7.65,0,0.58 293 | 292,300,102,2,1.5,2,7.87,0,0.56 294 | 293,302,99,2,1,2,7.97,0,0.56 295 | 294,312,98,1,3.5,3,8.18,1,0.64 296 | 295,316,101,2,2.5,2,8.32,1,0.61 297 | 296,317,100,2,3,2.5,8.57,0,0.68 298 | 297,310,107,3,3.5,3.5,8.67,0,0.76 299 | 298,320,120,3,4,4.5,9.11,0,0.86 300 | 299,330,114,3,4.5,4.5,9.24,1,0.9 301 | 300,305,112,3,3,3.5,8.65,0,0.71 302 | 301,309,106,2,2.5,2.5,8,0,0.62 303 | 302,319,108,2,2.5,3,8.76,0,0.66 304 | 303,322,105,2,3,3,8.45,1,0.65 305 | 304,323,107,3,3.5,3.5,8.55,1,0.73 306 | 305,313,106,2,2.5,2,8.43,0,0.62 307 | 306,321,109,3,3.5,3.5,8.8,1,0.74 308 | 307,323,110,3,4,3.5,9.1,1,0.79 309 | 308,325,112,4,4,4,9,1,0.8 310 | 309,312,108,3,3.5,3,8.53,0,0.69 311 | 310,308,110,4,3.5,3,8.6,0,0.7 312 | 311,320,104,3,3,3.5,8.74,1,0.76 313 | 312,328,108,4,4.5,4,9.18,1,0.84 314 | 313,311,107,4,4.5,4.5,9,1,0.78 315 | 314,301,100,3,3.5,3,8.04,0,0.67 316 | 315,305,105,2,3,4,8.13,0,0.66 317 | 316,308,104,2,2.5,3,8.07,0,0.65 318 | 317,298,101,2,1.5,2,7.86,0,0.54 319 | 318,300,99,1,1,2.5,8.01,0,0.58 320 | 319,324,111,3,2.5,2,8.8,1,0.79 321 | 320,327,113,4,3.5,3,8.69,1,0.8 322 | 321,317,106,3,4,3.5,8.5,1,0.75 323 | 322,323,104,3,4,4,8.44,1,0.73 324 | 323,314,107,2,2.5,4,8.27,0,0.72 325 | 324,305,102,2,2,2.5,8.18,0,0.62 326 | 325,315,104,3,3,2.5,8.33,0,0.67 327 | 326,326,116,3,3.5,4,9.14,1,0.81 328 | 327,299,100,3,2,2,8.02,0,0.63 329 | 328,295,101,2,2.5,2,7.86,0,0.69 330 | 329,324,112,4,4,3.5,8.77,1,0.8 331 | 330,297,96,2,2.5,1.5,7.89,0,0.43 332 | 331,327,113,3,3.5,3,8.66,1,0.8 333 | 332,311,105,2,3,2,8.12,1,0.73 334 | 333,308,106,3,3.5,2.5,8.21,1,0.75 335 | 334,319,108,3,3,3.5,8.54,1,0.71 336 | 335,312,107,4,4.5,4,8.65,1,0.73 337 | 336,325,111,4,4,4.5,9.11,1,0.83 338 | 337,319,110,3,3,2.5,8.79,0,0.72 339 | 338,332,118,5,5,5,9.47,1,0.94 340 | 339,323,108,5,4,4,8.74,1,0.81 341 | 340,324,107,5,3.5,4,8.66,1,0.81 342 | 341,312,107,3,3,3,8.46,1,0.75 343 | 342,326,110,3,3.5,3.5,8.76,1,0.79 344 | 343,308,106,3,3,3,8.24,0,0.58 345 | 344,305,103,2,2.5,3.5,8.13,0,0.59 346 | 345,295,96,2,1.5,2,7.34,0,0.47 347 | 346,316,98,1,1.5,2,7.43,0,0.49 348 | 347,304,97,2,1.5,2,7.64,0,0.47 349 | 348,299,94,1,1,1,7.34,0,0.42 350 | 349,302,99,1,2,2,7.25,0,0.57 351 | 350,313,101,3,2.5,3,8.04,0,0.62 352 | 351,318,107,3,3,3.5,8.27,1,0.74 353 | 352,325,110,4,3.5,4,8.67,1,0.73 354 | 353,303,100,2,3,3.5,8.06,1,0.64 355 | 354,300,102,3,3.5,2.5,8.17,0,0.63 356 | 355,297,98,2,2.5,3,7.67,0,0.59 357 | 356,317,106,2,2,3.5,8.12,0,0.73 358 | 357,327,109,3,3.5,4,8.77,1,0.79 359 | 358,301,104,2,3.5,3.5,7.89,1,0.68 360 | 359,314,105,2,2.5,2,7.64,0,0.7 361 | 360,321,107,2,2,1.5,8.44,0,0.81 362 | 361,322,110,3,4,5,8.64,1,0.85 363 | 362,334,116,4,4,3.5,9.54,1,0.93 364 | 363,338,115,5,4.5,5,9.23,1,0.91 365 | 364,306,103,2,2.5,3,8.36,0,0.69 366 | 365,313,102,3,3.5,4,8.9,1,0.77 367 | 366,330,114,4,4.5,3,9.17,1,0.86 368 | 367,320,104,3,3.5,4.5,8.34,1,0.74 369 | 368,311,98,1,1,2.5,7.46,0,0.57 370 | 369,298,92,1,2,2,7.88,0,0.51 371 | 370,301,98,1,2,3,8.03,1,0.67 372 | 371,310,103,2,2.5,2.5,8.24,0,0.72 373 | 372,324,110,3,3.5,3,9.22,1,0.89 374 | 373,336,119,4,4.5,4,9.62,1,0.95 375 | 374,321,109,3,3,3,8.54,1,0.79 376 | 375,315,105,2,2,2.5,7.65,0,0.39 377 | 376,304,101,2,2,2.5,7.66,0,0.38 378 | 377,297,96,2,2.5,2,7.43,0,0.34 379 | 378,290,100,1,1.5,2,7.56,0,0.47 380 | 379,303,98,1,2,2.5,7.65,0,0.56 381 | 380,311,99,1,2.5,3,8.43,1,0.71 382 | 381,322,104,3,3.5,4,8.84,1,0.78 383 | 382,319,105,3,3,3.5,8.67,1,0.73 384 | 383,324,110,4,4.5,4,9.15,1,0.82 385 | 384,300,100,3,3,3.5,8.26,0,0.62 386 | 385,340,113,4,5,5,9.74,1,0.96 387 | 386,335,117,5,5,5,9.82,1,0.96 388 | 387,302,101,2,2.5,3.5,7.96,0,0.46 389 | 388,307,105,2,2,3.5,8.1,0,0.53 390 | 389,296,97,2,1.5,2,7.8,0,0.49 391 | 390,320,108,3,3.5,4,8.44,1,0.76 392 | 391,314,102,2,2,2.5,8.24,0,0.64 393 | 392,318,106,3,2,3,8.65,0,0.71 394 | 393,326,112,4,4,3.5,9.12,1,0.84 395 | 394,317,104,2,3,3,8.76,0,0.77 396 | 395,329,111,4,4.5,4,9.23,1,0.89 397 | 396,324,110,3,3.5,3.5,9.04,1,0.82 398 | 397,325,107,3,3,3.5,9.11,1,0.84 399 | 398,330,116,4,5,4.5,9.45,1,0.91 400 | 399,312,103,3,3.5,4,8.78,0,0.67 401 | 400,333,117,4,5,4,9.66,1,0.95 --------------------------------------------------------------------------------