├── .gitignore ├── LICENSE ├── README.md ├── codes ├── data │ ├── cifar │ │ └── generate_cifar.ipynb │ ├── fashion-mnist │ │ └── generate_fashion_mnist.ipynb │ ├── generate_all_data.py │ ├── misc │ │ ├── profit_population.txt │ │ └── profit_population_new.txt │ ├── mnist │ │ └── generate_mnist.ipynb │ └── ptb │ │ ├── data_raw │ │ ├── test.txt │ │ ├── train.txt │ │ └── valid.txt │ │ └── generate_ptb.ipynb ├── installation │ └── installation.ipynb ├── labs_lecture02 │ ├── lab01_pytorch │ │ └── pytorch_introduction.ipynb │ ├── lab02_pytorch_tensor1 │ │ ├── pytorch_tensor_part1_exercise.ipynb │ │ └── pytorch_tensor_part1_solution.ipynb │ └── lab03_pytorch_tensor2 │ │ ├── pytorch_tensor_part2_exercise.ipynb │ │ ├── pytorch_tensor_part2_solution.ipynb │ │ └── utils.py ├── labs_lecture03 │ ├── lab01_linear_module │ │ ├── linear_module_demo.ipynb │ │ ├── linear_module_exercise.ipynb │ │ └── linear_module_solution.ipynb │ ├── lab02_softmax │ │ └── softmax_demo.ipynb │ ├── lab03_vanilla_nn │ │ ├── vanilla_nn_demo.ipynb │ │ ├── vanilla_nn_exercise.ipynb │ │ └── vanilla_nn_solution.ipynb │ ├── lab04_train_vanilla_nn │ │ ├── train_vanilla_nn_demo.ipynb │ │ ├── train_vanilla_nn_exercise.ipynb │ │ ├── train_vanilla_nn_solution.ipynb │ │ └── utils.py │ └── lab05_minibatch_training │ │ ├── minibatch_training_demo.ipynb │ │ ├── minibatch_training_exercise.ipynb │ │ ├── minibatch_training_solution.ipynb │ │ └── utils.py ├── labs_lecture04 │ └── lab01_cross_entropy │ │ ├── cross_entropy_demo.ipynb │ │ ├── cross_entropy_exercise.ipynb │ │ ├── cross_entropy_solution.ipynb │ │ └── utils.py ├── labs_lecture05 │ ├── lab01_mlp │ │ ├── mlp_demo.ipynb │ │ ├── mlp_exercise.ipynb │ │ ├── mlp_solution.ipynb │ │ └── utils.py │ ├── lab02_epoch │ │ ├── epoch_demo.ipynb │ │ ├── epoch_exercise.ipynb │ │ ├── epoch_solution.ipynb │ │ └── utils.py │ ├── lab03_monitoring_loss │ │ ├── monitoring_loss_demo.ipynb │ │ ├── monitoring_loss_exercise.ipynb │ │ ├── monitoring_loss_solution.ipynb │ │ └── utils.py │ ├── lab04_test_set │ │ ├── test_set_demo.ipynb │ │ ├── test_set_exercise.ipynb │ │ ├── test_set_solution.ipynb │ │ └── utils.py │ └── lab05_final │ │ ├── final_demo.ipynb │ │ └── utils.py ├── labs_lecture06 │ ├── lab01_mnist_multilayer │ │ ├── mnist_multilayer_demo.ipynb │ │ └── utils.py │ └── lab02_cifar_multilayer │ │ ├── cifar_multilayer_exercise.ipynb │ │ ├── cifar_multilayer_solution.ipynb │ │ ├── gpu_demo.ipynb │ │ └── utils.py ├── labs_lecture08 │ ├── lab01_conv_layer │ │ └── conv_layer_demo.ipynb │ ├── lab02_pool_layer │ │ └── pool_layer_demo.ipynb │ ├── lab03_lenet5 │ │ ├── lenet5_exercise.ipynb │ │ ├── lenet5_solution.ipynb │ │ └── utils.py │ └── lab04_vgg │ │ ├── utils.py │ │ ├── vgg_exercise.ipynb │ │ └── vgg_solution.ipynb ├── labs_lecture10 │ ├── lab01_vrnn │ │ ├── utils.py │ │ └── vrnn_demo.ipynb │ └── lab02_lstm │ │ ├── lstm_exercise.ipynb │ │ ├── lstm_solution.ipynb │ │ └── utils.py ├── labs_lecture13 │ └── seq2seq_transformers_demo.ipynb ├── labs_lecture14 │ ├── lab01_ChebGCNs │ │ ├── 01_Traditional_ConvNets.ipynb │ │ ├── 02_ChebGCNs.ipynb │ │ └── lib │ │ │ ├── __pycache__ │ │ │ ├── coarsening.cpython-36.pyc │ │ │ ├── coarsening.cpython-37.pyc │ │ │ ├── grid_graph.cpython-36.pyc │ │ │ └── grid_graph.cpython-37.pyc │ │ │ ├── coarsening.py │ │ │ └── grid_graph.py │ ├── lab02_GatedGCNs │ │ ├── 01_GatedGCNs_subgraph_recognition.ipynb │ │ ├── 02_GatedGCNs_graph_clustering.ipynb │ │ ├── data │ │ │ ├── set_100_clustering_maps_p05_q01_size5_25_2017-10-31_10-25-00_.txt │ │ │ └── set_100_subgraphs_p05_size20_Voc3_2017-10-31_10-23-00_.txt │ │ └── util │ │ │ ├── __pycache__ │ │ │ ├── block.cpython-36.pyc │ │ │ ├── block.cpython-37.pyc │ │ │ ├── graph_generator.cpython-36.pyc │ │ │ └── graph_generator.cpython-37.pyc │ │ │ ├── block.py │ │ │ └── graph_generator.py │ ├── lab03_Molecules │ │ ├── 01_GatedGCNs_Molecule_Regression.ipynb │ │ ├── datasets │ │ │ └── dataQM9 │ │ │ │ ├── atom_dict.pickle │ │ │ │ ├── bond_dict.pickle │ │ │ │ ├── info_test.pickle │ │ │ │ ├── info_train.pickle │ │ │ │ ├── info_val.pickle │ │ │ │ ├── test.pickle │ │ │ │ ├── train.pickle │ │ │ │ └── val.pickle │ │ └── dictionaries.py │ ├── lab04_TSP │ │ ├── 01_GatedGCNs_TSP.ipynb │ │ ├── config.py │ │ ├── configs │ │ │ ├── default.json │ │ │ └── tsp10_small.json │ │ ├── data │ │ │ ├── tsp10_small_concorde_test.txt │ │ │ └── tsp10_small_concorde_train.txt │ │ ├── logs │ │ │ └── tsp10 │ │ │ │ ├── config.json │ │ │ │ ├── events.out.tfevents.1570518905.MML-XavierNB1.local │ │ │ │ ├── events.out.tfevents.1570521674.MML-XavierNB1.local │ │ │ │ └── events.out.tfevents.1571736764.MML-XavierNB1.local │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-36.pyc │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ ├── gcn_layers.cpython-36.pyc │ │ │ │ ├── gcn_layers.cpython-37.pyc │ │ │ │ ├── gcn_model.cpython-36.pyc │ │ │ │ └── gcn_model.cpython-37.pyc │ │ │ ├── gcn_layers.py │ │ │ └── gcn_model.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── beamsearch.cpython-36.pyc │ │ │ ├── beamsearch.cpython-37.pyc │ │ │ ├── google_tsp_reader.cpython-36.pyc │ │ │ ├── google_tsp_reader.cpython-37.pyc │ │ │ ├── graph_utils.cpython-36.pyc │ │ │ ├── graph_utils.cpython-37.pyc │ │ │ ├── model_utils.cpython-36.pyc │ │ │ ├── model_utils.cpython-37.pyc │ │ │ ├── plot_utils.cpython-36.pyc │ │ │ └── plot_utils.cpython-37.pyc │ │ │ ├── beamsearch.py │ │ │ ├── google_tsp_reader.py │ │ │ ├── graph_utils.py │ │ │ ├── model_utils.py │ │ │ └── plot_utils.py │ └── lab05_DGL │ │ ├── 01_GatedGCNs_DGL.ipynb │ │ └── data │ │ └── artificial_dataset.pickle └── labs_lecture15 │ ├── lab01_DQN │ └── DQN_demo.ipynb │ ├── lab02_Policy_REINFORCE │ └── policy_REINFORCE.ipynb │ ├── lab03_Policy_Global_Rewards │ └── policy_global_reward_demo.ipynb │ ├── lab04_QAC │ └── QAC_demo.ipynb │ └── lab05_AAC │ └── AAC_demo.ipynb └── environment.yml /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.DS_Store 2 | **/*.ipynb_checkpoints/ 3 | 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Xavier Bresson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CE7454_2019 2 | Deep learning course CE7454, 2019 3 | 4 | 5 |

6 | 7 | 8 | ### Cloud Machine #1 : Google Colab (Free GPU) 9 | 10 | * Follow this Notebook installation :
11 | https://colab.research.google.com/github/xbresson/CE7454_2019/blob/master/codes/installation/installation.ipynb 12 | 13 | * Open your Google Drive :
14 | https://www.google.com/drive 15 | 16 | * Open in Google Drive Folder 'CE7454_2019' and go to Folder 'CE7454_2019/codes/'
17 | Select the notebook 'file.ipynb' and open it with Google Colab using Control Click + Open With Colaboratory 18 | 19 | 20 | 21 |

22 | 23 | ### Cloud Machine #2 : Binder (No GPU) 24 | 25 | * Simply [click here] 26 | 27 | [Click here]: https://mybinder.org/v2/gh/xbresson/CE7454_2019/master 28 | 29 | 30 | 31 |

32 | 33 | ### Local Installation 34 | 35 | * Follow these instructions (easy steps) : 36 | 37 | 38 | ```sh 39 | # Conda installation 40 | curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh # Linux 41 | curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh # OSX 42 | chmod +x ~/miniconda.sh 43 | ./miniconda.sh 44 | source ~/.bashrc 45 | #install https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe # Windows 46 | 47 | # Clone GitHub repo 48 | git clone https://github.com/xbresson/CE7454_2019.git 49 | cd CE7454_2019 50 | 51 | # Install python libraries 52 | conda env create -f environment.yml 53 | source activate deeplearn_course 54 | 55 | # Run the notebooks 56 | jupyter notebook 57 | ``` 58 | 59 | 60 | 61 | 62 |





-------------------------------------------------------------------------------- /codes/data/generate_all_data.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # run the code : 4 | # python generate_all_data.py 5 | 6 | 7 | import torch 8 | import os.path 9 | import os 10 | 11 | 12 | ############################# 13 | # MNIST 14 | ############################# 15 | def check_mnist_dataset_exists(path_data='./'): 16 | flag_train_data = os.path.isfile(path_data + 'mnist/train_data.pt') 17 | flag_train_label = os.path.isfile(path_data + 'mnist/train_label.pt') 18 | flag_test_data = os.path.isfile(path_data + 'mnist/test_data.pt') 19 | flag_test_label = os.path.isfile(path_data + 'mnist/test_label.pt') 20 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 21 | print('MNIST dataset missing - downloading...') 22 | import torchvision 23 | import torchvision.transforms as transforms 24 | trainset = torchvision.datasets.MNIST(root=path_data + 'mnist/temp', train=True, 25 | download=True, transform=transforms.ToTensor()) 26 | testset = torchvision.datasets.MNIST(root=path_data + 'mnist/temp', train=False, 27 | download=True, transform=transforms.ToTensor()) 28 | train_data=torch.Tensor(60000,28,28) 29 | train_label=torch.LongTensor(60000) 30 | for idx , example in enumerate(trainset): 31 | train_data[idx]=example[0].squeeze() 32 | train_label[idx]=example[1] 33 | torch.save(train_data,path_data + 'mnist/train_data.pt') 34 | torch.save(train_label,path_data + 'mnist/train_label.pt') 35 | test_data=torch.Tensor(10000,28,28) 36 | test_label=torch.LongTensor(10000) 37 | for idx , example in enumerate(testset): 38 | test_data[idx]=example[0].squeeze() 39 | test_label[idx]=example[1] 40 | torch.save(test_data,path_data + 'mnist/test_data.pt') 41 | torch.save(test_label,path_data + 'mnist/test_label.pt') 42 | return path_data 43 | 44 | ############################# 45 | # FASHION-MNIST 46 | ############################# 47 | def check_fashion_mnist_dataset_exists(path_data='./'): 48 | flag_train_data = os.path.isfile(path_data + 'fashion-mnist/train_data.pt') 49 | flag_train_label = os.path.isfile(path_data + 'fashion-mnist/train_label.pt') 50 | flag_test_data = os.path.isfile(path_data + 'fashion-mnist/test_data.pt') 51 | flag_test_label = os.path.isfile(path_data + 'fashion-mnist/test_label.pt') 52 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 53 | print('FASHION-MNIST dataset missing - downloading...') 54 | import torchvision 55 | import torchvision.transforms as transforms 56 | trainset = torchvision.datasets.FashionMNIST(root=path_data + 'fashion-mnist/temp', train=True, 57 | download=True, transform=transforms.ToTensor()) 58 | testset = torchvision.datasets.FashionMNIST(root=path_data + 'fashion-mnist/temp', train=False, 59 | download=True, transform=transforms.ToTensor()) 60 | train_data=torch.Tensor(60000,28,28) 61 | train_label=torch.LongTensor(60000) 62 | for idx , example in enumerate(trainset): 63 | train_data[idx]=example[0].squeeze() 64 | train_label[idx]=example[1] 65 | torch.save(train_data,path_data + 'fashion-mnist/train_data.pt') 66 | torch.save(train_label,path_data + 'fashion-mnist/train_label.pt') 67 | test_data=torch.Tensor(10000,28,28) 68 | test_label=torch.LongTensor(10000) 69 | for idx , example in enumerate(testset): 70 | test_data[idx]=example[0].squeeze() 71 | test_label[idx]=example[1] 72 | torch.save(test_data,path_data + 'fashion-mnist/test_data.pt') 73 | torch.save(test_label,path_data + 'fashion-mnist/test_label.pt') 74 | return path_data 75 | 76 | ############################# 77 | # CIFAR 78 | ############################# 79 | def check_cifar_dataset_exists(path_data='./'): 80 | flag_train_data = os.path.isfile(path_data + 'cifar/train_data.pt') 81 | flag_train_label = os.path.isfile(path_data + 'cifar/train_label.pt') 82 | flag_test_data = os.path.isfile(path_data + 'cifar/test_data.pt') 83 | flag_test_label = os.path.isfile(path_data + 'cifar/test_label.pt') 84 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 85 | print('CIFAR dataset missing - downloading...') 86 | import torchvision 87 | import torchvision.transforms as transforms 88 | trainset = torchvision.datasets.CIFAR10(root=path_data + 'cifar/temp', train=True, 89 | download=True, transform=transforms.ToTensor()) 90 | testset = torchvision.datasets.CIFAR10(root=path_data + 'cifar/temp', train=False, 91 | download=True, transform=transforms.ToTensor()) 92 | train_data=torch.Tensor(50000,3,32,32) 93 | train_label=torch.LongTensor(50000) 94 | for idx , example in enumerate(trainset): 95 | train_data[idx]=example[0] 96 | train_label[idx]=example[1] 97 | torch.save(train_data,path_data + 'cifar/train_data.pt') 98 | torch.save(train_label,path_data + 'cifar/train_label.pt') 99 | test_data=torch.Tensor(10000,3,32,32) 100 | test_label=torch.LongTensor(10000) 101 | for idx , example in enumerate(testset): 102 | test_data[idx]=example[0] 103 | test_label[idx]=example[1] 104 | torch.save(test_data,path_data + 'cifar/test_data.pt') 105 | torch.save(test_label,path_data + 'cifar/test_label.pt') 106 | return path_data 107 | 108 | ############################# 109 | # PTB 110 | ############################# 111 | class Dictionary(object): 112 | def __init__(self): 113 | self.word2idx = {} 114 | self.idx2word = [] 115 | 116 | def add_word(self, word): 117 | if word not in self.word2idx: 118 | self.idx2word.append(word) 119 | self.word2idx[word] = len(self.idx2word) - 1 120 | return self.word2idx[word] 121 | 122 | def __len__(self): 123 | return len(self.idx2word) 124 | 125 | class Corpus(object): 126 | def __init__(self, path): 127 | self.dictionary = Dictionary() 128 | self.train = self.tokenize(os.path.join(path, 'train.txt')) 129 | self.valid = self.tokenize(os.path.join(path, 'valid.txt')) 130 | self.test = self.tokenize(os.path.join(path, 'test.txt')) 131 | 132 | def tokenize(self, path): 133 | """Tokenizes a text file.""" 134 | assert os.path.exists(path) 135 | # Add words to the dictionary 136 | with open(path, 'r') as f: 137 | tokens = 0 138 | for line in f: 139 | words = line.split() + [''] 140 | tokens += len(words) 141 | for word in words: 142 | self.dictionary.add_word(word) 143 | 144 | # Tokenize file content 145 | with open(path, 'r') as f: 146 | ids = torch.LongTensor(tokens) 147 | token = 0 148 | for line in f: 149 | words = line.split() + [''] 150 | for word in words: 151 | ids[token] = self.dictionary.word2idx[word] 152 | token += 1 153 | 154 | return ids 155 | 156 | def batchify(data, bsz): 157 | # Work out how cleanly we can divide the dataset into bsz parts. 158 | nbatch = data.size(0) // bsz 159 | # Trim off any extra elements that wouldn't cleanly fit (remainders). 160 | data = data.narrow(0, 0, nbatch * bsz) 161 | # Evenly divide the data across the bsz batches. 162 | data = data.view(bsz, -1).t().contiguous() 163 | return data 164 | 165 | def check_ptb_dataset_exists(path_data='./'): 166 | flag_idx2word = os.path.isfile(path_data + 'ptb/idx2word.pt') 167 | flag_test_data = os.path.isfile(path_data + 'ptb/test_data.pt') 168 | flag_train_data = os.path.isfile(path_data + 'ptb/train_data.pt') 169 | flag_word2idx = os.path.isfile(path_data + 'ptb/word2idx.pt') 170 | if flag_idx2word==False or flag_test_data==False or flag_train_data==False or flag_word2idx==False: 171 | print('PTB dataset missing - generating...') 172 | data_folder = 'ptb/data_raw' 173 | corpus = Corpus(path_data+data_folder) 174 | batch_size=20 175 | train_data = batchify(corpus.train, batch_size) 176 | val_data = batchify(corpus.valid, batch_size) 177 | test_data = batchify(corpus.test, batch_size) 178 | vocab_size = len(corpus.dictionary) 179 | torch.save(train_data,path_data + 'ptb/train_data.pt') 180 | torch.save(test_data,path_data + 'ptb/test_data.pt') 181 | torch.save(corpus.dictionary.idx2word,path_data + 'ptb/idx2word.pt') 182 | torch.save(corpus.dictionary.word2idx,path_data + 'ptb/word2idx.pt') 183 | return path_data 184 | 185 | 186 | 187 | 188 | 189 | _ = check_mnist_dataset_exists() 190 | _ = check_fashion_mnist_dataset_exists() 191 | _ = check_cifar_dataset_exists() 192 | _ = check_ptb_dataset_exists() 193 | 194 | 195 | -------------------------------------------------------------------------------- /codes/data/misc/profit_population.txt: -------------------------------------------------------------------------------- 1 | 6.1101,17.592 2 | 5.5277,9.1302 3 | 8.5186,13.662 4 | 7.0032,11.854 5 | 5.8598,6.8233 6 | 8.3829,11.886 7 | 7.4764,4.3483 8 | 8.5781,12 9 | 6.4862,6.5987 10 | 5.0546,3.8166 11 | 5.7107,3.2522 12 | 14.164,15.505 13 | 5.734,3.1551 14 | 8.4084,7.2258 15 | 5.6407,0.71618 16 | 5.3794,3.5129 17 | 6.3654,5.3048 18 | 5.1301,0.56077 19 | 6.4296,3.6518 20 | 7.0708,5.3893 21 | 6.1891,3.1386 22 | 20.27,21.767 23 | 5.4901,4.263 24 | 6.3261,5.1875 25 | 5.5649,3.0825 26 | 18.945,22.638 27 | 12.828,13.501 28 | 10.957,7.0467 29 | 13.176,14.692 30 | 22.203,24.147 31 | 5.2524,-1.22 32 | 6.5894,5.9966 33 | 9.2482,12.134 34 | 5.8918,1.8495 35 | 8.2111,6.5426 36 | 7.9334,4.5623 37 | 8.0959,4.1164 38 | 5.6063,3.3928 39 | 12.836,10.117 40 | 6.3534,5.4974 41 | 5.4069,0.55657 42 | 6.8825,3.9115 43 | 11.708,5.3854 44 | 5.7737,2.4406 45 | 7.8247,6.7318 46 | 7.0931,1.0463 47 | 5.0702,5.1337 48 | 5.8014,1.844 49 | 11.7,8.0043 50 | 5.5416,1.0179 51 | 7.5402,6.7504 52 | 5.3077,1.8396 53 | 7.4239,4.2885 54 | 7.6031,4.9981 55 | 6.3328,1.4233 56 | 6.3589,-1.4211 57 | 6.2742,2.4756 58 | 5.6397,4.6042 59 | 9.3102,3.9624 60 | 9.4536,5.4141 61 | 8.8254,5.1694 62 | 5.1793,-0.74279 63 | 21.279,17.929 64 | 14.908,12.054 65 | 18.959,17.054 66 | 7.2182,4.8852 67 | 8.2951,5.7442 68 | 10.236,7.7754 69 | 5.4994,1.0173 70 | 20.341,20.992 71 | 10.136,6.6799 72 | 7.3345,4.0259 73 | 6.0062,1.2784 74 | 7.2259,3.3411 75 | 5.0269,-2.6807 76 | 6.5479,0.29678 77 | 7.5386,3.8845 78 | 5.0365,5.7014 79 | 10.274,6.7526 80 | 5.1077,2.0576 81 | 5.7292,0.47953 82 | 5.1884,0.20421 83 | 6.3557,0.67861 84 | 9.7687,7.5435 85 | 6.5159,5.3436 86 | 8.5172,4.2415 87 | 9.1802,6.7981 88 | 6.002,0.92695 89 | 5.5204,0.152 90 | 5.0594,2.8214 91 | 5.7077,1.8451 92 | 7.6366,4.2959 93 | 5.8707,7.2029 94 | 5.3054,1.9869 95 | 8.2934,0.14454 96 | 13.394,9.0551 97 | 5.4369,0.61705 98 | -------------------------------------------------------------------------------- /codes/data/misc/profit_population_new.txt: -------------------------------------------------------------------------------- 1 | 12.22020,35.18400 2 | 11.05540,18.26040 3 | 17.03720,27.32400 4 | 14.00640,23.70800 5 | 11.71960,13.64660 6 | 16.76580,23.77200 7 | 14.95280,8.69660 8 | 17.15620,24.00000 9 | 12.97240,13.19740 10 | 10.10920,7.63320 11 | 11.42140,6.50440 12 | 28.32800,31.01000 13 | 11.46800,6.31020 14 | 16.81680,14.45160 15 | 11.28140,1.43236 16 | 10.75880,7.02580 17 | 12.73080,10.60960 18 | 10.26020,1.12154 19 | 12.85920,7.30360 20 | 14.14160,10.77860 21 | 12.37820,6.27720 22 | 40.54000,43.53400 23 | 10.98020,8.52600 24 | 12.65220,10.37500 25 | 11.12980,6.16500 26 | 37.89000,45.27600 27 | 25.65600,27.00200 28 | 21.91400,14.09340 29 | 26.35200,29.38400 30 | 44.40600,48.29400 31 | 10.50480,-2.44000 32 | 13.17880,11.99320 33 | 18.49640,24.26800 34 | 11.78360,3.69900 35 | 16.42220,13.08520 36 | 15.86680,9.12460 37 | 16.19180,8.23280 38 | 11.21260,6.78560 39 | 25.67200,20.23400 40 | 12.70680,10.99480 41 | 10.81380,1.11314 42 | 13.76500,7.82300 43 | 23.41600,10.77080 44 | 11.54740,4.88120 45 | 15.64940,13.46360 46 | 14.18620,2.09260 47 | 10.14040,10.26740 48 | 11.60280,3.68800 49 | 23.40000,16.00860 50 | 11.08320,2.03580 51 | 15.08040,13.50080 52 | 10.61540,3.67920 53 | 14.84780,8.57700 54 | 15.20620,9.99620 55 | 12.66560,2.84660 56 | 12.71780,-2.84220 57 | 12.54840,4.95120 58 | 11.27940,9.20840 59 | 18.62040,7.92480 60 | 18.90720,10.82820 61 | 17.65080,10.33880 62 | 10.35860,-1.48558 63 | 42.55800,35.85800 64 | 29.81600,24.10800 65 | 37.91800,34.10800 66 | 14.43640,9.77040 67 | 16.59020,11.48840 68 | 20.47200,15.55080 69 | 10.99880,2.03460 70 | 40.68200,41.98400 71 | 20.27200,13.35980 72 | 14.66900,8.05180 73 | 12.01240,2.55680 74 | 14.45180,6.68220 75 | 10.05380,-5.36140 76 | 13.09580,0.59356 77 | 15.07720,7.76900 78 | 10.07300,11.40280 79 | 20.54800,13.50520 80 | 10.21540,4.11520 81 | 11.45840,0.95906 82 | 10.37680,0.40842 83 | 12.71140,1.35722 84 | 19.53740,15.08700 85 | 13.03180,10.68720 86 | 17.03440,8.48300 87 | 18.36040,13.59620 88 | 12.00400,1.85390 89 | 11.04080,0.30400 90 | 10.11880,5.64280 91 | 11.41540,3.69020 92 | 15.27320,8.59180 93 | 11.74140,14.40580 94 | 10.61080,3.97380 95 | 16.58680,0.28908 96 | 26.78800,18.11020 97 | 10.87380,1.23410 98 | -------------------------------------------------------------------------------- /codes/data/mnist/generate_mnist.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Download MNIST" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'generate_mnist.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torchvision\n", 38 | "import torchvision.transforms as transforms\n", 39 | "import numpy as np\n", 40 | "import matplotlib.pyplot as plt" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "trainset = torchvision.datasets.MNIST(root='./temp', train=True,\n", 50 | " download=True, transform=transforms.ToTensor())\n", 51 | "testset = torchvision.datasets.MNIST(root='./temp', train=False,\n", 52 | " download=True, transform=transforms.ToTensor())" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": { 59 | "scrolled": true 60 | }, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADaFJREFUeJzt3X+oHfWZx/HPR239lYqGJDZYXZsYymoQu150QTEuq9FdiqZKNYJLjKUpUmULFZQgNqCCLP2x/mMhxpCIqWkktolS1gZZjYESvIrU1NhGQ7a9m5BYUlGDIibP/nEny63e852T82tO8rxfIPecec7MPBzzuTPnfs/M1xEhAPkc13QDAJpB+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJHXCIHdmm68TAn0WEW7ndV0d+W1fa/sPtt+2fW832wIwWO70u/22j5f0R0lXSxqT9IqkWyLizcI6HPmBPhvEkf8SSW9HxM6I+ETSWknXd7E9AAPUTfjPkvTnCc/HqmV/w/YS26O2R7vYF4Ae6+YPfpOdWnzutD4ilktaLnHaDwyTbo78Y5LOnvD8K5J2d9cOgEHpJvyvSJpj+6u2vyhpoaSNvWkLQL91fNofEZ/avlPS85KOl7QyIn7fs84A9FXHQ30d7YzP/EDfDeRLPgCOXoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8k1fEU3ZJke5ekDyQdlPRpRIz0oikA/ddV+Cv/FBF/6cF2AAwQp/1AUt2GPyT9xvartpf0oiEAg9Htaf9lEbHb9gxJm2y/FRGbJ76g+qXALwZgyDgierMhe5mkDyPiR4XX9GZnAFqKCLfzuo5P+22favtLhx9Lmi9pW6fbAzBY3Zz2nynpl7YPb+fnEfFfPekKQN/17LS/rZ1x2g/0Xd9P+wEc3Qg/kBThB5Ii/EBShB9IivADSfXiqj4MsUsvvbRYv/XWW4v1efPmFesXXHDBEfd02N13312s7969u1i//PLLi/Unn3yyZW3r1q3FdTPgyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSXFJ7zHg5ptvbll75JFHiutOmzatWK/u19DSiy++WKxPnz69Ze38888vrlunrrenn366ZW3hwoVd7XuYcUkvgCLCDyRF+IGkCD+QFOEHkiL8QFKEH0iK6/mHwAknlP83jIyUZz5/7LHHWtZOOeWU4rqbN28u1h944IFifcuWLcX6iSee2LK2bt264rrz588v1uuMjo52tf6xjiM/kBThB5Ii/EBShB9IivADSRF+ICnCDyRVO85ve6Wkb0jaFxFzq2VTJf1C0rmSdkm6KSL+2r82j211985fsWJFx9vetGlTsV66F4Akvf/++x3vu2773Y7jj42NFeurV6/uavvHunaO/KskXfuZZfdKeiEi5kh6oXoO4ChSG/6I2Cxp/2cWXy/p8K/V1ZIW9LgvAH3W6Wf+MyNijyRVP2f0riUAg9D37/bbXiJpSb/3A+DIdHrk32t7piRVP/e1emFELI+IkYgoX50CYKA6Df9GSYuqx4skbehNOwAGpTb8tp+S9FtJX7M9Zvvbkh6WdLXtHZKurp4DOIpw3/4BqLsmfunSpcV63f+jRx99tGXtvvvuK67b7Th+ne3bt7eszZkzp6tt33jjjcX6hg05T0i5bz+AIsIPJEX4gaQIP5AU4QeSIvxAUty6uwfuv//+Yr1uKO+TTz4p1p9//vli/Z577mlZ++ijj4rr1jnppJOK9brLcs8555yWtbopth988MFiPetQXq9w5AeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpLikt02nn356y9pbb71VXHfatGnF+nPPPVesL1jQv/ujnnfeecX6mjVrivWLL764432vX7++WL/99tuL9QMHDnS872MZl/QCKCL8QFKEH0iK8ANJEX4gKcIPJEX4gaQY52/TjBmtpyPcvXt3V9ueNWtWsf7xxx8X64sXL25Zu+6664rrzp07t1ifMmVKsV7376dUv+GGG4rrPvvss8U6Jsc4P4Aiwg8kRfiBpAg/kBThB5Ii/EBShB9Iqnac3/ZKSd+QtC8i5lbLlkn6jqR3q5ctjYhf1+7sKB7nL13PX5qGWpKmT59erNfdv76f38Wo+45CXW8zZ84s1t99992Wtbp10ZlejvOvknTtJMt/GhEXVf/VBh/AcKkNf0RslrR/AL0AGKBuPvPfaft3tlfaPqNnHQEYiE7D/zNJsyVdJGmPpB+3eqHtJbZHbY92uC8AfdBR+CNib0QcjIhDkh6TdEnhtcsjYiQiRjptEkDvdRR+2xP/TPtNSdt60w6AQamdotv2U5KulDTN9pikH0q60vZFkkLSLknf7WOPAPqgNvwRccskix/vQy9D7b333mtZq7uvft19+adOnVqsv/POO8V6aZ76VatWFdfdv788kLN27dpivW6svm59NIdv+AFJEX4gKcIPJEX4gaQIP5AU4QeSqh3qQ72tW7cW63WX9DbpiiuuKNbnzZtXrB86dKhY37lz5xH3hMHgyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSTHOn9zJJ59crNeN49fdVpxLeocXR34gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSKp2iu6e7uwonqI7q4MHDxbrdf9+Srf2Lk3fjc71copuAMcgwg8kRfiBpAg/kBThB5Ii/EBShB9IqvZ6fttnS3pC0pclHZK0PCIesT1V0i8knStpl6SbIuKv/WsV/XDNNdc03QIa0s6R/1NJP4iIv5f0j5K+Z/t8SfdKeiEi5kh6oXoO4ChRG/6I2BMRr1WPP5C0XdJZkq6XtLp62WpJC/rVJIDeO6LP/LbPlfR1SVslnRkRe6TxXxCSZvS6OQD90/Y9/GxPkbRe0vcj4n27ra8Py/YSSUs6aw9Av7R15Lf9BY0Hf01EPFMt3mt7ZlWfKWnfZOtGxPKIGImIkV40DKA3asPv8UP845K2R8RPJpQ2SlpUPV4kaUPv2wPQL+2c9l8m6d8kvWH79WrZUkkPS1pn+9uS/iTpW/1pEf00a9aspltAQ2rDHxFbJLX6gP/PvW0HwKDwDT8gKcIPJEX4gaQIP5AU4QeSIvxAUkzRndzLL79crB93XPn4UDeFN4YXR34gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIpx/uS2bdtWrO/YsaNYr7sfwOzZs1vWmKK7WRz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApR8TgdmYPbmfoidtuu61YX7FiRbH+0ksvtazdddddxXXffPPNYh2Ti4i25tLjyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSdWO89s+W9ITkr4s6ZCk5RHxiO1lkr4j6fBF2Usj4tc122Kc/yhz2mmnFevr1q0r1q+66qqWtWeeeaa47uLFi4v1AwcOFOtZtTvO387NPD6V9IOIeM32lyS9antTVftpRPyo0yYBNKc2/BGxR9Ke6vEHtrdLOqvfjQHoryP6zG/7XElfl7S1WnSn7d/ZXmn7jBbrLLE9anu0q04B9FTb4bc9RdJ6Sd+PiPcl/UzSbEkXafzM4MeTrRcRyyNiJCJGetAvgB5pK/y2v6Dx4K+JiGckKSL2RsTBiDgk6TFJl/SvTQC9Vht+25b0uKTtEfGTCctnTnjZNyWVbwMLYKi0M9R3uaSXJb2h8aE+SVoq6RaNn/KHpF2Svlv9cbC0LYb6jjF1Q4EPPfRQy9odd9xRXPfCCy8s1rnkd3I9G+qLiC2SJttYcUwfwHDjG35AUoQfSIrwA0kRfiApwg8kRfiBpLh1N3CM4dbdAIoIP5AU4QeSIvxAUoQfSIrwA0kRfiCpdu7e20t/kfQ/E55Pq5YNo2HtbVj7kuitU73s7e/afeFAv+TzuZ3bo8N6b79h7W1Y+5LorVNN9cZpP5AU4QeSajr8yxvef8mw9jasfUn01qlGemv0Mz+A5jR95AfQkEbCb/ta23+w/bbte5vooRXbu2y/Yfv1pqcYq6ZB22d724RlU21vsr2j+jnpNGkN9bbM9v9W793rtv+1od7Otv3ftrfb/r3tf6+WN/reFfpq5H0b+Gm/7eMl/VHS1ZLGJL0i6ZaIGIqbsNveJWkkIhofE7Z9haQPJT0REXOrZf8haX9EPFz94jwjIu4Zkt6WSfqw6ZmbqwllZk6cWVrSAkm3qcH3rtDXTWrgfWviyH+JpLcjYmdEfCJpraTrG+hj6EXEZkn7P7P4ekmrq8erNf6PZ+Ba9DYUImJPRLxWPf5A0uGZpRt97wp9NaKJ8J8l6c8Tno9puKb8Dkm/sf2q7SVNNzOJMw/PjFT9nNFwP59VO3PzIH1mZumhee86mfG615oI/2S3GBqmIYfLIuIfJP2LpO9Vp7doT1szNw/KJDNLD4VOZ7zutSbCPybp7AnPvyJpdwN9TCoidlc/90n6pYZv9uG9hydJrX7ua7if/zdMMzdPNrO0huC9G6YZr5sI/yuS5tj+qu0vSlooaWMDfXyO7VOrP8TI9qmS5mv4Zh/eKGlR9XiRpA0N9vI3hmXm5lYzS6vh927YZrxu5Es+1VDGf0o6XtLKiGg9lesA2Z6l8aO9NH7F48+b7M32U5Ku1PhVX3sl/VDSryStk3SOpD9J+lZEDPwPby16u1JHOHNzn3prNbP0VjX43vVyxuue9MM3/ICc+IYfkBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk/g81Kx2HnWsInwAAAABJRU5ErkJggg==\n", 65 | "text/plain": [ 66 | "
" 67 | ] 68 | }, 69 | "metadata": {}, 70 | "output_type": "display_data" 71 | }, 72 | { 73 | "name": "stdout", 74 | "output_type": "stream", 75 | "text": [ 76 | "tensor(9)\n" 77 | ] 78 | } 79 | ], 80 | "source": [ 81 | "idx=4\n", 82 | "pic, label =trainset[idx]\n", 83 | "pic=pic.squeeze()\n", 84 | "plt.imshow(pic.numpy(), cmap='gray')\n", 85 | "plt.show()\n", 86 | "print(label)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 4, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [ 95 | "train_data=torch.Tensor(60000,28,28)\n", 96 | "train_label=torch.LongTensor(60000)\n", 97 | "for idx , example in enumerate(trainset):\n", 98 | " train_data[idx]=example[0].squeeze()\n", 99 | " train_label[idx]=example[1]\n", 100 | "torch.save(train_data,'train_data.pt')\n", 101 | "torch.save(train_label,'train_label.pt')" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 5, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "test_data=torch.Tensor(10000,28,28)\n", 111 | "test_label=torch.LongTensor(10000)\n", 112 | "for idx , example in enumerate(testset):\n", 113 | " test_data[idx]=example[0].squeeze()\n", 114 | " test_label[idx]=example[1]\n", 115 | "torch.save(test_data,'test_data.pt')\n", 116 | "torch.save(test_label,'test_label.pt')" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [] 132 | } 133 | ], 134 | "metadata": { 135 | "kernelspec": { 136 | "display_name": "Python 3", 137 | "language": "python", 138 | "name": "python3" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.6.8" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 2 155 | } 156 | -------------------------------------------------------------------------------- /codes/data/ptb/generate_ptb.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Download PTB" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'generate_ptb.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import os" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "class Dictionary(object):\n", 47 | " def __init__(self):\n", 48 | " self.word2idx = {}\n", 49 | " self.idx2word = []\n", 50 | "\n", 51 | " def add_word(self, word):\n", 52 | " if word not in self.word2idx:\n", 53 | " self.idx2word.append(word)\n", 54 | " self.word2idx[word] = len(self.idx2word) - 1\n", 55 | " return self.word2idx[word]\n", 56 | "\n", 57 | " def __len__(self):\n", 58 | " return len(self.idx2word)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 3, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "class Corpus(object):\n", 68 | " def __init__(self, path):\n", 69 | " self.dictionary = Dictionary()\n", 70 | " self.train = self.tokenize(os.path.join(path, 'train.txt'))\n", 71 | " self.valid = self.tokenize(os.path.join(path, 'valid.txt'))\n", 72 | " self.test = self.tokenize(os.path.join(path, 'test.txt'))\n", 73 | "\n", 74 | " def tokenize(self, path):\n", 75 | " \"\"\"Tokenizes a text file.\"\"\"\n", 76 | " assert os.path.exists(path)\n", 77 | " # Add words to the dictionary\n", 78 | " with open(path, 'r') as f:\n", 79 | " tokens = 0\n", 80 | " for line in f:\n", 81 | " words = line.split() + ['']\n", 82 | " tokens += len(words)\n", 83 | " for word in words:\n", 84 | " self.dictionary.add_word(word)\n", 85 | "\n", 86 | " # Tokenize file content\n", 87 | " with open(path, 'r') as f:\n", 88 | " ids = torch.LongTensor(tokens)\n", 89 | " token = 0\n", 90 | " for line in f:\n", 91 | " words = line.split() + ['']\n", 92 | " for word in words:\n", 93 | " ids[token] = self.dictionary.word2idx[word]\n", 94 | " token += 1\n", 95 | "\n", 96 | " return ids" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 4, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "data_folder='data_raw'\n", 106 | "corpus = Corpus(data_folder)" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 5, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "def batchify(data, bsz):\n", 116 | " # Work out how cleanly we can divide the dataset into bsz parts.\n", 117 | " nbatch = data.size(0) // bsz\n", 118 | " # Trim off any extra elements that wouldn't cleanly fit (remainders).\n", 119 | " data = data.narrow(0, 0, nbatch * bsz)\n", 120 | " # Evenly divide the data across the bsz batches.\n", 121 | " data = data.view(bsz, -1).t().contiguous()\n", 122 | " return data" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 6, 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "torch.Size([46479, 20])\n", 135 | "torch.Size([4121, 20])\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "batch_size=20\n", 141 | "\n", 142 | "train_data = batchify(corpus.train, batch_size)\n", 143 | "val_data = batchify(corpus.valid, batch_size)\n", 144 | "test_data = batchify(corpus.test, batch_size)\n", 145 | "\n", 146 | "vocab_size = len(corpus.dictionary)\n", 147 | "\n", 148 | "print(train_data.size())\n", 149 | "print(test_data.size())\n", 150 | "\n", 151 | "train_length = train_data.size(0)\n", 152 | "test_length = test_data.size(0)\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 7, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "torch.save(train_data,'train_data.pt')\n", 162 | "torch.save(test_data,'test_data.pt')\n", 163 | "torch.save(corpus.dictionary.idx2word,'idx2word.pt')\n", 164 | "torch.save(corpus.dictionary.word2idx,'word2idx.pt')" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": null, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [] 173 | } 174 | ], 175 | "metadata": { 176 | "kernelspec": { 177 | "display_name": "Python 3", 178 | "language": "python", 179 | "name": "python3" 180 | }, 181 | "language_info": { 182 | "codemirror_mode": { 183 | "name": "ipython", 184 | "version": 3 185 | }, 186 | "file_extension": ".py", 187 | "mimetype": "text/x-python", 188 | "name": "python", 189 | "nbconvert_exporter": "python", 190 | "pygments_lexer": "ipython3", 191 | "version": "3.6.8" 192 | } 193 | }, 194 | "nbformat": 4, 195 | "nbformat_minor": 2 196 | } 197 | -------------------------------------------------------------------------------- /codes/installation/installation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Installation Instructions" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# Git pull the codes from github to your colab machine\n", 17 | "# You will have this message:\n", 18 | " # Warning: This notebook was not authored by Google.\n", 19 | " # Click RUN ANYWAY\n", 20 | "!git clone https://github.com/xbresson/CE7454_2019.git" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "# Mount Google Drive to the colab machine\n", 30 | "# You will have this message:\n", 31 | " # Go to this URL in a browser: ---\n", 32 | " # Follow the URL, sign-in to Google login\n", 33 | " # ALLOW google drive to access to your google account \n", 34 | " # Copy-paste the code to the notebook\n", 35 | " # Enter your authorization code: ---\n", 36 | "from google.colab import drive\n", 37 | "drive.mount('/content/gdrive')\n", 38 | "\n", 39 | "# Copy github folder from colab machine to your google drive\n", 40 | "!mkdir /content/gdrive/My\\ Drive/CE7454_2019\n", 41 | "!cp -R /content/CE7454_2019 /content/gdrive/My\\ Drive\n", 42 | "!rm -R /content/CE7454_2019" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "# Installation is done.\n", 52 | "# You can close this notebook." 53 | ] 54 | } 55 | ], 56 | "metadata": { 57 | "kernelspec": { 58 | "display_name": "Python 3", 59 | "language": "python", 60 | "name": "python3" 61 | }, 62 | "language_info": { 63 | "codemirror_mode": { 64 | "name": "ipython", 65 | "version": 3 66 | }, 67 | "file_extension": ".py", 68 | "mimetype": "text/x-python", 69 | "name": "python", 70 | "nbconvert_exporter": "python", 71 | "pygments_lexer": "ipython3", 72 | "version": "3.7.4" 73 | } 74 | }, 75 | "nbformat": 4, 76 | "nbformat_minor": 2 77 | } 78 | -------------------------------------------------------------------------------- /codes/labs_lecture02/lab02_pytorch_tensor1/pytorch_tensor_part1_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02: Manipulate PyTorch Tensors -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'pytorch_tensor_part1_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "### Make the matrices A and B below. Add them together to obtain a matrix C. Print these three matrices.\n", 44 | "$$\n", 45 | "A =\\begin{bmatrix}\n", 46 | "1 & 2 \\\\ 3 & 4\n", 47 | "\\end{bmatrix} \n", 48 | "\\qquad \n", 49 | "B =\\begin{bmatrix}\n", 50 | "10 & 20 \\\\ 30 & 40\n", 51 | "\\end{bmatrix} \\qquad C=A+B =?\n", 52 | "$$" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "\n", 62 | "# write your code here\n", 63 | "\n" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "### Print the dimension, size and type of the matrix A. Remember, the commands are dim(), size() and type()" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "\n", 80 | "# write your code here\n", 81 | "\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "### Convert the matrix A to be an integer matrix (type LongTensor). Remember, the command is long(). Then print the type to check it was indeed converted." 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "\n", 98 | "# write your code here\n", 99 | "\n" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "### Make a random 5 x 2 x 3 Tensor. The command is torch.rand. Then do the following: 1) Print the tensor, 2) Print its type, 3) Print its dimension, 4) Print its size, 5) Print the size of its middle dimension." 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "\n", 116 | "# write your code here\n", 117 | "\n" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "### Make 2 x 3 x 4 x 5 tensor filled with zeros then print it. (The command is torch.zeros). See if you can make sense of the display." 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "\n", 134 | "# write your code here\n", 135 | "\n" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [] 144 | } 145 | ], 146 | "metadata": { 147 | "kernelspec": { 148 | "display_name": "Python 3", 149 | "language": "python", 150 | "name": "python3" 151 | }, 152 | "language_info": { 153 | "codemirror_mode": { 154 | "name": "ipython", 155 | "version": 3 156 | }, 157 | "file_extension": ".py", 158 | "mimetype": "text/x-python", 159 | "name": "python", 160 | "nbconvert_exporter": "python", 161 | "pygments_lexer": "ipython3", 162 | "version": "3.6.8" 163 | } 164 | }, 165 | "nbformat": 4, 166 | "nbformat_minor": 2 167 | } 168 | -------------------------------------------------------------------------------- /codes/labs_lecture02/lab02_pytorch_tensor1/pytorch_tensor_part1_solution.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02: Manipulate PyTorch Tensors -- solution" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'pytorch_tensor_part1_solution.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "### Make the matrices A and B below. Add them together to obtain a matrix C. Print these three matrices.\n", 44 | "$$\n", 45 | "A =\\begin{bmatrix}\n", 46 | "1 & 2 \\\\ 3 & 4\n", 47 | "\\end{bmatrix} \n", 48 | "\\qquad \n", 49 | "B =\\begin{bmatrix}\n", 50 | "10 & 20 \\\\ 30 & 40\n", 51 | "\\end{bmatrix} \\qquad C=A+B =?\n", 52 | "$$" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 2, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "tensor([[1., 2.],\n", 65 | " [3., 4.]])\n", 66 | "\n", 67 | "tensor([[10., 20.],\n", 68 | " [30., 40.]])\n", 69 | "\n", 70 | "tensor([[11., 22.],\n", 71 | " [33., 44.]])\n" 72 | ] 73 | } 74 | ], 75 | "source": [ 76 | "A=torch.Tensor( [ [1,2] , [3,4] ] )\n", 77 | "B=torch.Tensor( [ [10,20] , [30,40] ] )\n", 78 | "\n", 79 | "C=A+B\n", 80 | "\n", 81 | "print(A)\n", 82 | "print('')\n", 83 | "print(B)\n", 84 | "print('')\n", 85 | "print(C)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": {}, 91 | "source": [ 92 | "### Print the sizes, dimension, and type of the matrix A. Remember, the commands are size(), dim() and type()" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 3, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "name": "stdout", 102 | "output_type": "stream", 103 | "text": [ 104 | "torch.Size([2, 2])\n", 105 | "2\n", 106 | "torch.FloatTensor\n" 107 | ] 108 | } 109 | ], 110 | "source": [ 111 | "print( A.size() )\n", 112 | "print( A.dim() )\n", 113 | "print( A.type() )" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "### Convert the matrix A to be an integer matrix (type LongTensor). Remember, the command is long(). Then print the type to check it was indeed converted." 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": 4, 126 | "metadata": {}, 127 | "outputs": [ 128 | { 129 | "name": "stdout", 130 | "output_type": "stream", 131 | "text": [ 132 | "tensor([[1, 2],\n", 133 | " [3, 4]])\n", 134 | "torch.LongTensor\n" 135 | ] 136 | } 137 | ], 138 | "source": [ 139 | "A=A.long()\n", 140 | "print(A)\n", 141 | "print(A.type())" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "### Make a random 5 x 2 x 3 Tensor. The command is torch.rand. Then do the following: 1) Print the tensor, 2) Print its type, 3) Print its dimension, 4) Print its size, 5) Print the size of its middle dimension." 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 5, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "tensor([[[0.1654, 0.1702, 0.6423],\n", 161 | " [0.9256, 0.4278, 0.7653]],\n", 162 | "\n", 163 | " [[0.7499, 0.0527, 0.4733],\n", 164 | " [0.0441, 0.1477, 0.4260]],\n", 165 | "\n", 166 | " [[0.8478, 0.5063, 0.0653],\n", 167 | " [0.4906, 0.5045, 0.5422]],\n", 168 | "\n", 169 | " [[0.3924, 0.0418, 0.2341],\n", 170 | " [0.1129, 0.3110, 0.2656]],\n", 171 | "\n", 172 | " [[0.9444, 0.8174, 0.4218],\n", 173 | " [0.2645, 0.3379, 0.3014]]])\n", 174 | "torch.FloatTensor\n", 175 | "3\n", 176 | "torch.Size([5, 2, 3])\n", 177 | "2\n" 178 | ] 179 | } 180 | ], 181 | "source": [ 182 | "A=torch.rand(5,2,3)\n", 183 | "print(A)\n", 184 | "print(A.type())\n", 185 | "print(A.dim())\n", 186 | "print(A.size())\n", 187 | "print(A.size(1))" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "metadata": {}, 193 | "source": [ 194 | "### Make 2 x 3 x 4 x 5 tensor filled with zeros then print it. (The command is torch.zeros). See if you can make sense of the display." 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 6, 200 | "metadata": {}, 201 | "outputs": [ 202 | { 203 | "name": "stdout", 204 | "output_type": "stream", 205 | "text": [ 206 | "tensor([[[[0., 0., 0., 0., 0.],\n", 207 | " [0., 0., 0., 0., 0.],\n", 208 | " [0., 0., 0., 0., 0.],\n", 209 | " [0., 0., 0., 0., 0.]],\n", 210 | "\n", 211 | " [[0., 0., 0., 0., 0.],\n", 212 | " [0., 0., 0., 0., 0.],\n", 213 | " [0., 0., 0., 0., 0.],\n", 214 | " [0., 0., 0., 0., 0.]],\n", 215 | "\n", 216 | " [[0., 0., 0., 0., 0.],\n", 217 | " [0., 0., 0., 0., 0.],\n", 218 | " [0., 0., 0., 0., 0.],\n", 219 | " [0., 0., 0., 0., 0.]]],\n", 220 | "\n", 221 | "\n", 222 | " [[[0., 0., 0., 0., 0.],\n", 223 | " [0., 0., 0., 0., 0.],\n", 224 | " [0., 0., 0., 0., 0.],\n", 225 | " [0., 0., 0., 0., 0.]],\n", 226 | "\n", 227 | " [[0., 0., 0., 0., 0.],\n", 228 | " [0., 0., 0., 0., 0.],\n", 229 | " [0., 0., 0., 0., 0.],\n", 230 | " [0., 0., 0., 0., 0.]],\n", 231 | "\n", 232 | " [[0., 0., 0., 0., 0.],\n", 233 | " [0., 0., 0., 0., 0.],\n", 234 | " [0., 0., 0., 0., 0.],\n", 235 | " [0., 0., 0., 0., 0.]]]])\n" 236 | ] 237 | } 238 | ], 239 | "source": [ 240 | "A=torch.zeros(2,3,4,5)\n", 241 | "print(A)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [] 250 | } 251 | ], 252 | "metadata": { 253 | "kernelspec": { 254 | "display_name": "Python 3", 255 | "language": "python", 256 | "name": "python3" 257 | }, 258 | "language_info": { 259 | "codemirror_mode": { 260 | "name": "ipython", 261 | "version": 3 262 | }, 263 | "file_extension": ".py", 264 | "mimetype": "text/x-python", 265 | "name": "python", 266 | "nbconvert_exporter": "python", 267 | "pygments_lexer": "ipython3", 268 | "version": "3.6.8" 269 | } 270 | }, 271 | "nbformat": 4, 272 | "nbformat_minor": 2 273 | } 274 | -------------------------------------------------------------------------------- /codes/labs_lecture02/lab03_pytorch_tensor2/pytorch_tensor_part2_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 03: Resizing and slicing in PyTorch -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'pytorch_tensor_part2_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import utils" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### Make a 10 x 2 matrix random matrix A. Then store its third row (index = 2) in to a vector v. Then store the first 5 rows (index 0 to index 4) into a submatrix B. The important information is that B has a total of five rows. Print A, v and B." 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "\n", 54 | "# write your code here\n" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "### Extract entry (0,0) of the matrix A and store it into a PYTHON NUMBER x" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "\n", 71 | "# write your code here\n" 72 | ] 73 | }, 74 | { 75 | "cell_type": "markdown", 76 | "metadata": {}, 77 | "source": [ 78 | "### Let's download 60,000 gray scale pictures as well as their label. Each picture is 28 by 28 pixels." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "from utils import check_mnist_dataset_exists\n", 88 | "data_path=check_mnist_dataset_exists()\n", 89 | "\n", 90 | "data=torch.load(data_path+'mnist/train_data.pt')\n", 91 | "label=torch.load(data_path+'mnist/train_label.pt')" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Find the size of these two tensors" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "\n", 108 | "# write your code here\n" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "### Print the first picture by slicing the data tensor. You will see the intensity of each pixel (a value between 0 and 1)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": { 122 | "scrolled": false 123 | }, 124 | "outputs": [], 125 | "source": [ 126 | "\n", 127 | "# write your code here\n" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "### The function show() from the \"utils\" package will display the picture:" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "utils.show(data[10])" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "### Print the first entry of the label vector. The label is 5 telling you that this is the picture of a five." 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "\n", 160 | "# write your code here\n" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": {}, 166 | "source": [ 167 | "### Display picture 20 of the dataset and print its label" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "\n", 177 | "# write your code here\n" 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": {}, 183 | "source": [ 184 | "### Print the label corresponding to picture 10,000 10,001 10,002 10,003 and 10,004. So you need to extract 5 entries starting from entry 10,000." 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": null, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "\n", 194 | "# write your code here\n" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "### Display the two pictures that have label 9" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "\n", 211 | "# write your code here\n" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "### Lets now play with the CIFAR data set. These are RGB pictures" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "from utils import check_cifar_dataset_exists\n", 228 | "data_path=check_cifar_dataset_exists()\n", 229 | "\n", 230 | "data=torch.load(data_path+'cifar/train_data.pt')\n", 231 | "label=torch.load(data_path+'cifar/train_label.pt')" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": {}, 237 | "source": [ 238 | "### Find the size of these two tensors. How many pictures? How many pixels? Note that it is a 4-dimensional Tensor. Dimension 0 gives you the index of the picture, dimension 1 gives you the chanel (R, G or B) and the last two dimension gives you the pixel location." 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "\n", 248 | "# write your code here\n" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": {}, 254 | "source": [ 255 | "### Extract the first picture (a 3 x 32 x 32 Tensor) and check its size." 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "metadata": {}, 262 | "outputs": [], 263 | "source": [ 264 | "\n", 265 | "# write your code here\n" 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": {}, 271 | "source": [ 272 | "### Display picture 7, 40 and 100 of the data set with utils.show() and print its label. For CIFAR, the label are:\n", 273 | "0) Airplane \n", 274 | "1) Automobile \n", 275 | "2) Bird \n", 276 | "3) Cat \n", 277 | "4) Deer \n", 278 | "5) Dog \n", 279 | "6) Frog \n", 280 | "7) Horse \n", 281 | "8) Ship \n", 282 | "9) Truck\n", 283 | "\n", 284 | "For example, a picture of a dog will have label 5." 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": null, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "\n", 294 | "# write your code here\n" 295 | ] 296 | } 297 | ], 298 | "metadata": { 299 | "kernelspec": { 300 | "display_name": "Python 3", 301 | "language": "python", 302 | "name": "python3" 303 | }, 304 | "language_info": { 305 | "codemirror_mode": { 306 | "name": "ipython", 307 | "version": 3 308 | }, 309 | "file_extension": ".py", 310 | "mimetype": "text/x-python", 311 | "name": "python", 312 | "nbconvert_exporter": "python", 313 | "pygments_lexer": "ipython3", 314 | "version": "3.6.8" 315 | } 316 | }, 317 | "nbformat": 4, 318 | "nbformat_minor": 2 319 | } 320 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab01_linear_module/linear_module_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 01 : Linear module -- demo" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'linear_module_demo.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### Make a _Linear Module_ that takes input of size 5 and return output of size 3" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "name": "stdout", 54 | "output_type": "stream", 55 | "text": [ 56 | "Linear(in_features=5, out_features=3, bias=True)\n" 57 | ] 58 | } 59 | ], 60 | "source": [ 61 | "mod = nn.Linear(5,3)\n", 62 | "print(mod)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "### Let's make a random tensor of size 5:" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "tensor([0.5148, 0.6442, 0.5563, 0.4040, 0.9193])\n", 82 | "torch.Size([5])\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "x=torch.rand(5)\n", 88 | "print(x)\n", 89 | "print(x.size())" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "### Feed it to the module:" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 8, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "tensor([ 0.3300, -0.1137, 0.6271], grad_fn=)\n" 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "y=mod(x)\n", 114 | "print(y)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "### The output y is computed according to the formula:\n", 122 | "$$\n", 123 | "\\begin{bmatrix}\n", 124 | "y_1\\\\ y_2 \\\\y_3 \n", 125 | "\\end{bmatrix} =\n", 126 | "\\begin{bmatrix}\n", 127 | "w_{11} & w_{12} & w_{13}& w_{14}& w_{15} \\\\\n", 128 | "w_{21} & w_{22} & w_{23}& w_{24}& w_{25} \\\\\n", 129 | "w_{31} & w_{32} & w_{33}& w_{34}& w_{35} \\\\\n", 130 | "\\end{bmatrix}\n", 131 | "\\begin{bmatrix}\n", 132 | "x_1\\\\ x_2 \\\\x_3 \\\\ x_4 \\\\x_5\n", 133 | "\\end{bmatrix}\n", 134 | "+\n", 135 | "\\begin{bmatrix}\n", 136 | "b_1\\\\ b_2 \\\\b_3 \n", 137 | "\\end{bmatrix}\n", 138 | "$$\n", 139 | "### were the $w_{ij}$'s are the weight parameters and the $b_i$'s are the bias parameters. These internal parameters can be access as follow:" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 9, 145 | "metadata": {}, 146 | "outputs": [ 147 | { 148 | "name": "stdout", 149 | "output_type": "stream", 150 | "text": [ 151 | "Parameter containing:\n", 152 | "tensor([[ 0.2652, -0.4428, 0.1893, 0.3214, -0.0462],\n", 153 | " [-0.3800, 0.4037, 0.0042, 0.4156, -0.4236],\n", 154 | " [ 0.2434, 0.2813, 0.1570, -0.4028, 0.0869]], requires_grad=True)\n", 155 | "torch.Size([3, 5])\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "print(mod.weight)\n", 161 | "print(mod.weight.size())" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 10, 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "name": "stdout", 171 | "output_type": "stream", 172 | "text": [ 173 | "Parameter containing:\n", 174 | "tensor([0.2860, 0.0411, 0.3160], requires_grad=True)\n", 175 | "torch.Size([3])\n" 176 | ] 177 | } 178 | ], 179 | "source": [ 180 | "print(mod.bias)\n", 181 | "print(mod.bias.size())" 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "### If we want we can change the internal parameters of the module:" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 11, 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "name": "stdout", 198 | "output_type": "stream", 199 | "text": [ 200 | "Parameter containing:\n", 201 | "tensor([[ 0.0000, 1.0000, 2.0000, 0.3214, -0.0462],\n", 202 | " [-0.3800, 0.4037, 0.0042, 0.4156, -0.4236],\n", 203 | " [ 0.2434, 0.2813, 0.1570, -0.4028, 0.0869]], grad_fn=)\n" 204 | ] 205 | } 206 | ], 207 | "source": [ 208 | "mod.weight[0,0]=0\n", 209 | "mod.weight[0,1]=1\n", 210 | "mod.weight[0,2]=2\n", 211 | "print(mod.weight)" 212 | ] 213 | }, 214 | { 215 | "cell_type": "markdown", 216 | "metadata": {}, 217 | "source": [ 218 | "### We can also make a Linear module without bias:" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 12, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stdout", 228 | "output_type": "stream", 229 | "text": [ 230 | "Linear(in_features=5, out_features=3, bias=False)\n" 231 | ] 232 | } 233 | ], 234 | "source": [ 235 | "mod2 = nn.Linear(5,3,bias=False)\n", 236 | "print(mod2)" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 13, 242 | "metadata": {}, 243 | "outputs": [ 244 | { 245 | "name": "stdout", 246 | "output_type": "stream", 247 | "text": [ 248 | "Parameter containing:\n", 249 | "tensor([[ 0.1703, 0.1601, 0.3649, -0.1387, -0.3961],\n", 250 | " [ 0.4339, 0.2803, 0.0350, -0.3152, 0.3601],\n", 251 | " [-0.0434, 0.4186, 0.1819, 0.0771, 0.1898]], requires_grad=True)\n" 252 | ] 253 | } 254 | ], 255 | "source": [ 256 | "print(mod2.weight)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 14, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "None\n" 269 | ] 270 | } 271 | ], 272 | "source": [ 273 | "print(mod2.bias)" 274 | ] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": null, 279 | "metadata": {}, 280 | "outputs": [], 281 | "source": [] 282 | } 283 | ], 284 | "metadata": { 285 | "kernelspec": { 286 | "display_name": "Python 3", 287 | "language": "python", 288 | "name": "python3" 289 | }, 290 | "language_info": { 291 | "codemirror_mode": { 292 | "name": "ipython", 293 | "version": 3 294 | }, 295 | "file_extension": ".py", 296 | "mimetype": "text/x-python", 297 | "name": "python", 298 | "nbconvert_exporter": "python", 299 | "pygments_lexer": "ipython3", 300 | "version": "3.6.8" 301 | } 302 | }, 303 | "nbformat": 4, 304 | "nbformat_minor": 2 305 | } 306 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab01_linear_module/linear_module_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 01 : Linear module -- exercise\n", 8 | "\n", 9 | "# Inspecting a linear module" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "# For Google Colaboratory\n", 19 | "import sys, os\n", 20 | "if 'google.colab' in sys.modules:\n", 21 | " from google.colab import drive\n", 22 | " drive.mount('/content/gdrive')\n", 23 | " file_name = 'linear_module_exercise.ipynb'\n", 24 | " import subprocess\n", 25 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 26 | " print(path_to_file)\n", 27 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 28 | " os.chdir(path_to_file)\n", 29 | " !pwd" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import torch\n", 39 | "import torch.nn as nn" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "### Make a linear module WITHOUT bias that takes input of size 2 and return output of size 3. " 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "mod= # complete here\n", 56 | "print(mod)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "### Print the internal parameters of the module. Try to print the bias and double check that it return you \"None\"" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "print( # complete here )\n", 73 | "print( # complete here )" 74 | ] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": {}, 79 | "source": [ 80 | "### Make a vector $x=\\begin{bmatrix}1\\\\1 \\end{bmatrix}$ and feed it to your linear module. What is the output?" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "x= # complete here \n", 90 | "print(x)\n", 91 | "y= # complete here \n", 92 | "print(y)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "### Change the internal parameters of your module so that the weights become $\n", 100 | "W=\\begin{bmatrix}\n", 101 | "1&2 \\\\ 3&4 \\\\ 5&6\n", 102 | "\\end{bmatrix}\n", 103 | "$. You need to write 6 lines, one per entry to be changed." 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | " # complete here \n", 113 | " # complete here \n", 114 | " # complete here \n", 115 | " # complete here \n", 116 | " # complete here \n", 117 | " # complete here \n", 118 | "print(mod.weight)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "### Feed the vector x to your module with updated parameters. What is the output? " 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "y= # complete here \n", 135 | "print(y)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "### Compute by hand $y=Wx$ and check that pytorch got it correctly. \n", 143 | "\n", 144 | "$$\n", 145 | "\\begin{bmatrix}\n", 146 | "1&2 \\\\ 3&4 \\\\ 5&6\n", 147 | "\\end{bmatrix}\n", 148 | "\\begin{bmatrix}1\\\\1 \\end{bmatrix}=\n", 149 | "\\begin{bmatrix}?\\\\? \\\\ ? \\end{bmatrix}\n", 150 | "$$" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [] 159 | } 160 | ], 161 | "metadata": { 162 | "kernelspec": { 163 | "display_name": "Python 3", 164 | "language": "python", 165 | "name": "python3" 166 | }, 167 | "language_info": { 168 | "codemirror_mode": { 169 | "name": "ipython", 170 | "version": 3 171 | }, 172 | "file_extension": ".py", 173 | "mimetype": "text/x-python", 174 | "name": "python", 175 | "nbconvert_exporter": "python", 176 | "pygments_lexer": "ipython3", 177 | "version": "3.6.8" 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 2 182 | } 183 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab01_linear_module/linear_module_solution.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 01 : Linear module -- solution\n", 8 | "\n", 9 | "# Inspecting a linear module" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "# For Google Colaboratory\n", 19 | "import sys, os\n", 20 | "if 'google.colab' in sys.modules:\n", 21 | " from google.colab import drive\n", 22 | " drive.mount('/content/gdrive')\n", 23 | " file_name = 'linear_module_solution.ipynb'\n", 24 | " import subprocess\n", 25 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 26 | " print(path_to_file)\n", 27 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 28 | " os.chdir(path_to_file)\n", 29 | " !pwd" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 1, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import torch\n", 39 | "import torch.nn as nn\n", 40 | "import torch.nn.functional as F" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "### Make a linear module WITHOUT bias that takes input of size 2 and return output of size 3. " 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 2, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "Linear(in_features=2, out_features=3, bias=False)\n" 60 | ] 61 | } 62 | ], 63 | "source": [ 64 | "mod=nn.Linear(2,3,bias=False)\n", 65 | "print(mod)" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "### Print the internal parameters of the module. Try to print the bias and double check that it return you \"None\"" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 3, 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "name": "stdout", 82 | "output_type": "stream", 83 | "text": [ 84 | "Parameter containing:\n", 85 | "tensor([[-0.5430, -0.3259],\n", 86 | " [-0.5779, 0.5816],\n", 87 | " [-0.6248, -0.3755]], requires_grad=True)\n", 88 | "None\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "print(mod.weight)\n", 94 | "print(mod.bias)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "### Make a vector $x=\\begin{bmatrix}1\\\\1 \\end{bmatrix}$ and feed it to your linear module. What is the output?" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 4, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "tensor([1., 1.])\n", 114 | "tensor([-0.8690, 0.0037, -1.0003], grad_fn=)\n" 115 | ] 116 | } 117 | ], 118 | "source": [ 119 | "x=torch.Tensor([1,1])\n", 120 | "print(x)\n", 121 | "y=mod(x)\n", 122 | "print(y)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "### Change the internal parameters of your module so that the weights become $\n", 130 | "W=\\begin{bmatrix}\n", 131 | "1&2 \\\\ 3&4 \\\\ 5&6\n", 132 | "\\end{bmatrix}\n", 133 | "$. You need to write 6 lines, one per entry to be changed." 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 5, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "name": "stdout", 143 | "output_type": "stream", 144 | "text": [ 145 | "Parameter containing:\n", 146 | "tensor([[1., 2.],\n", 147 | " [3., 4.],\n", 148 | " [5., 6.]], grad_fn=)\n" 149 | ] 150 | } 151 | ], 152 | "source": [ 153 | "mod.weight[0,0]=1\n", 154 | "mod.weight[0,1]=2\n", 155 | "mod.weight[1,0]=3\n", 156 | "mod.weight[1,1]=4\n", 157 | "mod.weight[2,0]=5\n", 158 | "mod.weight[2,1]=6\n", 159 | "print(mod.weight)" 160 | ] 161 | }, 162 | { 163 | "cell_type": "markdown", 164 | "metadata": {}, 165 | "source": [ 166 | "### Feed the vector x to your module with updated parameters. What is the output? " 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 6, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "tensor([ 3., 7., 11.], grad_fn=)\n" 179 | ] 180 | } 181 | ], 182 | "source": [ 183 | "y=mod(x)\n", 184 | "print(y)" 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "metadata": {}, 190 | "source": [ 191 | "### Compute by hand $y=Wx$ and check that pytorch got it correctly. \n", 192 | "\n", 193 | "$$\n", 194 | "\\begin{bmatrix}\n", 195 | "1&2 \\\\ 3&4 \\\\ 5&6\n", 196 | "\\end{bmatrix}\n", 197 | "\\begin{bmatrix}1\\\\1 \\end{bmatrix}=\n", 198 | "\\begin{bmatrix}?\\\\? \\\\ ? \\end{bmatrix}\n", 199 | "$$" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [] 208 | } 209 | ], 210 | "metadata": { 211 | "kernelspec": { 212 | "display_name": "Python 3", 213 | "language": "python", 214 | "name": "python3" 215 | }, 216 | "language_info": { 217 | "codemirror_mode": { 218 | "name": "ipython", 219 | "version": 3 220 | }, 221 | "file_extension": ".py", 222 | "mimetype": "text/x-python", 223 | "name": "python", 224 | "nbconvert_exporter": "python", 225 | "pygments_lexer": "ipython3", 226 | "version": "3.6.8" 227 | } 228 | }, 229 | "nbformat": 4, 230 | "nbformat_minor": 2 231 | } 232 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab02_softmax/softmax_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02 : Softmax function -- demo" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'softmax_demo.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn.functional as F" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### Make a vector" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "name": "stdout", 54 | "output_type": "stream", 55 | "text": [ 56 | "tensor([-2.0000, -0.5000, 2.0000, 1.5000])\n", 57 | "torch.Size([4])\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "x=torch.Tensor([ -2 , -0.5 , 2.0 , 1.5 ])\n", 63 | "print(x)\n", 64 | "print(x.size())" 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "### Feed it to the softmax" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 6, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "name": "stdout", 81 | "output_type": "stream", 82 | "text": [ 83 | "tensor([0.0107, 0.0481, 0.5858, 0.3553])\n", 84 | "torch.Size([4])\n" 85 | ] 86 | } 87 | ], 88 | "source": [ 89 | "p=F.softmax(x , dim=0)\n", 90 | "print(p)\n", 91 | "print(p.size())" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Check that it sums to one" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 4, 104 | "metadata": {}, 105 | "outputs": [ 106 | { 107 | "name": "stdout", 108 | "output_type": "stream", 109 | "text": [ 110 | "tensor(1.)\n" 111 | ] 112 | } 113 | ], 114 | "source": [ 115 | "print( p.sum() )" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": {}, 121 | "source": [ 122 | "### Make a matrix a matrix" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 5, 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "tensor([[-2.0000, -0.5000, 2.0000, 1.5000],\n", 135 | " [-2.0000, -0.5000, 7.0000, 1.5000]])\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "A=torch.Tensor( [ [ -2 , -0.5 , 2.0 , 1.5 ] , [ -2 , -0.5 , 7.0 , 1.5 ] ])\n", 141 | "print(A)" 142 | ] 143 | }, 144 | { 145 | "cell_type": "markdown", 146 | "metadata": {}, 147 | "source": [ 148 | "### Take the softmax over the rows (dim=1)" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 6, 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "name": "stdout", 158 | "output_type": "stream", 159 | "text": [ 160 | "tensor([[0.0107, 0.0481, 0.5858, 0.3553],\n", 161 | " [0.0001, 0.0006, 0.9953, 0.0041]])\n" 162 | ] 163 | } 164 | ], 165 | "source": [ 166 | "p = F.softmax(A , dim=1)\n", 167 | "print(p)" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "### Take the softmax over the columns (dim=0)" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 7, 180 | "metadata": {}, 181 | "outputs": [ 182 | { 183 | "name": "stdout", 184 | "output_type": "stream", 185 | "text": [ 186 | "tensor([[0.5000, 0.5000, 0.0067, 0.5000],\n", 187 | " [0.5000, 0.5000, 0.9933, 0.5000]])\n" 188 | ] 189 | } 190 | ], 191 | "source": [ 192 | "p = F.softmax(A , dim=0)\n", 193 | "print(p)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [] 202 | } 203 | ], 204 | "metadata": { 205 | "kernelspec": { 206 | "display_name": "Python 3", 207 | "language": "python", 208 | "name": "python3" 209 | }, 210 | "language_info": { 211 | "codemirror_mode": { 212 | "name": "ipython", 213 | "version": 3 214 | }, 215 | "file_extension": ".py", 216 | "mimetype": "text/x-python", 217 | "name": "python", 218 | "nbconvert_exporter": "python", 219 | "pygments_lexer": "ipython3", 220 | "version": "3.6.8" 221 | } 222 | }, 223 | "nbformat": 4, 224 | "nbformat_minor": 2 225 | } 226 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab03_vanilla_nn/vanilla_nn_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 03 : Vanilla neural networks -- exercise\n", 8 | "\n", 9 | "# Creating a one-layer network" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "# For Google Colaboratory\n", 19 | "import sys, os\n", 20 | "if 'google.colab' in sys.modules:\n", 21 | " from google.colab import drive\n", 22 | " drive.mount('/content/gdrive')\n", 23 | " file_name = 'vanilla_nn_exercise.ipynb'\n", 24 | " import subprocess\n", 25 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 26 | " print(path_to_file)\n", 27 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 28 | " os.chdir(path_to_file)\n", 29 | " !pwd" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import torch\n", 39 | "import torch.nn as nn\n", 40 | "import torch.nn.functional as F" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "### Make a class for a one layer network. Let's call the layer \"mylayer\". And let's give it a bias." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "class one_layer_net(nn.Module):\n", 57 | "\n", 58 | " def __init__(self, input_size, output_size ):\n", 59 | " super(one_layer_net , self).__init__()\n", 60 | " \n", 61 | " # complete here\n", 62 | " \n", 63 | " \n", 64 | " def forward(self, x):\n", 65 | " \n", 66 | " # complete here\n", 67 | " # complete here\n", 68 | " \n", 69 | " return p" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "### Create an instance of a one layer net that take input of size 2 and return output of size 2" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "net= # complete here\n", 86 | "print(net)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "### Make a vector $x=\\begin{bmatrix}1\\\\1 \\end{bmatrix}$ and feed it to the network. What is the output probability? Check that it sums to one." 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "x= # complete here\n", 103 | "print(x)\n", 104 | "\n", 105 | "p = # complete here\n", 106 | "print(p)\n", 107 | "\n", 108 | "print(p.sum() )" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "### Print the weights as well as the bias of the unique layer of this network. Be careful to use the correct name for this unique layer " 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "print( # complete here)\n", 125 | "print( # complete here)" 126 | ] 127 | }, 128 | { 129 | "cell_type": "markdown", 130 | "metadata": {}, 131 | "source": [ 132 | "### Change the internal parameters of your network so that the weights are now equal to $\n", 133 | "W=\\begin{bmatrix}\n", 134 | "1&2 \\\\ 3&4 \n", 135 | "\\end{bmatrix}\n", 136 | "$ and the bias is equal to $\n", 137 | "b=\\begin{bmatrix}\n", 138 | "1 \\\\ 1 \n", 139 | "\\end{bmatrix}\n", 140 | "$ " 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "# CHANGE THE WEIGHTS\n", 150 | "# complete here\n", 151 | "# complete here\n", 152 | "# complete here\n", 153 | "# complete here\n", 154 | "\n", 155 | "# CHANGE THE BIAS\n", 156 | "# complete here\n", 157 | "# complete here\n", 158 | "\n", 159 | "print(net.mylayer.weight)\n", 160 | "print(net.mylayer.bias)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": {}, 166 | "source": [ 167 | "### Feed the vector x to your network with updated parameters. What is the output? (you should get p= [2% 98%])" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "p= # complete here\n", 177 | "print(p)" 178 | ] 179 | } 180 | ], 181 | "metadata": { 182 | "kernelspec": { 183 | "display_name": "Python 3", 184 | "language": "python", 185 | "name": "python3" 186 | }, 187 | "language_info": { 188 | "codemirror_mode": { 189 | "name": "ipython", 190 | "version": 3 191 | }, 192 | "file_extension": ".py", 193 | "mimetype": "text/x-python", 194 | "name": "python", 195 | "nbconvert_exporter": "python", 196 | "pygments_lexer": "ipython3", 197 | "version": "3.6.8" 198 | } 199 | }, 200 | "nbformat": 4, 201 | "nbformat_minor": 2 202 | } 203 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab03_vanilla_nn/vanilla_nn_solution.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 03 : Vanilla neural networks -- solution\n", 8 | "\n", 9 | "# Creating a one-layer network" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "# For Google Colaboratory\n", 19 | "import sys, os\n", 20 | "if 'google.colab' in sys.modules:\n", 21 | " from google.colab import drive\n", 22 | " drive.mount('/content/gdrive')\n", 23 | " file_name = 'vanilla_nn_solution.ipynb'\n", 24 | " import subprocess\n", 25 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 26 | " print(path_to_file)\n", 27 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 28 | " os.chdir(path_to_file)\n", 29 | " !pwd" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 1, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import torch\n", 39 | "import torch.nn as nn\n", 40 | "import torch.nn.functional as F" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "### Make a class for a one layer network. Let's call the layer \"mylayer\". And let's give it a bias." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 2, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "class one_layer_net(nn.Module):\n", 57 | "\n", 58 | " def __init__(self, input_size, output_size ):\n", 59 | " super(one_layer_net , self).__init__()\n", 60 | " \n", 61 | " self.mylayer=nn.Linear(input_size,output_size, bias=True)\n", 62 | " \n", 63 | " \n", 64 | " def forward(self, x):\n", 65 | " \n", 66 | " x = self.mylayer(x)\n", 67 | " p = F.softmax(x,dim=0)\n", 68 | " \n", 69 | " return p" 70 | ] 71 | }, 72 | { 73 | "cell_type": "markdown", 74 | "metadata": {}, 75 | "source": [ 76 | "### Create an instance of a one layer net that take input of size 2 and return output of size 2" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 3, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "one_layer_net(\n", 89 | " (mylayer): Linear(in_features=2, out_features=2, bias=True)\n", 90 | ")\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "net= one_layer_net(2,2)\n", 96 | "print(net)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "### Make a vector $x=\\begin{bmatrix}1\\\\1 \\end{bmatrix}$ and feed it to the network. What is the output probability?" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": 4, 109 | "metadata": {}, 110 | "outputs": [ 111 | { 112 | "name": "stdout", 113 | "output_type": "stream", 114 | "text": [ 115 | "tensor([1., 1.])\n", 116 | "tensor([0.5447, 0.4553], grad_fn=)\n", 117 | "tensor(1., grad_fn=)\n" 118 | ] 119 | } 120 | ], 121 | "source": [ 122 | "x= torch.Tensor([1,1])\n", 123 | "print(x)\n", 124 | "p = net(x)\n", 125 | "print(p)\n", 126 | "print(p.sum() )" 127 | ] 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "metadata": {}, 132 | "source": [ 133 | "### Print the weights as well as the bias of the unique layer of this network. Be careful to use the correct name for this unique layer " 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 5, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "name": "stdout", 143 | "output_type": "stream", 144 | "text": [ 145 | "Parameter containing:\n", 146 | "tensor([[-0.3148, 0.5843],\n", 147 | " [ 0.3549, -0.1694]], requires_grad=True)\n", 148 | "Parameter containing:\n", 149 | "tensor([0.3936, 0.2985], requires_grad=True)\n" 150 | ] 151 | } 152 | ], 153 | "source": [ 154 | "print(net.mylayer.weight)\n", 155 | "print(net.mylayer.bias)" 156 | ] 157 | }, 158 | { 159 | "cell_type": "markdown", 160 | "metadata": {}, 161 | "source": [ 162 | "### Change the internal parameters of your network so that the weights are now equal to $\n", 163 | "W=\\begin{bmatrix}\n", 164 | "1&2 \\\\ 3&4 \n", 165 | "\\end{bmatrix}\n", 166 | "$ and the bias is equal to $\n", 167 | "b=\\begin{bmatrix}\n", 168 | "1 \\\\ 1 \n", 169 | "\\end{bmatrix}\n", 170 | "$ " 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 6, 176 | "metadata": {}, 177 | "outputs": [ 178 | { 179 | "name": "stdout", 180 | "output_type": "stream", 181 | "text": [ 182 | "Parameter containing:\n", 183 | "tensor([[1., 2.],\n", 184 | " [3., 4.]], grad_fn=)\n", 185 | "Parameter containing:\n", 186 | "tensor([1., 1.], grad_fn=)\n" 187 | ] 188 | } 189 | ], 190 | "source": [ 191 | "net.mylayer.weight[0,0]=1\n", 192 | "net.mylayer.weight[0,1]=2\n", 193 | "net.mylayer.weight[1,0]=3\n", 194 | "net.mylayer.weight[1,1]=4\n", 195 | "\n", 196 | "net.mylayer.bias[0]=1\n", 197 | "net.mylayer.bias[1]=1\n", 198 | "\n", 199 | "print(net.mylayer.weight)\n", 200 | "print(net.mylayer.bias)" 201 | ] 202 | }, 203 | { 204 | "cell_type": "markdown", 205 | "metadata": {}, 206 | "source": [ 207 | "### Feed the vector x to your network with updated parameters. What is the output? (you should get p= [2% 98%])" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": 7, 213 | "metadata": {}, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "tensor([0.0180, 0.9820], grad_fn=)\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "p=net(x)\n", 225 | "print(p)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [] 241 | } 242 | ], 243 | "metadata": { 244 | "kernelspec": { 245 | "display_name": "Python 3", 246 | "language": "python", 247 | "name": "python3" 248 | }, 249 | "language_info": { 250 | "codemirror_mode": { 251 | "name": "ipython", 252 | "version": 3 253 | }, 254 | "file_extension": ".py", 255 | "mimetype": "text/x-python", 256 | "name": "python", 257 | "nbconvert_exporter": "python", 258 | "pygments_lexer": "ipython3", 259 | "version": "3.6.8" 260 | } 261 | }, 262 | "nbformat": 4, 263 | "nbformat_minor": 2 264 | } 265 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab04_train_vanilla_nn/train_vanilla_nn_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 04 : Train vanilla neural network -- exercise\n", 8 | "\n", 9 | "\n", 10 | "# Training a one-layer net on FASHION-MNIST" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "# For Google Colaboratory\n", 20 | "import sys, os\n", 21 | "if 'google.colab' in sys.modules:\n", 22 | " from google.colab import drive\n", 23 | " drive.mount('/content/gdrive')\n", 24 | " file_name = 'train_vanilla_nn_exercise.ipynb'\n", 25 | " import subprocess\n", 26 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 27 | " print(path_to_file)\n", 28 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 29 | " os.chdir(path_to_file)\n", 30 | " !pwd" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "import torch\n", 40 | "import torch.nn as nn\n", 41 | "import torch.nn.functional as F\n", 42 | "import torch.optim as optim\n", 43 | "from random import randint\n", 44 | "import utils" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "### Download the TRAINING SET (data+labels)" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "from utils import check_fashion_mnist_dataset_exists\n", 61 | "data_path=check_fashion_mnist_dataset_exists()\n", 62 | "\n", 63 | "train_data=torch.load(data_path+'fashion-mnist/train_data.pt')\n", 64 | "train_label=torch.load(data_path+'fashion-mnist/train_label.pt')\n", 65 | "print(train_data.size())\n", 66 | "print(train_label.size())" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Download the TEST SET (data only)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": {}, 80 | "outputs": [], 81 | "source": [ 82 | "test_data=torch.load(data_path+'fashion-mnist/test_data.pt')\n", 83 | "print(test_data.size())" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "### Make a one layer net class" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "class one_layer_net(nn.Module):\n", 100 | "\n", 101 | " def __init__(self, input_size, output_size):\n", 102 | " super(one_layer_net , self).__init__()\n", 103 | " \n", 104 | " # complete here\n", 105 | " \n", 106 | " def forward(self, x):\n", 107 | " \n", 108 | " x = # complete here\n", 109 | " p = # complete here\n", 110 | " \n", 111 | " return p" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | "### Build the net" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "net=one_layer_net(784,10)\n", 128 | "print(net)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "### Take the 4th image of the test set:" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "im= # complete here\n", 145 | "utils.show(im)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "markdown", 150 | "metadata": {}, 151 | "source": [ 152 | "### And feed it to the UNTRAINED network:" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "p = # complete here\n", 162 | "print(p)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": {}, 168 | "source": [ 169 | "### Display visually the confidence scores" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "utils.show_prob_fashion_mnist(p)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "### Train the network (only 5000 iterations) on the train set" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "criterion = nn.NLLLoss()\n", 195 | "optimizer=torch.optim.SGD(net.parameters() , lr=0.01 )\n", 196 | "\n", 197 | "for iter in range(1,5000):\n", 198 | " \n", 199 | " # choose a random integer between 0 and 59,999 \n", 200 | " # extract the corresponding picture and label\n", 201 | " # and reshape them to fit the network\n", 202 | "\n", 203 | " # complete here\n", 204 | " # complete here\n", 205 | " # complete here\n", 206 | "\n", 207 | "\n", 208 | " # feed the input to the net \n", 209 | " input.requires_grad_() # for backprobagation -- we will discuss it later\n", 210 | " # complete here \n", 211 | " \n", 212 | " # update the weights (all the magic happens here -- we will discuss it later)\n", 213 | " log_prob=torch.log(prob)\n", 214 | " loss = criterion(log_prob, label) \n", 215 | " optimizer.zero_grad() \n", 216 | " loss.backward()\n", 217 | " optimizer.step()" 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "metadata": {}, 223 | "source": [ 224 | "### Take the 34th image of the test set:" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": null, 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "im= # complete here\n", 234 | "utils.show(im)" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "### Feed it to the TRAINED net:" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "metadata": {}, 248 | "outputs": [], 249 | "source": [ 250 | "p = # complete here\n", 251 | "print(p)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": {}, 257 | "source": [ 258 | "### Display visually the confidence scores" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "utils.show_prob_fashion_mnist(prob)" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": {}, 273 | "source": [ 274 | "### Choose image at random from the test set and see how good/bad are the predictions" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "# choose a picture at random\n", 284 | "idx=randint(0, 10000-1)\n", 285 | "im=test_data[idx]\n", 286 | "\n", 287 | "# diplay the picture\n", 288 | "utils.show(im)\n", 289 | "\n", 290 | "# feed it to the net and display the confidence scores\n", 291 | "prob = net( im.view(1,784)) \n", 292 | "utils.show_prob_fashion_mnist(prob)" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "metadata": {}, 299 | "outputs": [], 300 | "source": [] 301 | } 302 | ], 303 | "metadata": { 304 | "kernelspec": { 305 | "display_name": "Python 3", 306 | "language": "python", 307 | "name": "python3" 308 | }, 309 | "language_info": { 310 | "codemirror_mode": { 311 | "name": "ipython", 312 | "version": 3 313 | }, 314 | "file_extension": ".py", 315 | "mimetype": "text/x-python", 316 | "name": "python", 317 | "nbconvert_exporter": "python", 318 | "pygments_lexer": "ipython3", 319 | "version": "3.6.8" 320 | } 321 | }, 322 | "nbformat": 4, 323 | "nbformat_minor": 2 324 | } 325 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab05_minibatch_training/minibatch_training_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 05 : Train with mini-batches -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'minibatch_training_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import torch.nn.functional as F\n", 39 | "import torch.optim as optim\n", 40 | "from random import randint\n", 41 | "import utils" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "### Download the data and print the sizes" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "from utils import check_fashion_mnist_dataset_exists\n", 58 | "data_path=check_fashion_mnist_dataset_exists()" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "train_data=torch.load(data_path+'fashion-mnist/train_data.pt')\n", 68 | "\n", 69 | "print(train_data.size())" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "train_label=torch.load(data_path+'fashion-mnist/train_label.pt')\n", 79 | "\n", 80 | "print(train_label.size())" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "test_data=torch.load(data_path+'fashion-mnist/test_data.pt')\n", 90 | "\n", 91 | "print(test_data.size())" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Make a one layer net class " 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "class one_layer_net(nn.Module):\n", 108 | "\n", 109 | " def __init__(self, input_size, output_size):\n", 110 | " super(one_layer_net , self).__init__()\n", 111 | " \n", 112 | " self.linear_layer = # complete here\n", 113 | " \n", 114 | " def forward(self, x):\n", 115 | " \n", 116 | " y = # complete here\n", 117 | " prob = # complete here\n", 118 | " return prob" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "metadata": {}, 124 | "source": [ 125 | "### Build the net" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "net=one_layer_net(784,10)\n", 135 | "print(net)" 136 | ] 137 | }, 138 | { 139 | "cell_type": "markdown", 140 | "metadata": {}, 141 | "source": [ 142 | "### Choose the size of the mini-batches " 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "bs= # complete here" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "### Train the network (only 5000 iterations) on the train set" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "criterion = nn.NLLLoss()\n", 168 | "optimizer=torch.optim.SGD(net.parameters() , lr=0.01 )\n", 169 | "\n", 170 | "for iter in range(1,5000):\n", 171 | " \n", 172 | " # create a minibatch\n", 173 | " indices= # complete here\n", 174 | " minibatch_data = # complete here\n", 175 | " minibatch_label= # complete here \n", 176 | " \n", 177 | " #reshape them to fit the network\n", 178 | " inputs= # complete here\n", 179 | "\n", 180 | " # feed the input to the net \n", 181 | " inputs.requires_grad_()\n", 182 | " prob= # complete here\n", 183 | " \n", 184 | " \n", 185 | " # update the weights (all the magic happens here -- we will discuss it later)\n", 186 | " log_prob=torch.log(prob)\n", 187 | " loss = criterion(log_prob, minibatch_label) \n", 188 | " optimizer.zero_grad() \n", 189 | " loss.backward()\n", 190 | " optimizer.step()" 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": {}, 196 | "source": [ 197 | "### Choose image at random from the test set and see how good/bad are the predictions" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "# choose a picture at random\n", 207 | "idx=randint(0, 10000-1)\n", 208 | "im=test_data[idx]\n", 209 | "\n", 210 | "# diplay the picture\n", 211 | "utils.show(im)\n", 212 | "\n", 213 | "# feed it to the net and display the confidence scores\n", 214 | "prob = net( im.view(1,784)) \n", 215 | "utils.show_prob_fashion_mnist(prob)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [] 224 | } 225 | ], 226 | "metadata": { 227 | "kernelspec": { 228 | "display_name": "Python 3", 229 | "language": "python", 230 | "name": "python3" 231 | }, 232 | "language_info": { 233 | "codemirror_mode": { 234 | "name": "ipython", 235 | "version": 3 236 | }, 237 | "file_extension": ".py", 238 | "mimetype": "text/x-python", 239 | "name": "python", 240 | "nbconvert_exporter": "python", 241 | "pygments_lexer": "ipython3", 242 | "version": "3.6.8" 243 | } 244 | }, 245 | "nbformat": 4, 246 | "nbformat_minor": 2 247 | } 248 | -------------------------------------------------------------------------------- /codes/labs_lecture03/lab05_minibatch_training/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | def show(X): 6 | if X.dim() == 3 and X.size(0) == 3: 7 | plt.imshow( np.transpose( X.numpy() , (1, 2, 0)) ) 8 | plt.show() 9 | elif X.dim() == 2: 10 | plt.imshow( X.numpy() , cmap='gray' ) 11 | plt.show() 12 | else: 13 | print('WRONG TENSOR SIZE') 14 | 15 | 16 | 17 | 18 | def show_prob_mnist(p): 19 | 20 | p=p.data.squeeze().numpy() 21 | 22 | ft=15 23 | label = ('zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight','nine') 24 | #p=p.data.squeeze().numpy() 25 | y_pos = np.arange(len(p))*1.2 26 | target=2 27 | width=0.9 28 | col= 'blue' 29 | #col='darkgreen' 30 | 31 | plt.rcdefaults() 32 | fig, ax = plt.subplots() 33 | 34 | # the plot 35 | ax.barh(y_pos, p, width , align='center', color=col) 36 | 37 | ax.set_xlim([0, 1.3]) 38 | #ax.set_ylim([-0.8, len(p)*1.2-1+0.8]) 39 | 40 | # y label 41 | ax.set_yticks(y_pos) 42 | ax.set_yticklabels(label, fontsize=ft) 43 | ax.invert_yaxis() 44 | #ax.set_xlabel('Performance') 45 | #ax.set_title('How fast do you want to go today?') 46 | 47 | # x label 48 | ax.set_xticklabels([]) 49 | ax.set_xticks([]) 50 | #x_pos=np.array([0, 0.25 , 0.5 , 0.75 , 1]) 51 | #ax.set_xticks(x_pos) 52 | #ax.set_xticklabels( [0, 0.25 , 0.5 , 0.75 , 1] , fontsize=15) 53 | 54 | ax.spines['right'].set_visible(False) 55 | ax.spines['top'].set_visible(False) 56 | ax.spines['bottom'].set_visible(False) 57 | ax.spines['left'].set_linewidth(4) 58 | 59 | 60 | for i in range(len(p)): 61 | str_nb="{0:.2f}".format(p[i]) 62 | ax.text( p[i] + 0.05 , y_pos[i] ,str_nb , 63 | horizontalalignment='left', verticalalignment='center', 64 | transform=ax.transData, color= col,fontsize=ft) 65 | 66 | 67 | 68 | plt.show() 69 | #fig.savefig('pic/prob', dpi=96, bbox_inches="tight") 70 | 71 | 72 | 73 | 74 | 75 | 76 | def show_prob_fashion_mnist(p): 77 | 78 | 79 | p=p.data.squeeze().numpy() 80 | 81 | ft=15 82 | label = ('T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag','Boot') 83 | #p=p.data.squeeze().numpy() 84 | y_pos = np.arange(len(p))*1.2 85 | target=2 86 | width=0.9 87 | col= 'blue' 88 | #col='darkgreen' 89 | 90 | plt.rcdefaults() 91 | fig, ax = plt.subplots() 92 | 93 | # the plot 94 | ax.barh(y_pos, p, width , align='center', color=col) 95 | 96 | ax.set_xlim([0, 1.3]) 97 | #ax.set_ylim([-0.8, len(p)*1.2-1+0.8]) 98 | 99 | # y label 100 | ax.set_yticks(y_pos) 101 | ax.set_yticklabels(label, fontsize=ft) 102 | ax.invert_yaxis() 103 | #ax.set_xlabel('Performance') 104 | #ax.set_title('How fast do you want to go today?') 105 | 106 | # x label 107 | ax.set_xticklabels([]) 108 | ax.set_xticks([]) 109 | #x_pos=np.array([0, 0.25 , 0.5 , 0.75 , 1]) 110 | #ax.set_xticks(x_pos) 111 | #ax.set_xticklabels( [0, 0.25 , 0.5 , 0.75 , 1] , fontsize=15) 112 | 113 | ax.spines['right'].set_visible(False) 114 | ax.spines['top'].set_visible(False) 115 | ax.spines['bottom'].set_visible(False) 116 | ax.spines['left'].set_linewidth(4) 117 | 118 | 119 | for i in range(len(p)): 120 | str_nb="{0:.2f}".format(p[i]) 121 | ax.text( p[i] + 0.05 , y_pos[i] ,str_nb , 122 | horizontalalignment='left', verticalalignment='center', 123 | transform=ax.transData, color= col,fontsize=ft) 124 | 125 | 126 | 127 | plt.show() 128 | #fig.savefig('pic/prob', dpi=96, bbox_inches="tight") 129 | 130 | 131 | 132 | import os.path 133 | def check_mnist_dataset_exists(path_data='../../data/'): 134 | flag_train_data = os.path.isfile(path_data + 'mnist/train_data.pt') 135 | flag_train_label = os.path.isfile(path_data + 'mnist/train_label.pt') 136 | flag_test_data = os.path.isfile(path_data + 'mnist/test_data.pt') 137 | flag_test_label = os.path.isfile(path_data + 'mnist/test_label.pt') 138 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 139 | print('MNIST dataset missing - downloading...') 140 | import torchvision 141 | import torchvision.transforms as transforms 142 | trainset = torchvision.datasets.MNIST(root=path_data + 'mnist/temp', train=True, 143 | download=True, transform=transforms.ToTensor()) 144 | testset = torchvision.datasets.MNIST(root=path_data + 'mnist/temp', train=False, 145 | download=True, transform=transforms.ToTensor()) 146 | train_data=torch.Tensor(60000,28,28) 147 | train_label=torch.LongTensor(60000) 148 | for idx , example in enumerate(trainset): 149 | train_data[idx]=example[0].squeeze() 150 | train_label[idx]=example[1] 151 | torch.save(train_data,path_data + 'mnist/train_data.pt') 152 | torch.save(train_label,path_data + 'mnist/train_label.pt') 153 | test_data=torch.Tensor(10000,28,28) 154 | test_label=torch.LongTensor(10000) 155 | for idx , example in enumerate(testset): 156 | test_data[idx]=example[0].squeeze() 157 | test_label[idx]=example[1] 158 | torch.save(test_data,path_data + 'mnist/test_data.pt') 159 | torch.save(test_label,path_data + 'mnist/test_label.pt') 160 | return path_data 161 | 162 | def check_fashion_mnist_dataset_exists(path_data='../../data/'): 163 | flag_train_data = os.path.isfile(path_data + 'fashion-mnist/train_data.pt') 164 | flag_train_label = os.path.isfile(path_data + 'fashion-mnist/train_label.pt') 165 | flag_test_data = os.path.isfile(path_data + 'fashion-mnist/test_data.pt') 166 | flag_test_label = os.path.isfile(path_data + 'fashion-mnist/test_label.pt') 167 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 168 | print('FASHION-MNIST dataset missing - downloading...') 169 | import torchvision 170 | import torchvision.transforms as transforms 171 | trainset = torchvision.datasets.FashionMNIST(root=path_data + 'fashion-mnist/temp', train=True, 172 | download=True, transform=transforms.ToTensor()) 173 | testset = torchvision.datasets.FashionMNIST(root=path_data + 'fashion-mnist/temp', train=False, 174 | download=True, transform=transforms.ToTensor()) 175 | train_data=torch.Tensor(60000,28,28) 176 | train_label=torch.LongTensor(60000) 177 | for idx , example in enumerate(trainset): 178 | train_data[idx]=example[0].squeeze() 179 | train_label[idx]=example[1] 180 | torch.save(train_data,path_data + 'fashion-mnist/train_data.pt') 181 | torch.save(train_label,path_data + 'fashion-mnist/train_label.pt') 182 | test_data=torch.Tensor(10000,28,28) 183 | test_label=torch.LongTensor(10000) 184 | for idx , example in enumerate(testset): 185 | test_data[idx]=example[0].squeeze() 186 | test_label[idx]=example[1] 187 | torch.save(test_data,path_data + 'fashion-mnist/test_data.pt') 188 | torch.save(test_label,path_data + 'fashion-mnist/test_label.pt') 189 | return path_data 190 | 191 | def check_cifar_dataset_exists(path_data='../../data/'): 192 | flag_train_data = os.path.isfile(path_data + 'cifar/train_data.pt') 193 | flag_train_label = os.path.isfile(path_data + 'cifar/train_label.pt') 194 | flag_test_data = os.path.isfile(path_data + 'cifar/test_data.pt') 195 | flag_test_label = os.path.isfile(path_data + 'cifar/test_label.pt') 196 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 197 | print('CIFAR dataset missing - downloading...') 198 | import torchvision 199 | import torchvision.transforms as transforms 200 | trainset = torchvision.datasets.CIFAR10(root=path_data + 'cifar/temp', train=True, 201 | download=True, transform=transforms.ToTensor()) 202 | testset = torchvision.datasets.CIFAR10(root=path_data + 'cifar/temp', train=False, 203 | download=True, transform=transforms.ToTensor()) 204 | train_data=torch.Tensor(50000,3,32,32) 205 | train_label=torch.LongTensor(50000) 206 | for idx , example in enumerate(trainset): 207 | train_data[idx]=example[0] 208 | train_label[idx]=example[1] 209 | torch.save(train_data,path_data + 'cifar/train_data.pt') 210 | torch.save(train_label,path_data + 'cifar/train_label.pt') 211 | test_data=torch.Tensor(10000,3,32,32) 212 | test_label=torch.LongTensor(10000) 213 | for idx , example in enumerate(testset): 214 | test_data[idx]=example[0] 215 | test_label[idx]=example[1] 216 | torch.save(test_data,path_data + 'cifar/test_data.pt') 217 | torch.save(test_label,path_data + 'cifar/test_label.pt') 218 | return path_data 219 | 220 | 221 | -------------------------------------------------------------------------------- /codes/labs_lecture04/lab01_cross_entropy/cross_entropy_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 01 : Cross-entropy loss -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'cross_entropy_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import utils" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "### Make a Cross Entropy Criterion and call it criterion. The command is nn.CrossEntropyLoss()." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "criterion= # COMPLETE HERE\n", 55 | "\n", 56 | "print(criterion)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "### Suppose that there only two classes (class 0 and class 1).\n", 64 | "### Suppose we have a batch of three data points: \n", 65 | "### ${\\bf x^{(0)}}$ belongs to class 0\n", 66 | "### ${\\bf x^{(1)}}$belongs to class 1\n", 67 | "### ${\\bf x^{(2)}}$ belongs to class 1\n", 68 | "### Put the labels of each of these point a LongTensor:" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "labels = # COMPLETE HERE\n", 78 | "\n", 79 | "print(labels,labels.type())" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "### Make a batch of scores: each row corresponds to the scores associated with a data point. So your batch of scores should look likes something like:\n", 87 | "\n", 88 | "$$\n", 89 | "\\text{scores} \\;\\; = \\;\\; \\begin{bmatrix}\n", 90 | "s_0^{(0)} & s_1^{(0)} & \\\\\n", 91 | "s_0^{(1)} & s_1^{(1)} & \\\\\n", 92 | "s_0^{(2)} & s_1^{(2)} & \\\\\n", 93 | "\\end{bmatrix}\n", 94 | "$$\n", 95 | "\n", 96 | "### You will need to create a tensor of the form torch.Tensor( [ [ ], [ ], [ ] ] ). Don't forget the extra square brackets!\n", 97 | "\n", 98 | "### Choose scores that will leads to a loss very close to zero, let say around or smaller than 0.05 (indicating that the scores are very good with respect to the labels). " 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "scores = # COMPLETE HERE\n", 108 | "\n", 109 | "print(scores)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "### Display your batch of scores" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": { 123 | "scrolled": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "utils.display_scores(scores)" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "metadata": {}, 133 | "source": [ 134 | "### Use the criterion to compute the average loss on this batch -- it needs to be around or smaller than 0.05" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "average_loss = # COMPLETE HERE\n", 144 | "\n", 145 | "print(average_loss.item())" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [] 154 | } 155 | ], 156 | "metadata": { 157 | "kernelspec": { 158 | "display_name": "Python 3", 159 | "language": "python", 160 | "name": "python3" 161 | }, 162 | "language_info": { 163 | "codemirror_mode": { 164 | "name": "ipython", 165 | "version": 3 166 | }, 167 | "file_extension": ".py", 168 | "mimetype": "text/x-python", 169 | "name": "python", 170 | "nbconvert_exporter": "python", 171 | "pygments_lexer": "ipython3", 172 | "version": "3.6.8" 173 | } 174 | }, 175 | "nbformat": 4, 176 | "nbformat_minor": 2 177 | } 178 | -------------------------------------------------------------------------------- /codes/labs_lecture04/lab01_cross_entropy/utils.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import matplotlib 3 | import numpy as np 4 | 5 | 6 | import matplotlib.pyplot as plt 7 | import matplotlib 8 | import numpy as np 9 | 10 | def display_scores(sc): 11 | 12 | 13 | ft=10 14 | ft_label=12 15 | 16 | bs=sc.size(0) 17 | nb_class=sc.size(1) 18 | 19 | f, ax = plt.subplots(1, bs) 20 | 21 | if bs ==2: 22 | f.set_size_inches(8, 8) 23 | f.subplots_adjust(left=None, bottom=None, right=None, top=None, 24 | wspace=None, hspace=0.5) 25 | else: 26 | f.set_size_inches(12, 12) 27 | f.subplots_adjust(left=None, bottom=None, right=None, top=None, 28 | wspace=None, hspace=0.25) 29 | 30 | max_score= sc.max().item() 31 | min_score=sc.min().item() 32 | 33 | label_pos= min_score-8 34 | xmin=-5.5 35 | xmax=5.5 36 | 37 | 38 | label=[] 39 | for i in range(nb_class): 40 | str_nb="{0:.0f}".format(i) 41 | mystr='class '+ str_nb 42 | label.append(mystr) 43 | 44 | y_pos = np.arange(nb_class)*1.2 45 | 46 | 47 | 48 | for count in range(bs): 49 | 50 | ax[count].set_title('data point '+ "{0:.0f}".format(count)) 51 | 52 | scores=sc[count].numpy() 53 | 54 | width=0.9 55 | col='darkgreen' 56 | 57 | # plt.rcdefaults() 58 | 59 | # line in the middle 60 | ax[count].plot([0,0], [y_pos[0]-1,y_pos[-1]+1], color='k',linewidth=4) 61 | 62 | 63 | # the plot 64 | barlist=ax[count].barh(y_pos, scores, width , align='center', color=col) 65 | 66 | for idx,bar in enumerate(barlist): 67 | if scores[idx]<0: 68 | bar.set_color('r') 69 | 70 | ax[count].set_xlim([xmin, xmax]) 71 | ax[count].invert_yaxis() 72 | 73 | # no y label 74 | ax[count].set_yticklabels([]) 75 | ax[count].set_yticks([]) 76 | 77 | # x label 78 | ax[count].set_xticklabels([]) 79 | ax[count].set_xticks([]) 80 | 81 | 82 | ax[count].spines['right'].set_visible(False) 83 | ax[count].spines['top'].set_visible(False) 84 | ax[count].spines['bottom'].set_visible(False) 85 | ax[count].spines['left'].set_visible(False) 86 | 87 | ax[count].set_aspect('equal') 88 | 89 | 90 | for i in range(len(scores)): 91 | 92 | str_nb="{0:.1f}".format(scores[i]) 93 | if scores[i]>=0: 94 | ax[count].text( scores[i] + 0.05 , y_pos[i] ,str_nb , 95 | horizontalalignment='left', verticalalignment='center', 96 | transform=ax[count].transData, color= col,fontsize=ft) 97 | else: 98 | ax[count].text( scores[i] - 0.05 , y_pos[i] ,str_nb , 99 | horizontalalignment='right', verticalalignment='center', 100 | transform=ax[count].transData, color= 'r',fontsize=ft) 101 | 102 | if count ==0: 103 | ax[0].text( label_pos , y_pos[i] , label[i] , 104 | horizontalalignment='left', verticalalignment='center', 105 | transform=ax[0].transData, color= 'black',fontsize=ft_label) 106 | 107 | 108 | plt.show() 109 | 110 | 111 | 112 | import os.path 113 | def check_mnist_dataset_exists(): 114 | flag_train_data = os.path.isfile('../data/mnist/train_data.pt') 115 | flag_train_label = os.path.isfile('../data/mnist/train_label.pt') 116 | flag_test_data = os.path.isfile('../data/mnist/test_data.pt') 117 | flag_test_label = os.path.isfile('../data/mnist/test_label.pt') 118 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 119 | print('MNIST dataset missing - downloading...') 120 | import torchvision 121 | import torchvision.transforms as transforms 122 | trainset = torchvision.datasets.MNIST(root='../data/mnist/temp', train=True, 123 | download=True, transform=transforms.ToTensor()) 124 | testset = torchvision.datasets.MNIST(root='../data/mnist/temp', train=False, 125 | download=True, transform=transforms.ToTensor()) 126 | train_data=torch.Tensor(60000,28,28) 127 | train_label=torch.LongTensor(60000) 128 | for idx , example in enumerate(trainset): 129 | train_data[idx]=example[0].squeeze() 130 | train_label[idx]=example[1] 131 | torch.save(train_data,'../data/mnist/train_data.pt') 132 | torch.save(train_label,'../data/mnist/train_label.pt') 133 | test_data=torch.Tensor(10000,28,28) 134 | test_label=torch.LongTensor(10000) 135 | for idx , example in enumerate(testset): 136 | test_data[idx]=example[0].squeeze() 137 | test_label[idx]=example[1] 138 | torch.save(test_data,'../data/mnist/test_data.pt') 139 | torch.save(test_label,'../data/mnist/test_label.pt') 140 | 141 | def check_fashion_mnist_dataset_exists(): 142 | flag_train_data = os.path.isfile('../data/fashion-mnist/train_data.pt') 143 | flag_train_label = os.path.isfile('../data/fashion-mnist/train_label.pt') 144 | flag_test_data = os.path.isfile('../data/fashion-mnist/test_data.pt') 145 | flag_test_label = os.path.isfile('../data/fashion-mnist/test_label.pt') 146 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 147 | print('FASHION-MNIST dataset missing - downloading...') 148 | import torchvision 149 | import torchvision.transforms as transforms 150 | trainset = torchvision.datasets.FashionMNIST(root='../data/fashion-mnist/temp', train=True, 151 | download=True, transform=transforms.ToTensor()) 152 | testset = torchvision.datasets.FashionMNIST(root='../data/fashion-mnist/temp', train=False, 153 | download=True, transform=transforms.ToTensor()) 154 | train_data=torch.Tensor(60000,28,28) 155 | train_label=torch.LongTensor(60000) 156 | for idx , example in enumerate(trainset): 157 | train_data[idx]=example[0].squeeze() 158 | train_label[idx]=example[1] 159 | torch.save(train_data,'../data/fashion-mnist/train_data.pt') 160 | torch.save(train_label,'../data/fashion-mnist/train_label.pt') 161 | test_data=torch.Tensor(10000,28,28) 162 | test_label=torch.LongTensor(10000) 163 | for idx , example in enumerate(testset): 164 | test_data[idx]=example[0].squeeze() 165 | test_label[idx]=example[1] 166 | torch.save(test_data,'../data/fashion-mnist/test_data.pt') 167 | torch.save(test_label,'../data/fashion-mnist/test_label.pt') 168 | 169 | def check_cifar_dataset_exists(): 170 | flag_train_data = os.path.isfile('../data/cifar/train_data.pt') 171 | flag_train_label = os.path.isfile('../data/cifar/train_label.pt') 172 | flag_test_data = os.path.isfile('../data/cifar/test_data.pt') 173 | flag_test_label = os.path.isfile('../data/cifar/test_label.pt') 174 | if flag_train_data==False or flag_train_label==False or flag_test_data==False or flag_test_label==False: 175 | print('CIFAR dataset missing - downloading... (5min)') 176 | import torchvision 177 | import torchvision.transforms as transforms 178 | trainset = torchvision.datasets.CIFAR10(root='../data/cifar/temp', train=True, 179 | download=True, transform=transforms.ToTensor()) 180 | testset = torchvision.datasets.CIFAR10(root='../data/cifar/temp', train=False, 181 | download=True, transform=transforms.ToTensor()) 182 | train_data=torch.Tensor(50000,3,32,32) 183 | train_label=torch.LongTensor(50000) 184 | for idx , example in enumerate(trainset): 185 | train_data[idx]=example[0] 186 | train_label[idx]=example[1] 187 | torch.save(train_data,'../data/cifar/train_data.pt') 188 | torch.save(train_label,'../data/cifar/train_label.pt') 189 | test_data=torch.Tensor(10000,3,32,32) 190 | test_label=torch.LongTensor(10000) 191 | for idx , example in enumerate(testset): 192 | test_data[idx]=example[0] 193 | test_label[idx]=example[1] 194 | torch.save(test_data,'../data/cifar/test_data.pt') 195 | torch.save(test_label,'../data/cifar/test_label.pt') -------------------------------------------------------------------------------- /codes/labs_lecture05/lab01_mlp/mlp_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 01 : MLP -- exercise\n", 8 | "\n", 9 | "# Understanding the training loop " 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "# For Google Colaboratory\n", 19 | "import sys, os\n", 20 | "if 'google.colab' in sys.modules:\n", 21 | " from google.colab import drive\n", 22 | " drive.mount('/content/gdrive')\n", 23 | " file_name = 'mlp_exercise.ipynb'\n", 24 | " import subprocess\n", 25 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 26 | " print(path_to_file)\n", 27 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 28 | " os.chdir(path_to_file)\n", 29 | " !pwd" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "import torch\n", 39 | "import torch.nn as nn\n", 40 | "import torch.nn.functional as F\n", 41 | "import torch.optim as optim\n", 42 | "from random import randint\n", 43 | "import utils" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": {}, 49 | "source": [ 50 | "### Download the data and print the sizes" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "from utils import check_fashion_mnist_dataset_exists\n", 60 | "data_path=check_fashion_mnist_dataset_exists()\n" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "train_data=torch.load(data_path+'fashion-mnist/train_data.pt')\n", 70 | "\n", 71 | "print(train_data.size())" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "train_label=torch.load(data_path+'fashion-mnist/train_label.pt')\n", 81 | "\n", 82 | "print(train_label.size())" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "test_data=torch.load(data_path+'fashion-mnist/test_data.pt')\n", 92 | "\n", 93 | "print(test_data.size())" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "### Make a ONE layer net class. The network output are the scores! No softmax needed! You have only one line to write in the forward function" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "class one_layer_net(nn.Module):\n", 110 | "\n", 111 | " def __init__(self, input_size, output_size):\n", 112 | " super(one_layer_net , self).__init__()\n", 113 | " self.linear_layer = # complete here\n", 114 | " \n", 115 | " def forward(self, x):\n", 116 | " scores = # complete here\n", 117 | " return scores" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "### Build the net" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "net= # complete here\n", 134 | "print(net)" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "### Choose the criterion and the optimizer: use the CHEAT SHEET to see the correct syntax. \n", 142 | "\n", 143 | "### Remember that the optimizer need to have access to the parameters of the network (net.parameters()).\n", 144 | "\n", 145 | "### Set the batchize and learning rate to be:\n", 146 | "### batchize = 50\n", 147 | "### learning rate = 0.01\n", 148 | "\n", 149 | "\n", 150 | "\n", 151 | "\n", 152 | "\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": null, 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "# make the criterion\n", 162 | "criterion = # complete here\n", 163 | "\n", 164 | "# make the SGD optimizer. \n", 165 | "optimizer=torch.optim.SGD(net.parameters(),lr=0.01)\n", 166 | "\n", 167 | "# set up the batch size \n", 168 | "bs=50" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "### Complete the training loop" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "for iter in range(1,5000):\n", 185 | " \n", 186 | " # Set dL/dU, dL/dV, dL/dW to be filled with zeros\n", 187 | " \n", 188 | " \n", 189 | " # create a minibatch\n", 190 | " \n", 191 | " \n", 192 | " # reshape the minibatch\n", 193 | " \n", 194 | " \n", 195 | " # tell Pytorch to start tracking all operations that will be done on \"inputs\"\n", 196 | " \n", 197 | "\n", 198 | " # forward the minibatch through the net \n", 199 | " \n", 200 | " \n", 201 | " # Compute the average of the losses of the data points in the minibatch\n", 202 | " \n", 203 | " \n", 204 | " # backward pass to compute dL/dU, dL/dV and dL/dW \n", 205 | " \n", 206 | " \n", 207 | " # do one step of stochastic gradient descent: U=U-lr(dL/dU), V=V-lr(dL/dU), ...\n", 208 | " \n", 209 | " " 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "### Choose image at random from the test set and see how good/bad are the predictions" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "# choose a picture at random\n", 226 | "idx=randint(0, 10000-1)\n", 227 | "im=test_data[idx]\n", 228 | "\n", 229 | "# diplay the picture\n", 230 | "utils.show(im)\n", 231 | "\n", 232 | "# feed it to the net and display the confidence scores\n", 233 | "scores = net( im.view(1,784)) \n", 234 | "probs= F.softmax(scores, dim=1)\n", 235 | "utils.show_prob_fashion_mnist(probs)" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [] 244 | } 245 | ], 246 | "metadata": { 247 | "kernelspec": { 248 | "display_name": "Python 3", 249 | "language": "python", 250 | "name": "python3" 251 | }, 252 | "language_info": { 253 | "codemirror_mode": { 254 | "name": "ipython", 255 | "version": 3 256 | }, 257 | "file_extension": ".py", 258 | "mimetype": "text/x-python", 259 | "name": "python", 260 | "nbconvert_exporter": "python", 261 | "pygments_lexer": "ipython3", 262 | "version": "3.6.8" 263 | } 264 | }, 265 | "nbformat": 4, 266 | "nbformat_minor": 2 267 | } 268 | -------------------------------------------------------------------------------- /codes/labs_lecture05/lab02_epoch/epoch_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02 : Training with epochs -- demo" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'epoch_demo.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "### Lets make an artificial training set of 10 images (28x28 pixels)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 2, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "name": "stdout", 53 | "output_type": "stream", 54 | "text": [ 55 | "torch.Size([10, 28, 28])\n" 56 | ] 57 | } 58 | ], 59 | "source": [ 60 | "train_data = torch.rand(10,28,28)\n", 61 | "\n", 62 | "print(train_data.size())" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "### Lets define a the random order in which we are going to visit these images:" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 3, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "tensor([3, 8, 0, 9, 7, 5, 1, 4, 2, 6])\n" 82 | ] 83 | } 84 | ], 85 | "source": [ 86 | "shuffled_indices = torch.randperm(10)\n", 87 | "print(shuffled_indices)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "markdown", 92 | "metadata": {}, 93 | "source": [ 94 | "### Visit the training set in this random order and do minibatch of size 2" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 4, 100 | "metadata": {}, 101 | "outputs": [ 102 | { 103 | "name": "stdout", 104 | "output_type": "stream", 105 | "text": [ 106 | "tensor([3, 8])\n", 107 | "tensor([0, 9])\n", 108 | "tensor([7, 5])\n", 109 | "tensor([1, 4])\n", 110 | "tensor([2, 6])\n" 111 | ] 112 | } 113 | ], 114 | "source": [ 115 | "bs=2\n", 116 | "\n", 117 | "for count in range(0,10,bs):\n", 118 | " \n", 119 | " batch_of_indices = shuffled_indices[count:count+bs]\n", 120 | " \n", 121 | " print(batch_of_indices)\n", 122 | " \n", 123 | " batch_of_images = train_data[ batch_of_indices ]" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [] 132 | } 133 | ], 134 | "metadata": { 135 | "kernelspec": { 136 | "display_name": "Python 3", 137 | "language": "python", 138 | "name": "python3" 139 | }, 140 | "language_info": { 141 | "codemirror_mode": { 142 | "name": "ipython", 143 | "version": 3 144 | }, 145 | "file_extension": ".py", 146 | "mimetype": "text/x-python", 147 | "name": "python", 148 | "nbconvert_exporter": "python", 149 | "pygments_lexer": "ipython3", 150 | "version": "3.6.8" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 2 155 | } 156 | -------------------------------------------------------------------------------- /codes/labs_lecture05/lab02_epoch/epoch_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02: Training with epochs -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'epoch_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import torch.nn.functional as F\n", 39 | "import torch.optim as optim\n", 40 | "from random import randint\n", 41 | "import utils" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "### Download the data and print the sizes" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "from utils import check_mnist_dataset_exists\n", 58 | "data_path=check_mnist_dataset_exists()\n", 59 | "\n", 60 | "train_data=torch.load(data_path+'mnist/train_data.pt')\n", 61 | "train_label=torch.load(data_path+'mnist/train_label.pt')\n", 62 | "test_data=torch.load(data_path+'mnist/test_data.pt')" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "### Make a ONE layer net class. " 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "class one_layer_net(nn.Module):\n", 79 | "\n", 80 | " def __init__(self, input_size, output_size):\n", 81 | " super(one_layer_net , self).__init__()\n", 82 | " self.linear_layer = nn.Linear( input_size, output_size , bias=False)\n", 83 | " \n", 84 | " def forward(self, x):\n", 85 | " scores = self.linear_layer(x)\n", 86 | " return scores" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "metadata": {}, 92 | "source": [ 93 | "### Build the net" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "net=one_layer_net(784,10)\n", 103 | "print(net)" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "### Choose the criterion, optimizer, batchsize, learning rate" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": null, 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "criterion = nn.CrossEntropyLoss()\n", 120 | "\n", 121 | "optimizer=torch.optim.SGD( net.parameters() , lr=0.01 )\n", 122 | "\n", 123 | "bs=50" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "metadata": {}, 129 | "source": [ 130 | "# You only have stuff to do in this cell" 131 | ] 132 | }, 133 | { 134 | "cell_type": "markdown", 135 | "metadata": {}, 136 | "source": [ 137 | "### Do 15 passes through the training set" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "metadata": {}, 144 | "outputs": [], 145 | "source": [ 146 | "for # COMPLETE\n", 147 | " \n", 148 | " # COMPLETE\n", 149 | " \n", 150 | " for # COMPLETE\n", 151 | " \n", 152 | " optimizer.zero_grad()\n", 153 | " \n", 154 | " # COMPLETE\n", 155 | " # COMPLETE\n", 156 | " # COMPLETE\n", 157 | "\n", 158 | " inputs = minibatch_data.view(bs,784)\n", 159 | " \n", 160 | " inputs.requires_grad_()\n", 161 | "\n", 162 | " scores=net( inputs ) \n", 163 | "\n", 164 | " loss = criterion( scores , minibatch_label) \n", 165 | " \n", 166 | " loss.backward()\n", 167 | "\n", 168 | " optimizer.step()" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "### Choose image at random from the test set and see how good/bad are the predictions" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "# choose a picture at random\n", 185 | "idx=randint(0, 10000-1)\n", 186 | "im=test_data[idx]\n", 187 | "\n", 188 | "# diplay the picture\n", 189 | "utils.show(im)\n", 190 | "\n", 191 | "# feed it to the net and display the confidence scores\n", 192 | "scores = net( im.view(1,784)) \n", 193 | "probs= F.softmax(scores, dim=1)\n", 194 | "utils.show_prob_mnist(probs)" 195 | ] 196 | } 197 | ], 198 | "metadata": { 199 | "kernelspec": { 200 | "display_name": "Python 3", 201 | "language": "python", 202 | "name": "python3" 203 | }, 204 | "language_info": { 205 | "codemirror_mode": { 206 | "name": "ipython", 207 | "version": 3 208 | }, 209 | "file_extension": ".py", 210 | "mimetype": "text/x-python", 211 | "name": "python", 212 | "nbconvert_exporter": "python", 213 | "pygments_lexer": "ipython3", 214 | "version": "3.6.8" 215 | } 216 | }, 217 | "nbformat": 4, 218 | "nbformat_minor": 2 219 | } 220 | -------------------------------------------------------------------------------- /codes/labs_lecture05/lab03_monitoring_loss/monitoring_loss_exercise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 03 : Loss and error rate -- exercise" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'monitoring_loss_exercise.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import torch.nn.functional as F\n", 39 | "import torch.optim as optim\n", 40 | "from random import randint\n", 41 | "import utils" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "### Download the CIFAR dataset -- check the size carefully!" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "torch.Size([50000, 3, 32, 32])\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "from utils import check_cifar_dataset_exists\n", 66 | "data_path=check_cifar_dataset_exists()\n", 67 | "\n", 68 | "train_data=torch.load(data_path+'cifar/train_data.pt')\n", 69 | "train_label=torch.load(data_path+'cifar/train_label.pt')\n", 70 | "test_data=torch.load(data_path+'cifar/test_data.pt')\n", 71 | "\n", 72 | "print(train_data.size())" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "### Make a ONE layer net class. " 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "class one_layer_net(nn.Module):\n", 89 | "\n", 90 | " def __init__(self, input_size, output_size):\n", 91 | " super(one_layer_net , self).__init__()\n", 92 | " self.linear_layer = nn.Linear( input_size, output_size , bias=True)\n", 93 | " \n", 94 | " def forward(self, x):\n", 95 | " scores = self.linear_layer(x)\n", 96 | " return scores" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "### Build the net. " 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "net= one_layer_net(3072,10)\n", 113 | "\n", 114 | "print(net)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "### Choose the criterion and optimizer. Also choose:\n", 122 | "\n", 123 | "### batchsize = 20\n", 124 | "\n", 125 | "### learning rate = 0.01" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "criterion = nn.CrossEntropyLoss()\n", 135 | "\n", 136 | "optimizer=torch.optim.SGD( net.parameters() , lr=0.01 )\n", 137 | "\n", 138 | "bs=20" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "# You only have to complete this cell\n", 146 | "\n", 147 | "### Do 40 passes through the training set (which contains 50,000 images -- not 60,000 like mnist!)" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "for epoch in range(40):\n", 157 | " \n", 158 | " running_loss= # COMPLETE HERE\n", 159 | " running_error= # COMPLETE HERE\n", 160 | " num_batches= # COMPLETE HERE\n", 161 | " \n", 162 | " shuffled_indices=torch.randperm(50000)\n", 163 | " \n", 164 | " for # COMPLETE HERE\n", 165 | " \n", 166 | " # Set the gradients to zeros\n", 167 | " optimizer.zero_grad()\n", 168 | " \n", 169 | " # create a minibatch \n", 170 | " indices=shuffled_indices[count:count+bs]\n", 171 | " minibatch_data = train_data[indices]\n", 172 | " minibatch_label= train_label[indices]\n", 173 | "\n", 174 | " # reshape the minibatch\n", 175 | " inputs = minibatch_data.view(bs,3072)\n", 176 | "\n", 177 | " # tell Pytorch to start tracking all operations that will be done on \"inputs\"\n", 178 | " inputs.requires_grad_()\n", 179 | "\n", 180 | " # forward the minibatch through the net \n", 181 | " scores=net( inputs ) \n", 182 | "\n", 183 | " # Compute the average of the losses of the data points in the minibatch\n", 184 | " loss = criterion( scores , minibatch_label) \n", 185 | " \n", 186 | " # backward pass to compute dL/dU, dL/dV and dL/dW \n", 187 | " loss.backward()\n", 188 | "\n", 189 | " # do one step of stochastic gradient descent: U=U-lr(dL/dU), V=V-lr(dL/dU), ...\n", 190 | " optimizer.step()\n", 191 | " \n", 192 | " # START COMPUTING STATS\n", 193 | " \n", 194 | " # add the loss of this batch to the running loss\n", 195 | " running_loss += # COMPLETE HERE\n", 196 | " \n", 197 | " # compute the error made on this batch and add it to the running error \n", 198 | " error = # COMPLETE HERE\n", 199 | " running_error += # COMPLETE HERE\n", 200 | " \n", 201 | " num_batches += # COMPLETE HERE\n", 202 | " \n", 203 | " \n", 204 | " # compute stats for the full training set\n", 205 | " total_loss = # COMPLETE HERE\n", 206 | " total_error = # COMPLETE HERE\n", 207 | " \n", 208 | " print('epoch=',epoch, '\\t loss=', total_loss , '\\t error=', total_error*100 ,'percent')\n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " " 213 | ] 214 | }, 215 | { 216 | "cell_type": "markdown", 217 | "metadata": {}, 218 | "source": [ 219 | "### Choose image at random from the test set and see how good/bad are the predictions" 220 | ] 221 | }, 222 | { 223 | "cell_type": "code", 224 | "execution_count": null, 225 | "metadata": {}, 226 | "outputs": [], 227 | "source": [ 228 | "# choose a picture at random\n", 229 | "idx=randint(0, 10000-1)\n", 230 | "im=test_data[idx]\n", 231 | "\n", 232 | "# diplay the picture\n", 233 | "utils.show(im)\n", 234 | "\n", 235 | "# feed it to the net and display the confidence scores\n", 236 | "scores = net( im.view(1,3072)) \n", 237 | "probs= F.softmax(scores, dim=1)\n", 238 | "utils.show_prob_cifar(probs)" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [] 247 | } 248 | ], 249 | "metadata": { 250 | "kernelspec": { 251 | "display_name": "Python 3", 252 | "language": "python", 253 | "name": "python3" 254 | }, 255 | "language_info": { 256 | "codemirror_mode": { 257 | "name": "ipython", 258 | "version": 3 259 | }, 260 | "file_extension": ".py", 261 | "mimetype": "text/x-python", 262 | "name": "python", 263 | "nbconvert_exporter": "python", 264 | "pygments_lexer": "ipython3", 265 | "version": "3.6.8" 266 | } 267 | }, 268 | "nbformat": 4, 269 | "nbformat_minor": 2 270 | } 271 | -------------------------------------------------------------------------------- /codes/labs_lecture06/lab02_cifar_multilayer/gpu_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02 : GPU demo" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'gpu_demo.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import torch.nn.functional as F" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "### Set the device variable to be either cuda or cpu" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 9, 51 | "metadata": {}, 52 | "outputs": [ 53 | { 54 | "name": "stdout", 55 | "output_type": "stream", 56 | "text": [ 57 | "cpu\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "device = torch.device(\"cuda\")\n", 63 | "\n", 64 | "#device = torch.device(\"cpu\")\n", 65 | "\n", 66 | "print(device)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Make a random tensor (by default the tensor is created on the CPU)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 10, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "tensor([[0.2126, 0.8232, 0.9466],\n", 86 | " [0.0370, 0.5176, 0.7032],\n", 87 | " [0.4544, 0.4457, 0.8515]])\n" 88 | ] 89 | } 90 | ], 91 | "source": [ 92 | "x=torch.rand(3,3)\n", 93 | "\n", 94 | "print(x)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "### Send the tensor to the device (which is going to be be the GPU if the device was set to be the GPU -- otherwise it will remain on the CPU)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 11, 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "tensor([[0.2126, 0.8232, 0.9466],\n", 114 | " [0.0370, 0.5176, 0.7032],\n", 115 | " [0.4544, 0.4457, 0.8515]])\n" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "x=x.to(device)\n", 121 | "\n", 122 | "print(x)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "metadata": {}, 128 | "source": [ 129 | "### Make a one layer net class" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 12, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "class one_layer_net(nn.Module):\n", 139 | "\n", 140 | " def __init__(self, input_size, output_size):\n", 141 | " super(one_layer_net , self).__init__()\n", 142 | " \n", 143 | " self.layer1 = nn.Linear( input_size, output_size , bias=False)\n", 144 | " \n", 145 | " def forward(self, x):\n", 146 | " scores = self.layer1(x)\n", 147 | " return scores" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "### Build the net" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 13, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "net=one_layer_net(5,2)" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "metadata": {}, 169 | "source": [ 170 | "### Send all the weights of the network to the GPU" 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 14, 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "net = net.to(device)" 180 | ] 181 | }, 182 | { 183 | "cell_type": "markdown", 184 | "metadata": {}, 185 | "source": [ 186 | "### Ckeck that the matrix is indeed on the GPU" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": 8, 192 | "metadata": {}, 193 | "outputs": [ 194 | { 195 | "name": "stdout", 196 | "output_type": "stream", 197 | "text": [ 198 | "Parameter containing:\n", 199 | "tensor([[-0.2646, 0.0491, 0.3317, 0.2717, 0.0483],\n", 200 | " [ 0.1272, -0.3401, 0.1400, 0.0687, 0.3442]], requires_grad=True)\n" 201 | ] 202 | } 203 | ], 204 | "source": [ 205 | "print(net.layer1.weight)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [] 214 | } 215 | ], 216 | "metadata": { 217 | "kernelspec": { 218 | "display_name": "Python 3", 219 | "language": "python", 220 | "name": "python3" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.6.8" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 2 237 | } 238 | -------------------------------------------------------------------------------- /codes/labs_lecture08/lab02_pool_layer/pool_layer_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Lab 02 : Pooling layer - demo" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "# For Google Colaboratory\n", 17 | "import sys, os\n", 18 | "if 'google.colab' in sys.modules:\n", 19 | " from google.colab import drive\n", 20 | " drive.mount('/content/gdrive')\n", 21 | " file_name = 'pool_layer_demo.ipynb'\n", 22 | " import subprocess\n", 23 | " path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode(\"utf-8\")\n", 24 | " print(path_to_file)\n", 25 | " path_to_file = path_to_file.replace(file_name,\"\").replace('\\n',\"\")\n", 26 | " os.chdir(path_to_file)\n", 27 | " !pwd" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import torch\n", 37 | "import torch.nn as nn" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "### Make a pooling module\n", 45 | "* inputs: activation maps of size n x n\n", 46 | "* output: activation maps of size n/p x n/p\n", 47 | "* p: pooling size" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 2, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "mod = nn.MaxPool2d(2,2)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "metadata": {}, 62 | "source": [ 63 | "### Make an input 2 x 6 x 6 (two channels, each one has 6 x 6 pixels )" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 8, 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "name": "stdout", 73 | "output_type": "stream", 74 | "text": [ 75 | "tensor([[[[0.6796, 0.0749, 0.5749, 0.5729, 0.3750, 0.9115],\n", 76 | " [0.0087, 0.2287, 0.1944, 0.5500, 0.9770, 0.0556],\n", 77 | " [0.2348, 0.4120, 0.8975, 0.8404, 0.1589, 0.7432],\n", 78 | " [0.7057, 0.1542, 0.8852, 0.1988, 0.2206, 0.2997],\n", 79 | " [0.6186, 0.6267, 0.1895, 0.5063, 0.0490, 0.9198],\n", 80 | " [0.1485, 0.1590, 0.0135, 0.9169, 0.1770, 0.9111]],\n", 81 | "\n", 82 | " [[0.9578, 0.7230, 0.8536, 0.0845, 0.8924, 0.4143],\n", 83 | " [0.8960, 0.1241, 0.8723, 0.3279, 0.6405, 0.8009],\n", 84 | " [0.4433, 0.3989, 0.8508, 0.8352, 0.5912, 0.4906],\n", 85 | " [0.7956, 0.3493, 0.9055, 0.5975, 0.6417, 0.7784],\n", 86 | " [0.8065, 0.9513, 0.0501, 0.2112, 0.8114, 0.8546],\n", 87 | " [0.3578, 0.3014, 0.4694, 0.0160, 0.3717, 0.1823]]]])\n", 88 | "torch.Size([1, 2, 6, 6])\n" 89 | ] 90 | } 91 | ], 92 | "source": [ 93 | "bs=1\n", 94 | "\n", 95 | "x=torch.rand(bs,2,6,6)\n", 96 | "\n", 97 | "\n", 98 | "print(x)\n", 99 | "print(x.size())" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "metadata": {}, 105 | "source": [ 106 | "### Feed it to the pooling layer: the output size should be divided by 2 " 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": 9, 112 | "metadata": {}, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "tensor([[[[0.6796, 0.5749, 0.9770],\n", 119 | " [0.7057, 0.8975, 0.7432],\n", 120 | " [0.6267, 0.9169, 0.9198]],\n", 121 | "\n", 122 | " [[0.9578, 0.8723, 0.8924],\n", 123 | " [0.7956, 0.9055, 0.7784],\n", 124 | " [0.9513, 0.4694, 0.8546]]]])\n", 125 | "torch.Size([1, 2, 3, 3])\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "y=mod(x)\n", 131 | "\n", 132 | "print(y)\n", 133 | "print(y.size())" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [] 149 | } 150 | ], 151 | "metadata": { 152 | "kernelspec": { 153 | "display_name": "Python 3", 154 | "language": "python", 155 | "name": "python3" 156 | }, 157 | "language_info": { 158 | "codemirror_mode": { 159 | "name": "ipython", 160 | "version": 3 161 | }, 162 | "file_extension": ".py", 163 | "mimetype": "text/x-python", 164 | "name": "python", 165 | "nbconvert_exporter": "python", 166 | "pygments_lexer": "ipython3", 167 | "version": "3.6.8" 168 | } 169 | }, 170 | "nbformat": 4, 171 | "nbformat_minor": 2 172 | } 173 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/coarsening.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/coarsening.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/coarsening.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/coarsening.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/grid_graph.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/grid_graph.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/grid_graph.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab01_ChebGCNs/lib/__pycache__/grid_graph.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab01_ChebGCNs/lib/grid_graph.py: -------------------------------------------------------------------------------- 1 | import sklearn 2 | import sklearn.metrics 3 | import scipy.sparse, scipy.sparse.linalg # scipy.spatial.distance 4 | import numpy as np 5 | 6 | 7 | def grid_graph(grid_side,number_edges,metric): 8 | """Generate graph of a grid""" 9 | z = grid(grid_side) 10 | dist, idx = distance_sklearn_metrics(z, k=number_edges, metric=metric) 11 | A = adjacency(dist, idx) 12 | print("nb edges: ",A.nnz) 13 | return A 14 | 15 | 16 | def grid(m, dtype=np.float32): 17 | """Return coordinates of grid points""" 18 | M = m**2 19 | x = np.linspace(0,1,m, dtype=dtype) 20 | y = np.linspace(0,1,m, dtype=dtype) 21 | xx, yy = np.meshgrid(x, y) 22 | z = np.empty((M,2), dtype) 23 | z[:,0] = xx.reshape(M) 24 | z[:,1] = yy.reshape(M) 25 | return z 26 | 27 | 28 | def distance_sklearn_metrics(z, k=4, metric='euclidean'): 29 | """Compute pairwise distances""" 30 | #d = sklearn.metrics.pairwise.pairwise_distances(z, metric=metric, n_jobs=-2) 31 | d = sklearn.metrics.pairwise.pairwise_distances(z, metric=metric, n_jobs=1) 32 | # k-NN 33 | idx = np.argsort(d)[:,1:k+1] 34 | d.sort() 35 | d = d[:,1:k+1] 36 | return d, idx 37 | 38 | 39 | def adjacency(dist, idx): 40 | """Return adjacency matrix of a kNN graph""" 41 | M, k = dist.shape 42 | assert M, k == idx.shape 43 | assert dist.min() >= 0 44 | assert dist.max() <= 1 45 | 46 | # Pairwise distances 47 | sigma2 = np.mean(dist[:,-1])**2 48 | dist = np.exp(- dist**2 / sigma2) 49 | 50 | # Weight matrix 51 | I = np.arange(0, M).repeat(k) 52 | J = idx.reshape(M*k) 53 | V = dist.reshape(M*k) 54 | W = scipy.sparse.coo_matrix((V, (I, J)), shape=(M, M)) 55 | 56 | # No self-connections 57 | W.setdiag(0) 58 | 59 | # Undirected graph 60 | bigger = W.T > W 61 | W = W - W.multiply(bigger) + W.T.multiply(bigger) 62 | 63 | assert W.nnz % 2 == 0 64 | assert np.abs(W - W.T).mean() < 1e-10 65 | assert type(W) is scipy.sparse.csr.csr_matrix 66 | return W 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/data/set_100_clustering_maps_p05_q01_size5_25_2017-10-31_10-25-00_.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/data/set_100_clustering_maps_p05_q01_size5_25_2017-10-31_10-25-00_.txt -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/data/set_100_subgraphs_p05_size20_Voc3_2017-10-31_10-23-00_.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/data/set_100_subgraphs_p05_size20_Voc3_2017-10-31_10-23-00_.txt -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/block.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/block.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/block.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/block.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/graph_generator.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/graph_generator.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/graph_generator.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab02_GatedGCNs/util/__pycache__/graph_generator.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/block.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def random_graph(n,p): 5 | W=np.zeros((n,n)) 6 | for i in range(n): 7 | for j in range(i+1,n): 8 | if np.random.binomial(1,p)==1: 9 | W[i,j]=1 10 | W[j,i]=1 11 | return W 12 | 13 | 14 | def block_model(c,p,q): 15 | n=len(c) 16 | W=np.zeros((n,n)) 17 | for i in range(n): 18 | for j in range(i+1,n): 19 | if c[i]==c[j]: 20 | prob=p 21 | else: 22 | prob=q 23 | if np.random.binomial(1,prob)==1: 24 | W[i,j]=1 25 | W[j,i]=1 26 | return W 27 | 28 | 29 | def balanced_block_model(nb_of_clust, clust_size , p, q): 30 | n = nb_of_clust*clust_size 31 | c=np.zeros(n) 32 | for r in range(nb_of_clust): 33 | start=r*clust_size 34 | c[start:start+clust_size]=r 35 | W=block_model(c,p,q) 36 | return W,c 37 | 38 | 39 | def unbalanced_block_model(nb_of_clust, clust_size_min, clust_size_max, p, q): 40 | c = [] 41 | for r in range(nb_of_clust): 42 | if clust_size_max==clust_size_min: 43 | clust_size_r = clust_size_max 44 | else: 45 | clust_size_r = np.random.randint(clust_size_min,clust_size_max,size=1)[0] 46 | val_r = np.repeat(r,clust_size_r,axis=0) 47 | c.append(val_r) 48 | c = np.concatenate(c) 49 | W = block_model(c,p,q) 50 | return W,c 51 | 52 | 53 | def add_a_block(W0,W,c,nb_of_clust,q): 54 | n=W.shape[0] 55 | n0=W0.shape[0] 56 | V=(np.random.rand(n0,n) < q).astype(float) 57 | W_up=np.concatenate( ( W , V.T ) , axis=1 ) 58 | W_low=np.concatenate( ( V , W0 ) , axis=1 ) 59 | W_new=np.concatenate( (W_up,W_low) , axis=0) 60 | c0=np.full(n0,nb_of_clust) 61 | c_new=np.concatenate( (c, c0),axis=0) 62 | return W_new,c_new 63 | 64 | 65 | def schuffle(W,c): 66 | # relabel the vertices at random 67 | idx=np.random.permutation( W.shape[0] ) 68 | W_new=W[idx,:] 69 | W_new=W_new[:,idx] 70 | c_new=c[idx] 71 | return W_new , c_new , idx 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab02_GatedGCNs/util/graph_generator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import block 3 | import torch 4 | import scipy.sparse as sp 5 | 6 | default_type='torch.cuda.FloatTensor' 7 | default_type='torch.FloatTensor' 8 | 9 | 10 | 11 | 12 | 13 | class variable_size_graph(): 14 | 15 | def __init__(self, task_parameters): 16 | 17 | # parameters 18 | vocab_size = task_parameters['Voc'] 19 | nb_of_clust = task_parameters['nb_clusters_target'] 20 | clust_size_min = task_parameters['size_min'] 21 | clust_size_max = task_parameters['size_max'] 22 | p = task_parameters['p'] 23 | q = task_parameters['q'] 24 | self_loop = True 25 | W0 = task_parameters['W0'] 26 | u0 = task_parameters['u0'] 27 | 28 | # create block model graph and put random signal on it 29 | W,c=block.unbalanced_block_model(nb_of_clust,clust_size_min,clust_size_max,p,q) 30 | u=np.random.randint(vocab_size,size=W.shape[0]) 31 | 32 | # add the subgraph to be detected 33 | W,c=block.add_a_block(W0,W,c,nb_of_clust,q) 34 | u=np.concatenate((u,u0),axis=0) 35 | 36 | # shuffle 37 | W,c,idx=block.schuffle(W,c) 38 | u=u[idx] 39 | u=torch.from_numpy(u) 40 | u=u.long() 41 | 42 | # add self loop 43 | if self_loop: 44 | for i in range(W.shape[0]): 45 | W[i,i]=1 46 | 47 | # create the target 48 | target= (c==nb_of_clust).astype(float) 49 | target=torch.from_numpy(target) 50 | target=target.long() 51 | 52 | # mapping matrices 53 | W_coo=sp.coo_matrix(W) 54 | nb_edges=W_coo.nnz 55 | nb_vertices=W.shape[0] 56 | edge_to_starting_vertex=sp.coo_matrix( ( np.ones(nb_edges) ,(np.arange(nb_edges), W_coo.row) ), 57 | shape=(nb_edges, nb_vertices) ) 58 | edge_to_ending_vertex=sp.coo_matrix( ( np.ones(nb_edges) ,(np.arange(nb_edges), W_coo.col) ), 59 | shape=(nb_edges, nb_vertices) ) 60 | 61 | # attribute 62 | #self.adj_matrix=torch.from_numpy(W).type(default_type) 63 | #self.edge_to_starting_vertex=torch.from_numpy(edge_to_starting_vertex.toarray()).type(default_type) 64 | #self.edge_to_ending_vertex=torch.from_numpy(edge_to_ending_vertex.toarray()).type(default_type) 65 | self.adj_matrix=W 66 | self.edge_to_starting_vertex=edge_to_starting_vertex 67 | self.edge_to_ending_vertex=edge_to_ending_vertex 68 | self.signal=u 69 | self.target=target 70 | 71 | 72 | class graph_semi_super_clu(): 73 | 74 | def __init__(self, task_parameters): 75 | 76 | # parameters 77 | vocab_size = task_parameters['Voc'] 78 | nb_of_clust = task_parameters['nb_clusters_target'] 79 | clust_size_min = task_parameters['size_min'] 80 | clust_size_max = task_parameters['size_max'] 81 | p = task_parameters['p'] 82 | q = task_parameters['q'] 83 | self_loop = True 84 | 85 | # block model 86 | W, c = block.unbalanced_block_model(nb_of_clust, clust_size_min, clust_size_max, p, q) 87 | 88 | # add self loop 89 | if self_loop: 90 | for i in range(W.shape[0]): 91 | W[i,i]=1 92 | 93 | # shuffle 94 | W,c,idx = block.schuffle(W,c) 95 | 96 | # signal on block model 97 | u = np.zeros(c.shape[0]) 98 | for r in range(nb_of_clust): 99 | cluster = np.where(c==r)[0] 100 | s = cluster[np.random.randint(cluster.shape[0])] 101 | u[s] = r+1 102 | 103 | # target 104 | target = c 105 | 106 | # convert to pytorch 107 | u = torch.from_numpy(u) 108 | u = u.long() 109 | target = torch.from_numpy(target) 110 | target = target.long() 111 | 112 | # mapping matrices 113 | W_coo=sp.coo_matrix(W) 114 | nb_edges=W_coo.nnz 115 | nb_vertices=W.shape[0] 116 | edge_to_starting_vertex=sp.coo_matrix( ( np.ones(nb_edges) ,(np.arange(nb_edges), W_coo.row) ), 117 | shape=(nb_edges, nb_vertices) ) 118 | edge_to_ending_vertex=sp.coo_matrix( ( np.ones(nb_edges) ,(np.arange(nb_edges), W_coo.col) ), 119 | shape=(nb_edges, nb_vertices) ) 120 | 121 | # attribute 122 | #self.adj_matrix=torch.from_numpy(W).type(default_type) 123 | #self.edge_to_starting_vertex=torch.from_numpy(edge_to_starting_vertex.toarray()).type(default_type) 124 | #self.edge_to_ending_vertex=torch.from_numpy(edge_to_ending_vertex.toarray()).type(default_type) 125 | self.adj_matrix=W 126 | self.edge_to_starting_vertex=edge_to_starting_vertex 127 | self.edge_to_ending_vertex=edge_to_ending_vertex 128 | self.signal=u 129 | self.target=target 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/atom_dict.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/atom_dict.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/bond_dict.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/bond_dict.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_test.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_test.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_train.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_train.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_val.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/info_val.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/test.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/test.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/train.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/train.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/val.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab03_Molecules/datasets/dataQM9/val.pickle -------------------------------------------------------------------------------- /codes/labs_lecture14/lab03_Molecules/dictionaries.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | from rdkit import Chem 4 | 5 | 6 | class Dictionary(object): 7 | """ 8 | worddidx is a dictionary 9 | idx2word is a list 10 | """ 11 | 12 | 13 | def __init__(self): 14 | self.word2idx = {} 15 | self.idx2word = [] 16 | self.word2num_occurence = {} 17 | self.idx2num_occurence = [] 18 | 19 | def add_word(self, word): 20 | if word not in self.word2idx: 21 | # dictionaries 22 | self.idx2word.append(word) 23 | self.word2idx[word] = len(self.idx2word) - 1 24 | # stats 25 | self.idx2num_occurence.append(0) 26 | self.word2num_occurence[word] = 0 27 | 28 | # increase counters 29 | self.word2num_occurence[word]+=1 30 | self.idx2num_occurence[ self.word2idx[word] ] += 1 31 | 32 | def __len__(self): 33 | return len(self.idx2word) 34 | 35 | 36 | 37 | 38 | def augment_dictionary(atom_dict, bond_dict, list_of_mol ): 39 | 40 | """ 41 | take a lists of rdkit molecules and use it to augment existing atom and bond dictionaries 42 | """ 43 | for idx,mol in enumerate(list_of_mol): 44 | 45 | for atom in mol.GetAtoms(): 46 | atom_dict.add_word( atom.GetSymbol() ) 47 | 48 | for bond in mol.GetBonds(): 49 | bond_dict.add_word( str(bond.GetBondType()) ) 50 | 51 | # compute the number of edges of type 'None' 52 | N=mol.GetNumAtoms() 53 | if N>2: 54 | E=N+math.factorial(N)/(math.factorial(2)*math.factorial(N-2)) # including self loop 55 | num_NONE_bonds = E-mol.GetNumBonds() 56 | bond_dict.word2num_occurence['NONE']+=num_NONE_bonds 57 | bond_dict.idx2num_occurence[0]+=num_NONE_bonds 58 | 59 | 60 | 61 | 62 | def make_dictionary(list_of_mol_train, list_of_mol_val, list_of_mol_test): 63 | 64 | """ 65 | take three lists of smiles (train, val and test) and build atoms and bond dictionaries 66 | """ 67 | atom_dict=Dictionary() 68 | bond_dict=Dictionary() 69 | bond_dict.add_word('NONE') 70 | print('train') 71 | augment_dictionary(atom_dict, bond_dict, list_of_mol_train ) 72 | print('val') 73 | augment_dictionary(atom_dict, bond_dict, list_of_mol_val ) 74 | print('test') 75 | augment_dictionary(atom_dict, bond_dict, list_of_mol_test ) 76 | return atom_dict, bond_dict 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/config.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class Settings(dict): 5 | """Experiment configuration options. 6 | 7 | Wrapper around in-built dict class to access members through the dot operation. 8 | 9 | Experiment parameters: 10 | "expt_name": Description of experiment, used for logging. 11 | "gpu_id": Available GPU ID(s) 12 | "train_filepath": Training set path 13 | "val_filepath": Validation set path 14 | "test_filepath": Test set path 15 | "num_nodes": Number of nodes in TSP tours 16 | "node_dim": Number of dimensions for each node 17 | "voc_nodes_in": Input node signal vocabulary size 18 | "voc_edges": Edge signal vocabulary size 19 | "beam_size": Beam size for beamsearch procedure (-1 for disabling beamsearch) 20 | "hidden_dim": Dimension of GCN hidden state 21 | "num_layers": Number of layers in GCN 22 | "max_epochs": Maximum training epochs 23 | "batch_size": Batch size 24 | "batches_per_epoch": Batches per epoch (-1 for using full set) 25 | "accumulation_steps": Number of steps for gradient accumulation 26 | "learning_rate": Initial learning rate 27 | "decay_rate": Rate of learning rate decay 28 | """ 29 | 30 | def __init__(self, config_dict): 31 | super().__init__() 32 | for key in config_dict: 33 | self[key] = config_dict[key] 34 | 35 | def __getattr__(self, attr): 36 | return self[attr] 37 | 38 | def __setitem__(self, key, value): 39 | return super().__setitem__(key, value) 40 | 41 | def __setattr__(self, key, value): 42 | return self.__setitem__(key, value) 43 | 44 | __delattr__ = dict.__delitem__ 45 | 46 | 47 | def get_default_config(): 48 | """Returns default settings object. 49 | """ 50 | return Settings(json.load(open("./configs/default.json"))) 51 | 52 | 53 | def get_config(filepath): 54 | """Returns settings from json file. 55 | """ 56 | config = get_default_config() 57 | config.update(Settings(json.load(open(filepath)))) 58 | return config 59 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/configs/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "expt_name": "default", 3 | "gpu_id": "1", 4 | 5 | "train_filepath": "./data/tsp5_train.txt", 6 | "val_filepath": "./data/tsp5_val.txt", 7 | "test_filepath": "./data/tsp5_test.txt", 8 | 9 | "num_nodes": 5, 10 | "node_dim": 2, 11 | "voc_nodes_in": 2, 12 | "voc_edges": 2, 13 | 14 | "beam_size": 10, 15 | 16 | "hidden_dim": 50, 17 | "num_layers": 2, 18 | 19 | "max_epochs": 10, 20 | "batch_size": 50, 21 | "batches_per_epoch": 1000, 22 | "accumulation_steps": 10, 23 | 24 | "learning_rate": 0.001, 25 | "decay_rate": 1.1 26 | } 27 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/configs/tsp10_small.json: -------------------------------------------------------------------------------- 1 | { 2 | "expt_name": "tsp10", 3 | "gpu_id": "0", 4 | 5 | "train_filepath": "./data/tsp10_small_concorde_train.txt", 6 | "test_filepath": "./data/tsp10_small_concorde_test.txt", 7 | 8 | "num_nodes": 10, 9 | "node_dim": 2, 10 | "voc_nodes_in": 2, 11 | "voc_edges": 2, 12 | 13 | "beam_size": 100, 14 | 15 | "hidden_dim": 200, 16 | "num_layers": 10, 17 | 18 | "max_epochs": 20, 19 | "batch_size": 100, 20 | "batches_per_epoch": -1, 21 | 22 | "learning_rate": 0.001, 23 | "decay_rate": 1.1 24 | } 25 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/logs/tsp10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "expt_name": "tsp10", 3 | "gpu_id": "0", 4 | "train_filepath": "./data/tsp10_small_concorde_train.txt", 5 | "val_filepath": "./data/tsp5_val.txt", 6 | "test_filepath": "./data/tsp10_small_concorde_test.txt", 7 | "num_nodes": 10, 8 | "node_dim": 2, 9 | "voc_nodes_in": 2, 10 | "voc_edges": 2, 11 | "beam_size": 100, 12 | "hidden_dim": 200, 13 | "num_layers": 10, 14 | "max_epochs": 20, 15 | "batch_size": 100, 16 | "batches_per_epoch": -1, 17 | "accumulation_steps": 10, 18 | "learning_rate": 0.001, 19 | "decay_rate": 1.1 20 | } -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1570518905.MML-XavierNB1.local: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1570518905.MML-XavierNB1.local -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1570521674.MML-XavierNB1.local: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1570521674.MML-XavierNB1.local -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1571736764.MML-XavierNB1.local: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/logs/tsp10/events.out.tfevents.1571736764.MML-XavierNB1.local -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__init__.py -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_layers.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_layers.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_layers.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_layers.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_model.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/models/__pycache__/gcn_model.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/gcn_layers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | 6 | class BatchNormNode(nn.Module): 7 | """Batch normalization for node features. 8 | """ 9 | 10 | def __init__(self, hidden_dim): 11 | super(BatchNormNode, self).__init__() 12 | self.batch_norm = nn.BatchNorm1d(hidden_dim, track_running_stats=False) 13 | 14 | def forward(self, x): 15 | """ 16 | Args: 17 | x: Node features (batch_size, num_nodes, hidden_dim) 18 | 19 | Returns: 20 | x_bn: Node features after batch normalization (batch_size, num_nodes, hidden_dim) 21 | """ 22 | x_trans = x.transpose(1, 2).contiguous() # Reshape input: (batch_size, hidden_dim, num_nodes) 23 | x_trans_bn = self.batch_norm(x_trans) 24 | x_bn = x_trans_bn.transpose(1, 2).contiguous() # Reshape to original shape 25 | return x_bn 26 | 27 | 28 | class BatchNormEdge(nn.Module): 29 | """Batch normalization for edge features. 30 | """ 31 | 32 | def __init__(self, hidden_dim): 33 | super(BatchNormEdge, self).__init__() 34 | self.batch_norm = nn.BatchNorm2d(hidden_dim, track_running_stats=False) 35 | 36 | def forward(self, e): 37 | """ 38 | Args: 39 | e: Edge features (batch_size, num_nodes, num_nodes, hidden_dim) 40 | 41 | Returns: 42 | e_bn: Edge features after batch normalization (batch_size, num_nodes, num_nodes, hidden_dim) 43 | """ 44 | e_trans = e.transpose(1, 3).contiguous() # Reshape input: (batch_size, num_nodes, num_nodes, hidden_dim) 45 | e_trans_bn = self.batch_norm(e_trans) 46 | e_bn = e_trans_bn.transpose(1, 3).contiguous() # Reshape to original 47 | return e_bn 48 | 49 | 50 | class NodeFeatures(nn.Module): 51 | """Convnet features for nodes. 52 | 53 | x_i = U*x_i + sum_j [ gate_ij * (V*x_j) ] 54 | """ 55 | def __init__(self, hidden_dim): 56 | super(NodeFeatures, self).__init__() 57 | self.U = nn.Linear(hidden_dim, hidden_dim, True) 58 | self.V = nn.Linear(hidden_dim, hidden_dim, True) 59 | 60 | def forward(self, x, edge_gate): 61 | """ 62 | Args: 63 | x: Node features (batch_size, num_nodes, hidden_dim) 64 | edge_gate: Edge gate values (batch_size, num_nodes, num_nodes, hidden_dim) 65 | 66 | Returns: 67 | x_new: Convolved node features (batch_size, num_nodes, hidden_dim) 68 | """ 69 | Ux = self.U(x) # B x V x H 70 | Vx = self.V(x) # B x V x H 71 | Vx = Vx.unsqueeze(1) # extend Vx from "B x V x H" to "B x 1 x V x H" 72 | gateVx = edge_gate * Vx # B x V x V x H 73 | x_new = Ux + torch.sum(gateVx, dim=2) # B x V x H 74 | return x_new 75 | 76 | 77 | class EdgeFeatures(nn.Module): 78 | """Convnet features for edges. 79 | 80 | e_ij = U*e_ij + V*x_i + W*x_j 81 | """ 82 | 83 | def __init__(self, hidden_dim): 84 | super(EdgeFeatures, self).__init__() 85 | self.U = nn.Linear(hidden_dim, hidden_dim, True) 86 | self.V = nn.Linear(hidden_dim, hidden_dim, True) 87 | 88 | def forward(self, x, e): 89 | """ 90 | Args: 91 | x: Node features (batch_size, num_nodes, hidden_dim) 92 | e: Edge features (batch_size, num_nodes, num_nodes, hidden_dim) 93 | 94 | Returns: 95 | e_new: Convolved edge features (batch_size, num_nodes, num_nodes, hidden_dim) 96 | """ 97 | Ue = self.U(e) 98 | Vx = self.V(x) 99 | Wx = Vx.unsqueeze(1) # Extend Vx from "B x V x H" to "B x V x 1 x H" 100 | Vx = Vx.unsqueeze(2) # extend Vx from "B x V x H" to "B x 1 x V x H" 101 | e_new = Ue + Vx + Wx 102 | return e_new 103 | 104 | 105 | class ResidualGatedGCNLayer(nn.Module): 106 | """Convnet layer for updating node and edge features with gating and residual connection. 107 | """ 108 | 109 | def __init__(self, hidden_dim): 110 | super(ResidualGatedGCNLayer, self).__init__() 111 | self.node_feat = NodeFeatures(hidden_dim) 112 | self.edge_feat = EdgeFeatures(hidden_dim) 113 | self.bn_node = BatchNormNode(hidden_dim) 114 | self.bn_edge = BatchNormEdge(hidden_dim) 115 | 116 | def forward(self, x, e): 117 | """ 118 | Args: 119 | x: Node features (batch_size, num_nodes, hidden_dim) 120 | e: Edge features (batch_size, num_nodes, num_nodes, hidden_dim) 121 | 122 | Returns: 123 | x_new: Convolved node features (batch_size, num_nodes, hidden_dim) 124 | e_new: Convolved edge features (batch_size, num_nodes, num_nodes, hidden_dim) 125 | """ 126 | e_in = e 127 | x_in = x 128 | # Edge convolution 129 | e_tmp = self.edge_feat(x_in, e_in) # B x V x V x H 130 | # Compute edge gates 131 | edge_gate = F.sigmoid(e_tmp) 132 | # Node convolution 133 | x_tmp = self.node_feat(x_in, edge_gate) 134 | # Batch normalization 135 | e_tmp = self.bn_edge(e_tmp) 136 | x_tmp = self.bn_node(x_tmp) 137 | # ReLU Activation 138 | e = F.relu(e_tmp) 139 | x = F.relu(x_tmp) 140 | # Residual connection 141 | x_new = x_in + x 142 | e_new = e_in + e 143 | return x_new, e_new 144 | 145 | 146 | class MLP(nn.Module): 147 | """Multi-layer Perceptron layer for output prediction. 148 | """ 149 | 150 | def __init__(self, hidden_dim, output_dim, L=2): 151 | super(MLP, self).__init__() 152 | self.L = L 153 | U = [] 154 | for layer in range(self.L - 1): 155 | U.append(nn.Linear(hidden_dim, hidden_dim, True)) 156 | self.U = nn.ModuleList(U) 157 | self.V = nn.Linear(hidden_dim, output_dim, True) 158 | 159 | def forward(self, x): 160 | """ 161 | Args: 162 | x: Input features (batch_size, hidden_dim) 163 | 164 | Returns: 165 | y: Output predictions (batch_size, output_dim) 166 | """ 167 | Ux = x 168 | for U_i in self.U: 169 | Ux = U_i(Ux) # B x H 170 | Ux = F.relu(Ux) # B x H 171 | y = self.V(Ux) # B x O 172 | return y 173 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/models/gcn_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | import torch.nn as nn 4 | 5 | from models.gcn_layers import ResidualGatedGCNLayer, MLP 6 | from utils.model_utils import * 7 | 8 | 9 | class ResidualGatedGCNModel(nn.Module): 10 | """Residual Gated GCN Model for outputting predictions as edge adjacency matrices. 11 | 12 | References: 13 | Paper: https://arxiv.org/pdf/1711.07553v2.pdf 14 | Code: https://github.com/xbresson/spatial_graph_convnets 15 | """ 16 | 17 | def __init__(self, config, dtypeFloat, dtypeLong): 18 | super(ResidualGatedGCNModel, self).__init__() 19 | self.dtypeFloat = dtypeFloat 20 | self.dtypeLong = dtypeLong 21 | # Define net parameters 22 | self.num_nodes = config.num_nodes 23 | self.node_dim = config.node_dim 24 | self.voc_nodes_in = config['voc_nodes_in'] 25 | self.voc_nodes_out = config['num_nodes'] 26 | self.voc_edges = config['voc_edges'] 27 | self.hidden_dim = config['hidden_dim'] 28 | self.num_layers = config['num_layers'] 29 | # Node and edge embedding layers/lookups 30 | self.nodes_coord_embedding = nn.Linear(self.node_dim, self.hidden_dim, bias=False) 31 | self.edges_values_embedding = nn.Linear(1, self.hidden_dim, bias=False) 32 | # Define GCN Layers 33 | gcn_layers = [] 34 | for layer in range(self.num_layers): 35 | gcn_layers.append(ResidualGatedGCNLayer(self.hidden_dim)) 36 | self.gcn_layers = nn.ModuleList(gcn_layers) 37 | # Define MLP classifiers 38 | self.mlp_edges = MLP(self.hidden_dim, self.voc_edges) 39 | # self.mlp_nodes = MLP(self.hidden_dim, self.voc_nodes_out) 40 | 41 | def forward(self, x_edges, x_edges_values, x_nodes, x_nodes_coord, y_edges, edge_cw): 42 | """ 43 | Args: 44 | x_edges: Input edge adjacency matrix (batch_size, num_nodes, num_nodes) 45 | x_edges_values: Input edge distance matrix (batch_size, num_nodes, num_nodes) 46 | x_nodes: Input nodes (batch_size, num_nodes) 47 | x_nodes_coord: Input node coordinates (batch_size, num_nodes, node_dim) 48 | y_edges: Targets for edges (batch_size, num_nodes, num_nodes) 49 | edge_cw: Class weights for edges loss 50 | # y_nodes: Targets for nodes (batch_size, num_nodes, num_nodes) 51 | # node_cw: Class weights for nodes loss 52 | 53 | Returns: 54 | y_pred_edges: Predictions for edges (batch_size, num_nodes, num_nodes) 55 | # y_pred_nodes: Predictions for nodes (batch_size, num_nodes) 56 | loss: Value of loss function 57 | """ 58 | # Node and edge embedding 59 | e = self.edges_values_embedding(x_edges_values.unsqueeze(3)) # B x V x V x H 60 | x = self.nodes_coord_embedding(x_nodes_coord) # B x V x H 61 | # GCN layers 62 | for layer in range(self.num_layers): 63 | x, e = self.gcn_layers[layer](x, e) # B x V x H, B x V x V x H 64 | # MLP classifier 65 | y_pred_edges = self.mlp_edges(e) # B x V x V x voc_edges 66 | # y_pred_nodes = self.mlp_nodes(x) # B x V x voc_nodes 67 | 68 | # Compute loss 69 | edge_cw = torch.Tensor(edge_cw).type(self.dtypeFloat) # Convert to tensors 70 | loss = loss_edges(y_pred_edges, y_edges, edge_cw) 71 | 72 | return y_pred_edges, loss 73 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__init__.py -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/beamsearch.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/beamsearch.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/beamsearch.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/beamsearch.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/google_tsp_reader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/google_tsp_reader.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/google_tsp_reader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/google_tsp_reader.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/graph_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/graph_utils.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/graph_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/graph_utils.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/model_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/model_utils.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/model_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/model_utils.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/plot_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/plot_utils.cpython-36.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/__pycache__/plot_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab04_TSP/utils/__pycache__/plot_utils.cpython-37.pyc -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/beamsearch.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | 5 | class Beamsearch(object): 6 | """Class for managing internals of beamsearch procedure. 7 | 8 | References: 9 | General: https://github.com/OpenNMT/OpenNMT-py/blob/master/onmt/translate/beam.py 10 | For TSP: https://github.com/alexnowakvila/QAP_pt/blob/master/src/tsp/beam_search.py 11 | """ 12 | 13 | def __init__(self, beam_size, batch_size, num_nodes, 14 | dtypeFloat=torch.FloatTensor, dtypeLong=torch.LongTensor, 15 | probs_type='raw', random_start=False): 16 | """ 17 | Args: 18 | beam_size: Beam size 19 | batch_size: Batch size 20 | num_nodes: Number of nodes in TSP tours 21 | dtypeFloat: Float data type (for GPU/CPU compatibility) 22 | dtypeLong: Long data type (for GPU/CPU compatibility) 23 | probs_type: Type of probability values being handled by beamsearch (either 'raw'/'logits'/'argmax'(TODO)) 24 | random_start: Flag for using fixed (at node 0) vs. random starting points for beamsearch 25 | """ 26 | # Beamsearch parameters 27 | self.batch_size = batch_size 28 | self.beam_size = beam_size 29 | self.num_nodes = num_nodes 30 | self.probs_type = probs_type 31 | # Set data types 32 | self.dtypeFloat = dtypeFloat 33 | self.dtypeLong = dtypeLong 34 | # Set beamsearch starting nodes 35 | self.start_nodes = torch.zeros(batch_size, beam_size).type(self.dtypeLong) 36 | if random_start == True: 37 | # Random starting nodes 38 | self.start_nodes = torch.randint(0, num_nodes, (batch_size, beam_size)).type(self.dtypeLong) 39 | # Mask for constructing valid hypothesis 40 | self.mask = torch.ones(batch_size, beam_size, num_nodes).type(self.dtypeFloat) 41 | self.update_mask(self.start_nodes) # Mask the starting node of the beam search 42 | # Score for each translation on the beam 43 | self.scores = torch.zeros(batch_size, beam_size).type(self.dtypeFloat) 44 | self.all_scores = [] 45 | # Backpointers at each time-step 46 | self.prev_Ks = [] 47 | # Outputs at each time-step 48 | self.next_nodes = [self.start_nodes] 49 | 50 | def get_current_state(self): 51 | """Get the output of the beam at the current timestep. 52 | """ 53 | current_state = (self.next_nodes[-1].unsqueeze(2) 54 | .expand(self.batch_size, self.beam_size, self.num_nodes)) 55 | return current_state 56 | 57 | def get_current_origin(self): 58 | """Get the backpointers for the current timestep. 59 | """ 60 | return self.prev_Ks[-1] 61 | 62 | def advance(self, trans_probs): 63 | """Advances the beam based on transition probabilities. 64 | 65 | Args: 66 | trans_probs: Probabilities of advancing from the previous step (batch_size, beam_size, num_nodes) 67 | """ 68 | # Compound the previous scores (summing logits == multiplying probabilities) 69 | if len(self.prev_Ks) > 0: 70 | if self.probs_type == 'raw': 71 | beam_lk = trans_probs * self.scores.unsqueeze(2).expand_as(trans_probs) 72 | elif self.probs_type == 'logits': 73 | beam_lk = trans_probs + self.scores.unsqueeze(2).expand_as(trans_probs) 74 | else: 75 | beam_lk = trans_probs 76 | # Only use the starting nodes from the beam 77 | if self.probs_type == 'raw': 78 | beam_lk[:, 1:] = torch.zeros(beam_lk[:, 1:].size()).type(self.dtypeFloat) 79 | elif self.probs_type == 'logits': 80 | beam_lk[:, 1:] = -1e20 * torch.ones(beam_lk[:, 1:].size()).type(self.dtypeFloat) 81 | # Multiply by mask 82 | beam_lk = beam_lk * self.mask 83 | beam_lk = beam_lk.view(self.batch_size, -1) # (batch_size, beam_size * num_nodes) 84 | # Get top k scores and indexes (k = beam_size) 85 | bestScores, bestScoresId = beam_lk.topk(self.beam_size, 1, True, True) 86 | # Update scores 87 | self.scores = bestScores 88 | # Update backpointers 89 | prev_k = bestScoresId / self.num_nodes 90 | self.prev_Ks.append(prev_k) 91 | # Update outputs 92 | new_nodes = bestScoresId - prev_k * self.num_nodes 93 | self.next_nodes.append(new_nodes) 94 | # Re-index mask 95 | perm_mask = prev_k.unsqueeze(2).expand_as(self.mask) # (batch_size, beam_size, num_nodes) 96 | self.mask = self.mask.gather(1, perm_mask) 97 | # Mask newly added nodes 98 | self.update_mask(new_nodes) 99 | 100 | def update_mask(self, new_nodes): 101 | """Sets new_nodes to zero in mask. 102 | """ 103 | arr = (torch.arange(0, self.num_nodes).unsqueeze(0).unsqueeze(1) 104 | .expand_as(self.mask).type(self.dtypeLong)) 105 | new_nodes = new_nodes.unsqueeze(2).expand_as(self.mask) 106 | update_mask = 1 - torch.eq(arr, new_nodes).type(self.dtypeFloat) 107 | self.mask = self.mask * update_mask 108 | if self.probs_type == 'logits': 109 | # Convert 0s in mask to inf 110 | self.mask[self.mask == 0] = 1e20 111 | 112 | def sort_best(self): 113 | """Sort the beam. 114 | """ 115 | return torch.sort(self.scores, 0, True) 116 | 117 | def get_best(self): 118 | """Get the score and index of the best hypothesis in the beam. 119 | """ 120 | scores, ids = self.sort_best() 121 | return scores[1], ids[1] 122 | 123 | def get_hypothesis(self, k): 124 | """Walk back to construct the full hypothesis. 125 | 126 | Args: 127 | k: Position in the beam to construct (usually 0s for most probable hypothesis) 128 | """ 129 | assert self.num_nodes == len(self.prev_Ks) + 1 130 | 131 | hyp = -1 * torch.ones(self.batch_size, self.num_nodes).type(self.dtypeLong) 132 | for j in range(len(self.prev_Ks) - 1, -2, -1): 133 | hyp[:, j + 1] = self.next_nodes[j + 1].gather(1, k).view(1, self.batch_size) 134 | k = self.prev_Ks[j].gather(1, k) 135 | return hyp 136 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/google_tsp_reader.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | from scipy.spatial.distance import pdist, squareform 4 | from sklearn.utils import shuffle 5 | 6 | 7 | class DotDict(dict): 8 | """Wrapper around in-built dict class to access members through the dot operation. 9 | """ 10 | 11 | def __init__(self, **kwds): 12 | self.update(kwds) 13 | self.__dict__ = self 14 | 15 | 16 | class GoogleTSPReader(object): 17 | """Iterator that reads TSP dataset files and yields mini-batches. 18 | 19 | Format expected as in Vinyals et al., 2015: https://arxiv.org/abs/1506.03134, http://goo.gl/NDcOIG 20 | """ 21 | 22 | def __init__(self, num_nodes, batch_size, filepath): 23 | """ 24 | Args: 25 | num_nodes: Number of nodes in TSP tours 26 | batch_size: Batch size 27 | filepath: Path to dataset file (.txt file) 28 | """ 29 | self.num_nodes = num_nodes 30 | self.batch_size = batch_size 31 | self.filepath = filepath 32 | self.filedata = shuffle(open(filepath, "r").readlines()) # Always shuffle upon reading data 33 | self.max_iter = (len(self.filedata) // batch_size) 34 | 35 | def __iter__(self): 36 | for batch in range(self.max_iter): 37 | start_idx = batch * self.batch_size 38 | end_idx = (batch + 1) * self.batch_size 39 | yield self.process_batch(self.filedata[start_idx:end_idx]) 40 | 41 | def process_batch(self, lines): 42 | """Helper function to convert raw lines into a mini-batch as a DotDict. 43 | """ 44 | batch_edges = [] 45 | batch_edges_values = [] 46 | batch_edges_target = [] # Binary classification targets (0/1) 47 | batch_nodes = [] 48 | batch_nodes_target = [] # Multi-class classification targets (`num_nodes` classes) 49 | batch_nodes_coord = [] 50 | batch_tour_nodes = [] 51 | batch_tour_len = [] 52 | 53 | for line_num, line in enumerate(lines): 54 | 55 | line = line.split(" ") # Split into list 56 | 57 | # Compute signal on nodes 58 | nodes = np.ones(self.num_nodes) # All 1s for TSP... 59 | # Convert node coordinates to required format 60 | nodes_coord = [] 61 | for idx in range(0, 2 * self.num_nodes, 2): 62 | nodes_coord.append([float(line[idx]), float(line[idx + 1])]) 63 | 64 | # Compute distance matrix 65 | W_val = squareform(pdist(nodes_coord, metric='euclidean')) 66 | # Convert tour nodes to required format 67 | # Don't add final connection for tour/cycle 68 | 69 | tour_nodes = [int(node) - 1 for node in line[ line.index('output')+1:-1] ][ :-1 ] 70 | #tour_nodes = [int(node) - 1 for node in line[ line.index('output')+1:line.index('nearest')-1] ] 71 | #print('tour_nodes',tour_nodes) 72 | 73 | # Compute adjacency matrix 74 | W = np.ones((self.num_nodes, self.num_nodes)) 75 | np.fill_diagonal(W, 0) # No self connections 76 | 77 | # Compute node and edge representation of tour + tour_len 78 | tour_len = 0 79 | nodes_target = np.zeros(self.num_nodes) 80 | edges_target = np.zeros((self.num_nodes, self.num_nodes)) 81 | for idx in range(len(tour_nodes) - 1): 82 | i = tour_nodes[idx] 83 | j = tour_nodes[idx + 1] 84 | nodes_target[i] = idx # node targets: ordering of nodes in tour 85 | edges_target[i][j] = 1 86 | edges_target[j][i] = 1 87 | tour_len += W_val[i][j] 88 | # Add final connection of tour in edge target 89 | nodes_target[j] = len(tour_nodes) - 1 90 | edges_target[j][tour_nodes[0]] = 1 91 | edges_target[tour_nodes[0]][j] = 1 92 | tour_len += W_val[j][tour_nodes[0]] 93 | # Concatenate the data 94 | 95 | batch_edges.append(W) 96 | #batch_edges.append(edges_target) ###### TEST 97 | 98 | batch_edges_values.append(W_val) 99 | batch_edges_target.append(edges_target) 100 | batch_nodes.append(nodes) 101 | batch_nodes_target.append(nodes_target) 102 | batch_nodes_coord.append(nodes_coord) 103 | batch_tour_nodes.append(tour_nodes) 104 | batch_tour_len.append(tour_len) 105 | # From list to tensors as a DotDict 106 | batch = DotDict() 107 | batch.edges = np.stack(batch_edges, axis=0) 108 | batch.edges_values = np.stack(batch_edges_values, axis=0) 109 | batch.edges_target = np.stack(batch_edges_target, axis=0) 110 | batch.nodes = np.stack(batch_nodes, axis=0) 111 | batch.nodes_target = np.stack(batch_nodes_target, axis=0) 112 | batch.nodes_coord = np.stack(batch_nodes_coord, axis=0) 113 | batch.tour_nodes = np.stack(batch_tour_nodes, axis=0) 114 | batch.tour_len = np.stack(batch_tour_len, axis=0) 115 | return batch 116 | 117 | 118 | if __name__ == "__main__": 119 | num_nodes = 5 120 | batch_size = 50 121 | filepath = "./data/tsp5.txt" 122 | 123 | dataset = GoogleTSPReader(num_nodes, batch_size, filepath) 124 | print("Number of batches of size {}: {}".format(batch_size, dataset.max_iter)) 125 | 126 | t = time.time() 127 | batch = next(iter(dataset)) # Generate a batch of TSPs 128 | print("Batch generation took: {} sec".format(time.time() - t)) 129 | 130 | print("Batch shapes: ") 131 | print(batch.edges.shape) 132 | print(batch.edges_values.shape) 133 | print(batch.edges_target.shape) 134 | print(batch.nodes.shape) 135 | print(batch.nodes_target.shape) 136 | print(batch.nodes_coord.shape) 137 | print(batch.tour_nodes.shape) 138 | print(batch.tour_len.shape) 139 | 140 | print("Sample individual entry: ") 141 | idx = 0 142 | print(batch.edges[idx]) 143 | print(batch.edges_values[idx]) 144 | print(batch.edges_target[idx]) 145 | print(batch.nodes[idx]) 146 | print(batch.nodes_target[idx]) 147 | print(batch.nodes_coord[idx]) 148 | print(batch.tour_nodes[idx]) 149 | print(batch.tour_len[idx]) 150 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab04_TSP/utils/graph_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | import numpy as np 5 | 6 | 7 | def tour_nodes_to_W(nodes): 8 | """Helper function to convert ordered list of tour nodes to edge adjacency matrix. 9 | """ 10 | W = np.zeros((len(nodes), len(nodes))) 11 | for idx in range(len(nodes) - 1): 12 | i = int(nodes[idx]) 13 | j = int(nodes[idx + 1]) 14 | W[i][j] = 1 15 | W[j][i] = 1 16 | # Add final connection of tour in edge target 17 | W[j][int(nodes[0])] = 1 18 | W[int(nodes[0])][j] = 1 19 | return W 20 | 21 | 22 | def tour_nodes_to_tour_len(nodes, W_values): 23 | """Helper function to calculate tour length from ordered list of tour nodes. 24 | """ 25 | tour_len = 0 26 | for idx in range(len(nodes) - 1): 27 | i = nodes[idx] 28 | j = nodes[idx + 1] 29 | tour_len += W_values[i][j] 30 | # Add final connection of tour in edge target 31 | tour_len += W_values[j][nodes[0]] 32 | return tour_len 33 | 34 | 35 | def W_to_tour_len(W, W_values): 36 | """Helper function to calculate tour length from edge adjacency matrix. 37 | """ 38 | tour_len = 0 39 | for i in range(W.shape[0]): 40 | for j in range(W.shape[1]): 41 | if W[i][j] == 1: 42 | tour_len += W_values[i][j] 43 | tour_len /= 2 # Divide by 2 because adjacency matrices are symmetric 44 | return tour_len 45 | 46 | 47 | def is_valid_tour(nodes, num_nodes): 48 | """Sanity check: tour visits all nodes given. 49 | """ 50 | return sorted(nodes) == [i for i in range(num_nodes)] 51 | 52 | 53 | def mean_tour_len_edges(x_edges_values, y_pred_edges): 54 | """ 55 | Computes mean tour length for given batch prediction as edge adjacency matrices (for PyTorch tensors). 56 | 57 | Args: 58 | x_edges_values: Edge values (distance) matrix (batch_size, num_nodes, num_nodes) 59 | y_pred_edges: Edge predictions (batch_size, num_nodes, num_nodes, voc_edges) 60 | 61 | Returns: 62 | mean_tour_len: Mean tour length over batch 63 | """ 64 | y = F.softmax(y_pred_edges, dim=3) # B x V x V x voc_edges 65 | y = y.argmax(dim=3) # B x V x V 66 | # Divide by 2 because edges_values is symmetric 67 | tour_lens = (y.float() * x_edges_values.float()).sum(dim=1).sum(dim=1) / 2 68 | mean_tour_len = tour_lens.sum().to(dtype=torch.float).item() / tour_lens.numel() 69 | return mean_tour_len 70 | 71 | 72 | def mean_tour_len_nodes(x_edges_values, bs_nodes): 73 | """ 74 | Computes mean tour length for given batch prediction as node ordering after beamsearch (for Pytorch tensors). 75 | 76 | Args: 77 | x_edges_values: Edge values (distance) matrix (batch_size, num_nodes, num_nodes) 78 | bs_nodes: Node orderings (batch_size, num_nodes) 79 | 80 | Returns: 81 | mean_tour_len: Mean tour length over batch 82 | """ 83 | y = bs_nodes.cpu().numpy() 84 | W_val = x_edges_values.cpu().numpy() 85 | running_tour_len = 0 86 | for batch_idx in range(y.shape[0]): 87 | for y_idx in range(y[batch_idx].shape[0] - 1): 88 | i = y[batch_idx][y_idx] 89 | j = y[batch_idx][y_idx + 1] 90 | running_tour_len += W_val[batch_idx][i][j] 91 | running_tour_len += W_val[batch_idx][j][0] # Add final connection to tour/cycle 92 | return running_tour_len / y.shape[0] 93 | -------------------------------------------------------------------------------- /codes/labs_lecture14/lab05_DGL/data/artificial_dataset.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xbresson/CE7454_2019/b30c28096ef6a70ff98006ec9e71e349506c9212/codes/labs_lecture14/lab05_DGL/data/artificial_dataset.pickle -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | # Xavier Bresson 2 | 3 | name: deeplearn_course 4 | 5 | 6 | channels: 7 | 8 | - pytorch # DL 9 | - https://conda.anaconda.org/rdkit # Molecule 10 | - dglteam # DGL 11 | 12 | dependencies: 13 | 14 | # basic 15 | - python=3.6 16 | - numpy 17 | - mkl 18 | - ipython 19 | - jupyter 20 | - scipy 21 | - pandas 22 | - matplotlib 23 | - scikit-learn 24 | - networkx 25 | - seaborn 26 | 27 | - pytorch=1.1 # DL 28 | - rdkit # Molecule 29 | - dgl=0.3 # DGL 30 | 31 | - pip: 32 | - torchvision # DL 33 | - gym # RL 34 | - tensorboardx # TSP 35 | - fastprogress # TSP --------------------------------------------------------------------------------