├── PROJECT_5_Face_Generation ├── floyd_requirements.txt ├── .floydexpt ├── .DS_Store ├── .floydignore ├── problem_unittests.py ├── helper.py └── .ipynb_checkpoints │ └── dlnd_face_generation-checkpoint.ipynb ├── .DS_Store ├── README.md ├── PROJECT_2_Image_Classification ├── .DS_Store ├── helper.py └── problem_unittests.py ├── PROJECT_3_TV_Script_Generation ├── .DS_Store ├── helper.py ├── problem_unittests.py ├── dlnd_tv_script_generation.ipynb └── .ipynb_checkpoints │ └── dlnd_tv_script_generation-checkpoint.ipynb └── PROJECT_4_Language_Translation ├── helper.py └── problem_unittests.py /PROJECT_5_Face_Generation/floyd_requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm==4.11.2 2 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdbourke/Udacity_DLND_Projects/HEAD/.DS_Store -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/.floydexpt: -------------------------------------------------------------------------------- 1 | {"family_id": "dvmarS9c5Q5dTE3biUq88d", "name": "face_generation"} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Udacity DLND Projects 2 | All of my projects from the Udacity Deep Learning Foundations Nanodegree. 3 | -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdbourke/Udacity_DLND_Projects/HEAD/PROJECT_5_Face_Generation/.DS_Store -------------------------------------------------------------------------------- /PROJECT_2_Image_Classification/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdbourke/Udacity_DLND_Projects/HEAD/PROJECT_2_Image_Classification/.DS_Store -------------------------------------------------------------------------------- /PROJECT_3_TV_Script_Generation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrdbourke/Udacity_DLND_Projects/HEAD/PROJECT_3_TV_Script_Generation/.DS_Store -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/.floydignore: -------------------------------------------------------------------------------- 1 | 2 | # Directories and files to ignore when uploading code to floyd 3 | 4 | .git 5 | .eggs 6 | eggs 7 | lib 8 | lib64 9 | parts 10 | sdist 11 | var 12 | -------------------------------------------------------------------------------- /PROJECT_3_TV_Script_Generation/helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | 4 | 5 | def load_data(path): 6 | """ 7 | Load Dataset from File 8 | """ 9 | input_file = os.path.join(path) 10 | with open(input_file, "r") as f: 11 | data = f.read() 12 | 13 | return data 14 | 15 | 16 | def preprocess_and_save_data(dataset_path, token_lookup, create_lookup_tables): 17 | """ 18 | Preprocess Text Data 19 | """ 20 | text = load_data(dataset_path) 21 | 22 | # Ignore notice, since we don't use it for analysing the data 23 | text = text[81:] 24 | 25 | token_dict = token_lookup() 26 | for key, token in token_dict.items(): 27 | text = text.replace(key, ' {} '.format(token)) 28 | 29 | text = text.lower() 30 | text = text.split() 31 | 32 | vocab_to_int, int_to_vocab = create_lookup_tables(text) 33 | int_text = [vocab_to_int[word] for word in text] 34 | pickle.dump((int_text, vocab_to_int, int_to_vocab, token_dict), open('preprocess.p', 'wb')) 35 | 36 | 37 | def load_preprocess(): 38 | """ 39 | Load the Preprocessed Training data and return them in batches of or less 40 | """ 41 | return pickle.load(open('preprocess.p', mode='rb')) 42 | 43 | 44 | def save_params(params): 45 | """ 46 | Save parameters to file 47 | """ 48 | pickle.dump(params, open('params.p', 'wb')) 49 | 50 | 51 | def load_params(): 52 | """ 53 | Load parameters from file 54 | """ 55 | return pickle.load(open('params.p', mode='rb')) 56 | -------------------------------------------------------------------------------- /PROJECT_4_Language_Translation/helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import copy 4 | import numpy as np 5 | 6 | 7 | CODES = {'': 0, '': 1, '': 2, '': 3 } 8 | 9 | 10 | def load_data(path): 11 | """ 12 | Load Dataset from File 13 | """ 14 | input_file = os.path.join(path) 15 | with open(input_file, 'r', encoding='utf-8') as f: 16 | return f.read() 17 | 18 | 19 | def preprocess_and_save_data(source_path, target_path, text_to_ids): 20 | """ 21 | Preprocess Text Data. Save to to file. 22 | """ 23 | # Preprocess 24 | source_text = load_data(source_path) 25 | target_text = load_data(target_path) 26 | 27 | source_text = source_text.lower() 28 | target_text = target_text.lower() 29 | 30 | source_vocab_to_int, source_int_to_vocab = create_lookup_tables(source_text) 31 | target_vocab_to_int, target_int_to_vocab = create_lookup_tables(target_text) 32 | 33 | source_text, target_text = text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int) 34 | 35 | # Save Data 36 | with open('preprocess.p', 'wb') as out_file: 37 | pickle.dump(( 38 | (source_text, target_text), 39 | (source_vocab_to_int, target_vocab_to_int), 40 | (source_int_to_vocab, target_int_to_vocab)), out_file) 41 | 42 | 43 | def load_preprocess(): 44 | """ 45 | Load the Preprocessed Training data and return them in batches of or less 46 | """ 47 | with open('preprocess.p', mode='rb') as in_file: 48 | return pickle.load(in_file) 49 | 50 | 51 | def create_lookup_tables(text): 52 | """ 53 | Create lookup tables for vocabulary 54 | """ 55 | vocab = set(text.split()) 56 | vocab_to_int = copy.copy(CODES) 57 | 58 | for v_i, v in enumerate(vocab, len(CODES)): 59 | vocab_to_int[v] = v_i 60 | 61 | int_to_vocab = {v_i: v for v, v_i in vocab_to_int.items()} 62 | 63 | return vocab_to_int, int_to_vocab 64 | 65 | 66 | def save_params(params): 67 | """ 68 | Save parameters to file 69 | """ 70 | with open('params.p', 'wb') as out_file: 71 | pickle.dump(params, out_file) 72 | 73 | 74 | def load_params(): 75 | """ 76 | Load parameters from file 77 | """ 78 | with open('params.p', mode='rb') as in_file: 79 | return pickle.load(in_file) 80 | 81 | 82 | def batch_data(source, target, batch_size): 83 | """ 84 | Batch source and target together 85 | """ 86 | for batch_i in range(0, len(source)//batch_size): 87 | start_i = batch_i * batch_size 88 | source_batch = source[start_i:start_i + batch_size] 89 | target_batch = target[start_i:start_i + batch_size] 90 | yield np.array(pad_sentence_batch(source_batch)), np.array(pad_sentence_batch(target_batch)) 91 | 92 | 93 | def pad_sentence_batch(sentence_batch): 94 | """ 95 | Pad sentence with id 96 | """ 97 | max_sentence = max([len(sentence) for sentence in sentence_batch]) 98 | return [sentence + [CODES['']] * (max_sentence - len(sentence)) 99 | for sentence in sentence_batch] 100 | -------------------------------------------------------------------------------- /PROJECT_2_Image_Classification/helper.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | from sklearn.preprocessing import LabelBinarizer 5 | 6 | 7 | def _load_label_names(): 8 | """ 9 | Load the label names from file 10 | """ 11 | return ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] 12 | 13 | 14 | def load_cfar10_batch(cifar10_dataset_folder_path, batch_id): 15 | """ 16 | Load a batch of the dataset 17 | """ 18 | with open(cifar10_dataset_folder_path + '/data_batch_' + str(batch_id), mode='rb') as file: 19 | batch = pickle.load(file, encoding='latin1') 20 | 21 | features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1) 22 | labels = batch['labels'] 23 | 24 | return features, labels 25 | 26 | 27 | def display_stats(cifar10_dataset_folder_path, batch_id, sample_id): 28 | """ 29 | Display Stats of the the dataset 30 | """ 31 | batch_ids = list(range(1, 6)) 32 | 33 | if batch_id not in batch_ids: 34 | print('Batch Id out of Range. Possible Batch Ids: {}'.format(batch_ids)) 35 | return None 36 | 37 | features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_id) 38 | 39 | if not (0 <= sample_id < len(features)): 40 | print('{} samples in batch {}. {} is out of range.'.format(len(features), batch_id, sample_id)) 41 | return None 42 | 43 | print('\nStats of batch {}:'.format(batch_id)) 44 | print('Samples: {}'.format(len(features))) 45 | print('Label Counts: {}'.format(dict(zip(*np.unique(labels, return_counts=True))))) 46 | print('First 20 Labels: {}'.format(labels[:20])) 47 | 48 | sample_image = features[sample_id] 49 | sample_label = labels[sample_id] 50 | label_names = _load_label_names() 51 | 52 | print('\nExample of Image {}:'.format(sample_id)) 53 | print('Image - Min Value: {} Max Value: {}'.format(sample_image.min(), sample_image.max())) 54 | print('Image - Shape: {}'.format(sample_image.shape)) 55 | print('Label - Label Id: {} Name: {}'.format(sample_label, label_names[sample_label])) 56 | plt.axis('off') 57 | plt.imshow(sample_image) 58 | 59 | 60 | def _preprocess_and_save(normalize, one_hot_encode, features, labels, filename): 61 | """ 62 | Preprocess data and save it to file 63 | """ 64 | features = normalize(features) 65 | labels = one_hot_encode(labels) 66 | 67 | pickle.dump((features, labels), open(filename, 'wb')) 68 | 69 | 70 | def preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode): 71 | """ 72 | Preprocess Training and Validation Data 73 | """ 74 | n_batches = 5 75 | valid_features = [] 76 | valid_labels = [] 77 | 78 | for batch_i in range(1, n_batches + 1): 79 | features, labels = load_cfar10_batch(cifar10_dataset_folder_path, batch_i) 80 | validation_count = int(len(features) * 0.1) 81 | 82 | # Prprocess and save a batch of training data 83 | _preprocess_and_save( 84 | normalize, 85 | one_hot_encode, 86 | features[:-validation_count], 87 | labels[:-validation_count], 88 | 'preprocess_batch_' + str(batch_i) + '.p') 89 | 90 | # Use a portion of training batch for validation 91 | valid_features.extend(features[-validation_count:]) 92 | valid_labels.extend(labels[-validation_count:]) 93 | 94 | # Preprocess and Save all validation data 95 | _preprocess_and_save( 96 | normalize, 97 | one_hot_encode, 98 | np.array(valid_features), 99 | np.array(valid_labels), 100 | 'preprocess_validation.p') 101 | 102 | with open(cifar10_dataset_folder_path + '/test_batch', mode='rb') as file: 103 | batch = pickle.load(file, encoding='latin1') 104 | 105 | # load the training data 106 | test_features = batch['data'].reshape((len(batch['data']), 3, 32, 32)).transpose(0, 2, 3, 1) 107 | test_labels = batch['labels'] 108 | 109 | # Preprocess and Save all training data 110 | _preprocess_and_save( 111 | normalize, 112 | one_hot_encode, 113 | np.array(test_features), 114 | np.array(test_labels), 115 | 'preprocess_training.p') 116 | 117 | 118 | def batch_features_labels(features, labels, batch_size): 119 | """ 120 | Split features and labels into batches 121 | """ 122 | for start in range(0, len(features), batch_size): 123 | end = min(start + batch_size, len(features)) 124 | yield features[start:end], labels[start:end] 125 | 126 | 127 | def load_preprocess_training_batch(batch_id, batch_size): 128 | """ 129 | Load the Preprocessed Training data and return them in batches of or less 130 | """ 131 | filename = 'preprocess_batch_' + str(batch_id) + '.p' 132 | features, labels = pickle.load(open(filename, mode='rb')) 133 | 134 | # Return the training data in batches of size or less 135 | return batch_features_labels(features, labels, batch_size) 136 | 137 | 138 | def display_image_predictions(features, labels, predictions): 139 | n_classes = 10 140 | label_names = _load_label_names() 141 | label_binarizer = LabelBinarizer() 142 | label_binarizer.fit(range(n_classes)) 143 | label_ids = label_binarizer.inverse_transform(np.array(labels)) 144 | 145 | fig, axies = plt.subplots(nrows=4, ncols=2) 146 | fig.tight_layout() 147 | fig.suptitle('Softmax Predictions', fontsize=20, y=1.1) 148 | 149 | n_predictions = 3 150 | margin = 0.05 151 | ind = np.arange(n_predictions) 152 | width = (1. - 2. * margin) / n_predictions 153 | 154 | for image_i, (feature, label_id, pred_indicies, pred_values) in enumerate(zip(features, label_ids, predictions.indices, predictions.values)): 155 | pred_names = [label_names[pred_i] for pred_i in pred_indicies] 156 | correct_name = label_names[label_id] 157 | 158 | axies[image_i][0].imshow(feature) 159 | axies[image_i][0].set_title(correct_name) 160 | axies[image_i][0].set_axis_off() 161 | 162 | axies[image_i][1].barh(ind + margin, pred_values[::-1], width) 163 | axies[image_i][1].set_yticks(ind + margin) 164 | axies[image_i][1].set_yticklabels(pred_names[::-1]) 165 | axies[image_i][1].set_xticks([0, 0.5, 1.0]) 166 | -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/problem_unittests.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from unittest import mock 3 | import tensorflow as tf 4 | 5 | 6 | def test_safe(func): 7 | """ 8 | Isolate tests 9 | """ 10 | def func_wrapper(*args): 11 | with tf.Graph().as_default(): 12 | result = func(*args) 13 | print('Tests Passed') 14 | return result 15 | 16 | return func_wrapper 17 | 18 | 19 | def _assert_tensor_shape(tensor, shape, display_name): 20 | assert tf.assert_rank(tensor, len(shape), message='{} has wrong rank'.format(display_name)) 21 | 22 | tensor_shape = tensor.get_shape().as_list() if len(shape) else [] 23 | 24 | wrong_dimension = [ten_dim for ten_dim, cor_dim in zip(tensor_shape, shape) 25 | if cor_dim is not None and ten_dim != cor_dim] 26 | assert not wrong_dimension, \ 27 | '{} has wrong shape. Found {}'.format(display_name, tensor_shape) 28 | 29 | 30 | def _check_input(tensor, shape, display_name, tf_name=None): 31 | assert tensor.op.type == 'Placeholder', \ 32 | '{} is not a Placeholder.'.format(display_name) 33 | 34 | _assert_tensor_shape(tensor, shape, 'Real Input') 35 | 36 | if tf_name: 37 | assert tensor.name == tf_name, \ 38 | '{} has bad name. Found name {}'.format(display_name, tensor.name) 39 | 40 | 41 | class TmpMock(): 42 | """ 43 | Mock a attribute. Restore attribute when exiting scope. 44 | """ 45 | def __init__(self, module, attrib_name): 46 | self.original_attrib = deepcopy(getattr(module, attrib_name)) 47 | setattr(module, attrib_name, mock.MagicMock()) 48 | self.module = module 49 | self.attrib_name = attrib_name 50 | 51 | def __enter__(self): 52 | return getattr(self.module, self.attrib_name) 53 | 54 | def __exit__(self, type, value, traceback): 55 | setattr(self.module, self.attrib_name, self.original_attrib) 56 | 57 | 58 | @test_safe 59 | def test_model_inputs(model_inputs): 60 | image_width = 28 61 | image_height = 28 62 | image_channels = 3 63 | z_dim = 100 64 | input_real, input_z, learn_rate = model_inputs(image_width, image_height, image_channels, z_dim) 65 | 66 | _check_input(input_real, [None, image_width, image_height, image_channels], 'Real Input') 67 | _check_input(input_z, [None, z_dim], 'Z Input') 68 | _check_input(learn_rate, [], 'Learning Rate') 69 | 70 | 71 | @test_safe 72 | def test_discriminator(discriminator, tf_module): 73 | with TmpMock(tf_module, 'variable_scope') as mock_variable_scope: 74 | image = tf.placeholder(tf.float32, [None, 28, 28, 3]) 75 | 76 | output, logits = discriminator(image) 77 | _assert_tensor_shape(output, [None, 1], 'Discriminator Training(reuse=false) output') 78 | _assert_tensor_shape(logits, [None, 1], 'Discriminator Training(reuse=false) Logits') 79 | assert mock_variable_scope.called,\ 80 | 'tf.variable_scope not called in Discriminator Training(reuse=false)' 81 | assert mock_variable_scope.call_args == mock.call('discriminator', reuse=False), \ 82 | 'tf.variable_scope called with wrong arguments in Discriminator Training(reuse=false)' 83 | 84 | mock_variable_scope.reset_mock() 85 | 86 | output_reuse, logits_reuse = discriminator(image, True) 87 | _assert_tensor_shape(output_reuse, [None, 1], 'Discriminator Inference(reuse=True) output') 88 | _assert_tensor_shape(logits_reuse, [None, 1], 'Discriminator Inference(reuse=True) Logits') 89 | assert mock_variable_scope.called, \ 90 | 'tf.variable_scope not called in Discriminator Inference(reuse=True)' 91 | assert mock_variable_scope.call_args == mock.call('discriminator', reuse=True), \ 92 | 'tf.variable_scope called with wrong arguments in Discriminator Inference(reuse=True)' 93 | 94 | 95 | @test_safe 96 | def test_generator(generator, tf_module): 97 | with TmpMock(tf_module, 'variable_scope') as mock_variable_scope: 98 | z = tf.placeholder(tf.float32, [None, 100]) 99 | out_channel_dim = 5 100 | 101 | output = generator(z, out_channel_dim) 102 | _assert_tensor_shape(output, [None, 28, 28, out_channel_dim], 'Generator output (is_train=True)') 103 | assert mock_variable_scope.called, \ 104 | 'tf.variable_scope not called in Generator Training(reuse=false)' 105 | assert mock_variable_scope.call_args == mock.call('generator', reuse=False), \ 106 | 'tf.variable_scope called with wrong arguments in Generator Training(reuse=false)' 107 | 108 | mock_variable_scope.reset_mock() 109 | output = generator(z, out_channel_dim, False) 110 | _assert_tensor_shape(output, [None, 28, 28, out_channel_dim], 'Generator output (is_train=False)') 111 | assert mock_variable_scope.called, \ 112 | 'tf.variable_scope not called in Generator Inference(reuse=True)' 113 | assert mock_variable_scope.call_args == mock.call('generator', reuse=True), \ 114 | 'tf.variable_scope called with wrong arguments in Generator Inference(reuse=True)' 115 | 116 | 117 | @test_safe 118 | def test_model_loss(model_loss): 119 | out_channel_dim = 4 120 | input_real = tf.placeholder(tf.float32, [None, 28, 28, out_channel_dim]) 121 | input_z = tf.placeholder(tf.float32, [None, 100]) 122 | 123 | d_loss, g_loss = model_loss(input_real, input_z, out_channel_dim) 124 | 125 | _assert_tensor_shape(d_loss, [], 'Discriminator Loss') 126 | _assert_tensor_shape(d_loss, [], 'Generator Loss') 127 | 128 | 129 | @test_safe 130 | def test_model_opt(model_opt, tf_module): 131 | with TmpMock(tf_module, 'trainable_variables') as mock_trainable_variables: 132 | with tf.variable_scope('discriminator'): 133 | discriminator_logits = tf.Variable(tf.zeros([3, 3])) 134 | with tf.variable_scope('generator'): 135 | generator_logits = tf.Variable(tf.zeros([3, 3])) 136 | 137 | mock_trainable_variables.return_value = [discriminator_logits, generator_logits] 138 | d_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( 139 | logits=discriminator_logits, 140 | labels=[[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])) 141 | g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( 142 | logits=generator_logits, 143 | labels=[[0.0, 0.0, 1.0], [0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])) 144 | learning_rate = 0.001 145 | beta1 = 0.9 146 | 147 | d_train_opt, g_train_opt = model_opt(d_loss, g_loss, learning_rate, beta1) 148 | assert mock_trainable_variables.called,\ 149 | 'tf.mock_trainable_variables not called' 150 | 151 | 152 | -------------------------------------------------------------------------------- /PROJECT_2_Image_Classification/problem_unittests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | import random 5 | from unittest.mock import MagicMock 6 | 7 | 8 | def _print_success_message(): 9 | return print('Tests Passed') 10 | 11 | 12 | def test_folder_path(cifar10_dataset_folder_path): 13 | assert cifar10_dataset_folder_path is not None,\ 14 | 'Cifar-10 data folder not set.' 15 | assert cifar10_dataset_folder_path[-1] != '/',\ 16 | 'The "/" shouldn\'t be added to the end of the path.' 17 | assert os.path.exists(cifar10_dataset_folder_path),\ 18 | 'Path not found.' 19 | assert os.path.isdir(cifar10_dataset_folder_path),\ 20 | '{} is not a folder.'.format(os.path.basename(cifar10_dataset_folder_path)) 21 | 22 | train_files = [cifar10_dataset_folder_path + '/data_batch_' + str(batch_id) for batch_id in range(1, 6)] 23 | other_files = [cifar10_dataset_folder_path + '/batches.meta', cifar10_dataset_folder_path + '/test_batch'] 24 | missing_files = [path for path in train_files + other_files if not os.path.exists(path)] 25 | 26 | assert not missing_files,\ 27 | 'Missing files in directory: {}'.format(missing_files) 28 | 29 | print('All files found!') 30 | 31 | 32 | def test_normalize(normalize): 33 | test_shape = (np.random.choice(range(1000)), 32, 32, 3) 34 | test_numbers = np.random.choice(range(256), test_shape) 35 | normalize_out = normalize(test_numbers) 36 | 37 | assert type(normalize_out).__module__ == np.__name__,\ 38 | 'Not Numpy Object' 39 | 40 | assert normalize_out.shape == test_shape,\ 41 | 'Incorrect Shape. {} shape found'.format(normalize_out.shape) 42 | 43 | assert normalize_out.max() <= 1 and normalize_out.min() >= 0,\ 44 | 'Incorect Range. {} to {} found'.format(normalize_out.min(), normalize_out.max()) 45 | 46 | _print_success_message() 47 | 48 | 49 | def test_one_hot_encode(one_hot_encode): 50 | test_shape = np.random.choice(range(1000)) 51 | test_numbers = np.random.choice(range(10), test_shape) 52 | one_hot_out = one_hot_encode(test_numbers) 53 | 54 | assert type(one_hot_out).__module__ == np.__name__,\ 55 | 'Not Numpy Object' 56 | 57 | assert one_hot_out.shape == (test_shape, 10),\ 58 | 'Incorrect Shape. {} shape found'.format(one_hot_out.shape) 59 | 60 | n_encode_tests = 5 61 | test_pairs = list(zip(test_numbers, one_hot_out)) 62 | test_indices = np.random.choice(len(test_numbers), n_encode_tests) 63 | labels = [test_pairs[test_i][0] for test_i in test_indices] 64 | enc_labels = np.array([test_pairs[test_i][1] for test_i in test_indices]) 65 | new_enc_labels = one_hot_encode(labels) 66 | 67 | assert np.array_equal(enc_labels, new_enc_labels),\ 68 | 'Encodings returned different results for the same numbers.\n' \ 69 | 'For the first call it returned:\n' \ 70 | '{}\n' \ 71 | 'For the second call it returned\n' \ 72 | '{}\n' \ 73 | 'Make sure you save the map of labels to encodings outside of the function.'.format(enc_labels, new_enc_labels) 74 | 75 | _print_success_message() 76 | 77 | 78 | def test_nn_image_inputs(neural_net_image_input): 79 | image_shape = (32, 32, 3) 80 | nn_inputs_out_x = neural_net_image_input(image_shape) 81 | 82 | assert nn_inputs_out_x.get_shape().as_list() == [None, image_shape[0], image_shape[1], image_shape[2]],\ 83 | 'Incorrect Image Shape. Found {} shape'.format(nn_inputs_out_x.get_shape().as_list()) 84 | 85 | assert nn_inputs_out_x.op.type == 'Placeholder',\ 86 | 'Incorrect Image Type. Found {} type'.format(nn_inputs_out_x.op.type) 87 | 88 | assert nn_inputs_out_x.name == 'x:0', \ 89 | 'Incorrect Name. Found {}'.format(nn_inputs_out_x.name) 90 | 91 | print('Image Input Tests Passed.') 92 | 93 | 94 | def test_nn_label_inputs(neural_net_label_input): 95 | n_classes = 10 96 | nn_inputs_out_y = neural_net_label_input(n_classes) 97 | 98 | assert nn_inputs_out_y.get_shape().as_list() == [None, n_classes],\ 99 | 'Incorrect Label Shape. Found {} shape'.format(nn_inputs_out_y.get_shape().as_list()) 100 | 101 | assert nn_inputs_out_y.op.type == 'Placeholder',\ 102 | 'Incorrect Label Type. Found {} type'.format(nn_inputs_out_y.op.type) 103 | 104 | assert nn_inputs_out_y.name == 'y:0', \ 105 | 'Incorrect Name. Found {}'.format(nn_inputs_out_y.name) 106 | 107 | print('Label Input Tests Passed.') 108 | 109 | 110 | def test_nn_keep_prob_inputs(neural_net_keep_prob_input): 111 | nn_inputs_out_k = neural_net_keep_prob_input() 112 | 113 | assert nn_inputs_out_k.get_shape().ndims is None,\ 114 | 'Too many dimensions found for keep prob. Found {} dimensions. It should be a scalar (0-Dimension Tensor).'.format(nn_inputs_out_k.get_shape().ndims) 115 | 116 | assert nn_inputs_out_k.op.type == 'Placeholder',\ 117 | 'Incorrect keep prob Type. Found {} type'.format(nn_inputs_out_k.op.type) 118 | 119 | assert nn_inputs_out_k.name == 'keep_prob:0', \ 120 | 'Incorrect Name. Found {}'.format(nn_inputs_out_k.name) 121 | 122 | print('Keep Prob Tests Passed.') 123 | 124 | 125 | def test_con_pool(conv2d_maxpool): 126 | test_x = tf.placeholder(tf.float32, [None, 32, 32, 5]) 127 | test_num_outputs = 10 128 | test_con_k = (2, 2) 129 | test_con_s = (4, 4) 130 | test_pool_k = (2, 2) 131 | test_pool_s = (2, 2) 132 | 133 | conv2d_maxpool_out = conv2d_maxpool(test_x, test_num_outputs, test_con_k, test_con_s, test_pool_k, test_pool_s) 134 | 135 | assert conv2d_maxpool_out.get_shape().as_list() == [None, 4, 4, 10],\ 136 | 'Incorrect Shape. Found {} shape'.format(conv2d_maxpool_out.get_shape().as_list()) 137 | 138 | _print_success_message() 139 | 140 | 141 | def test_flatten(flatten): 142 | test_x = tf.placeholder(tf.float32, [None, 10, 30, 6]) 143 | flat_out = flatten(test_x) 144 | 145 | assert flat_out.get_shape().as_list() == [None, 10*30*6],\ 146 | 'Incorrect Shape. Found {} shape'.format(flat_out.get_shape().as_list()) 147 | 148 | _print_success_message() 149 | 150 | 151 | def test_fully_conn(fully_conn): 152 | test_x = tf.placeholder(tf.float32, [None, 128]) 153 | test_num_outputs = 40 154 | 155 | fc_out = fully_conn(test_x, test_num_outputs) 156 | 157 | assert fc_out.get_shape().as_list() == [None, 40],\ 158 | 'Incorrect Shape. Found {} shape'.format(fc_out.get_shape().as_list()) 159 | 160 | _print_success_message() 161 | 162 | 163 | def test_output(output): 164 | test_x = tf.placeholder(tf.float32, [None, 128]) 165 | test_num_outputs = 40 166 | 167 | output_out = output(test_x, test_num_outputs) 168 | 169 | assert output_out.get_shape().as_list() == [None, 40],\ 170 | 'Incorrect Shape. Found {} shape'.format(output_out.get_shape().as_list()) 171 | 172 | _print_success_message() 173 | 174 | 175 | def test_conv_net(conv_net): 176 | test_x = tf.placeholder(tf.float32, [None, 32, 32, 3]) 177 | test_k = tf.placeholder(tf.float32) 178 | 179 | logits_out = conv_net(test_x, test_k) 180 | 181 | assert logits_out.get_shape().as_list() == [None, 10],\ 182 | 'Incorrect Model Output. Found {}'.format(logits_out.get_shape().as_list()) 183 | 184 | print('Neural Network Built!') 185 | 186 | 187 | def test_train_nn(train_neural_network): 188 | mock_session = tf.Session() 189 | test_x = np.random.rand(128, 32, 32, 3) 190 | test_y = np.random.rand(128, 10) 191 | test_k = np.random.rand(1) 192 | test_optimizer = tf.train.AdamOptimizer() 193 | 194 | mock_session.run = MagicMock() 195 | train_neural_network(mock_session, test_optimizer, test_k, test_x, test_y) 196 | 197 | assert mock_session.run.called, 'Session not used' 198 | 199 | _print_success_message() 200 | -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/helper.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | import hashlib 4 | from urllib.request import urlretrieve 5 | import zipfile 6 | import gzip 7 | import shutil 8 | 9 | import numpy as np 10 | from PIL import Image 11 | from tqdm import tqdm 12 | 13 | 14 | def _read32(bytestream): 15 | """ 16 | Read 32-bit integer from bytesteam 17 | :param bytestream: A bytestream 18 | :return: 32-bit integer 19 | """ 20 | dt = np.dtype(np.uint32).newbyteorder('>') 21 | return np.frombuffer(bytestream.read(4), dtype=dt)[0] 22 | 23 | 24 | def _unzip(save_path, _, database_name, data_path): 25 | """ 26 | Unzip wrapper with the same interface as _ungzip 27 | :param save_path: The path of the gzip files 28 | :param database_name: Name of database 29 | :param data_path: Path to extract to 30 | :param _: HACK - Used to have to same interface as _ungzip 31 | """ 32 | print('Extracting {}...'.format(database_name)) 33 | with zipfile.ZipFile(save_path) as zf: 34 | zf.extractall(data_path) 35 | 36 | 37 | def _ungzip(save_path, extract_path, database_name, _): 38 | """ 39 | Unzip a gzip file and extract it to extract_path 40 | :param save_path: The path of the gzip files 41 | :param extract_path: The location to extract the data to 42 | :param database_name: Name of database 43 | :param _: HACK - Used to have to same interface as _unzip 44 | """ 45 | # Get data from save_path 46 | with open(save_path, 'rb') as f: 47 | with gzip.GzipFile(fileobj=f) as bytestream: 48 | magic = _read32(bytestream) 49 | if magic != 2051: 50 | raise ValueError('Invalid magic number {} in file: {}'.format(magic, f.name)) 51 | num_images = _read32(bytestream) 52 | rows = _read32(bytestream) 53 | cols = _read32(bytestream) 54 | buf = bytestream.read(rows * cols * num_images) 55 | data = np.frombuffer(buf, dtype=np.uint8) 56 | data = data.reshape(num_images, rows, cols) 57 | 58 | # Save data to extract_path 59 | for image_i, image in enumerate( 60 | tqdm(data, unit='File', unit_scale=True, miniters=1, desc='Extracting {}'.format(database_name))): 61 | Image.fromarray(image, 'L').save(os.path.join(extract_path, 'image_{}.jpg'.format(image_i))) 62 | 63 | 64 | def get_image(image_path, width, height, mode): 65 | """ 66 | Read image from image_path 67 | :param image_path: Path of image 68 | :param width: Width of image 69 | :param height: Height of image 70 | :param mode: Mode of image 71 | :return: Image data 72 | """ 73 | image = Image.open(image_path) 74 | 75 | if image.size != (width, height): # HACK - Check if image is from the CELEBA dataset 76 | # Remove most pixels that aren't part of a face 77 | face_width = face_height = 108 78 | j = (image.size[0] - face_width) // 2 79 | i = (image.size[1] - face_height) // 2 80 | image = image.crop([j, i, j + face_width, i + face_height]) 81 | image = image.resize([width, height], Image.BILINEAR) 82 | 83 | return np.array(image.convert(mode)) 84 | 85 | 86 | def get_batch(image_files, width, height, mode): 87 | data_batch = np.array( 88 | [get_image(sample_file, width, height, mode) for sample_file in image_files]).astype(np.float32) 89 | 90 | # Make sure the images are in 4 dimensions 91 | if len(data_batch.shape) < 4: 92 | data_batch = data_batch.reshape(data_batch.shape + (1,)) 93 | 94 | return data_batch 95 | 96 | 97 | def images_square_grid(images, mode): 98 | """ 99 | Save images as a square grid 100 | :param images: Images to be used for the grid 101 | :param mode: The mode to use for images 102 | :return: Image of images in a square grid 103 | """ 104 | # Get maximum size for square grid of images 105 | save_size = math.floor(np.sqrt(images.shape[0])) 106 | 107 | # Scale to 0-255 108 | images = (((images - images.min()) * 255) / (images.max() - images.min())).astype(np.uint8) 109 | 110 | # Put images in a square arrangement 111 | images_in_square = np.reshape( 112 | images[:save_size*save_size], 113 | (save_size, save_size, images.shape[1], images.shape[2], images.shape[3])) 114 | if mode == 'L': 115 | images_in_square = np.squeeze(images_in_square, 4) 116 | 117 | # Combine images to grid image 118 | new_im = Image.new(mode, (images.shape[1] * save_size, images.shape[2] * save_size)) 119 | for col_i, col_images in enumerate(images_in_square): 120 | for image_i, image in enumerate(col_images): 121 | im = Image.fromarray(image, mode) 122 | new_im.paste(im, (col_i * images.shape[1], image_i * images.shape[2])) 123 | 124 | return new_im 125 | 126 | 127 | def download_extract(database_name, data_path): 128 | """ 129 | Download and extract database 130 | :param database_name: Database name 131 | """ 132 | DATASET_CELEBA_NAME = 'celeba' 133 | DATASET_MNIST_NAME = 'mnist' 134 | 135 | if database_name == DATASET_CELEBA_NAME: 136 | url = 'https://s3-us-west-1.amazonaws.com/udacity-dlnfd/datasets/celeba.zip' 137 | hash_code = '00d2c5bc6d35e252742224ab0c1e8fcb' 138 | extract_path = os.path.join(data_path, 'img_align_celeba') 139 | save_path = os.path.join(data_path, 'celeba.zip') 140 | extract_fn = _unzip 141 | elif database_name == DATASET_MNIST_NAME: 142 | url = 'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz' 143 | hash_code = 'f68b3c2dcbeaaa9fbdd348bbdeb94873' 144 | extract_path = os.path.join(data_path, 'mnist') 145 | save_path = os.path.join(data_path, 'train-images-idx3-ubyte.gz') 146 | extract_fn = _ungzip 147 | 148 | if os.path.exists(extract_path): 149 | print('Found {} Data'.format(database_name)) 150 | return 151 | 152 | if not os.path.exists(data_path): 153 | os.makedirs(data_path) 154 | 155 | if not os.path.exists(save_path): 156 | with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Downloading {}'.format(database_name)) as pbar: 157 | urlretrieve( 158 | url, 159 | save_path, 160 | pbar.hook) 161 | 162 | assert hashlib.md5(open(save_path, 'rb').read()).hexdigest() == hash_code, \ 163 | '{} file is corrupted. Remove the file and try again.'.format(save_path) 164 | 165 | os.makedirs(extract_path) 166 | try: 167 | extract_fn(save_path, extract_path, database_name, data_path) 168 | except Exception as err: 169 | shutil.rmtree(extract_path) # Remove extraction folder if there is an error 170 | raise err 171 | 172 | # Remove compressed data 173 | os.remove(save_path) 174 | 175 | 176 | class Dataset(object): 177 | """ 178 | Dataset 179 | """ 180 | def __init__(self, dataset_name, data_files): 181 | """ 182 | Initalize the class 183 | :param dataset_name: Database name 184 | :param data_files: List of files in the database 185 | """ 186 | DATASET_CELEBA_NAME = 'celeba' 187 | DATASET_MNIST_NAME = 'mnist' 188 | IMAGE_WIDTH = 28 189 | IMAGE_HEIGHT = 28 190 | 191 | if dataset_name == DATASET_CELEBA_NAME: 192 | self.image_mode = 'RGB' 193 | image_channels = 3 194 | 195 | elif dataset_name == DATASET_MNIST_NAME: 196 | self.image_mode = 'L' 197 | image_channels = 1 198 | 199 | self.data_files = data_files 200 | self.shape = len(data_files), IMAGE_WIDTH, IMAGE_HEIGHT, image_channels 201 | 202 | def get_batches(self, batch_size): 203 | """ 204 | Generate batches 205 | :param batch_size: Batch Size 206 | :return: Batches of data 207 | """ 208 | IMAGE_MAX_VALUE = 255 209 | 210 | current_index = 0 211 | while current_index + batch_size <= self.shape[0]: 212 | data_batch = get_batch( 213 | self.data_files[current_index:current_index + batch_size], 214 | *self.shape[1:3], 215 | self.image_mode) 216 | 217 | current_index += batch_size 218 | 219 | yield data_batch / IMAGE_MAX_VALUE - 0.5 220 | 221 | 222 | class DLProgress(tqdm): 223 | """ 224 | Handle Progress Bar while Downloading 225 | """ 226 | last_block = 0 227 | 228 | def hook(self, block_num=1, block_size=1, total_size=None): 229 | """ 230 | A hook function that will be called once on establishment of the network connection and 231 | once after each block read thereafter. 232 | :param block_num: A count of blocks transferred so far 233 | :param block_size: Block size in bytes 234 | :param total_size: The total size of the file. This may be -1 on older FTP servers which do not return 235 | a file size in response to a retrieval request. 236 | """ 237 | self.total = total_size 238 | self.update((block_num - self.last_block) * block_size) 239 | self.last_block = block_num 240 | -------------------------------------------------------------------------------- /PROJECT_4_Language_Translation/problem_unittests.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import itertools 4 | import collections 5 | import helper 6 | 7 | 8 | def _print_success_message(): 9 | print('Tests Passed') 10 | 11 | 12 | def test_text_to_ids(text_to_ids): 13 | test_source_text = 'new jersey is sometimes quiet during autumn , and it is snowy in april .\nthe united states is usually chilly during july , and it is usually freezing in november .\ncalifornia is usually quiet during march , and it is usually hot in june .\nthe united states is sometimes mild during june , and it is cold in september .' 14 | test_target_text = 'new jersey est parfois calme pendant l\' automne , et il est neigeux en avril .\nles états-unis est généralement froid en juillet , et il gèle habituellement en novembre .\ncalifornia est généralement calme en mars , et il est généralement chaud en juin .\nles états-unis est parfois légère en juin , et il fait froid en septembre .' 15 | 16 | test_source_text = test_source_text.lower() 17 | test_target_text = test_target_text.lower() 18 | 19 | source_vocab_to_int, source_int_to_vocab = helper.create_lookup_tables(test_source_text) 20 | target_vocab_to_int, target_int_to_vocab = helper.create_lookup_tables(test_target_text) 21 | 22 | test_source_id_seq, test_target_id_seq = text_to_ids(test_source_text, test_target_text, source_vocab_to_int, target_vocab_to_int) 23 | 24 | assert len(test_source_id_seq) == len(test_source_text.split('\n')),\ 25 | 'source_id_text has wrong length, it should be {}.'.format(len(test_source_text.split('\n'))) 26 | assert len(test_target_id_seq) == len(test_target_text.split('\n')), \ 27 | 'target_id_text has wrong length, it should be {}.'.format(len(test_target_text.split('\n'))) 28 | 29 | target_not_iter = [type(x) for x in test_source_id_seq if not isinstance(x, collections.Iterable)] 30 | assert not target_not_iter,\ 31 | 'Element in source_id_text is not iteratable. Found type {}'.format(target_not_iter[0]) 32 | target_not_iter = [type(x) for x in test_target_id_seq if not isinstance(x, collections.Iterable)] 33 | assert not target_not_iter, \ 34 | 'Element in target_id_text is not iteratable. Found type {}'.format(target_not_iter[0]) 35 | 36 | source_changed_length = [(words, word_ids) 37 | for words, word_ids in zip(test_source_text.split('\n'), test_source_id_seq) 38 | if len(words.split()) != len(word_ids)] 39 | assert not source_changed_length,\ 40 | 'Source text changed in size from {} word(s) to {} id(s): {}'.format( 41 | len(source_changed_length[0][0].split()), len(source_changed_length[0][1]), source_changed_length[0][1]) 42 | 43 | target_missing_end = [word_ids for word_ids in test_target_id_seq if word_ids[-1] != target_vocab_to_int['']] 44 | assert not target_missing_end,\ 45 | 'Missing id at the end of {}'.format(target_missing_end[0]) 46 | 47 | target_bad_size = [(words.split(), word_ids) 48 | for words, word_ids in zip(test_target_text.split('\n'), test_target_id_seq) 49 | if len(word_ids) != len(words.split()) + 1] 50 | assert not target_bad_size,\ 51 | 'Target text incorrect size. {} should be length {}'.format( 52 | target_bad_size[0][1], len(target_bad_size[0][0]) + 1) 53 | 54 | source_bad_id = [(word, word_id) 55 | for word, word_id in zip( 56 | [word for sentence in test_source_text.split('\n') for word in sentence.split()], 57 | itertools.chain.from_iterable(test_source_id_seq)) 58 | if source_vocab_to_int[word] != word_id] 59 | assert not source_bad_id,\ 60 | 'Source word incorrectly converted from {} to id {}.'.format(source_bad_id[0][0], source_bad_id[0][1]) 61 | 62 | target_bad_id = [(word, word_id) 63 | for word, word_id in zip( 64 | [word for sentence in test_target_text.split('\n') for word in sentence.split()], 65 | [word_id for word_ids in test_target_id_seq for word_id in word_ids[:-1]]) 66 | if target_vocab_to_int[word] != word_id] 67 | assert not target_bad_id,\ 68 | 'Target word incorrectly converted from {} to id {}.'.format(target_bad_id[0][0], target_bad_id[0][1]) 69 | 70 | _print_success_message() 71 | 72 | 73 | def test_model_inputs(model_inputs): 74 | with tf.Graph().as_default(): 75 | input_data, targets, lr, keep_prob = model_inputs() 76 | 77 | # Check type 78 | assert input_data.op.type == 'Placeholder',\ 79 | 'Input is not a Placeholder.' 80 | assert targets.op.type == 'Placeholder',\ 81 | 'Targets is not a Placeholder.' 82 | assert lr.op.type == 'Placeholder',\ 83 | 'Learning Rate is not a Placeholder.' 84 | assert keep_prob.op.type == 'Placeholder', \ 85 | 'Keep Probability is not a Placeholder.' 86 | 87 | # Check name 88 | assert input_data.name == 'input:0',\ 89 | 'Input has bad name. Found name {}'.format(input_data.name) 90 | assert keep_prob.name == 'keep_prob:0', \ 91 | 'Keep Probability has bad name. Found name {}'.format(keep_prob.name) 92 | 93 | assert tf.assert_rank(input_data, 2, message='Input data has wrong rank') 94 | assert tf.assert_rank(targets, 2, message='Targets has wrong rank') 95 | assert tf.assert_rank(lr, 0, message='Learning Rate has wrong rank') 96 | assert tf.assert_rank(keep_prob, 0, message='Keep Probability has wrong rank') 97 | 98 | _print_success_message() 99 | 100 | 101 | def test_encoding_layer(encoding_layer): 102 | rnn_size = 512 103 | batch_size = 64 104 | num_layers = 3 105 | 106 | with tf.Graph().as_default(): 107 | rnn_inputs = tf.placeholder(tf.float32, [batch_size, 22, 1000]) 108 | keep_prob = tf.placeholder(tf.float32) 109 | states = encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob) 110 | 111 | assert len(states) == num_layers,\ 112 | 'Found {} state(s). It should be {} states.'.format(len(states), num_layers) 113 | 114 | bad_types = [type(state) for state in states if not isinstance(state, tf.contrib.rnn.LSTMStateTuple)] 115 | assert not bad_types,\ 116 | 'Found wrong type: {}'.format(bad_types[0]) 117 | 118 | bad_shapes = [state_tensor.get_shape() 119 | for state in states 120 | for state_tensor in state 121 | if state_tensor.get_shape().as_list() not in [[None, rnn_size], [batch_size, rnn_size]]] 122 | assert not bad_shapes,\ 123 | 'Found wrong shape: {}'.format(bad_shapes[0]) 124 | 125 | _print_success_message() 126 | 127 | 128 | def test_decoding_layer(decoding_layer): 129 | batch_size = 64 130 | vocab_size = 1000 131 | embedding_size = 200 132 | sequence_length = 22 133 | rnn_size = 512 134 | num_layers = 3 135 | target_vocab_to_int = {'': 1, '': 3} 136 | 137 | with tf.Graph().as_default(): 138 | dec_embed_input = tf.placeholder(tf.float32, [batch_size, 22, embedding_size]) 139 | dec_embeddings = tf.placeholder(tf.float32, [vocab_size, embedding_size]) 140 | keep_prob = tf.placeholder(tf.float32) 141 | state = tf.contrib.rnn.LSTMStateTuple( 142 | tf.placeholder(tf.float32, [None, rnn_size]), 143 | tf.placeholder(tf.float32, [None, rnn_size])) 144 | encoder_state = (state, state, state) 145 | 146 | train_output, inf_output = decoding_layer(dec_embed_input, dec_embeddings, encoder_state, vocab_size, 147 | sequence_length, rnn_size, num_layers, target_vocab_to_int, keep_prob) 148 | 149 | assert isinstance(train_output, tf.Tensor),\ 150 | 'Train Logits is wrong type: {}'.format(type(train_output)) 151 | assert isinstance(inf_output, tf.Tensor), \ 152 | 'Inference Logits is wrong type: {}'.format(type(inf_output)) 153 | 154 | assert train_output.get_shape().as_list() == [batch_size, None, vocab_size],\ 155 | 'Train Logits is the wrong shape: {}'.format(train_output.get_shape()) 156 | assert inf_output.get_shape().as_list() == [None, None, vocab_size], \ 157 | 'Inference Logits is the wrong shape: {}'.format(inf_output.get_shape()) 158 | 159 | _print_success_message() 160 | 161 | 162 | def test_seq2seq_model(seq2seq_model): 163 | batch_size = 64 164 | target_vocab_size = 300 165 | sequence_length = 22 166 | rnn_size = 512 167 | num_layers = 3 168 | target_vocab_to_int = {'': 1, '': 3} 169 | 170 | with tf.Graph().as_default(): 171 | input_data = tf.placeholder(tf.int32, [64, 22]) 172 | target_data = tf.placeholder(tf.int32, [64, 22]) 173 | keep_prob = tf.placeholder(tf.float32) 174 | train_output, inf_output = seq2seq_model(input_data, target_data, keep_prob, batch_size, sequence_length, 175 | 200, target_vocab_size, 64, 80, rnn_size, num_layers, target_vocab_to_int) 176 | 177 | assert isinstance(train_output, tf.Tensor),\ 178 | 'Train Logits is wrong type: {}'.format(type(train_output)) 179 | assert isinstance(inf_output, tf.Tensor), \ 180 | 'Inference Logits is wrong type: {}'.format(type(inf_output)) 181 | 182 | assert train_output.get_shape().as_list() == [batch_size, None, target_vocab_size],\ 183 | 'Train Logits is the wrong shape: {}'.format(train_output.get_shape()) 184 | assert inf_output.get_shape().as_list() == [None, None, target_vocab_size], \ 185 | 'Inference Logits is the wrong shape: {}'.format(inf_output.get_shape()) 186 | 187 | 188 | _print_success_message() 189 | 190 | 191 | def test_sentence_to_seq(sentence_to_seq): 192 | sentence = 'this is a test sentence' 193 | vocab_to_int = {'': 0, '': 1, '': 2, 'this': 3, 'is': 6, 'a': 5, 'sentence': 4} 194 | 195 | output = sentence_to_seq(sentence, vocab_to_int) 196 | 197 | assert len(output) == 5,\ 198 | 'Wrong length. Found a length of {}'.format(len(output)) 199 | 200 | assert output[3] == 2,\ 201 | 'Missing id.' 202 | 203 | assert np.array_equal(output, [3, 6, 5, 2, 4]),\ 204 | 'Incorrect ouput. Found {}'.format(output) 205 | 206 | _print_success_message() 207 | 208 | 209 | def test_process_decoding_input(process_decoding_input): 210 | batch_size = 2 211 | seq_length = 3 212 | target_vocab_to_int = {'': 3} 213 | with tf.Graph().as_default(): 214 | target_data = tf.placeholder(tf.int32, [batch_size, seq_length]) 215 | dec_input = process_decoding_input(target_data, target_vocab_to_int, batch_size) 216 | 217 | assert dec_input.get_shape() == (batch_size, seq_length),\ 218 | 'Wrong shape returned. Found {}'.format(dec_input.get_shape()) 219 | 220 | test_target_data = [[10, 20, 30], [40, 18, 23]] 221 | with tf.Session() as sess: 222 | test_dec_input = sess.run(dec_input, {target_data: test_target_data}) 223 | 224 | assert test_dec_input[0][0] == target_vocab_to_int[''] and\ 225 | test_dec_input[1][0] == target_vocab_to_int[''],\ 226 | 'Missing GO Id.' 227 | 228 | _print_success_message() 229 | 230 | 231 | def test_decoding_layer_train(decoding_layer_train): 232 | batch_size = 64 233 | vocab_size = 1000 234 | embedding_size = 200 235 | sequence_length = 22 236 | rnn_size = 512 237 | num_layers = 3 238 | 239 | with tf.Graph().as_default(): 240 | with tf.variable_scope("decoding") as decoding_scope: 241 | dec_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.BasicLSTMCell(rnn_size)] * num_layers) 242 | output_fn = lambda x: tf.contrib.layers.fully_connected(x, vocab_size, None, scope=decoding_scope) 243 | dec_embed_input = tf.placeholder(tf.float32, [batch_size, 22, embedding_size]) 244 | keep_prob = tf.placeholder(tf.float32) 245 | state = tf.contrib.rnn.LSTMStateTuple( 246 | tf.placeholder(tf.float32, [None, rnn_size]), 247 | tf.placeholder(tf.float32, [None, rnn_size])) 248 | encoder_state = (state, state, state) 249 | 250 | train_logits = decoding_layer_train(encoder_state, dec_cell, dec_embed_input, sequence_length, 251 | decoding_scope, output_fn, keep_prob) 252 | 253 | assert train_logits.get_shape().as_list() == [batch_size, None, vocab_size], \ 254 | 'Wrong shape returned. Found {}'.format(train_logits.get_shape()) 255 | 256 | _print_success_message() 257 | 258 | 259 | def test_decoding_layer_infer(decoding_layer_infer): 260 | vocab_size = 1000 261 | sequence_length = 22 262 | embedding_size = 200 263 | rnn_size = 512 264 | num_layers = 3 265 | 266 | with tf.Graph().as_default(): 267 | with tf.variable_scope("decoding") as decoding_scope: 268 | dec_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.BasicLSTMCell(rnn_size)] * num_layers) 269 | output_fn = lambda x: tf.contrib.layers.fully_connected(x, vocab_size, None, scope=decoding_scope) 270 | dec_embeddings = tf.placeholder(tf.float32, [vocab_size, embedding_size]) 271 | keep_prob = tf.placeholder(tf.float32) 272 | state = tf.contrib.rnn.LSTMStateTuple( 273 | tf.placeholder(tf.float32, [None, rnn_size]), 274 | tf.placeholder(tf.float32, [None, rnn_size])) 275 | encoder_state = (state, state, state) 276 | 277 | infer_logits = decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, 10, 20, 278 | sequence_length, vocab_size, decoding_scope, output_fn, keep_prob) 279 | 280 | assert infer_logits.get_shape().as_list() == [None, None, vocab_size], \ 281 | 'Wrong shape returned. Found {}'.format(infer_logits.get_shape()) 282 | 283 | _print_success_message() 284 | -------------------------------------------------------------------------------- /PROJECT_3_TV_Script_Generation/problem_unittests.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.contrib import rnn 4 | 5 | 6 | def _print_success_message(): 7 | print('Tests Passed') 8 | 9 | 10 | def test_create_lookup_tables(create_lookup_tables): 11 | with tf.Graph().as_default(): 12 | test_text = ''' 13 | Moe_Szyslak Moe's Tavern Where the elite meet to drink 14 | Bart_Simpson Eh yeah hello is Mike there Last name Rotch 15 | Moe_Szyslak Hold on I'll check Mike Rotch Mike Rotch Hey has anybody seen Mike Rotch lately 16 | Moe_Szyslak Listen you little puke One of these days I'm gonna catch you and I'm gonna carve my name on your back with an ice pick 17 | Moe_Szyslak Whats the matter Homer You're not your normal effervescent self 18 | Homer_Simpson I got my problems Moe Give me another one 19 | Moe_Szyslak Homer hey you should not drink to forget your problems 20 | Barney_Gumble Yeah you should only drink to enhance your social skills''' 21 | 22 | test_text = test_text.lower() 23 | test_text = test_text.split() 24 | 25 | vocab_to_int, int_to_vocab = create_lookup_tables(test_text) 26 | 27 | # Check types 28 | assert isinstance(vocab_to_int, dict),\ 29 | 'vocab_to_int is not a dictionary.' 30 | assert isinstance(int_to_vocab, dict),\ 31 | 'int_to_vocab is not a dictionary.' 32 | 33 | # Compare lengths of dicts 34 | assert len(vocab_to_int) == len(int_to_vocab),\ 35 | 'Length of vocab_to_int and int_to_vocab don\'t match. ' \ 36 | 'vocab_to_int is length {}. int_to_vocab is length {}'.format(len(vocab_to_int), len(int_to_vocab)) 37 | 38 | # Make sure the dicts have the same words 39 | vocab_to_int_word_set = set(vocab_to_int.keys()) 40 | int_to_vocab_word_set = set(int_to_vocab.values()) 41 | 42 | assert not (vocab_to_int_word_set - int_to_vocab_word_set),\ 43 | 'vocab_to_int and int_to_vocab don\'t have the same words.' \ 44 | '{} found in vocab_to_int, but not in int_to_vocab'.format(vocab_to_int_word_set - int_to_vocab_word_set) 45 | assert not (int_to_vocab_word_set - vocab_to_int_word_set),\ 46 | 'vocab_to_int and int_to_vocab don\'t have the same words.' \ 47 | '{} found in int_to_vocab, but not in vocab_to_int'.format(int_to_vocab_word_set - vocab_to_int_word_set) 48 | 49 | # Make sure the dicts have the same word ids 50 | vocab_to_int_word_id_set = set(vocab_to_int.values()) 51 | int_to_vocab_word_id_set = set(int_to_vocab.keys()) 52 | 53 | assert not (vocab_to_int_word_id_set - int_to_vocab_word_id_set),\ 54 | 'vocab_to_int and int_to_vocab don\'t contain the same word ids.' \ 55 | '{} found in vocab_to_int, but not in int_to_vocab'.format(vocab_to_int_word_id_set - int_to_vocab_word_id_set) 56 | assert not (int_to_vocab_word_id_set - vocab_to_int_word_id_set),\ 57 | 'vocab_to_int and int_to_vocab don\'t contain the same word ids.' \ 58 | '{} found in int_to_vocab, but not in vocab_to_int'.format(int_to_vocab_word_id_set - vocab_to_int_word_id_set) 59 | 60 | # Make sure the dicts make the same lookup 61 | missmatches = [(word, id, id, int_to_vocab[id]) for word, id in vocab_to_int.items() if int_to_vocab[id] != word] 62 | 63 | assert not missmatches,\ 64 | 'Found {} missmatche(s). First missmatch: vocab_to_int[{}] = {} and int_to_vocab[{}] = {}'.format( 65 | len(missmatches), 66 | *missmatches[0]) 67 | 68 | assert len(vocab_to_int) > len(set(test_text))/2,\ 69 | 'The length of vocab seems too small. Found a length of {}'.format(len(vocab_to_int)) 70 | 71 | _print_success_message() 72 | 73 | 74 | def test_get_batches(get_batches): 75 | with tf.Graph().as_default(): 76 | test_batch_size = 128 77 | test_seq_length = 5 78 | test_int_text = list(range(1000*test_seq_length)) 79 | batches = get_batches(test_int_text, test_batch_size, test_seq_length) 80 | 81 | # Check type 82 | assert isinstance(batches, np.ndarray),\ 83 | 'Batches is not a Numpy array' 84 | 85 | # Check shape 86 | assert batches.shape == (7, 2, 128, 5),\ 87 | 'Batches returned wrong shape. Found {}'.format(batches.shape) 88 | 89 | for x in range(batches.shape[2]): 90 | assert np.array_equal(batches[0,0,x], np.array(range(x * 35, x * 35 + batches.shape[3]))),\ 91 | 'Batches returned wrong contents. For example, input sequence {} in the first batch was {}'.format(x, batches[0,0,x]) 92 | assert np.array_equal(batches[0,1,x], np.array(range(x * 35 + 1, x * 35 + 1 + batches.shape[3]))),\ 93 | 'Batches returned wrong contents. For example, target sequence {} in the first batch was {}'.format(x, batches[0,1,x]) 94 | 95 | 96 | last_seq_target = (test_batch_size-1) * 35 + 31 97 | last_seq = np.array(range(last_seq_target, last_seq_target+ batches.shape[3])) 98 | last_seq[-1] = batches[0,0,0,0] 99 | 100 | assert np.array_equal(batches[-1,1,-1], last_seq),\ 101 | 'The last target of the last batch should be the first input of the first batch. Found {} but expected {}'.format(batches[-1,1,-1], last_seq) 102 | 103 | _print_success_message() 104 | 105 | 106 | def test_tokenize(token_lookup): 107 | with tf.Graph().as_default(): 108 | symbols = set(['.', ',', '"', ';', '!', '?', '(', ')', '--', '\n']) 109 | token_dict = token_lookup() 110 | 111 | # Check type 112 | assert isinstance(token_dict, dict), \ 113 | 'Returned type is {}.'.format(type(token_dict)) 114 | 115 | # Check symbols 116 | missing_symbols = symbols - set(token_dict.keys()) 117 | unknown_symbols = set(token_dict.keys()) - symbols 118 | 119 | assert not missing_symbols, \ 120 | 'Missing symbols: {}'.format(missing_symbols) 121 | assert not unknown_symbols, \ 122 | 'Unknown symbols: {}'.format(unknown_symbols) 123 | 124 | # Check values type 125 | bad_value_type = [type(val) for val in token_dict.values() if not isinstance(val, str)] 126 | 127 | assert not bad_value_type,\ 128 | 'Found token as {} type.'.format(bad_value_type[0]) 129 | 130 | # Check for spaces 131 | key_has_spaces = [k for k in token_dict.keys() if ' ' in k] 132 | val_has_spaces = [val for val in token_dict.values() if ' ' in val] 133 | 134 | assert not key_has_spaces,\ 135 | 'The key "{}" includes spaces. Remove spaces from keys and values'.format(key_has_spaces[0]) 136 | assert not val_has_spaces,\ 137 | 'The value "{}" includes spaces. Remove spaces from keys and values'.format(val_has_spaces[0]) 138 | 139 | # Check for symbols in values 140 | symbol_val = () 141 | for symbol in symbols: 142 | for val in token_dict.values(): 143 | if symbol in val: 144 | symbol_val = (symbol, val) 145 | 146 | assert not symbol_val,\ 147 | 'Don\'t use a symbol that will be replaced in your tokens. Found the symbol {} in value {}'.format(*symbol_val) 148 | 149 | _print_success_message() 150 | 151 | 152 | def test_get_inputs(get_inputs): 153 | with tf.Graph().as_default(): 154 | input_data, targets, lr = get_inputs() 155 | 156 | # Check type 157 | assert input_data.op.type == 'Placeholder',\ 158 | 'Input not a Placeholder.' 159 | assert targets.op.type == 'Placeholder',\ 160 | 'Targets not a Placeholder.' 161 | assert lr.op.type == 'Placeholder',\ 162 | 'Learning Rate not a Placeholder.' 163 | 164 | # Check name 165 | assert input_data.name == 'input:0',\ 166 | 'Input has bad name. Found name {}'.format(input_data.name) 167 | 168 | # Check rank 169 | input_rank = 0 if input_data.get_shape() == None else len(input_data.get_shape()) 170 | targets_rank = 0 if targets.get_shape() == None else len(targets.get_shape()) 171 | lr_rank = 0 if lr.get_shape() == None else len(lr.get_shape()) 172 | 173 | assert input_rank == 2,\ 174 | 'Input has wrong rank. Rank {} found.'.format(input_rank) 175 | assert targets_rank == 2,\ 176 | 'Targets has wrong rank. Rank {} found.'.format(targets_rank) 177 | assert lr_rank == 0,\ 178 | 'Learning Rate has wrong rank. Rank {} found'.format(lr_rank) 179 | 180 | _print_success_message() 181 | 182 | 183 | def test_get_init_cell(get_init_cell): 184 | with tf.Graph().as_default(): 185 | test_batch_size_ph = tf.placeholder(tf.int32) 186 | test_rnn_size = 256 187 | 188 | cell, init_state = get_init_cell(test_batch_size_ph, test_rnn_size) 189 | 190 | # Check type 191 | assert isinstance(cell, tf.contrib.rnn.MultiRNNCell),\ 192 | 'Cell is wrong type. Found {} type'.format(type(cell)) 193 | 194 | # Check for name attribute 195 | assert hasattr(init_state, 'name'),\ 196 | 'Initial state doesn\'t have the "name" attribute. Try using `tf.identity` to set the name.' 197 | 198 | # Check name 199 | assert init_state.name == 'initial_state:0',\ 200 | 'Initial state doesn\'t have the correct name. Found the name {}'.format(init_state.name) 201 | 202 | _print_success_message() 203 | 204 | 205 | def test_get_embed(get_embed): 206 | with tf.Graph().as_default(): 207 | embed_shape = [50, 5, 256] 208 | test_input_data = tf.placeholder(tf.int32, embed_shape[:2]) 209 | test_vocab_size = 27 210 | test_embed_dim = embed_shape[2] 211 | 212 | embed = get_embed(test_input_data, test_vocab_size, test_embed_dim) 213 | 214 | # Check shape 215 | assert embed.shape == embed_shape,\ 216 | 'Wrong shape. Found shape {}'.format(embed.shape) 217 | 218 | _print_success_message() 219 | 220 | 221 | def test_build_rnn(build_rnn): 222 | with tf.Graph().as_default(): 223 | test_rnn_size = 256 224 | test_rnn_layer_size = 2 225 | test_cell = rnn.MultiRNNCell([rnn.BasicLSTMCell(test_rnn_size) for _ in range(test_rnn_layer_size)]) 226 | 227 | test_inputs = tf.placeholder(tf.float32, [None, None, test_rnn_size]) 228 | outputs, final_state = build_rnn(test_cell, test_inputs) 229 | 230 | # Check name 231 | assert hasattr(final_state, 'name'),\ 232 | 'Final state doesn\'t have the "name" attribute. Try using `tf.identity` to set the name.' 233 | assert final_state.name == 'final_state:0',\ 234 | 'Final state doesn\'t have the correct name. Found the name {}'.format(final_state.name) 235 | 236 | # Check shape 237 | assert outputs.get_shape().as_list() == [None, None, test_rnn_size],\ 238 | 'Outputs has wrong shape. Found shape {}'.format(outputs.get_shape()) 239 | assert final_state.get_shape().as_list() == [test_rnn_layer_size, 2, None, test_rnn_size],\ 240 | 'Final state wrong shape. Found shape {}'.format(final_state.get_shape()) 241 | 242 | _print_success_message() 243 | 244 | 245 | def test_build_nn(build_nn): 246 | with tf.Graph().as_default(): 247 | test_input_data_shape = [128, 5] 248 | test_input_data = tf.placeholder(tf.int32, test_input_data_shape) 249 | test_rnn_size = 256 250 | test_embed_dim = 300 251 | test_rnn_layer_size = 2 252 | test_vocab_size = 27 253 | test_cell = rnn.MultiRNNCell([rnn.BasicLSTMCell(test_rnn_size) for _ in range(test_rnn_layer_size)]) 254 | 255 | logits, final_state = build_nn(test_cell, test_rnn_size, test_input_data, test_vocab_size, test_embed_dim) 256 | 257 | # Check name 258 | assert hasattr(final_state, 'name'), \ 259 | 'Final state doesn\'t have the "name" attribute. Are you using build_rnn?' 260 | assert final_state.name == 'final_state:0', \ 261 | 'Final state doesn\'t have the correct name. Found the name {}. Are you using build_rnn?'.format(final_state.name) 262 | 263 | # Check Shape 264 | assert logits.get_shape().as_list() == test_input_data_shape + [test_vocab_size], \ 265 | 'Outputs has wrong shape. Found shape {}'.format(logits.get_shape()) 266 | assert final_state.get_shape().as_list() == [test_rnn_layer_size, 2, None, test_rnn_size], \ 267 | 'Final state wrong shape. Found shape {}'.format(final_state.get_shape()) 268 | 269 | _print_success_message() 270 | 271 | 272 | def test_get_tensors(get_tensors): 273 | test_graph = tf.Graph() 274 | with test_graph.as_default(): 275 | test_input = tf.placeholder(tf.int32, name='input') 276 | test_initial_state = tf.placeholder(tf.int32, name='initial_state') 277 | test_final_state = tf.placeholder(tf.int32, name='final_state') 278 | test_probs = tf.placeholder(tf.float32, name='probs') 279 | 280 | input_text, initial_state, final_state, probs = get_tensors(test_graph) 281 | 282 | # Check correct tensor 283 | assert input_text == test_input,\ 284 | 'Test input is wrong tensor' 285 | assert initial_state == test_initial_state, \ 286 | 'Initial state is wrong tensor' 287 | assert final_state == test_final_state, \ 288 | 'Final state is wrong tensor' 289 | assert probs == test_probs, \ 290 | 'Probabilities is wrong tensor' 291 | 292 | _print_success_message() 293 | 294 | 295 | def test_pick_word(pick_word): 296 | with tf.Graph().as_default(): 297 | test_probabilities = np.array([0.1, 0.8, 0.05, 0.05]) 298 | test_int_to_vocab = {word_i: word for word_i, word in enumerate(['this', 'is', 'a', 'test'])} 299 | 300 | pred_word = pick_word(test_probabilities, test_int_to_vocab) 301 | 302 | # Check type 303 | assert isinstance(pred_word, str),\ 304 | 'Predicted word is wrong type. Found {} type.'.format(type(pred_word)) 305 | 306 | # Check word is from vocab 307 | assert pred_word in test_int_to_vocab.values(),\ 308 | 'Predicted word not found in int_to_vocab.' 309 | 310 | 311 | _print_success_message() 312 | 313 | -------------------------------------------------------------------------------- /PROJECT_5_Face_Generation/.ipynb_checkpoints/dlnd_face_generation-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# Face Generation\n", 11 | "In this project, you'll use generative adversarial networks to generate new images of faces.\n", 12 | "### Get the Data\n", 13 | "You'll be using two datasets in this project:\n", 14 | "- MNIST\n", 15 | "- CelebA\n", 16 | "\n", 17 | "Since the celebA dataset is complex and you're doing GANs in a project for the first time, we want you to test your neural network on MNIST before CelebA. Running the GANs on MNIST will allow you to see how well your model trains sooner.\n", 18 | "\n", 19 | "If you're using [FloydHub](https://www.floydhub.com/), set `data_dir` to \"/input\" and use the [FloydHub data ID](http://docs.floydhub.com/home/using_datasets/) \"R5KrjnANiKVhLWAkpXhNBe\"." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": { 26 | "collapsed": false, 27 | "deletable": true, 28 | "editable": true 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "data_dir = './data'\n", 33 | "\n", 34 | "# FloydHub - Use with data ID \"R5KrjnANiKVhLWAkpXhNBe\"\n", 35 | "#data_dir = '/input'\n", 36 | "\n", 37 | "\n", 38 | "\"\"\"\n", 39 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 40 | "\"\"\"\n", 41 | "import helper\n", 42 | "\n", 43 | "helper.download_extract('mnist', data_dir)\n", 44 | "helper.download_extract('celeba', data_dir)" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": { 50 | "deletable": true, 51 | "editable": true 52 | }, 53 | "source": [ 54 | "## Explore the Data\n", 55 | "### MNIST\n", 56 | "As you're aware, the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset contains images of handwritten digits. You can view the first number of examples by changing `show_n_images`. " 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": { 63 | "collapsed": false, 64 | "deletable": true, 65 | "editable": true 66 | }, 67 | "outputs": [], 68 | "source": [ 69 | "show_n_images = 25\n", 70 | "\n", 71 | "\"\"\"\n", 72 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 73 | "\"\"\"\n", 74 | "%matplotlib inline\n", 75 | "import os\n", 76 | "from glob import glob\n", 77 | "from matplotlib import pyplot\n", 78 | "\n", 79 | "mnist_images = helper.get_batch(glob(os.path.join(data_dir, 'mnist/*.jpg'))[:show_n_images], 28, 28, 'L')\n", 80 | "pyplot.imshow(helper.images_square_grid(mnist_images, 'L'), cmap='gray')" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": { 86 | "deletable": true, 87 | "editable": true 88 | }, 89 | "source": [ 90 | "### CelebA\n", 91 | "The [CelebFaces Attributes Dataset (CelebA)](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) dataset contains over 200,000 celebrity images with annotations. Since you're going to be generating faces, you won't need the annotations. You can view the first number of examples by changing `show_n_images`." 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": { 98 | "collapsed": false, 99 | "deletable": true, 100 | "editable": true 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "show_n_images = 25\n", 105 | "\n", 106 | "\"\"\"\n", 107 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 108 | "\"\"\"\n", 109 | "mnist_images = helper.get_batch(glob(os.path.join(data_dir, 'img_align_celeba/*.jpg'))[:show_n_images], 28, 28, 'RGB')\n", 110 | "pyplot.imshow(helper.images_square_grid(mnist_images, 'RGB'))" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": { 116 | "deletable": true, 117 | "editable": true 118 | }, 119 | "source": [ 120 | "## Preprocess the Data\n", 121 | "Since the project's main focus is on building the GANs, we'll preprocess the data for you. The values of the MNIST and CelebA dataset will be in the range of -0.5 to 0.5 of 28x28 dimensional images. The CelebA images will be cropped to remove parts of the image that don't include a face, then resized down to 28x28.\n", 122 | "\n", 123 | "The MNIST images are black and white images with a single [color channel](https://en.wikipedia.org/wiki/Channel_(digital_image%29) while the CelebA images have [3 color channels (RGB color channel)](https://en.wikipedia.org/wiki/Channel_(digital_image%29#RGB_Images).\n", 124 | "## Build the Neural Network\n", 125 | "You'll build the components necessary to build a GANs by implementing the following functions below:\n", 126 | "- `model_inputs`\n", 127 | "- `discriminator`\n", 128 | "- `generator`\n", 129 | "- `model_loss`\n", 130 | "- `model_opt`\n", 131 | "- `train`\n", 132 | "\n", 133 | "### Check the Version of TensorFlow and Access to GPU\n", 134 | "This will check to make sure you have the correct version of TensorFlow and access to a GPU" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": { 141 | "collapsed": false, 142 | "deletable": true, 143 | "editable": true 144 | }, 145 | "outputs": [], 146 | "source": [ 147 | "\"\"\"\n", 148 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 149 | "\"\"\"\n", 150 | "from distutils.version import LooseVersion\n", 151 | "import warnings\n", 152 | "import tensorflow as tf\n", 153 | "\n", 154 | "# Check TensorFlow Version\n", 155 | "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer. You are using {}'.format(tf.__version__)\n", 156 | "print('TensorFlow Version: {}'.format(tf.__version__))\n", 157 | "\n", 158 | "# Check for a GPU\n", 159 | "if not tf.test.gpu_device_name():\n", 160 | " warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n", 161 | "else:\n", 162 | " print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))" 163 | ] 164 | }, 165 | { 166 | "cell_type": "markdown", 167 | "metadata": { 168 | "deletable": true, 169 | "editable": true 170 | }, 171 | "source": [ 172 | "### Input\n", 173 | "Implement the `model_inputs` function to create TF Placeholders for the Neural Network. It should create the following placeholders:\n", 174 | "- Real input images placeholder with rank 4 using `image_width`, `image_height`, and `image_channels`.\n", 175 | "- Z input placeholder with rank 2 using `z_dim`.\n", 176 | "- Learning rate placeholder with rank 0.\n", 177 | "\n", 178 | "Return the placeholders in the following the tuple (tensor of real input images, tensor of z data)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": { 185 | "collapsed": false, 186 | "deletable": true, 187 | "editable": true 188 | }, 189 | "outputs": [], 190 | "source": [ 191 | "import problem_unittests as tests\n", 192 | "\n", 193 | "def model_inputs(image_width, image_height, image_channels, z_dim):\n", 194 | " \"\"\"\n", 195 | " Create the model inputs\n", 196 | " :param image_width: The input image width\n", 197 | " :param image_height: The input image height\n", 198 | " :param image_channels: The number of image channels\n", 199 | " :param z_dim: The dimension of Z\n", 200 | " :return: Tuple of (tensor of real input images, tensor of z data, learning rate)\n", 201 | " \"\"\"\n", 202 | " # TODO: Implement Function\n", 203 | "\n", 204 | " # Create real input images with rank 4 using image_width, image_height and image_channels\n", 205 | " # Z input placeholder with rank 2 using z_dim (this may be for fake images)\n", 206 | " # Learning rate placeholder with rank 0\n", 207 | " \n", 208 | " return None, None, None\n", 209 | "\n", 210 | " ### EXAMPLE MODEL INPUTS CODE ####\n", 211 | " \n", 212 | " def model_inputs(real_dim, z_dim):\n", 213 | " # inputs_real feeds the actual images that come from the dataset\n", 214 | " inputs_real = tf.placeholder(tf.float32, (None, *real_dim), name='input_real')\n", 215 | " # inputs_z feeds the random noise used to guide the generator\n", 216 | " inputs_z = tf.placeholder(tf.float32, (None, z_dim), name='input_z')\n", 217 | " # y is the labels which specifies which of the ten digit classes each \n", 218 | " # example belongs to\n", 219 | " y = tf.placeholder(tf.int32, (None), name='y')\n", 220 | " # specificies which images we use the labels on \n", 221 | " label_mask = tf.placeholder(tf.int32, (None), name='label_mask')\n", 222 | " \n", 223 | " return inputs_real, inputs_z, y, label_mask\n", 224 | "\n", 225 | " ### END EXAMPLE MODEL INPUTS ###\n", 226 | "\n", 227 | "\n", 228 | "\"\"\"\n", 229 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 230 | "\"\"\"\n", 231 | "tests.test_model_inputs(model_inputs)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "markdown", 236 | "metadata": { 237 | "deletable": true, 238 | "editable": true 239 | }, 240 | "source": [ 241 | "### Discriminator\n", 242 | "Implement `discriminator` to create a discriminator neural network that discriminates on `images`. This function should be able to reuse the variables in the neural network. Use [`tf.variable_scope`](https://www.tensorflow.org/api_docs/python/tf/variable_scope) with a scope name of \"discriminator\" to allow the variables to be reused. The function should return a tuple of (tensor output of the discriminator, tensor logits of the discriminator)." 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": null, 248 | "metadata": { 249 | "collapsed": false, 250 | "deletable": true, 251 | "editable": true 252 | }, 253 | "outputs": [], 254 | "source": [ 255 | "def discriminator(images, reuse=False):\n", 256 | " \"\"\"\n", 257 | " Create the discriminator network\n", 258 | " :param images: Tensor of input image(s)\n", 259 | " :param reuse: Boolean if the weights should be reused\n", 260 | " :return: Tuple of (tensor output of the discriminator, tensor logits of the discriminator)\n", 261 | " \"\"\"\n", 262 | " # TODO: Implement Function\n", 263 | " \n", 264 | " # Create a discriminator network to discrimate images. \n", 265 | " # Ensure that the variables are reusable using tf.variable_scope\n", 266 | "\n", 267 | " return None, None\n", 268 | "\n", 269 | " ### EXAMPLE DISCRIMINATOR NETWORK ###\n", 270 | " \n", 271 | " def discriminator(x, reuse=False, alpha=0.2, drop_rate=0., num_classes=10, size_mult=64):\n", 272 | " with tf.variable_scope('discriminator', reuse=reuse):\n", 273 | " x = tf.layers.dropout(x, rate=drop_rate/2.5)\n", 274 | " \n", 275 | " # Input layer is 32x32x3\n", 276 | " x1 = tf.layers.conv2d(x, size_mult, 3, strides=2, padding='same')\n", 277 | " # Leaky relu's are used all through the network\n", 278 | " # Leaky relu's always have a gradient that pass down to the next layer of the network\n", 279 | " relu1 = tf.maximum(alpha * x1, x1)\n", 280 | " # Dropout is used frequently throughout the network as an optimization technique \n", 281 | " relu1 = tf.layers.dropout(relu1, rate=drop_rate)\n", 282 | " \n", 283 | " x2 = tf.layers.conv2d(relu1, size_mult, 3, strides=2, padding='same')\n", 284 | " bn2 = tf.layers.batch_normalization(x2, training=True)\n", 285 | " relu2 = tf.maximum(alpha * x2, x2)\n", 286 | " \n", 287 | " \n", 288 | " x3 = tf.layers.conv2d(relu2, size_mult, 3, strides=2, padding='same')\n", 289 | " bn3 = tf.layers.batch_normalization(x3, training=True)\n", 290 | " relu3 = tf.maximum(alpha * bn3, bn3)\n", 291 | " relu3 = tf.layers.dropout(relu3, rate=drop_rate)\n", 292 | " \n", 293 | " x4 = tf.layers.conv2d(relu3, 2 * size_mult, 3, strides=1, padding='same')\n", 294 | " bn4 = tf.layers.batch_normalization(x4, training=True)\n", 295 | " relu4 = tf.maximum(alpha * bn4, bn4)\n", 296 | " \n", 297 | " x5 = tf.layers.conv2d(relu4, 2 * size_mult, 3, strides=1, padding='same')\n", 298 | " bn5 = tf.layers.batch_normalization(x5, training=True)\n", 299 | " relu5 = tf.maximum(alpha * bn5, bn5)\n", 300 | " \n", 301 | " x6 = tf.layers.conv2d(relu5, 2 * size_mult, 3, strides=2, padding='same')\n", 302 | " bn6 = tf.layers.batch_normalization(x6, training=True)\n", 303 | " relu6 = tf.maximum(alpha * bn6, bn6)\n", 304 | " relu6 = tf.layers.dropout(relu6, rate=drop_rate)\n", 305 | " \n", 306 | " x7 = tf.layers.conv2d(relu5, 2 * size_mult, 3, strides=1, padding='valid')\n", 307 | " # Don't use bn on this layer, because bn would set the mean of each feature\n", 308 | " # to the bn mu parameter.\n", 309 | " # This layer is used for the feature matching loss, which only works if\n", 310 | " # the means can be different when the discriminator is run on the data than\n", 311 | " # when the discriminator is run on the generator samples.\n", 312 | " relu7 = tf.maximum(alpha * x7, x7)\n", 313 | " \n", 314 | " # Flatten it by global average pooling\n", 315 | " # Take the average across the spatial dimensions of the last layer of features\n", 316 | " # Take the average using reduce_mean\n", 317 | " # Axis 0 is the batch, axis 1 is the height, axis 2 is the width, axis 3 is \n", 318 | " # channels of the feature map. \n", 319 | " # Take the average over axis 1 and axis 2\n", 320 | " features = tf.reduce_mean(relu7, (1, 2))\n", 321 | " \n", 322 | " # Set class_logits to be the inputs to a softmax distribution over the different classes\n", 323 | " # This is actual classifier of the network \n", 324 | " # Used a fully connected layer that outputs a distribution over the different classes\n", 325 | " class_logits = tf.layers.dense(features, num_classes + extra_classes)\n", 326 | " \n", 327 | " # Set gan_logits such that P(input is real | input) = sigmoid(gan_logits).\n", 328 | " # Keep in mind that class_logits gives you the probability distribution over all the real\n", 329 | " # classes and the fake class. You need to work out how to transform this multiclass softmax\n", 330 | " # distribution into a binary real-vs-fake decision that can be described with a sigmoid.\n", 331 | " # Numerical stability is very important.\n", 332 | " # You'll probably need to use this numerical stability trick:\n", 333 | " # log sum_i exp a_i = m + log sum_i exp(a_i - m).\n", 334 | " # This is numerically stable when m = max_i a_i.\n", 335 | " # (It helps to think about what goes wrong when...\n", 336 | " # 1. One value of a_i is very large\n", 337 | " # 2. All the values of a_i are very negative\n", 338 | " # This trick and this value of m fix both those cases, but the naive implementation and\n", 339 | " # other values of m encounter various problems)\n", 340 | " \n", 341 | " if extra_class:\n", 342 | " # Using 11 different classes (10 real, 1 fake)\n", 343 | " # 10 logits giving the distribution of the real classes\n", 344 | " real_class_logits, fake_class_logits = tf.split(class_logits, [num_classes, 1], 1)\n", 345 | " # 1 extra logit for the fake class\n", 346 | " assert fake_class_logits.get_shape()[1] == 1, fake_class_logits.get_shape()\n", 347 | " fake_class_logits = tf.squeeze(fake_class_logits)\n", 348 | " else:\n", 349 | " # if you wanted to only use 10 classes, could do it just like this \n", 350 | " real_class_logits = class_logits\n", 351 | " fake_class_logits = 0.\n", 352 | " \n", 353 | " mx = tf.reduce_max(real_class_logits, 1, keep_dims=True)\n", 354 | " stable_real_class_logits = real_class_logits - mx\n", 355 | " \n", 356 | " # This is a log-sum-exp function (tf.log(tf.reduce_sum(tf.exp())))\n", 357 | " gan_logits = tf.log(tf.reduce_sum(tf.exp(stable_real_class_logits), 1)) + tf.squeeze(mx) - fake_class_logits\n", 358 | " \n", 359 | " out = tf.nn.softmax(class_logits)\n", 360 | " \n", 361 | " return out, class_logits, gan_logits, features\n", 362 | " \n", 363 | " ### END EXAMPLE DISCRIMINATOR NETWORK ###\n", 364 | "\n", 365 | "\"\"\"\n", 366 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 367 | "\"\"\"\n", 368 | "tests.test_discriminator(discriminator, tf)" 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "metadata": { 374 | "deletable": true, 375 | "editable": true 376 | }, 377 | "source": [ 378 | "### Generator\n", 379 | "Implement `generator` to generate an image using `z`. This function should be able to reuse the variables in the neural network. Use [`tf.variable_scope`](https://www.tensorflow.org/api_docs/python/tf/variable_scope) with a scope name of \"generator\" to allow the variables to be reused. The function should return the generated 28 x 28 x `out_channel_dim` images." 380 | ] 381 | }, 382 | { 383 | "cell_type": "code", 384 | "execution_count": null, 385 | "metadata": { 386 | "collapsed": false, 387 | "deletable": true, 388 | "editable": true 389 | }, 390 | "outputs": [], 391 | "source": [ 392 | "def generator(z, out_channel_dim, is_train=True):\n", 393 | " \"\"\"\n", 394 | " Create the generator network\n", 395 | " :param z: Input z\n", 396 | " :param out_channel_dim: The number of channels in the output image\n", 397 | " :param is_train: Boolean if generator is being used for training\n", 398 | " :return: The tensor output of the generator\n", 399 | " \"\"\"\n", 400 | " # TODO: Implement Function\n", 401 | " \n", 402 | " # Use the placeholder 'z' to generate an image. \n", 403 | " # The function should be able to resuse variables, use tf.variable_scope with a name \"generator\" for this.\n", 404 | " # The function should return a generated 28 * 28 * out_channel_dim image\n", 405 | " \n", 406 | " return None\n", 407 | " \n", 408 | " ### EXAMPLE GENERATOR NETWORK ###\n", 409 | " \n", 410 | " with tf.variable_scope('generator', reuse=reuse):\n", 411 | " # First fully connected layer\n", 412 | " x1 = tf.layers.dense(z, 4 * 4 * size_mult * 4)\n", 413 | " # Reshape it to start the convolutional stack\n", 414 | " x1 = tf.reshape(x1, (-1, 4, 4, size_mult * 4))\n", 415 | " x1 = tf.layers.batch_normalization(x1, training=training)\n", 416 | " x1 = tf.maximum(alpha * x1, x1)\n", 417 | " \n", 418 | " x2 = tf.layers.conv2d_transpose(x1, size_mult * 2, 5, strides=2, padding='same')\n", 419 | " x2 = tf.layers.batch_normalization(x2, training=training)\n", 420 | " x2 = tf.maximum(alpha * x2, x2)\n", 421 | " \n", 422 | " x3 = tf.layers.conv2d_transpose(x2, size_mult, 5, strides=2, padding='same')\n", 423 | " x3 = tf.layers.batch_normalization(x3, training=training)\n", 424 | " x3 = tf.maximum(alpha * x3, x3)\n", 425 | " \n", 426 | " # Output layer\n", 427 | " logits = tf.layers.conv2d_transpose(x3, output_dim, 5, strides=2, padding='same')\n", 428 | " \n", 429 | " out = tf.tanh(logits)\n", 430 | " \n", 431 | " return out\n", 432 | " \n", 433 | " ### END EXAMPLE GENERATOR NETWORK ###\n", 434 | " \n", 435 | "\n", 436 | "\n", 437 | "\"\"\"\n", 438 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 439 | "\"\"\"\n", 440 | "tests.test_generator(generator, tf)" 441 | ] 442 | }, 443 | { 444 | "cell_type": "markdown", 445 | "metadata": { 446 | "deletable": true, 447 | "editable": true 448 | }, 449 | "source": [ 450 | "### Loss\n", 451 | "Implement `model_loss` to build the GANs for training and calculate the loss. The function should return a tuple of (discriminator loss, generator loss). Use the following functions you implemented:\n", 452 | "- `discriminator(images, reuse=False)`\n", 453 | "- `generator(z, out_channel_dim, is_train=True)`" 454 | ] 455 | }, 456 | { 457 | "cell_type": "code", 458 | "execution_count": null, 459 | "metadata": { 460 | "collapsed": false, 461 | "deletable": true, 462 | "editable": true 463 | }, 464 | "outputs": [], 465 | "source": [ 466 | "def model_loss(input_real, input_z, out_channel_dim):\n", 467 | " \"\"\"\n", 468 | " Get the loss for the discriminator and generator\n", 469 | " :param input_real: Images from the real dataset\n", 470 | " :param input_z: Z input\n", 471 | " :param out_channel_dim: The number of channels in the output image\n", 472 | " :return: A tuple of (discriminator loss, generator loss)\n", 473 | " \"\"\"\n", 474 | " # TODO: Implement Function\n", 475 | " \n", 476 | " # Use previously created functions \"discriminator\" and \"generator\" \n", 477 | " # to build the GAN for training.\n", 478 | " # Return the discriminator loss and generator loss. \n", 479 | " \n", 480 | " return None, None\n", 481 | "\n", 482 | " ### EXAMPLE MODEL LOSS CODE ###\n", 483 | " \n", 484 | " def model_loss(input_real, input_z, output_dim, y, num_classes, label_mask, alpha=0.2, drop_rate=0.):\n", 485 | " \"\"\"\n", 486 | " Get the loss for the discriminator and generator\n", 487 | " :param input_real: Images from the real dataset\n", 488 | " :param input_z: Z input\n", 489 | " :param output_dim: The number of channels in the output image\n", 490 | " :param y: Integer class labels\n", 491 | " :param num_classes: The number of classes\n", 492 | " :param alpha: The slope of the left half of leaky ReLU activation\n", 493 | " :param drop_rate: The probability of dropping a hidden unit\n", 494 | " :return: A tuple of (discriminator loss, generator loss)\n", 495 | " \"\"\"\n", 496 | " \n", 497 | " # These numbers multiply the size of each layer of the generator and the discriminator,\n", 498 | " # respectively. You can reduce them to run your code faster for debugging purposes.\n", 499 | " g_size_mult = 32\n", 500 | " d_size_mult = 64\n", 501 | " \n", 502 | " # Here we run the generator and the discriminator\n", 503 | " # Run the generator on input_z (random noise) to get some samples for the discriminator later on\n", 504 | " g_model = generator(input_z, output_dim, alpha=alpha, size_mult=g_size_mult)\n", 505 | " # Run the discriminator on the real data \n", 506 | " d_on_data = discriminator(input_real, alpha=alpha, drop_rate=drop_rate, size_mult=d_size_mult)\n", 507 | " d_model_real, class_logits_on_data, gan_logits_on_data, data_features = d_on_data\n", 508 | " # Run the discriminator again but this time run it on the samples (g_model) rather than real data\n", 509 | " # Note in this function, reuse is set to True \n", 510 | " d_on_samples = discriminator(g_model, reuse=True, alpha=alpha, drop_rate=drop_rate, size_mult=d_size_mult)\n", 511 | " d_model_fake, class_logits_on_samples, gan_logits_on_samples, sample_features = d_on_samples\n", 512 | " \n", 513 | " \n", 514 | " # Here we compute `d_loss`, the loss for the discriminator.\n", 515 | " # This should combine two different losses:\n", 516 | " # 1. The loss for the GAN problem, where we minimize the cross-entropy for the binary\n", 517 | " # real-vs-fake classification problem.\n", 518 | " # 2. The loss for the SVHN digit classification problem, where we minimize the cross-entropy\n", 519 | " # for the multi-class softmax. For this one we use the labels. Don't forget to ignore\n", 520 | " # use `label_mask` to ignore the examples that we are pretending are unlabeled for the\n", 521 | " # semi-supervised learning problem\n", 522 | " \n", 523 | " #Loss on real data and fake data for the unsupervised portion of the model\n", 524 | " d_loss_real = tf.reduce_mean(\n", 525 | " tf.nn.sigmoid_cross_entropy_with_logits(logits=gan_logits_on_data,\n", 526 | " labels=tf.ones_like(gan_logits_on_data)))\n", 527 | " d_loss_fake = tf.reduce_mean(\n", 528 | " tf.nn.sigmoid_cross_entropy_with_logits(logits=gan_logits_on_samples,\n", 529 | " labels=tf.zeros_like(gan_logits_on_samples)))\n", 530 | " \n", 531 | " #Loss on the supervised portion of the model\n", 532 | " y = tf.squeeze(y)\n", 533 | " class_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=class_logits_on_data,\n", 534 | " labels=tf.one_hot(y, num_classes + extra_class,\n", 535 | " dtype=tf.float32))\n", 536 | " class_cross_entropy = tf.squeeze(class_cross_entropy)\n", 537 | " label_mask = tf.squeeze(tf.to_float(label_mask))\n", 538 | " d_loss_class = tf.reduce_sum(label_mask * class_cross_entropy) / tf.maximum(1., tf.reduce_sum(label_mask))\n", 539 | " \n", 540 | " #Total loss (combining the supervised loss and unsupervised loss)\n", 541 | " d_loss = d_loss_class + d_loss_real + d_loss_fake\n", 542 | " \n", 543 | " # Here we set `g_loss` to the \"feature matching\" loss invented by Tim Salimans at OpenAI.\n", 544 | " # This loss consists of minimizing the absolute difference between the expected features\n", 545 | " # on the data and the expected features on the generated samples.\n", 546 | " # This loss works better for semi-supervised learning than the tradition GAN losses.\n", 547 | " \n", 548 | " data_moments = tf.reduce_mean(data_features, axis=0)\n", 549 | " sample_moments = tf.reduce_mean(sample_features, axis=0)\n", 550 | " # Compute the mean absolute loss between the moments and use this for the generator loss. \n", 551 | " g_loss = tf.reduce_mean(tf.abs(data_moments - sample_moments))\n", 552 | "\n", 553 | " pred_class = tf.cast(tf.argmax(class_logits_on_data, 1), tf.int32)\n", 554 | " eq = tf.equal(tf.squeeze(y), pred_class)\n", 555 | " correct = tf.reduce_sum(tf.to_float(eq))\n", 556 | " masked_correct = tf.reduce_sum(label_mask * tf.to_float(eq))\n", 557 | " \n", 558 | " return d_loss, g_loss, correct, masked_correct, g_model\n", 559 | "\n", 560 | " ### END EXAMPLE MODEL LOSS ###\n", 561 | "\n", 562 | "\"\"\"\n", 563 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 564 | "\"\"\"\n", 565 | "tests.test_model_loss(model_loss)" 566 | ] 567 | }, 568 | { 569 | "cell_type": "markdown", 570 | "metadata": { 571 | "deletable": true, 572 | "editable": true 573 | }, 574 | "source": [ 575 | "### Optimization\n", 576 | "Implement `model_opt` to create the optimization operations for the GANs. Use [`tf.trainable_variables`](https://www.tensorflow.org/api_docs/python/tf/trainable_variables) to get all the trainable variables. Filter the variables with names that are in the discriminator and generator scope names. The function should return a tuple of (discriminator training operation, generator training operation)." 577 | ] 578 | }, 579 | { 580 | "cell_type": "code", 581 | "execution_count": null, 582 | "metadata": { 583 | "collapsed": false, 584 | "deletable": true, 585 | "editable": true 586 | }, 587 | "outputs": [], 588 | "source": [ 589 | "def model_opt(d_loss, g_loss, learning_rate, beta1):\n", 590 | " \"\"\"\n", 591 | " Get optimization operations\n", 592 | " :param d_loss: Discriminator loss Tensor\n", 593 | " :param g_loss: Generator loss Tensor\n", 594 | " :param learning_rate: Learning Rate Placeholder\n", 595 | " :param beta1: The exponential decay rate for the 1st moment in the optimizer\n", 596 | " :return: A tuple of (discriminator training operation, generator training operation)\n", 597 | " \"\"\"\n", 598 | " # TODO: Implement Function\n", 599 | " \n", 600 | " # Create optimzation operation for the GANs. \n", 601 | " # Use \"tf.trainable_variables\" to get all the trainable variables. \n", 602 | " # Filter the trainable variables using the \"discriminator\" and \"generator\" name scopes.\n", 603 | " # Return the discriminator training operation and generator training operation. \n", 604 | " \n", 605 | " return None, None\n", 606 | "\n", 607 | " ### EXAMPLE MODEL OPT CODE ###\n", 608 | " \n", 609 | " def model_opt(d_loss, g_loss, learning_rate, beta1):\n", 610 | " \"\"\"\n", 611 | " Get optimization operations\n", 612 | " :param d_loss: Discriminator loss Tensor\n", 613 | " :param g_loss: Generator loss Tensor\n", 614 | " :param learning_rate: Learning Rate Placeholder\n", 615 | " :param beta1: The exponential decay rate for the 1st moment in the optimizer\n", 616 | " :return: A tuple of (discriminator training operation, generator training operation)\n", 617 | " \"\"\"\n", 618 | " # Get weights and biases to update. Get them separately for the discriminator and the generator\n", 619 | " \n", 620 | " t_vars = tf.trainable_variables()\n", 621 | " \n", 622 | " # Because variable scopes are used, all of the variables for the \n", 623 | " # discriminator begin with 'discriminator' and all of the variables for\n", 624 | " # the generator begin with 'generator'. \n", 625 | " d_vars = [var for var in t_vars if var.name.startswith('discriminator')]\n", 626 | " g_vars = [var for var in t_vars if var.name.startswith('generator')]\n", 627 | " for t in t_vars:\n", 628 | " # Use assert to make sure all of the training variables belong to \n", 629 | " # one of the above lists or the other. \n", 630 | " assert t in d_vars or t in g_vars\n", 631 | "\n", 632 | " # Minimize both players' costs simultaneously\n", 633 | " # It's important each of the optimized variables are optimized for their specific network\n", 634 | " # eg, discriminator is optimized with discriminator variables. \n", 635 | " d_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(d_loss, var_list=d_vars)\n", 636 | " g_train_opt = tf.train.AdamOptimizer(learning_rate, beta1=beta1).minimize(g_loss, var_list=g_vars)\n", 637 | " shrink_lr = tf.assign(learning_rate, learning_rate * 0.9)\n", 638 | " \n", 639 | " return d_train_opt, g_train_opt, shrink_lr\n", 640 | "\n", 641 | " ### END MODEL OPT EXAMPLE ###\n", 642 | "\n", 643 | "\n", 644 | "\"\"\"\n", 645 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 646 | "\"\"\"\n", 647 | "tests.test_model_opt(model_opt, tf)" 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "metadata": { 653 | "deletable": true, 654 | "editable": true 655 | }, 656 | "source": [ 657 | "## Neural Network Training\n", 658 | "### Show Output\n", 659 | "Use this function to show the current output of the generator during training. It will help you determine how well the GANs is training." 660 | ] 661 | }, 662 | { 663 | "cell_type": "code", 664 | "execution_count": null, 665 | "metadata": { 666 | "collapsed": true, 667 | "deletable": true, 668 | "editable": true 669 | }, 670 | "outputs": [], 671 | "source": [ 672 | "\"\"\"\n", 673 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 674 | "\"\"\"\n", 675 | "import numpy as np\n", 676 | "\n", 677 | "def show_generator_output(sess, n_images, input_z, out_channel_dim, image_mode):\n", 678 | " \"\"\"\n", 679 | " Show example output for the generator\n", 680 | " :param sess: TensorFlow session\n", 681 | " :param n_images: Number of Images to display\n", 682 | " :param input_z: Input Z Tensor\n", 683 | " :param out_channel_dim: The number of channels in the output image\n", 684 | " :param image_mode: The mode to use for images (\"RGB\" or \"L\")\n", 685 | " \"\"\"\n", 686 | " cmap = None if image_mode == 'RGB' else 'gray'\n", 687 | " z_dim = input_z.get_shape().as_list()[-1]\n", 688 | " example_z = np.random.uniform(-1, 1, size=[n_images, z_dim])\n", 689 | "\n", 690 | " samples = sess.run(\n", 691 | " generator(input_z, out_channel_dim, False),\n", 692 | " feed_dict={input_z: example_z})\n", 693 | "\n", 694 | " images_grid = helper.images_square_grid(samples, image_mode)\n", 695 | " pyplot.imshow(images_grid, cmap=cmap)\n", 696 | " pyplot.show()" 697 | ] 698 | }, 699 | { 700 | "cell_type": "markdown", 701 | "metadata": { 702 | "deletable": true, 703 | "editable": true 704 | }, 705 | "source": [ 706 | "### Train\n", 707 | "Implement `train` to build and train the GANs. Use the following functions you implemented:\n", 708 | "- `model_inputs(image_width, image_height, image_channels, z_dim)`\n", 709 | "- `model_loss(input_real, input_z, out_channel_dim)`\n", 710 | "- `model_opt(d_loss, g_loss, learning_rate, beta1)`\n", 711 | "\n", 712 | "Use the `show_generator_output` to show `generator` output while you train. Running `show_generator_output` for every batch will drastically increase training time and increase the size of the notebook. It's recommended to print the `generator` output every 100 batches." 713 | ] 714 | }, 715 | { 716 | "cell_type": "code", 717 | "execution_count": null, 718 | "metadata": { 719 | "collapsed": true, 720 | "deletable": true, 721 | "editable": true 722 | }, 723 | "outputs": [], 724 | "source": [ 725 | "def train(epoch_count, batch_size, z_dim, learning_rate, beta1, get_batches, data_shape, data_image_mode):\n", 726 | " \"\"\"\n", 727 | " Train the GAN\n", 728 | " :param epoch_count: Number of epochs\n", 729 | " :param batch_size: Batch Size\n", 730 | " :param z_dim: Z dimension\n", 731 | " :param learning_rate: Learning Rate\n", 732 | " :param beta1: The exponential decay rate for the 1st moment in the optimizer\n", 733 | " :param get_batches: Function to get batches\n", 734 | " :param data_shape: Shape of the data\n", 735 | " :param data_image_mode: The image mode to use for images (\"RGB\" or \"L\")\n", 736 | " \"\"\"\n", 737 | " # TODO: Build Model\n", 738 | " \n", 739 | " # Use the previously created functions to build and train the GANs. \n", 740 | " # Show the output of the generator using 'show_generator_output'. - [FIND THIS]\n", 741 | " # Show the output every 100 batches, anything more will slow down training. \n", 742 | " \n", 743 | " with tf.Session() as sess:\n", 744 | " sess.run(tf.global_variables_initializer())\n", 745 | " for epoch_i in range(epoch_count):\n", 746 | " for batch_images in get_batches(batch_size):\n", 747 | " # TODO: Train Model\n", 748 | " \n", 749 | " \n", 750 | " ### EXAMPLE TRAIN FUNCTION CODE ###\n", 751 | " \n", 752 | " # This section is simplifying the training process a little bit,\n", 753 | "# For a real application it would need to be improved.\n", 754 | "def train(net, dataset, epochs, batch_size, figsize=(5,5)):\n", 755 | " \n", 756 | " saver = tf.train.Saver()\n", 757 | " sample_z = np.random.normal(0, 1, size=(50, z_size))\n", 758 | "\n", 759 | " samples, train_accuracies, test_accuracies = [], [], []\n", 760 | " steps = 0\n", 761 | "\n", 762 | " with tf.Session() as sess:\n", 763 | " sess.run(tf.global_variables_initializer())\n", 764 | " for e in range(epochs):\n", 765 | " print(\"Epoch\",e)\n", 766 | " \n", 767 | " t1e = time.time()\n", 768 | " num_examples = 0\n", 769 | " num_correct = 0\n", 770 | " for x, y, label_mask in dataset.batches(batch_size):\n", 771 | " assert 'int' in str(y.dtype)\n", 772 | " steps += 1\n", 773 | " num_examples += label_mask.sum()\n", 774 | "\n", 775 | " # Sample random noise for G\n", 776 | " batch_z = np.random.normal(0, 1, size=(batch_size, z_size))\n", 777 | "\n", 778 | " # Run optimizers\n", 779 | " t1 = time.time()\n", 780 | " _, _, correct = sess.run([net.d_opt, net.g_opt, net.masked_correct],\n", 781 | " feed_dict={net.input_real: x, net.input_z: batch_z,\n", 782 | " net.y : y, net.label_mask : label_mask})\n", 783 | " t2 = time.time()\n", 784 | " num_correct += correct\n", 785 | "\n", 786 | " sess.run([net.shrink_lr])\n", 787 | " \n", 788 | " \n", 789 | " train_accuracy = num_correct / float(num_examples)\n", 790 | " \n", 791 | " print(\"\\t\\tClassifier train accuracy: \", train_accuracy)\n", 792 | " \n", 793 | " num_examples = 0\n", 794 | " num_correct = 0\n", 795 | " for x, y in dataset.batches(batch_size, which_set=\"test\"):\n", 796 | " assert 'int' in str(y.dtype)\n", 797 | " num_examples += x.shape[0]\n", 798 | "\n", 799 | " correct, = sess.run([net.correct], feed_dict={net.input_real: x,\n", 800 | " net.y : y,\n", 801 | " net.drop_rate: 0.})\n", 802 | " num_correct += correct\n", 803 | " \n", 804 | " test_accuracy = num_correct / float(num_examples)\n", 805 | " print(\"\\t\\tClassifier test accuracy\", test_accuracy)\n", 806 | " print(\"\\t\\tStep time: \", t2 - t1)\n", 807 | " t2e = time.time()\n", 808 | " print(\"\\t\\tEpoch time: \", t2e - t1e)\n", 809 | " \n", 810 | " \n", 811 | " gen_samples = sess.run(\n", 812 | " net.samples,\n", 813 | " feed_dict={net.input_z: sample_z})\n", 814 | " samples.append(gen_samples)\n", 815 | " _ = view_samples(-1, samples, 5, 10, figsize=figsize)\n", 816 | " plt.show()\n", 817 | " \n", 818 | " \n", 819 | " # Save history of accuracies to view after training\n", 820 | " train_accuracies.append(train_accuracy)\n", 821 | " test_accuracies.append(test_accuracy)\n", 822 | " \n", 823 | "\n", 824 | " saver.save(sess, './checkpoints/generator.ckpt')\n", 825 | "\n", 826 | " with open('samples.pkl', 'wb') as f:\n", 827 | " pkl.dump(samples, f)\n", 828 | " \n", 829 | " return train_accuracies, test_accuracies, samples\n", 830 | "\n", 831 | " ### END TRAIN FUNCTION EXAMPLE ###\n", 832 | " " 833 | ] 834 | }, 835 | { 836 | "cell_type": "markdown", 837 | "metadata": { 838 | "deletable": true, 839 | "editable": true 840 | }, 841 | "source": [ 842 | "### MNIST\n", 843 | "Test your GANs architecture on MNIST. After 2 epochs, the GANs should be able to generate images that look like handwritten digits. Make sure the loss of the generator is lower than the loss of the discriminator or close to 0." 844 | ] 845 | }, 846 | { 847 | "cell_type": "code", 848 | "execution_count": null, 849 | "metadata": { 850 | "collapsed": false, 851 | "deletable": true, 852 | "editable": true, 853 | "scrolled": true 854 | }, 855 | "outputs": [], 856 | "source": [ 857 | "batch_size = None\n", 858 | "z_dim = None\n", 859 | "learning_rate = None\n", 860 | "beta1 = None\n", 861 | "\n", 862 | "\n", 863 | "\"\"\"\n", 864 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 865 | "\"\"\"\n", 866 | "epochs = 2\n", 867 | "\n", 868 | "mnist_dataset = helper.Dataset('mnist', glob(os.path.join(data_dir, 'mnist/*.jpg')))\n", 869 | "with tf.Graph().as_default():\n", 870 | " train(epochs, batch_size, z_dim, learning_rate, beta1, mnist_dataset.get_batches,\n", 871 | " mnist_dataset.shape, mnist_dataset.image_mode)" 872 | ] 873 | }, 874 | { 875 | "cell_type": "markdown", 876 | "metadata": { 877 | "deletable": true, 878 | "editable": true 879 | }, 880 | "source": [ 881 | "### CelebA\n", 882 | "Run your GANs on CelebA. It will take around 20 minutes on the average GPU to run one epoch. You can run the whole epoch or stop when it starts to generate realistic faces." 883 | ] 884 | }, 885 | { 886 | "cell_type": "code", 887 | "execution_count": null, 888 | "metadata": { 889 | "collapsed": false, 890 | "deletable": true, 891 | "editable": true, 892 | "scrolled": true 893 | }, 894 | "outputs": [], 895 | "source": [ 896 | "batch_size = None\n", 897 | "z_dim = None\n", 898 | "learning_rate = None\n", 899 | "beta1 = None\n", 900 | "\n", 901 | "\n", 902 | "\"\"\"\n", 903 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 904 | "\"\"\"\n", 905 | "epochs = 1\n", 906 | "\n", 907 | "celeba_dataset = helper.Dataset('celeba', glob(os.path.join(data_dir, 'img_align_celeba/*.jpg')))\n", 908 | "with tf.Graph().as_default():\n", 909 | " train(epochs, batch_size, z_dim, learning_rate, beta1, celeba_dataset.get_batches,\n", 910 | " celeba_dataset.shape, celeba_dataset.image_mode)" 911 | ] 912 | }, 913 | { 914 | "cell_type": "markdown", 915 | "metadata": { 916 | "deletable": true, 917 | "editable": true 918 | }, 919 | "source": [ 920 | "### Submitting This Project\n", 921 | "When submitting this project, make sure to run all the cells before saving the notebook. Save the notebook file as \"dlnd_face_generation.ipynb\" and save it as a HTML file under \"File\" -> \"Download as\". Include the \"helper.py\" and \"problem_unittests.py\" files in your submission." 922 | ] 923 | } 924 | ], 925 | "metadata": { 926 | "kernelspec": { 927 | "display_name": "Python 3", 928 | "language": "python", 929 | "name": "python3" 930 | }, 931 | "language_info": { 932 | "codemirror_mode": { 933 | "name": "ipython", 934 | "version": 3 935 | }, 936 | "file_extension": ".py", 937 | "mimetype": "text/x-python", 938 | "name": "python", 939 | "nbconvert_exporter": "python", 940 | "pygments_lexer": "ipython3", 941 | "version": "3.6.0" 942 | } 943 | }, 944 | "nbformat": 4, 945 | "nbformat_minor": 0 946 | } 947 | -------------------------------------------------------------------------------- /PROJECT_3_TV_Script_Generation/dlnd_tv_script_generation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# TV Script Generation\n", 11 | "In this project, you'll generate your own [Simpsons](https://en.wikipedia.org/wiki/The_Simpsons) TV scripts using RNNs. You'll be using part of the [Simpsons dataset](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data) of scripts from 27 seasons. The Neural Network you'll build will generate a new TV script for a scene at [Moe's Tavern](https://simpsonswiki.com/wiki/Moe's_Tavern).\n", 12 | "## Get the Data\n", 13 | "The data is already provided for you. You'll be using a subset of the original dataset. It consists of only the scenes in Moe's Tavern. This doesn't include other versions of the tavern, like \"Moe's Cavern\", \"Flaming Moe's\", \"Uncle Moe's Family Feed-Bag\", etc.." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "metadata": { 20 | "collapsed": false, 21 | "deletable": true, 22 | "editable": true 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "\"\"\"\n", 27 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 28 | "\"\"\"\n", 29 | "import helper\n", 30 | "\n", 31 | "data_dir = './data/simpsons/moes_tavern_lines.txt'\n", 32 | "text = helper.load_data(data_dir)\n", 33 | "# Ignore notice, since we don't use it for analysing the data\n", 34 | "text = text[81:]" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": { 40 | "deletable": true, 41 | "editable": true 42 | }, 43 | "source": [ 44 | "## Explore the Data\n", 45 | "Play around with `view_sentence_range` to view different parts of the data." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": { 52 | "collapsed": false, 53 | "deletable": true, 54 | "editable": true 55 | }, 56 | "outputs": [ 57 | { 58 | "name": "stdout", 59 | "output_type": "stream", 60 | "text": [ 61 | "Dataset Stats\n", 62 | "Roughly the number of unique words: 11492\n", 63 | "Number of scenes: 262\n", 64 | "Average number of sentences in each scene: 15.248091603053435\n", 65 | "Number of lines: 4257\n", 66 | "Average number of words in each line: 11.50434578341555\n", 67 | "\n", 68 | "The sentences 0 to 10:\n", 69 | "Moe_Szyslak: (INTO PHONE) Moe's Tavern. Where the elite meet to drink.\n", 70 | "Bart_Simpson: Eh, yeah, hello, is Mike there? Last name, Rotch.\n", 71 | "Moe_Szyslak: (INTO PHONE) Hold on, I'll check. (TO BARFLIES) Mike Rotch. Mike Rotch. Hey, has anybody seen Mike Rotch, lately?\n", 72 | "Moe_Szyslak: (INTO PHONE) Listen you little puke. One of these days I'm gonna catch you, and I'm gonna carve my name on your back with an ice pick.\n", 73 | "Moe_Szyslak: What's the matter Homer? You're not your normal effervescent self.\n", 74 | "Homer_Simpson: I got my problems, Moe. Give me another one.\n", 75 | "Moe_Szyslak: Homer, hey, you should not drink to forget your problems.\n", 76 | "Barney_Gumble: Yeah, you should only drink to enhance your social skills.\n", 77 | "\n", 78 | "\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "view_sentence_range = (0, 10)\n", 84 | "\n", 85 | "\"\"\"\n", 86 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 87 | "\"\"\"\n", 88 | "import numpy as np\n", 89 | "\n", 90 | "print('Dataset Stats')\n", 91 | "print('Roughly the number of unique words: {}'.format(len({word: None for word in text.split()})))\n", 92 | "scenes = text.split('\\n\\n')\n", 93 | "print('Number of scenes: {}'.format(len(scenes)))\n", 94 | "sentence_count_scene = [scene.count('\\n') for scene in scenes]\n", 95 | "print('Average number of sentences in each scene: {}'.format(np.average(sentence_count_scene)))\n", 96 | "\n", 97 | "sentences = [sentence for scene in scenes for sentence in scene.split('\\n')]\n", 98 | "print('Number of lines: {}'.format(len(sentences)))\n", 99 | "word_count_sentence = [len(sentence.split()) for sentence in sentences]\n", 100 | "print('Average number of words in each line: {}'.format(np.average(word_count_sentence)))\n", 101 | "\n", 102 | "print()\n", 103 | "print('The sentences {} to {}:'.format(*view_sentence_range))\n", 104 | "print('\\n'.join(text.split('\\n')[view_sentence_range[0]:view_sentence_range[1]]))" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": { 110 | "deletable": true, 111 | "editable": true 112 | }, 113 | "source": [ 114 | "## Implement Preprocessing Functions\n", 115 | "The first thing to do to any dataset is preprocessing. Implement the following preprocessing functions below:\n", 116 | "- Lookup Table\n", 117 | "- Tokenize Punctuation\n", 118 | "\n", 119 | "### Lookup Table\n", 120 | "To create a word embedding, you first need to transform the words to ids. In this function, create two dictionaries:\n", 121 | "- Dictionary to go from the words to an id, we'll call `vocab_to_int`\n", 122 | "- Dictionary to go from the id to word, we'll call `int_to_vocab`\n", 123 | "\n", 124 | "Return these dictionaries in the following tuple `(vocab_to_int, int_to_vocab)`" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 3, 130 | "metadata": { 131 | "collapsed": false, 132 | "deletable": true, 133 | "editable": true 134 | }, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "Tests Passed\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "import numpy as np\n", 146 | "import problem_unittests as tests\n", 147 | "from collections import Counter\n", 148 | "\n", 149 | "def create_lookup_tables(text):\n", 150 | " \"\"\"\n", 151 | " Create lookup tables for vocabulary\n", 152 | " :param text: The text of tv scripts split into words\n", 153 | " :return: A tuple of dicts (vocab_to_int, int_to_vocab)\n", 154 | " \"\"\"\n", 155 | " # TODO: Implement Function\n", 156 | " '''NEW METHOD - Simpler'''\n", 157 | " vocab = set(text)\n", 158 | " #create dict of vocab to ints\n", 159 | " vocab_to_int = dict((word, index) for index, word in enumerate(vocab))\n", 160 | " int_to_vocab = dict((index, word) for index, word in enumerate(vocab))\n", 161 | " return vocab_to_int, int_to_vocab\n", 162 | " \n", 163 | " '''OLD METHOD'''\n", 164 | " '''\n", 165 | " counts = Counter(text)\n", 166 | " vocab = sorted(counts, key=counts.get, reverse=True)\n", 167 | " #create a dictionary to go from word to id\n", 168 | " vocab_to_int = {word: i for i, word in enumerate(vocab)}\n", 169 | " #create a dictionary to go from id to word\n", 170 | " int_to_vocab = {v: k for k, v in vocab_to_int.items()} \n", 171 | " return (vocab_to_int, int_to_vocab)\n", 172 | " '''\n", 173 | "\n", 174 | "\n", 175 | "\"\"\"\n", 176 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 177 | "\"\"\"\n", 178 | "tests.test_create_lookup_tables(create_lookup_tables)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": { 184 | "deletable": true, 185 | "editable": true 186 | }, 187 | "source": [ 188 | "### Tokenize Punctuation\n", 189 | "We'll be splitting the script into a word array using spaces as delimiters. However, punctuations like periods and exclamation marks make it hard for the neural network to distinguish between the word \"bye\" and \"bye!\".\n", 190 | "\n", 191 | "Implement the function `token_lookup` to return a dict that will be used to tokenize symbols like \"!\" into \"||Exclamation_Mark||\". Create a dictionary for the following symbols where the symbol is the key and value is the token:\n", 192 | "- Period ( . )\n", 193 | "- Comma ( , )\n", 194 | "- Quotation Mark ( \" )\n", 195 | "- Semicolon ( ; )\n", 196 | "- Exclamation mark ( ! )\n", 197 | "- Question mark ( ? )\n", 198 | "- Left Parentheses ( ( )\n", 199 | "- Right Parentheses ( ) )\n", 200 | "- Dash ( -- )\n", 201 | "- Return ( \\n )\n", 202 | "\n", 203 | "This dictionary will be used to token the symbols and add the delimiter (space) around it. This separates the symbols as it's own word, making it easier for the neural network to predict on the next word. Make sure you don't use a token that could be confused as a word. Instead of using the token \"dash\", try using something like \"||dash||\"." 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 4, 209 | "metadata": { 210 | "collapsed": false, 211 | "deletable": true, 212 | "editable": true 213 | }, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "Tests Passed\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "def token_lookup():\n", 225 | " \"\"\"\n", 226 | " Generate a dict to turn punctuation into a token.\n", 227 | " :return: Tokenize dictionary where the key is the punctuation and the value is the token\n", 228 | " \"\"\"\n", 229 | " #creating two arrays for punctuation marks and their relevant tokens\n", 230 | " punc_mark = ['.', ',', '\"', \";\", \"!\", \"?\", \"(\", \")\", \"--\", \"\\n\"]\n", 231 | " punc_token = [\"||period||\", \"||comma||\", \"||quotation||\", \"||semicolon||\",\n", 232 | " \"||exclamation||\", \"||question_mark||\", \"||left_bracket||\",\n", 233 | " \"||right_bracket||\", \"||dash||\", \"||return||\"]\n", 234 | " punc_token_dict = dict(zip(punc_mark, punc_token))\n", 235 | " return punc_token_dict\n", 236 | "\n", 237 | "\n", 238 | "\"\"\"\n", 239 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 240 | "\"\"\"\n", 241 | "tests.test_tokenize(token_lookup)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": { 247 | "deletable": true, 248 | "editable": true 249 | }, 250 | "source": [ 251 | "## Preprocess all the data and save it\n", 252 | "Running the code cell below will preprocess all the data and save it to file." 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 5, 258 | "metadata": { 259 | "collapsed": false, 260 | "deletable": true, 261 | "editable": true 262 | }, 263 | "outputs": [], 264 | "source": [ 265 | "\"\"\"\n", 266 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 267 | "\"\"\"\n", 268 | "# Preprocess Training, Validation, and Testing Data\n", 269 | "helper.preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "metadata": { 275 | "deletable": true, 276 | "editable": true 277 | }, 278 | "source": [ 279 | "# Check Point\n", 280 | "This is your first checkpoint. If you ever decide to come back to this notebook or have to restart the notebook, you can start from here. The preprocessed data has been saved to disk." 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": 6, 286 | "metadata": { 287 | "collapsed": false, 288 | "deletable": true, 289 | "editable": true 290 | }, 291 | "outputs": [], 292 | "source": [ 293 | "\"\"\"\n", 294 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 295 | "\"\"\"\n", 296 | "import helper\n", 297 | "import numpy as np\n", 298 | "import problem_unittests as tests\n", 299 | "\n", 300 | "int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": { 306 | "deletable": true, 307 | "editable": true 308 | }, 309 | "source": [ 310 | "## Build the Neural Network\n", 311 | "You'll build the components necessary to build a RNN by implementing the following functions below:\n", 312 | "- get_inputs\n", 313 | "- get_init_cell\n", 314 | "- get_embed\n", 315 | "- build_rnn\n", 316 | "- build_nn\n", 317 | "- get_batches\n", 318 | "\n", 319 | "### Check the Version of TensorFlow and Access to GPU" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": 7, 325 | "metadata": { 326 | "collapsed": false, 327 | "deletable": true, 328 | "editable": true 329 | }, 330 | "outputs": [ 331 | { 332 | "name": "stdout", 333 | "output_type": "stream", 334 | "text": [ 335 | "TensorFlow Version: 1.0.0\n", 336 | "Default GPU Device: /gpu:0\n" 337 | ] 338 | } 339 | ], 340 | "source": [ 341 | "\"\"\"\n", 342 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 343 | "\"\"\"\n", 344 | "from distutils.version import LooseVersion\n", 345 | "import warnings\n", 346 | "import tensorflow as tf\n", 347 | "\n", 348 | "# Check TensorFlow Version\n", 349 | "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'\n", 350 | "print('TensorFlow Version: {}'.format(tf.__version__))\n", 351 | "\n", 352 | "# Check for a GPU\n", 353 | "if not tf.test.gpu_device_name():\n", 354 | " warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n", 355 | "else:\n", 356 | " print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))" 357 | ] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "metadata": { 362 | "deletable": true, 363 | "editable": true 364 | }, 365 | "source": [ 366 | "### Input\n", 367 | "Implement the `get_inputs()` function to create TF Placeholders for the Neural Network. It should create the following placeholders:\n", 368 | "- Input text placeholder named \"input\" using the [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder) `name` parameter.\n", 369 | "- Targets placeholder\n", 370 | "- Learning Rate placeholder\n", 371 | "\n", 372 | "Return the placeholders in the following tuple `(Input, Targets, LearningRate)`" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 8, 378 | "metadata": { 379 | "collapsed": false, 380 | "deletable": true, 381 | "editable": true 382 | }, 383 | "outputs": [ 384 | { 385 | "name": "stdout", 386 | "output_type": "stream", 387 | "text": [ 388 | "Tests Passed\n" 389 | ] 390 | } 391 | ], 392 | "source": [ 393 | "def get_inputs():\n", 394 | " \"\"\"\n", 395 | " Create TF Placeholders for input, targets, and learning rate.\n", 396 | " :return: Tuple (input, targets, learning rate)\n", 397 | " \"\"\"\n", 398 | " # TODO: Implement Function\n", 399 | " inputs_ = tf.placeholder(tf.int32, [None, None], name = 'input')\n", 400 | " targets_ = tf.placeholder(tf.int32, [None, None], name = 'targets')\n", 401 | " learning_rate_ = tf.placeholder(tf.float32, name = 'learning_rate')\n", 402 | " return (inputs_, targets_, learning_rate_)\n", 403 | "\n", 404 | "\n", 405 | "\"\"\"\n", 406 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 407 | "\"\"\"\n", 408 | "tests.test_get_inputs(get_inputs)" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": { 414 | "deletable": true, 415 | "editable": true 416 | }, 417 | "source": [ 418 | "### Build RNN Cell and Initialize\n", 419 | "Stack one or more [`BasicLSTMCells`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/BasicLSTMCell) in a [`MultiRNNCell`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell).\n", 420 | "- The Rnn size should be set using `rnn_size`\n", 421 | "- Initalize Cell State using the MultiRNNCell's [`zero_state()`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell#zero_state) function\n", 422 | " - Apply the name \"initial_state\" to the initial state using [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity)\n", 423 | "\n", 424 | "Return the cell and initial state in the following tuple `(Cell, InitialState)`" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 9, 430 | "metadata": { 431 | "collapsed": false, 432 | "deletable": true, 433 | "editable": true 434 | }, 435 | "outputs": [ 436 | { 437 | "name": "stdout", 438 | "output_type": "stream", 439 | "text": [ 440 | "Tests Passed\n" 441 | ] 442 | } 443 | ], 444 | "source": [ 445 | "def get_init_cell(batch_size, rnn_size):\n", 446 | " \"\"\"\n", 447 | " Create an RNN Cell and initialize it.\n", 448 | " :param batch_size: Size of batches\n", 449 | " :param rnn_size: Size of RNNs\n", 450 | " :return: Tuple (cell, initialize state)\n", 451 | " \"\"\"\n", 452 | " # TODO: Implement Function\n", 453 | " cell = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n", 454 | " #not sure whether or not to keep the dropout layer\n", 455 | " #drop = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob = 0.6)\n", 456 | " cell_stack = tf.contrib.rnn.MultiRNNCell([cell], rnn_size)\n", 457 | " initial_state = cell_stack.zero_state(batch_size, tf.float32)\n", 458 | " initial_state = tf.identity(initial_state, name =\"initial_state\")\n", 459 | " return (cell_stack, initial_state)\n", 460 | "\n", 461 | "\n", 462 | "\"\"\"\n", 463 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 464 | "\"\"\"\n", 465 | "tests.test_get_init_cell(get_init_cell)" 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": { 471 | "deletable": true, 472 | "editable": true 473 | }, 474 | "source": [ 475 | "### Word Embedding\n", 476 | "Apply embedding to `input_data` using TensorFlow. Return the embedded sequence." 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 10, 482 | "metadata": { 483 | "collapsed": false, 484 | "deletable": true, 485 | "editable": true 486 | }, 487 | "outputs": [ 488 | { 489 | "name": "stdout", 490 | "output_type": "stream", 491 | "text": [ 492 | "Tests Passed\n" 493 | ] 494 | } 495 | ], 496 | "source": [ 497 | "def get_embed(input_data, vocab_size, embed_dim):\n", 498 | " \"\"\"\n", 499 | " Create embedding for .\n", 500 | " :param input_data: TF placeholder for text input.\n", 501 | " :param vocab_size: Number of words in vocabulary.\n", 502 | " :param embed_dim: Number of embedding dimensions\n", 503 | " :return: Embedded input.\n", 504 | " \"\"\"\n", 505 | " # TODO: Implement Function\n", 506 | " embedding = tf.Variable(tf.random_uniform((vocab_size, embed_dim), -1, 1))\n", 507 | " embed = tf.nn.embedding_lookup(embedding, input_data)\n", 508 | " return embed\n", 509 | "\n", 510 | "\n", 511 | "\"\"\"\n", 512 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 513 | "\"\"\"\n", 514 | "tests.test_get_embed(get_embed)" 515 | ] 516 | }, 517 | { 518 | "cell_type": "markdown", 519 | "metadata": { 520 | "deletable": true, 521 | "editable": true 522 | }, 523 | "source": [ 524 | "### Build RNN\n", 525 | "You created a RNN Cell in the `get_init_cell()` function. Time to use the cell to create a RNN.\n", 526 | "- Build the RNN using the [`tf.nn.dynamic_rnn()`](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn)\n", 527 | " - Apply the name \"final_state\" to the final state using [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity)\n", 528 | "\n", 529 | "Return the outputs and final_state state in the following tuple `(Outputs, FinalState)` " 530 | ] 531 | }, 532 | { 533 | "cell_type": "code", 534 | "execution_count": 11, 535 | "metadata": { 536 | "collapsed": false, 537 | "deletable": true, 538 | "editable": true 539 | }, 540 | "outputs": [ 541 | { 542 | "name": "stdout", 543 | "output_type": "stream", 544 | "text": [ 545 | "Tests Passed\n" 546 | ] 547 | } 548 | ], 549 | "source": [ 550 | "def build_rnn(cell, inputs):\n", 551 | " \"\"\"\n", 552 | " Create a RNN using a RNN Cell\n", 553 | " :param cell: RNN Cell\n", 554 | " :param inputs: Input text data\n", 555 | " :return: Tuple (Outputs, Final State)\n", 556 | " \"\"\"\n", 557 | " # TODO: Implement Function\n", 558 | " #creating the outputs and final_state using tf.nn.dynamic_rnn\n", 559 | " outputs, final_state = tf.nn.dynamic_rnn(cell, inputs = inputs, dtype=tf.float32)\n", 560 | " #using tf.identity to apply the name \"final_state\" to final_state\n", 561 | " final_state = tf.identity(final_state, \"final_state\")\n", 562 | " return (outputs, final_state)\n", 563 | "\n", 564 | "\n", 565 | "\"\"\"\n", 566 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 567 | "\"\"\"\n", 568 | "tests.test_build_rnn(build_rnn)" 569 | ] 570 | }, 571 | { 572 | "cell_type": "markdown", 573 | "metadata": { 574 | "deletable": true, 575 | "editable": true 576 | }, 577 | "source": [ 578 | "### Build the Neural Network\n", 579 | "Apply the functions you implemented above to:\n", 580 | "- Apply embedding to `input_data` using your `get_embed(input_data, vocab_size, embed_dim)` function.\n", 581 | "- Build RNN using `cell` and your `build_rnn(cell, inputs)` function.\n", 582 | "- Apply a fully connected layer with a linear activation and `vocab_size` as the number of outputs.\n", 583 | "\n", 584 | "Return the logits and final state in the following tuple (Logits, FinalState) " 585 | ] 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": 12, 590 | "metadata": { 591 | "collapsed": false, 592 | "deletable": true, 593 | "editable": true 594 | }, 595 | "outputs": [ 596 | { 597 | "name": "stdout", 598 | "output_type": "stream", 599 | "text": [ 600 | "Tests Passed\n" 601 | ] 602 | } 603 | ], 604 | "source": [ 605 | "def build_nn(cell, rnn_size, input_data, vocab_size, embed_dim):\n", 606 | " \"\"\"\n", 607 | " Build part of the neural network\n", 608 | " :param cell: RNN cell\n", 609 | " :param rnn_size: Size of rnns\n", 610 | " :param input_data: Input data\n", 611 | " :param vocab_size: Vocabulary size\n", 612 | " :param embed_dim: Number of embedding dimensions\n", 613 | " :return: Tuple (Logits, FinalState)\n", 614 | " \"\"\"\n", 615 | " # TODO: Implement Function\n", 616 | " inputs = get_embed(input_data, vocab_size, embed_dim)\n", 617 | " outputs, final_state = build_rnn(cell, inputs)\n", 618 | " logits = tf.contrib.layers.fully_connected(outputs, vocab_size, activation_fn=None,\n", 619 | " weights_initializer=tf.truncated_normal_initializer\n", 620 | " (mean=0.0, stddev=0.1),\n", 621 | " biases_initializer=tf.zeros_initializer())\n", 622 | " return(logits, final_state)\n", 623 | "\n", 624 | "\n", 625 | "\"\"\"\n", 626 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 627 | "\"\"\"\n", 628 | "tests.test_build_nn(build_nn)" 629 | ] 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": { 634 | "deletable": true, 635 | "editable": true 636 | }, 637 | "source": [ 638 | "### Batches\n", 639 | "Implement `get_batches` to create batches of input and targets using `int_text`. The batches should be a Numpy array with the shape `(number of batches, 2, batch size, sequence length)`. Each batch contains two elements:\n", 640 | "- The first element is a single batch of **input** with the shape `[batch size, sequence length]`\n", 641 | "- The second element is a single batch of **targets** with the shape `[batch size, sequence length]`\n", 642 | "\n", 643 | "If you can't fill the last batch with enough data, drop the last batch.\n", 644 | "\n", 645 | "For exmple, `get_batches([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], 3, 2)` would return a Numpy array of the following:\n", 646 | "```\n", 647 | "[\n", 648 | " # First Batch\n", 649 | " [\n", 650 | " # Batch of Input\n", 651 | " [[ 1 2], [ 7 8], [13 14]]\n", 652 | " # Batch of targets\n", 653 | " [[ 2 3], [ 8 9], [14 15]]\n", 654 | " ]\n", 655 | "\n", 656 | " # Second Batch\n", 657 | " [\n", 658 | " # Batch of Input\n", 659 | " [[ 3 4], [ 9 10], [15 16]]\n", 660 | " # Batch of targets\n", 661 | " [[ 4 5], [10 11], [16 17]]\n", 662 | " ]\n", 663 | "\n", 664 | " # Third Batch\n", 665 | " [\n", 666 | " # Batch of Input\n", 667 | " [[ 5 6], [11 12], [17 18]]\n", 668 | " # Batch of targets\n", 669 | " [[ 6 7], [12 13], [18 1]]\n", 670 | " ]\n", 671 | "]\n", 672 | "```\n", 673 | "\n", 674 | "Notice that the last target value in the last batch is the first input value of the first batch. In this case, `1`. This is a common technique used when creating sequence batches, although it is rather unintuitive." 675 | ] 676 | }, 677 | { 678 | "cell_type": "code", 679 | "execution_count": 13, 680 | "metadata": { 681 | "collapsed": false, 682 | "deletable": true, 683 | "editable": true 684 | }, 685 | "outputs": [ 686 | { 687 | "name": "stdout", 688 | "output_type": "stream", 689 | "text": [ 690 | "Tests Passed\n" 691 | ] 692 | } 693 | ], 694 | "source": [ 695 | "def get_batches(int_text, batch_size, seq_length):\n", 696 | " \"\"\"\n", 697 | " Return batches of input and target\n", 698 | " :param int_text: Text with the words replaced by their ids\n", 699 | " :param batch_size: The size of batch\n", 700 | " :param seq_length: The length of sequence\n", 701 | " :return: Batches as a Numpy array\n", 702 | " \"\"\"\n", 703 | " # TODO: Implement Function\n", 704 | " \n", 705 | " #define number of batches\n", 706 | " text_len = len(int_text)\n", 707 | " n_batches = text_len // (batch_size * seq_length)\n", 708 | " #print(n_batches)\n", 709 | " \n", 710 | " #define inputs and targets as single row arrays\n", 711 | " inputs = np.array(int_text[: n_batches * batch_size * seq_length])\n", 712 | " #targets starts at 1 index instead of 0 and goes 1 step further\n", 713 | " #on seq_length\n", 714 | " targets = np.array(int_text[1: n_batches * batch_size * seq_length + 1])\n", 715 | " #change last element of targets to first element of inputs\n", 716 | " targets[-1] = inputs[0]\n", 717 | " #print(targets)\n", 718 | " #print(inputs)\n", 719 | "\n", 720 | " #reshape inputs and targets into batches\n", 721 | " inputs_batches = np.split(inputs.reshape(batch_size, -1), n_batches, 1)\n", 722 | " targets_batches = np.split(targets.reshape(batch_size, -1), n_batches, 1)\n", 723 | " #print(inputs_batches)\n", 724 | " #print(targets_batches)\n", 725 | " \n", 726 | " #combine inputs batches and targets batches into one array\n", 727 | " batches = np.array(list(zip(inputs_batches, targets_batches)))\n", 728 | " \n", 729 | " return batches\n", 730 | " \n", 731 | "\n", 732 | "\"\"\"\n", 733 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 734 | "\"\"\"\n", 735 | "tests.test_get_batches(get_batches)" 736 | ] 737 | }, 738 | { 739 | "cell_type": "markdown", 740 | "metadata": { 741 | "deletable": true, 742 | "editable": true 743 | }, 744 | "source": [ 745 | "## Neural Network Training\n", 746 | "### Hyperparameters\n", 747 | "Tune the following parameters:\n", 748 | "\n", 749 | "- Set `num_epochs` to the number of epochs.\n", 750 | "- Set `batch_size` to the batch size.\n", 751 | "- Set `rnn_size` to the size of the RNNs.\n", 752 | "- Set `embed_dim` to the size of the embedding.\n", 753 | "- Set `seq_length` to the length of sequence.\n", 754 | "- Set `learning_rate` to the learning rate.\n", 755 | "- Set `show_every_n_batches` to the number of batches the neural network should print progress." 756 | ] 757 | }, 758 | { 759 | "cell_type": "code", 760 | "execution_count": 17, 761 | "metadata": { 762 | "collapsed": true, 763 | "deletable": true, 764 | "editable": true 765 | }, 766 | "outputs": [], 767 | "source": [ 768 | "# Number of Epochs\n", 769 | "num_epochs = 100\n", 770 | "# Batch Size\n", 771 | "batch_size = 256\n", 772 | "# RNN Size\n", 773 | "rnn_size = 512\n", 774 | "# Embedding Dimension Size\n", 775 | "embed_dim = 300\n", 776 | "# Sequence Length\n", 777 | "seq_length = 12\n", 778 | "# Learning Rate\n", 779 | "learning_rate = 0.001\n", 780 | "# Show stats for every n number of batches\n", 781 | "show_every_n_batches = 25\n", 782 | "\n", 783 | "\"\"\"\n", 784 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 785 | "\"\"\"\n", 786 | "save_dir = './save'" 787 | ] 788 | }, 789 | { 790 | "cell_type": "markdown", 791 | "metadata": { 792 | "deletable": true, 793 | "editable": true 794 | }, 795 | "source": [ 796 | "### Build the Graph\n", 797 | "Build the graph using the neural network you implemented." 798 | ] 799 | }, 800 | { 801 | "cell_type": "code", 802 | "execution_count": 18, 803 | "metadata": { 804 | "collapsed": false, 805 | "deletable": true, 806 | "editable": true 807 | }, 808 | "outputs": [], 809 | "source": [ 810 | "\"\"\"\n", 811 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 812 | "\"\"\"\n", 813 | "from tensorflow.contrib import seq2seq\n", 814 | "\n", 815 | "train_graph = tf.Graph()\n", 816 | "with train_graph.as_default():\n", 817 | " vocab_size = len(int_to_vocab)\n", 818 | " input_text, targets, lr = get_inputs()\n", 819 | " input_data_shape = tf.shape(input_text)\n", 820 | " cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)\n", 821 | " logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size, embed_dim)\n", 822 | "\n", 823 | " # Probabilities for generating words\n", 824 | " probs = tf.nn.softmax(logits, name='probs')\n", 825 | "\n", 826 | " # Loss function\n", 827 | " cost = seq2seq.sequence_loss(\n", 828 | " logits,\n", 829 | " targets,\n", 830 | " tf.ones([input_data_shape[0], input_data_shape[1]]))\n", 831 | "\n", 832 | " # Optimizer\n", 833 | " optimizer = tf.train.AdamOptimizer(lr)\n", 834 | "\n", 835 | " # Gradient Clipping\n", 836 | " gradients = optimizer.compute_gradients(cost)\n", 837 | " capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]\n", 838 | " train_op = optimizer.apply_gradients(capped_gradients)" 839 | ] 840 | }, 841 | { 842 | "cell_type": "markdown", 843 | "metadata": { 844 | "deletable": true, 845 | "editable": true 846 | }, 847 | "source": [ 848 | "## Train\n", 849 | "Train the neural network on the preprocessed data. If you have a hard time getting a good loss, check the [forms](https://discussions.udacity.com/) to see if anyone is having the same problem." 850 | ] 851 | }, 852 | { 853 | "cell_type": "code", 854 | "execution_count": 19, 855 | "metadata": { 856 | "collapsed": false, 857 | "deletable": true, 858 | "editable": true 859 | }, 860 | "outputs": [ 861 | { 862 | "name": "stdout", 863 | "output_type": "stream", 864 | "text": [ 865 | "Epoch 0 Batch 0/22 train_loss = 8.825\n", 866 | "Epoch 1 Batch 3/22 train_loss = 5.726\n", 867 | "Epoch 2 Batch 6/22 train_loss = 5.259\n", 868 | "Epoch 3 Batch 9/22 train_loss = 5.073\n", 869 | "Epoch 4 Batch 12/22 train_loss = 4.756\n", 870 | "Epoch 5 Batch 15/22 train_loss = 4.485\n", 871 | "Epoch 6 Batch 18/22 train_loss = 4.373\n", 872 | "Epoch 7 Batch 21/22 train_loss = 4.217\n", 873 | "Epoch 9 Batch 2/22 train_loss = 3.889\n", 874 | "Epoch 10 Batch 5/22 train_loss = 3.734\n", 875 | "Epoch 11 Batch 8/22 train_loss = 3.597\n", 876 | "Epoch 12 Batch 11/22 train_loss = 3.435\n", 877 | "Epoch 13 Batch 14/22 train_loss = 3.262\n", 878 | "Epoch 14 Batch 17/22 train_loss = 3.085\n", 879 | "Epoch 15 Batch 20/22 train_loss = 2.899\n", 880 | "Epoch 17 Batch 1/22 train_loss = 2.682\n", 881 | "Epoch 18 Batch 4/22 train_loss = 2.543\n", 882 | "Epoch 19 Batch 7/22 train_loss = 2.450\n", 883 | "Epoch 20 Batch 10/22 train_loss = 2.339\n", 884 | "Epoch 21 Batch 13/22 train_loss = 2.210\n", 885 | "Epoch 22 Batch 16/22 train_loss = 2.054\n", 886 | "Epoch 23 Batch 19/22 train_loss = 1.877\n", 887 | "Epoch 25 Batch 0/22 train_loss = 1.837\n", 888 | "Epoch 26 Batch 3/22 train_loss = 1.738\n", 889 | "Epoch 27 Batch 6/22 train_loss = 1.611\n", 890 | "Epoch 28 Batch 9/22 train_loss = 1.507\n", 891 | "Epoch 29 Batch 12/22 train_loss = 1.448\n", 892 | "Epoch 30 Batch 15/22 train_loss = 1.355\n", 893 | "Epoch 31 Batch 18/22 train_loss = 1.334\n", 894 | "Epoch 32 Batch 21/22 train_loss = 1.199\n", 895 | "Epoch 34 Batch 2/22 train_loss = 1.170\n", 896 | "Epoch 35 Batch 5/22 train_loss = 1.077\n", 897 | "Epoch 36 Batch 8/22 train_loss = 0.972\n", 898 | "Epoch 37 Batch 11/22 train_loss = 0.921\n", 899 | "Epoch 38 Batch 14/22 train_loss = 0.913\n", 900 | "Epoch 39 Batch 17/22 train_loss = 0.851\n", 901 | "Epoch 40 Batch 20/22 train_loss = 0.815\n", 902 | "Epoch 42 Batch 1/22 train_loss = 0.733\n", 903 | "Epoch 43 Batch 4/22 train_loss = 0.736\n", 904 | "Epoch 44 Batch 7/22 train_loss = 0.659\n", 905 | "Epoch 45 Batch 10/22 train_loss = 0.660\n", 906 | "Epoch 46 Batch 13/22 train_loss = 0.612\n", 907 | "Epoch 47 Batch 16/22 train_loss = 0.595\n", 908 | "Epoch 48 Batch 19/22 train_loss = 0.562\n", 909 | "Epoch 50 Batch 0/22 train_loss = 0.548\n", 910 | "Epoch 51 Batch 3/22 train_loss = 0.506\n", 911 | "Epoch 52 Batch 6/22 train_loss = 0.493\n", 912 | "Epoch 53 Batch 9/22 train_loss = 0.461\n", 913 | "Epoch 54 Batch 12/22 train_loss = 0.477\n", 914 | "Epoch 55 Batch 15/22 train_loss = 0.445\n", 915 | "Epoch 56 Batch 18/22 train_loss = 0.430\n", 916 | "Epoch 57 Batch 21/22 train_loss = 0.383\n", 917 | "Epoch 59 Batch 2/22 train_loss = 0.427\n", 918 | "Epoch 60 Batch 5/22 train_loss = 0.393\n", 919 | "Epoch 61 Batch 8/22 train_loss = 0.368\n", 920 | "Epoch 62 Batch 11/22 train_loss = 0.365\n", 921 | "Epoch 63 Batch 14/22 train_loss = 0.362\n", 922 | "Epoch 64 Batch 17/22 train_loss = 0.350\n", 923 | "Epoch 65 Batch 20/22 train_loss = 0.344\n", 924 | "Epoch 67 Batch 1/22 train_loss = 0.359\n", 925 | "Epoch 68 Batch 4/22 train_loss = 0.355\n", 926 | "Epoch 69 Batch 7/22 train_loss = 0.336\n", 927 | "Epoch 70 Batch 10/22 train_loss = 0.355\n", 928 | "Epoch 71 Batch 13/22 train_loss = 0.334\n", 929 | "Epoch 72 Batch 16/22 train_loss = 0.337\n", 930 | "Epoch 73 Batch 19/22 train_loss = 0.333\n", 931 | "Epoch 75 Batch 0/22 train_loss = 0.329\n", 932 | "Epoch 76 Batch 3/22 train_loss = 0.323\n", 933 | "Epoch 77 Batch 6/22 train_loss = 0.326\n", 934 | "Epoch 78 Batch 9/22 train_loss = 0.317\n", 935 | "Epoch 79 Batch 12/22 train_loss = 0.324\n", 936 | "Epoch 80 Batch 15/22 train_loss = 0.317\n", 937 | "Epoch 81 Batch 18/22 train_loss = 0.303\n", 938 | "Epoch 82 Batch 21/22 train_loss = 0.285\n", 939 | "Epoch 84 Batch 2/22 train_loss = 0.322\n", 940 | "Epoch 85 Batch 5/22 train_loss = 0.304\n", 941 | "Epoch 86 Batch 8/22 train_loss = 0.293\n", 942 | "Epoch 87 Batch 11/22 train_loss = 0.303\n", 943 | "Epoch 88 Batch 14/22 train_loss = 0.303\n", 944 | "Epoch 89 Batch 17/22 train_loss = 0.295\n", 945 | "Epoch 90 Batch 20/22 train_loss = 0.290\n", 946 | "Epoch 92 Batch 1/22 train_loss = 0.314\n", 947 | "Epoch 93 Batch 4/22 train_loss = 0.312\n", 948 | "Epoch 94 Batch 7/22 train_loss = 0.294\n", 949 | "Epoch 95 Batch 10/22 train_loss = 0.314\n", 950 | "Epoch 96 Batch 13/22 train_loss = 0.295\n", 951 | "Epoch 97 Batch 16/22 train_loss = 0.302\n", 952 | "Epoch 98 Batch 19/22 train_loss = 0.301\n", 953 | "Model Trained and Saved\n" 954 | ] 955 | } 956 | ], 957 | "source": [ 958 | "\"\"\"\n", 959 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 960 | "\"\"\"\n", 961 | "batches = get_batches(int_text, batch_size, seq_length)\n", 962 | "\n", 963 | "with tf.Session(graph=train_graph) as sess:\n", 964 | " sess.run(tf.global_variables_initializer())\n", 965 | "\n", 966 | " for epoch_i in range(num_epochs):\n", 967 | " state = sess.run(initial_state, {input_text: batches[0][0]})\n", 968 | "\n", 969 | " for batch_i, (x, y) in enumerate(batches):\n", 970 | " feed = {\n", 971 | " input_text: x,\n", 972 | " targets: y,\n", 973 | " initial_state: state,\n", 974 | " lr: learning_rate}\n", 975 | " train_loss, state, _ = sess.run([cost, final_state, train_op], feed)\n", 976 | "\n", 977 | " # Show every batches\n", 978 | " if (epoch_i * len(batches) + batch_i) % show_every_n_batches == 0:\n", 979 | " print('Epoch {:>3} Batch {:>4}/{} train_loss = {:.3f}'.format(\n", 980 | " epoch_i,\n", 981 | " batch_i,\n", 982 | " len(batches),\n", 983 | " train_loss))\n", 984 | "\n", 985 | " # Save Model\n", 986 | " saver = tf.train.Saver()\n", 987 | " saver.save(sess, save_dir)\n", 988 | " print('Model Trained and Saved')" 989 | ] 990 | }, 991 | { 992 | "cell_type": "markdown", 993 | "metadata": { 994 | "deletable": true, 995 | "editable": true 996 | }, 997 | "source": [ 998 | "## Save Parameters\n", 999 | "Save `seq_length` and `save_dir` for generating a new TV script." 1000 | ] 1001 | }, 1002 | { 1003 | "cell_type": "code", 1004 | "execution_count": 20, 1005 | "metadata": { 1006 | "collapsed": false, 1007 | "deletable": true, 1008 | "editable": true 1009 | }, 1010 | "outputs": [], 1011 | "source": [ 1012 | "\"\"\"\n", 1013 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 1014 | "\"\"\"\n", 1015 | "# Save parameters for checkpoint\n", 1016 | "helper.save_params((seq_length, save_dir))" 1017 | ] 1018 | }, 1019 | { 1020 | "cell_type": "markdown", 1021 | "metadata": { 1022 | "deletable": true, 1023 | "editable": true 1024 | }, 1025 | "source": [ 1026 | "# Checkpoint" 1027 | ] 1028 | }, 1029 | { 1030 | "cell_type": "code", 1031 | "execution_count": 21, 1032 | "metadata": { 1033 | "collapsed": false, 1034 | "deletable": true, 1035 | "editable": true 1036 | }, 1037 | "outputs": [], 1038 | "source": [ 1039 | "\"\"\"\n", 1040 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 1041 | "\"\"\"\n", 1042 | "import tensorflow as tf\n", 1043 | "import numpy as np\n", 1044 | "import helper\n", 1045 | "import problem_unittests as tests\n", 1046 | "\n", 1047 | "_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()\n", 1048 | "seq_length, load_dir = helper.load_params()" 1049 | ] 1050 | }, 1051 | { 1052 | "cell_type": "markdown", 1053 | "metadata": { 1054 | "deletable": true, 1055 | "editable": true 1056 | }, 1057 | "source": [ 1058 | "## Implement Generate Functions\n", 1059 | "### Get Tensors\n", 1060 | "Get tensors from `loaded_graph` using the function [`get_tensor_by_name()`](https://www.tensorflow.org/api_docs/python/tf/Graph#get_tensor_by_name). Get the tensors using the following names:\n", 1061 | "- \"input:0\"\n", 1062 | "- \"initial_state:0\"\n", 1063 | "- \"final_state:0\"\n", 1064 | "- \"probs:0\"\n", 1065 | "\n", 1066 | "Return the tensors in the following tuple `(InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)` " 1067 | ] 1068 | }, 1069 | { 1070 | "cell_type": "code", 1071 | "execution_count": 22, 1072 | "metadata": { 1073 | "collapsed": false, 1074 | "deletable": true, 1075 | "editable": true 1076 | }, 1077 | "outputs": [ 1078 | { 1079 | "name": "stdout", 1080 | "output_type": "stream", 1081 | "text": [ 1082 | "Tests Passed\n" 1083 | ] 1084 | } 1085 | ], 1086 | "source": [ 1087 | "def get_tensors(loaded_graph):\n", 1088 | " \"\"\"\n", 1089 | " Get input, initial state, final state, and probabilities tensor from \n", 1090 | " :param loaded_graph: TensorFlow graph loaded from file\n", 1091 | " :return: Tuple (InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)\n", 1092 | " \"\"\"\n", 1093 | " # TODO: Implement Function\n", 1094 | " InputTensor = loaded_graph.get_tensor_by_name(\"input:0\")\n", 1095 | " InitialStateTensor = loaded_graph.get_tensor_by_name(\"initial_state:0\")\n", 1096 | " FinalStateTensor = loaded_graph.get_tensor_by_name(\"final_state:0\")\n", 1097 | " ProbsTensor = loaded_graph.get_tensor_by_name(\"probs:0\")\n", 1098 | " return InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor\n", 1099 | "\n", 1100 | "\n", 1101 | "\"\"\"\n", 1102 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1103 | "\"\"\"\n", 1104 | "tests.test_get_tensors(get_tensors)" 1105 | ] 1106 | }, 1107 | { 1108 | "cell_type": "markdown", 1109 | "metadata": { 1110 | "deletable": true, 1111 | "editable": true 1112 | }, 1113 | "source": [ 1114 | "### Choose Word\n", 1115 | "Implement the `pick_word()` function to select the next word using `probabilities`." 1116 | ] 1117 | }, 1118 | { 1119 | "cell_type": "code", 1120 | "execution_count": 23, 1121 | "metadata": { 1122 | "collapsed": false, 1123 | "deletable": true, 1124 | "editable": true 1125 | }, 1126 | "outputs": [ 1127 | { 1128 | "name": "stdout", 1129 | "output_type": "stream", 1130 | "text": [ 1131 | "Tests Passed\n" 1132 | ] 1133 | } 1134 | ], 1135 | "source": [ 1136 | "def pick_word(probabilities, int_to_vocab):\n", 1137 | " \"\"\"\n", 1138 | " Pick the next word in the generated text\n", 1139 | " :param probabilities: Probabilites of the next word\n", 1140 | " :param int_to_vocab: Dictionary of word ids as the keys and words as the values\n", 1141 | " :return: String of the predicted word\n", 1142 | " \"\"\"\n", 1143 | " # TODO: Implement Function\n", 1144 | " #use np.random.choice to generate next_word, p as probabilities\n", 1145 | " #turned int_to_vocab.values() into a list for 1D array\n", 1146 | " next_word = np.random.choice(list(int_to_vocab.values()), p=probabilities)\n", 1147 | " return next_word\n", 1148 | "\n", 1149 | "\n", 1150 | "\"\"\"\n", 1151 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1152 | "\"\"\"\n", 1153 | "tests.test_pick_word(pick_word)" 1154 | ] 1155 | }, 1156 | { 1157 | "cell_type": "markdown", 1158 | "metadata": { 1159 | "deletable": true, 1160 | "editable": true 1161 | }, 1162 | "source": [ 1163 | "## Generate TV Script\n", 1164 | "This will generate the TV script for you. Set `gen_length` to the length of TV script you want to generate." 1165 | ] 1166 | }, 1167 | { 1168 | "cell_type": "code", 1169 | "execution_count": 24, 1170 | "metadata": { 1171 | "collapsed": false, 1172 | "deletable": true, 1173 | "editable": true 1174 | }, 1175 | "outputs": [ 1176 | { 1177 | "name": "stdout", 1178 | "output_type": "stream", 1179 | "text": [ 1180 | "moe_szyslak: drinking will help us plan.\n", 1181 | "homer_simpson: this valentine's crap has to be a bar.(gets off) new_health_inspector: bar an idiot.\n", 1182 | "homer_simpson:(to self) sorry, i need i'm behind your foot.\n", 1183 | "moe_szyslak: but i suppose i got a two hundred and people all can use the kids.\n", 1184 | "homer_simpson: to be the best thing?\n", 1185 | "barney_gumble: 'cause only one i thought you said.\n", 1186 | "carl_carlson:(to self) someone's makes a little one, can i have a free? take this!(homer's sound)\n", 1187 | "homer_simpson: the one, but i did not going to find it out.\n", 1188 | "moe_szyslak:(sings) i just wanna tell my life till i'm on their go!\n", 1189 | "\n", 1190 | "\n", 1191 | "moe_szyslak: the guys are make around in the gentleman's of woman.\n", 1192 | "lenny_leonard: oh, you don't let me do being here? no, moe.\n", 1193 | "barney_gumble: you know, it's you, moe. the drinks are on you.\n", 1194 | "seymour_skinner:(sighs) isn't it eyes no more.\n", 1195 | "homer_simpson:(chuckles) all right.\n" 1196 | ] 1197 | } 1198 | ], 1199 | "source": [ 1200 | "gen_length = 200\n", 1201 | "# homer_simpson, moe_szyslak, or Barney_Gumble\n", 1202 | "prime_word = 'moe_szyslak'\n", 1203 | "\n", 1204 | "\"\"\"\n", 1205 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1206 | "\"\"\"\n", 1207 | "loaded_graph = tf.Graph()\n", 1208 | "with tf.Session(graph=loaded_graph) as sess:\n", 1209 | " # Load saved model\n", 1210 | " loader = tf.train.import_meta_graph(load_dir + '.meta')\n", 1211 | " loader.restore(sess, load_dir)\n", 1212 | "\n", 1213 | " # Get Tensors from loaded model\n", 1214 | " input_text, initial_state, final_state, probs = get_tensors(loaded_graph)\n", 1215 | "\n", 1216 | " # Sentences generation setup\n", 1217 | " gen_sentences = [prime_word + ':']\n", 1218 | " prev_state = sess.run(initial_state, {input_text: np.array([[1]])})\n", 1219 | "\n", 1220 | " # Generate sentences\n", 1221 | " for n in range(gen_length):\n", 1222 | " # Dynamic Input\n", 1223 | " dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]\n", 1224 | " dyn_seq_length = len(dyn_input[0])\n", 1225 | "\n", 1226 | " # Get Prediction\n", 1227 | " probabilities, prev_state = sess.run(\n", 1228 | " [probs, final_state],\n", 1229 | " {input_text: dyn_input, initial_state: prev_state})\n", 1230 | " \n", 1231 | " pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)\n", 1232 | "\n", 1233 | " gen_sentences.append(pred_word)\n", 1234 | " \n", 1235 | " # Remove tokens\n", 1236 | " tv_script = ' '.join(gen_sentences)\n", 1237 | " for key, token in token_dict.items():\n", 1238 | " ending = ' ' if key in ['\\n', '(', '\"'] else ''\n", 1239 | " tv_script = tv_script.replace(' ' + token.lower(), key)\n", 1240 | " tv_script = tv_script.replace('\\n ', '\\n')\n", 1241 | " tv_script = tv_script.replace('( ', '(')\n", 1242 | " \n", 1243 | " print(tv_script)" 1244 | ] 1245 | }, 1246 | { 1247 | "cell_type": "markdown", 1248 | "metadata": { 1249 | "deletable": true, 1250 | "editable": true 1251 | }, 1252 | "source": [ 1253 | "# The TV Script is Nonsensical\n", 1254 | "It's ok if the TV script doesn't make any sense. We trained on less than a megabyte of text. In order to get good results, you'll have to use a smaller vocabulary or get more data. Luckly there's more data! As we mentioned in the begging of this project, this is a subset of [another dataset](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data). We didn't have you train on all the data, because that would take too long. However, you are free to train your neural network on all the data. After you complete the project, of course.\n", 1255 | "# Submitting This Project\n", 1256 | "When submitting this project, make sure to run all the cells before saving the notebook. Save the notebook file as \"dlnd_tv_script_generation.ipynb\" and save it as a HTML file under \"File\" -> \"Download as\". Include the \"helper.py\" and \"problem_unittests.py\" files in your submission." 1257 | ] 1258 | } 1259 | ], 1260 | "metadata": { 1261 | "kernelspec": { 1262 | "display_name": "Python 3", 1263 | "language": "python", 1264 | "name": "python3" 1265 | }, 1266 | "language_info": { 1267 | "codemirror_mode": { 1268 | "name": "ipython", 1269 | "version": 3 1270 | }, 1271 | "file_extension": ".py", 1272 | "mimetype": "text/x-python", 1273 | "name": "python", 1274 | "nbconvert_exporter": "python", 1275 | "pygments_lexer": "ipython3", 1276 | "version": "3.6.0" 1277 | } 1278 | }, 1279 | "nbformat": 4, 1280 | "nbformat_minor": 0 1281 | } 1282 | -------------------------------------------------------------------------------- /PROJECT_3_TV_Script_Generation/.ipynb_checkpoints/dlnd_tv_script_generation-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "deletable": true, 7 | "editable": true 8 | }, 9 | "source": [ 10 | "# TV Script Generation\n", 11 | "In this project, you'll generate your own [Simpsons](https://en.wikipedia.org/wiki/The_Simpsons) TV scripts using RNNs. You'll be using part of the [Simpsons dataset](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data) of scripts from 27 seasons. The Neural Network you'll build will generate a new TV script for a scene at [Moe's Tavern](https://simpsonswiki.com/wiki/Moe's_Tavern).\n", 12 | "## Get the Data\n", 13 | "The data is already provided for you. You'll be using a subset of the original dataset. It consists of only the scenes in Moe's Tavern. This doesn't include other versions of the tavern, like \"Moe's Cavern\", \"Flaming Moe's\", \"Uncle Moe's Family Feed-Bag\", etc.." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "metadata": { 20 | "collapsed": false, 21 | "deletable": true, 22 | "editable": true 23 | }, 24 | "outputs": [], 25 | "source": [ 26 | "\"\"\"\n", 27 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 28 | "\"\"\"\n", 29 | "import helper\n", 30 | "\n", 31 | "data_dir = './data/simpsons/moes_tavern_lines.txt'\n", 32 | "text = helper.load_data(data_dir)\n", 33 | "# Ignore notice, since we don't use it for analysing the data\n", 34 | "text = text[81:]" 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": { 40 | "deletable": true, 41 | "editable": true 42 | }, 43 | "source": [ 44 | "## Explore the Data\n", 45 | "Play around with `view_sentence_range` to view different parts of the data." 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 2, 51 | "metadata": { 52 | "collapsed": false, 53 | "deletable": true, 54 | "editable": true 55 | }, 56 | "outputs": [ 57 | { 58 | "name": "stdout", 59 | "output_type": "stream", 60 | "text": [ 61 | "Dataset Stats\n", 62 | "Roughly the number of unique words: 11492\n", 63 | "Number of scenes: 262\n", 64 | "Average number of sentences in each scene: 15.248091603053435\n", 65 | "Number of lines: 4257\n", 66 | "Average number of words in each line: 11.50434578341555\n", 67 | "\n", 68 | "The sentences 0 to 10:\n", 69 | "Moe_Szyslak: (INTO PHONE) Moe's Tavern. Where the elite meet to drink.\n", 70 | "Bart_Simpson: Eh, yeah, hello, is Mike there? Last name, Rotch.\n", 71 | "Moe_Szyslak: (INTO PHONE) Hold on, I'll check. (TO BARFLIES) Mike Rotch. Mike Rotch. Hey, has anybody seen Mike Rotch, lately?\n", 72 | "Moe_Szyslak: (INTO PHONE) Listen you little puke. One of these days I'm gonna catch you, and I'm gonna carve my name on your back with an ice pick.\n", 73 | "Moe_Szyslak: What's the matter Homer? You're not your normal effervescent self.\n", 74 | "Homer_Simpson: I got my problems, Moe. Give me another one.\n", 75 | "Moe_Szyslak: Homer, hey, you should not drink to forget your problems.\n", 76 | "Barney_Gumble: Yeah, you should only drink to enhance your social skills.\n", 77 | "\n", 78 | "\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "view_sentence_range = (0, 10)\n", 84 | "\n", 85 | "\"\"\"\n", 86 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 87 | "\"\"\"\n", 88 | "import numpy as np\n", 89 | "\n", 90 | "print('Dataset Stats')\n", 91 | "print('Roughly the number of unique words: {}'.format(len({word: None for word in text.split()})))\n", 92 | "scenes = text.split('\\n\\n')\n", 93 | "print('Number of scenes: {}'.format(len(scenes)))\n", 94 | "sentence_count_scene = [scene.count('\\n') for scene in scenes]\n", 95 | "print('Average number of sentences in each scene: {}'.format(np.average(sentence_count_scene)))\n", 96 | "\n", 97 | "sentences = [sentence for scene in scenes for sentence in scene.split('\\n')]\n", 98 | "print('Number of lines: {}'.format(len(sentences)))\n", 99 | "word_count_sentence = [len(sentence.split()) for sentence in sentences]\n", 100 | "print('Average number of words in each line: {}'.format(np.average(word_count_sentence)))\n", 101 | "\n", 102 | "print()\n", 103 | "print('The sentences {} to {}:'.format(*view_sentence_range))\n", 104 | "print('\\n'.join(text.split('\\n')[view_sentence_range[0]:view_sentence_range[1]]))" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": { 110 | "deletable": true, 111 | "editable": true 112 | }, 113 | "source": [ 114 | "## Implement Preprocessing Functions\n", 115 | "The first thing to do to any dataset is preprocessing. Implement the following preprocessing functions below:\n", 116 | "- Lookup Table\n", 117 | "- Tokenize Punctuation\n", 118 | "\n", 119 | "### Lookup Table\n", 120 | "To create a word embedding, you first need to transform the words to ids. In this function, create two dictionaries:\n", 121 | "- Dictionary to go from the words to an id, we'll call `vocab_to_int`\n", 122 | "- Dictionary to go from the id to word, we'll call `int_to_vocab`\n", 123 | "\n", 124 | "Return these dictionaries in the following tuple `(vocab_to_int, int_to_vocab)`" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": 3, 130 | "metadata": { 131 | "collapsed": false, 132 | "deletable": true, 133 | "editable": true 134 | }, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "Tests Passed\n" 141 | ] 142 | } 143 | ], 144 | "source": [ 145 | "import numpy as np\n", 146 | "import problem_unittests as tests\n", 147 | "from collections import Counter\n", 148 | "\n", 149 | "def create_lookup_tables(text):\n", 150 | " \"\"\"\n", 151 | " Create lookup tables for vocabulary\n", 152 | " :param text: The text of tv scripts split into words\n", 153 | " :return: A tuple of dicts (vocab_to_int, int_to_vocab)\n", 154 | " \"\"\"\n", 155 | " # TODO: Implement Function\n", 156 | " '''NEW METHOD - Simpler'''\n", 157 | " vocab = set(text)\n", 158 | " #create dict of vocab to ints\n", 159 | " vocab_to_int = dict((word, index) for index, word in enumerate(vocab))\n", 160 | " int_to_vocab = dict((index, word) for index, word in enumerate(vocab))\n", 161 | " return vocab_to_int, int_to_vocab\n", 162 | " \n", 163 | " '''OLD METHOD'''\n", 164 | " '''\n", 165 | " counts = Counter(text)\n", 166 | " vocab = sorted(counts, key=counts.get, reverse=True)\n", 167 | " #create a dictionary to go from word to id\n", 168 | " vocab_to_int = {word: i for i, word in enumerate(vocab)}\n", 169 | " #create a dictionary to go from id to word\n", 170 | " int_to_vocab = {v: k for k, v in vocab_to_int.items()} \n", 171 | " return (vocab_to_int, int_to_vocab)\n", 172 | " '''\n", 173 | "\n", 174 | "\n", 175 | "\"\"\"\n", 176 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 177 | "\"\"\"\n", 178 | "tests.test_create_lookup_tables(create_lookup_tables)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": { 184 | "deletable": true, 185 | "editable": true 186 | }, 187 | "source": [ 188 | "### Tokenize Punctuation\n", 189 | "We'll be splitting the script into a word array using spaces as delimiters. However, punctuations like periods and exclamation marks make it hard for the neural network to distinguish between the word \"bye\" and \"bye!\".\n", 190 | "\n", 191 | "Implement the function `token_lookup` to return a dict that will be used to tokenize symbols like \"!\" into \"||Exclamation_Mark||\". Create a dictionary for the following symbols where the symbol is the key and value is the token:\n", 192 | "- Period ( . )\n", 193 | "- Comma ( , )\n", 194 | "- Quotation Mark ( \" )\n", 195 | "- Semicolon ( ; )\n", 196 | "- Exclamation mark ( ! )\n", 197 | "- Question mark ( ? )\n", 198 | "- Left Parentheses ( ( )\n", 199 | "- Right Parentheses ( ) )\n", 200 | "- Dash ( -- )\n", 201 | "- Return ( \\n )\n", 202 | "\n", 203 | "This dictionary will be used to token the symbols and add the delimiter (space) around it. This separates the symbols as it's own word, making it easier for the neural network to predict on the next word. Make sure you don't use a token that could be confused as a word. Instead of using the token \"dash\", try using something like \"||dash||\"." 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 4, 209 | "metadata": { 210 | "collapsed": false, 211 | "deletable": true, 212 | "editable": true 213 | }, 214 | "outputs": [ 215 | { 216 | "name": "stdout", 217 | "output_type": "stream", 218 | "text": [ 219 | "Tests Passed\n" 220 | ] 221 | } 222 | ], 223 | "source": [ 224 | "def token_lookup():\n", 225 | " \"\"\"\n", 226 | " Generate a dict to turn punctuation into a token.\n", 227 | " :return: Tokenize dictionary where the key is the punctuation and the value is the token\n", 228 | " \"\"\"\n", 229 | " #creating two arrays for punctuation marks and their relevant tokens\n", 230 | " punc_mark = ['.', ',', '\"', \";\", \"!\", \"?\", \"(\", \")\", \"--\", \"\\n\"]\n", 231 | " punc_token = [\"||period||\", \"||comma||\", \"||quotation||\", \"||semicolon||\",\n", 232 | " \"||exclamation||\", \"||question_mark||\", \"||left_bracket||\",\n", 233 | " \"||right_bracket||\", \"||dash||\", \"||return||\"]\n", 234 | " punc_token_dict = dict(zip(punc_mark, punc_token))\n", 235 | " return punc_token_dict\n", 236 | "\n", 237 | "\n", 238 | "\"\"\"\n", 239 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 240 | "\"\"\"\n", 241 | "tests.test_tokenize(token_lookup)" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": { 247 | "deletable": true, 248 | "editable": true 249 | }, 250 | "source": [ 251 | "## Preprocess all the data and save it\n", 252 | "Running the code cell below will preprocess all the data and save it to file." 253 | ] 254 | }, 255 | { 256 | "cell_type": "code", 257 | "execution_count": 5, 258 | "metadata": { 259 | "collapsed": false, 260 | "deletable": true, 261 | "editable": true 262 | }, 263 | "outputs": [], 264 | "source": [ 265 | "\"\"\"\n", 266 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 267 | "\"\"\"\n", 268 | "# Preprocess Training, Validation, and Testing Data\n", 269 | "helper.preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)" 270 | ] 271 | }, 272 | { 273 | "cell_type": "markdown", 274 | "metadata": { 275 | "deletable": true, 276 | "editable": true 277 | }, 278 | "source": [ 279 | "# Check Point\n", 280 | "This is your first checkpoint. If you ever decide to come back to this notebook or have to restart the notebook, you can start from here. The preprocessed data has been saved to disk." 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": 6, 286 | "metadata": { 287 | "collapsed": false, 288 | "deletable": true, 289 | "editable": true 290 | }, 291 | "outputs": [], 292 | "source": [ 293 | "\"\"\"\n", 294 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 295 | "\"\"\"\n", 296 | "import helper\n", 297 | "import numpy as np\n", 298 | "import problem_unittests as tests\n", 299 | "\n", 300 | "int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()" 301 | ] 302 | }, 303 | { 304 | "cell_type": "markdown", 305 | "metadata": { 306 | "deletable": true, 307 | "editable": true 308 | }, 309 | "source": [ 310 | "## Build the Neural Network\n", 311 | "You'll build the components necessary to build a RNN by implementing the following functions below:\n", 312 | "- get_inputs\n", 313 | "- get_init_cell\n", 314 | "- get_embed\n", 315 | "- build_rnn\n", 316 | "- build_nn\n", 317 | "- get_batches\n", 318 | "\n", 319 | "### Check the Version of TensorFlow and Access to GPU" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": 7, 325 | "metadata": { 326 | "collapsed": false, 327 | "deletable": true, 328 | "editable": true 329 | }, 330 | "outputs": [ 331 | { 332 | "name": "stdout", 333 | "output_type": "stream", 334 | "text": [ 335 | "TensorFlow Version: 1.0.0\n", 336 | "Default GPU Device: /gpu:0\n" 337 | ] 338 | } 339 | ], 340 | "source": [ 341 | "\"\"\"\n", 342 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 343 | "\"\"\"\n", 344 | "from distutils.version import LooseVersion\n", 345 | "import warnings\n", 346 | "import tensorflow as tf\n", 347 | "\n", 348 | "# Check TensorFlow Version\n", 349 | "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'\n", 350 | "print('TensorFlow Version: {}'.format(tf.__version__))\n", 351 | "\n", 352 | "# Check for a GPU\n", 353 | "if not tf.test.gpu_device_name():\n", 354 | " warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n", 355 | "else:\n", 356 | " print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))" 357 | ] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "metadata": { 362 | "deletable": true, 363 | "editable": true 364 | }, 365 | "source": [ 366 | "### Input\n", 367 | "Implement the `get_inputs()` function to create TF Placeholders for the Neural Network. It should create the following placeholders:\n", 368 | "- Input text placeholder named \"input\" using the [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder) `name` parameter.\n", 369 | "- Targets placeholder\n", 370 | "- Learning Rate placeholder\n", 371 | "\n", 372 | "Return the placeholders in the following tuple `(Input, Targets, LearningRate)`" 373 | ] 374 | }, 375 | { 376 | "cell_type": "code", 377 | "execution_count": 8, 378 | "metadata": { 379 | "collapsed": false, 380 | "deletable": true, 381 | "editable": true 382 | }, 383 | "outputs": [ 384 | { 385 | "name": "stdout", 386 | "output_type": "stream", 387 | "text": [ 388 | "Tests Passed\n" 389 | ] 390 | } 391 | ], 392 | "source": [ 393 | "def get_inputs():\n", 394 | " \"\"\"\n", 395 | " Create TF Placeholders for input, targets, and learning rate.\n", 396 | " :return: Tuple (input, targets, learning rate)\n", 397 | " \"\"\"\n", 398 | " # TODO: Implement Function\n", 399 | " inputs_ = tf.placeholder(tf.int32, [None, None], name = 'input')\n", 400 | " targets_ = tf.placeholder(tf.int32, [None, None], name = 'targets')\n", 401 | " learning_rate_ = tf.placeholder(tf.float32, name = 'learning_rate')\n", 402 | " return (inputs_, targets_, learning_rate_)\n", 403 | "\n", 404 | "\n", 405 | "\"\"\"\n", 406 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 407 | "\"\"\"\n", 408 | "tests.test_get_inputs(get_inputs)" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": { 414 | "deletable": true, 415 | "editable": true 416 | }, 417 | "source": [ 418 | "### Build RNN Cell and Initialize\n", 419 | "Stack one or more [`BasicLSTMCells`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/BasicLSTMCell) in a [`MultiRNNCell`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell).\n", 420 | "- The Rnn size should be set using `rnn_size`\n", 421 | "- Initalize Cell State using the MultiRNNCell's [`zero_state()`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell#zero_state) function\n", 422 | " - Apply the name \"initial_state\" to the initial state using [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity)\n", 423 | "\n", 424 | "Return the cell and initial state in the following tuple `(Cell, InitialState)`" 425 | ] 426 | }, 427 | { 428 | "cell_type": "code", 429 | "execution_count": 9, 430 | "metadata": { 431 | "collapsed": false, 432 | "deletable": true, 433 | "editable": true 434 | }, 435 | "outputs": [ 436 | { 437 | "name": "stdout", 438 | "output_type": "stream", 439 | "text": [ 440 | "Tests Passed\n" 441 | ] 442 | } 443 | ], 444 | "source": [ 445 | "def get_init_cell(batch_size, rnn_size):\n", 446 | " \"\"\"\n", 447 | " Create an RNN Cell and initialize it.\n", 448 | " :param batch_size: Size of batches\n", 449 | " :param rnn_size: Size of RNNs\n", 450 | " :return: Tuple (cell, initialize state)\n", 451 | " \"\"\"\n", 452 | " # TODO: Implement Function\n", 453 | " cell = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n", 454 | " #not sure whether or not to keep the dropout layer\n", 455 | " #drop = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob = 0.6)\n", 456 | " cell_stack = tf.contrib.rnn.MultiRNNCell([cell], rnn_size)\n", 457 | " initial_state = cell_stack.zero_state(batch_size, tf.float32)\n", 458 | " initial_state = tf.identity(initial_state, name =\"initial_state\")\n", 459 | " return (cell_stack, initial_state)\n", 460 | "\n", 461 | "\n", 462 | "\"\"\"\n", 463 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 464 | "\"\"\"\n", 465 | "tests.test_get_init_cell(get_init_cell)" 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": { 471 | "deletable": true, 472 | "editable": true 473 | }, 474 | "source": [ 475 | "### Word Embedding\n", 476 | "Apply embedding to `input_data` using TensorFlow. Return the embedded sequence." 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 10, 482 | "metadata": { 483 | "collapsed": false, 484 | "deletable": true, 485 | "editable": true 486 | }, 487 | "outputs": [ 488 | { 489 | "name": "stdout", 490 | "output_type": "stream", 491 | "text": [ 492 | "Tests Passed\n" 493 | ] 494 | } 495 | ], 496 | "source": [ 497 | "def get_embed(input_data, vocab_size, embed_dim):\n", 498 | " \"\"\"\n", 499 | " Create embedding for .\n", 500 | " :param input_data: TF placeholder for text input.\n", 501 | " :param vocab_size: Number of words in vocabulary.\n", 502 | " :param embed_dim: Number of embedding dimensions\n", 503 | " :return: Embedded input.\n", 504 | " \"\"\"\n", 505 | " # TODO: Implement Function\n", 506 | " embedding = tf.Variable(tf.random_uniform((vocab_size, embed_dim), -1, 1))\n", 507 | " embed = tf.nn.embedding_lookup(embedding, input_data)\n", 508 | " return embed\n", 509 | "\n", 510 | "\n", 511 | "\"\"\"\n", 512 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 513 | "\"\"\"\n", 514 | "tests.test_get_embed(get_embed)" 515 | ] 516 | }, 517 | { 518 | "cell_type": "markdown", 519 | "metadata": { 520 | "deletable": true, 521 | "editable": true 522 | }, 523 | "source": [ 524 | "### Build RNN\n", 525 | "You created a RNN Cell in the `get_init_cell()` function. Time to use the cell to create a RNN.\n", 526 | "- Build the RNN using the [`tf.nn.dynamic_rnn()`](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn)\n", 527 | " - Apply the name \"final_state\" to the final state using [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity)\n", 528 | "\n", 529 | "Return the outputs and final_state state in the following tuple `(Outputs, FinalState)` " 530 | ] 531 | }, 532 | { 533 | "cell_type": "code", 534 | "execution_count": 11, 535 | "metadata": { 536 | "collapsed": false, 537 | "deletable": true, 538 | "editable": true 539 | }, 540 | "outputs": [ 541 | { 542 | "name": "stdout", 543 | "output_type": "stream", 544 | "text": [ 545 | "Tests Passed\n" 546 | ] 547 | } 548 | ], 549 | "source": [ 550 | "def build_rnn(cell, inputs):\n", 551 | " \"\"\"\n", 552 | " Create a RNN using a RNN Cell\n", 553 | " :param cell: RNN Cell\n", 554 | " :param inputs: Input text data\n", 555 | " :return: Tuple (Outputs, Final State)\n", 556 | " \"\"\"\n", 557 | " # TODO: Implement Function\n", 558 | " #creating the outputs and final_state using tf.nn.dynamic_rnn\n", 559 | " outputs, final_state = tf.nn.dynamic_rnn(cell, inputs = inputs, dtype=tf.float32)\n", 560 | " #using tf.identity to apply the name \"final_state\" to final_state\n", 561 | " final_state = tf.identity(final_state, \"final_state\")\n", 562 | " return (outputs, final_state)\n", 563 | "\n", 564 | "\n", 565 | "\"\"\"\n", 566 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 567 | "\"\"\"\n", 568 | "tests.test_build_rnn(build_rnn)" 569 | ] 570 | }, 571 | { 572 | "cell_type": "markdown", 573 | "metadata": { 574 | "deletable": true, 575 | "editable": true 576 | }, 577 | "source": [ 578 | "### Build the Neural Network\n", 579 | "Apply the functions you implemented above to:\n", 580 | "- Apply embedding to `input_data` using your `get_embed(input_data, vocab_size, embed_dim)` function.\n", 581 | "- Build RNN using `cell` and your `build_rnn(cell, inputs)` function.\n", 582 | "- Apply a fully connected layer with a linear activation and `vocab_size` as the number of outputs.\n", 583 | "\n", 584 | "Return the logits and final state in the following tuple (Logits, FinalState) " 585 | ] 586 | }, 587 | { 588 | "cell_type": "code", 589 | "execution_count": 12, 590 | "metadata": { 591 | "collapsed": false, 592 | "deletable": true, 593 | "editable": true 594 | }, 595 | "outputs": [ 596 | { 597 | "name": "stdout", 598 | "output_type": "stream", 599 | "text": [ 600 | "Tests Passed\n" 601 | ] 602 | } 603 | ], 604 | "source": [ 605 | "def build_nn(cell, rnn_size, input_data, vocab_size, embed_dim):\n", 606 | " \"\"\"\n", 607 | " Build part of the neural network\n", 608 | " :param cell: RNN cell\n", 609 | " :param rnn_size: Size of rnns\n", 610 | " :param input_data: Input data\n", 611 | " :param vocab_size: Vocabulary size\n", 612 | " :param embed_dim: Number of embedding dimensions\n", 613 | " :return: Tuple (Logits, FinalState)\n", 614 | " \"\"\"\n", 615 | " # TODO: Implement Function\n", 616 | " inputs = get_embed(input_data, vocab_size, embed_dim)\n", 617 | " outputs, final_state = build_rnn(cell, inputs)\n", 618 | " logits = tf.contrib.layers.fully_connected(outputs, vocab_size, activation_fn=None,\n", 619 | " weights_initializer=tf.truncated_normal_initializer\n", 620 | " (mean=0.0, stddev=0.1),\n", 621 | " biases_initializer=tf.zeros_initializer())\n", 622 | " return(logits, final_state)\n", 623 | "\n", 624 | "\n", 625 | "\"\"\"\n", 626 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 627 | "\"\"\"\n", 628 | "tests.test_build_nn(build_nn)" 629 | ] 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": { 634 | "deletable": true, 635 | "editable": true 636 | }, 637 | "source": [ 638 | "### Batches\n", 639 | "Implement `get_batches` to create batches of input and targets using `int_text`. The batches should be a Numpy array with the shape `(number of batches, 2, batch size, sequence length)`. Each batch contains two elements:\n", 640 | "- The first element is a single batch of **input** with the shape `[batch size, sequence length]`\n", 641 | "- The second element is a single batch of **targets** with the shape `[batch size, sequence length]`\n", 642 | "\n", 643 | "If you can't fill the last batch with enough data, drop the last batch.\n", 644 | "\n", 645 | "For exmple, `get_batches([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], 3, 2)` would return a Numpy array of the following:\n", 646 | "```\n", 647 | "[\n", 648 | " # First Batch\n", 649 | " [\n", 650 | " # Batch of Input\n", 651 | " [[ 1 2], [ 7 8], [13 14]]\n", 652 | " # Batch of targets\n", 653 | " [[ 2 3], [ 8 9], [14 15]]\n", 654 | " ]\n", 655 | "\n", 656 | " # Second Batch\n", 657 | " [\n", 658 | " # Batch of Input\n", 659 | " [[ 3 4], [ 9 10], [15 16]]\n", 660 | " # Batch of targets\n", 661 | " [[ 4 5], [10 11], [16 17]]\n", 662 | " ]\n", 663 | "\n", 664 | " # Third Batch\n", 665 | " [\n", 666 | " # Batch of Input\n", 667 | " [[ 5 6], [11 12], [17 18]]\n", 668 | " # Batch of targets\n", 669 | " [[ 6 7], [12 13], [18 1]]\n", 670 | " ]\n", 671 | "]\n", 672 | "```\n", 673 | "\n", 674 | "Notice that the last target value in the last batch is the first input value of the first batch. In this case, `1`. This is a common technique used when creating sequence batches, although it is rather unintuitive." 675 | ] 676 | }, 677 | { 678 | "cell_type": "code", 679 | "execution_count": 13, 680 | "metadata": { 681 | "collapsed": false, 682 | "deletable": true, 683 | "editable": true 684 | }, 685 | "outputs": [ 686 | { 687 | "name": "stdout", 688 | "output_type": "stream", 689 | "text": [ 690 | "Tests Passed\n" 691 | ] 692 | } 693 | ], 694 | "source": [ 695 | "def get_batches(int_text, batch_size, seq_length):\n", 696 | " \"\"\"\n", 697 | " Return batches of input and target\n", 698 | " :param int_text: Text with the words replaced by their ids\n", 699 | " :param batch_size: The size of batch\n", 700 | " :param seq_length: The length of sequence\n", 701 | " :return: Batches as a Numpy array\n", 702 | " \"\"\"\n", 703 | " # TODO: Implement Function\n", 704 | " \n", 705 | " #define number of batches\n", 706 | " text_len = len(int_text)\n", 707 | " n_batches = text_len // (batch_size * seq_length)\n", 708 | " #print(n_batches)\n", 709 | " \n", 710 | " #define inputs and targets as single row arrays\n", 711 | " inputs = np.array(int_text[: n_batches * batch_size * seq_length])\n", 712 | " #targets starts at 1 index instead of 0 and goes 1 step further\n", 713 | " #on seq_length\n", 714 | " targets = np.array(int_text[1: n_batches * batch_size * seq_length + 1])\n", 715 | " #change last element of targets to first element of inputs\n", 716 | " targets[-1] = inputs[0]\n", 717 | " #print(targets)\n", 718 | " #print(inputs)\n", 719 | "\n", 720 | " #reshape inputs and targets into batches\n", 721 | " inputs_batches = np.split(inputs.reshape(batch_size, -1), n_batches, 1)\n", 722 | " targets_batches = np.split(targets.reshape(batch_size, -1), n_batches, 1)\n", 723 | " #print(inputs_batches)\n", 724 | " #print(targets_batches)\n", 725 | " \n", 726 | " #combine inputs batches and targets batches into one array\n", 727 | " batches = np.array(list(zip(inputs_batches, targets_batches)))\n", 728 | " \n", 729 | " return batches\n", 730 | " \n", 731 | "\n", 732 | "\"\"\"\n", 733 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 734 | "\"\"\"\n", 735 | "tests.test_get_batches(get_batches)" 736 | ] 737 | }, 738 | { 739 | "cell_type": "markdown", 740 | "metadata": { 741 | "deletable": true, 742 | "editable": true 743 | }, 744 | "source": [ 745 | "## Neural Network Training\n", 746 | "### Hyperparameters\n", 747 | "Tune the following parameters:\n", 748 | "\n", 749 | "- Set `num_epochs` to the number of epochs.\n", 750 | "- Set `batch_size` to the batch size.\n", 751 | "- Set `rnn_size` to the size of the RNNs.\n", 752 | "- Set `embed_dim` to the size of the embedding.\n", 753 | "- Set `seq_length` to the length of sequence.\n", 754 | "- Set `learning_rate` to the learning rate.\n", 755 | "- Set `show_every_n_batches` to the number of batches the neural network should print progress." 756 | ] 757 | }, 758 | { 759 | "cell_type": "code", 760 | "execution_count": 17, 761 | "metadata": { 762 | "collapsed": true, 763 | "deletable": true, 764 | "editable": true 765 | }, 766 | "outputs": [], 767 | "source": [ 768 | "# Number of Epochs\n", 769 | "num_epochs = 100\n", 770 | "# Batch Size\n", 771 | "batch_size = 256\n", 772 | "# RNN Size\n", 773 | "rnn_size = 512\n", 774 | "# Embedding Dimension Size\n", 775 | "embed_dim = 300\n", 776 | "# Sequence Length\n", 777 | "seq_length = 12\n", 778 | "# Learning Rate\n", 779 | "learning_rate = 0.001\n", 780 | "# Show stats for every n number of batches\n", 781 | "show_every_n_batches = 25\n", 782 | "\n", 783 | "\"\"\"\n", 784 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 785 | "\"\"\"\n", 786 | "save_dir = './save'" 787 | ] 788 | }, 789 | { 790 | "cell_type": "markdown", 791 | "metadata": { 792 | "deletable": true, 793 | "editable": true 794 | }, 795 | "source": [ 796 | "### Build the Graph\n", 797 | "Build the graph using the neural network you implemented." 798 | ] 799 | }, 800 | { 801 | "cell_type": "code", 802 | "execution_count": 18, 803 | "metadata": { 804 | "collapsed": false, 805 | "deletable": true, 806 | "editable": true 807 | }, 808 | "outputs": [], 809 | "source": [ 810 | "\"\"\"\n", 811 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 812 | "\"\"\"\n", 813 | "from tensorflow.contrib import seq2seq\n", 814 | "\n", 815 | "train_graph = tf.Graph()\n", 816 | "with train_graph.as_default():\n", 817 | " vocab_size = len(int_to_vocab)\n", 818 | " input_text, targets, lr = get_inputs()\n", 819 | " input_data_shape = tf.shape(input_text)\n", 820 | " cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)\n", 821 | " logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size, embed_dim)\n", 822 | "\n", 823 | " # Probabilities for generating words\n", 824 | " probs = tf.nn.softmax(logits, name='probs')\n", 825 | "\n", 826 | " # Loss function\n", 827 | " cost = seq2seq.sequence_loss(\n", 828 | " logits,\n", 829 | " targets,\n", 830 | " tf.ones([input_data_shape[0], input_data_shape[1]]))\n", 831 | "\n", 832 | " # Optimizer\n", 833 | " optimizer = tf.train.AdamOptimizer(lr)\n", 834 | "\n", 835 | " # Gradient Clipping\n", 836 | " gradients = optimizer.compute_gradients(cost)\n", 837 | " capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]\n", 838 | " train_op = optimizer.apply_gradients(capped_gradients)" 839 | ] 840 | }, 841 | { 842 | "cell_type": "markdown", 843 | "metadata": { 844 | "deletable": true, 845 | "editable": true 846 | }, 847 | "source": [ 848 | "## Train\n", 849 | "Train the neural network on the preprocessed data. If you have a hard time getting a good loss, check the [forms](https://discussions.udacity.com/) to see if anyone is having the same problem." 850 | ] 851 | }, 852 | { 853 | "cell_type": "code", 854 | "execution_count": 19, 855 | "metadata": { 856 | "collapsed": false, 857 | "deletable": true, 858 | "editable": true 859 | }, 860 | "outputs": [ 861 | { 862 | "name": "stdout", 863 | "output_type": "stream", 864 | "text": [ 865 | "Epoch 0 Batch 0/22 train_loss = 8.825\n", 866 | "Epoch 1 Batch 3/22 train_loss = 5.726\n", 867 | "Epoch 2 Batch 6/22 train_loss = 5.259\n", 868 | "Epoch 3 Batch 9/22 train_loss = 5.073\n", 869 | "Epoch 4 Batch 12/22 train_loss = 4.756\n", 870 | "Epoch 5 Batch 15/22 train_loss = 4.485\n", 871 | "Epoch 6 Batch 18/22 train_loss = 4.373\n", 872 | "Epoch 7 Batch 21/22 train_loss = 4.217\n", 873 | "Epoch 9 Batch 2/22 train_loss = 3.889\n", 874 | "Epoch 10 Batch 5/22 train_loss = 3.734\n", 875 | "Epoch 11 Batch 8/22 train_loss = 3.597\n", 876 | "Epoch 12 Batch 11/22 train_loss = 3.435\n", 877 | "Epoch 13 Batch 14/22 train_loss = 3.262\n", 878 | "Epoch 14 Batch 17/22 train_loss = 3.085\n", 879 | "Epoch 15 Batch 20/22 train_loss = 2.899\n", 880 | "Epoch 17 Batch 1/22 train_loss = 2.682\n", 881 | "Epoch 18 Batch 4/22 train_loss = 2.543\n", 882 | "Epoch 19 Batch 7/22 train_loss = 2.450\n", 883 | "Epoch 20 Batch 10/22 train_loss = 2.339\n", 884 | "Epoch 21 Batch 13/22 train_loss = 2.210\n", 885 | "Epoch 22 Batch 16/22 train_loss = 2.054\n", 886 | "Epoch 23 Batch 19/22 train_loss = 1.877\n", 887 | "Epoch 25 Batch 0/22 train_loss = 1.837\n", 888 | "Epoch 26 Batch 3/22 train_loss = 1.738\n", 889 | "Epoch 27 Batch 6/22 train_loss = 1.611\n", 890 | "Epoch 28 Batch 9/22 train_loss = 1.507\n", 891 | "Epoch 29 Batch 12/22 train_loss = 1.448\n", 892 | "Epoch 30 Batch 15/22 train_loss = 1.355\n", 893 | "Epoch 31 Batch 18/22 train_loss = 1.334\n", 894 | "Epoch 32 Batch 21/22 train_loss = 1.199\n", 895 | "Epoch 34 Batch 2/22 train_loss = 1.170\n", 896 | "Epoch 35 Batch 5/22 train_loss = 1.077\n", 897 | "Epoch 36 Batch 8/22 train_loss = 0.972\n", 898 | "Epoch 37 Batch 11/22 train_loss = 0.921\n", 899 | "Epoch 38 Batch 14/22 train_loss = 0.913\n", 900 | "Epoch 39 Batch 17/22 train_loss = 0.851\n", 901 | "Epoch 40 Batch 20/22 train_loss = 0.815\n", 902 | "Epoch 42 Batch 1/22 train_loss = 0.733\n", 903 | "Epoch 43 Batch 4/22 train_loss = 0.736\n", 904 | "Epoch 44 Batch 7/22 train_loss = 0.659\n", 905 | "Epoch 45 Batch 10/22 train_loss = 0.660\n", 906 | "Epoch 46 Batch 13/22 train_loss = 0.612\n", 907 | "Epoch 47 Batch 16/22 train_loss = 0.595\n", 908 | "Epoch 48 Batch 19/22 train_loss = 0.562\n", 909 | "Epoch 50 Batch 0/22 train_loss = 0.548\n", 910 | "Epoch 51 Batch 3/22 train_loss = 0.506\n", 911 | "Epoch 52 Batch 6/22 train_loss = 0.493\n", 912 | "Epoch 53 Batch 9/22 train_loss = 0.461\n", 913 | "Epoch 54 Batch 12/22 train_loss = 0.477\n", 914 | "Epoch 55 Batch 15/22 train_loss = 0.445\n", 915 | "Epoch 56 Batch 18/22 train_loss = 0.430\n", 916 | "Epoch 57 Batch 21/22 train_loss = 0.383\n", 917 | "Epoch 59 Batch 2/22 train_loss = 0.427\n", 918 | "Epoch 60 Batch 5/22 train_loss = 0.393\n", 919 | "Epoch 61 Batch 8/22 train_loss = 0.368\n", 920 | "Epoch 62 Batch 11/22 train_loss = 0.365\n", 921 | "Epoch 63 Batch 14/22 train_loss = 0.362\n", 922 | "Epoch 64 Batch 17/22 train_loss = 0.350\n", 923 | "Epoch 65 Batch 20/22 train_loss = 0.344\n", 924 | "Epoch 67 Batch 1/22 train_loss = 0.359\n", 925 | "Epoch 68 Batch 4/22 train_loss = 0.355\n", 926 | "Epoch 69 Batch 7/22 train_loss = 0.336\n", 927 | "Epoch 70 Batch 10/22 train_loss = 0.355\n", 928 | "Epoch 71 Batch 13/22 train_loss = 0.334\n", 929 | "Epoch 72 Batch 16/22 train_loss = 0.337\n", 930 | "Epoch 73 Batch 19/22 train_loss = 0.333\n", 931 | "Epoch 75 Batch 0/22 train_loss = 0.329\n", 932 | "Epoch 76 Batch 3/22 train_loss = 0.323\n", 933 | "Epoch 77 Batch 6/22 train_loss = 0.326\n", 934 | "Epoch 78 Batch 9/22 train_loss = 0.317\n", 935 | "Epoch 79 Batch 12/22 train_loss = 0.324\n", 936 | "Epoch 80 Batch 15/22 train_loss = 0.317\n", 937 | "Epoch 81 Batch 18/22 train_loss = 0.303\n", 938 | "Epoch 82 Batch 21/22 train_loss = 0.285\n", 939 | "Epoch 84 Batch 2/22 train_loss = 0.322\n", 940 | "Epoch 85 Batch 5/22 train_loss = 0.304\n", 941 | "Epoch 86 Batch 8/22 train_loss = 0.293\n", 942 | "Epoch 87 Batch 11/22 train_loss = 0.303\n", 943 | "Epoch 88 Batch 14/22 train_loss = 0.303\n", 944 | "Epoch 89 Batch 17/22 train_loss = 0.295\n", 945 | "Epoch 90 Batch 20/22 train_loss = 0.290\n", 946 | "Epoch 92 Batch 1/22 train_loss = 0.314\n", 947 | "Epoch 93 Batch 4/22 train_loss = 0.312\n", 948 | "Epoch 94 Batch 7/22 train_loss = 0.294\n", 949 | "Epoch 95 Batch 10/22 train_loss = 0.314\n", 950 | "Epoch 96 Batch 13/22 train_loss = 0.295\n", 951 | "Epoch 97 Batch 16/22 train_loss = 0.302\n", 952 | "Epoch 98 Batch 19/22 train_loss = 0.301\n", 953 | "Model Trained and Saved\n" 954 | ] 955 | } 956 | ], 957 | "source": [ 958 | "\"\"\"\n", 959 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 960 | "\"\"\"\n", 961 | "batches = get_batches(int_text, batch_size, seq_length)\n", 962 | "\n", 963 | "with tf.Session(graph=train_graph) as sess:\n", 964 | " sess.run(tf.global_variables_initializer())\n", 965 | "\n", 966 | " for epoch_i in range(num_epochs):\n", 967 | " state = sess.run(initial_state, {input_text: batches[0][0]})\n", 968 | "\n", 969 | " for batch_i, (x, y) in enumerate(batches):\n", 970 | " feed = {\n", 971 | " input_text: x,\n", 972 | " targets: y,\n", 973 | " initial_state: state,\n", 974 | " lr: learning_rate}\n", 975 | " train_loss, state, _ = sess.run([cost, final_state, train_op], feed)\n", 976 | "\n", 977 | " # Show every batches\n", 978 | " if (epoch_i * len(batches) + batch_i) % show_every_n_batches == 0:\n", 979 | " print('Epoch {:>3} Batch {:>4}/{} train_loss = {:.3f}'.format(\n", 980 | " epoch_i,\n", 981 | " batch_i,\n", 982 | " len(batches),\n", 983 | " train_loss))\n", 984 | "\n", 985 | " # Save Model\n", 986 | " saver = tf.train.Saver()\n", 987 | " saver.save(sess, save_dir)\n", 988 | " print('Model Trained and Saved')" 989 | ] 990 | }, 991 | { 992 | "cell_type": "markdown", 993 | "metadata": { 994 | "deletable": true, 995 | "editable": true 996 | }, 997 | "source": [ 998 | "## Save Parameters\n", 999 | "Save `seq_length` and `save_dir` for generating a new TV script." 1000 | ] 1001 | }, 1002 | { 1003 | "cell_type": "code", 1004 | "execution_count": 20, 1005 | "metadata": { 1006 | "collapsed": false, 1007 | "deletable": true, 1008 | "editable": true 1009 | }, 1010 | "outputs": [], 1011 | "source": [ 1012 | "\"\"\"\n", 1013 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 1014 | "\"\"\"\n", 1015 | "# Save parameters for checkpoint\n", 1016 | "helper.save_params((seq_length, save_dir))" 1017 | ] 1018 | }, 1019 | { 1020 | "cell_type": "markdown", 1021 | "metadata": { 1022 | "deletable": true, 1023 | "editable": true 1024 | }, 1025 | "source": [ 1026 | "# Checkpoint" 1027 | ] 1028 | }, 1029 | { 1030 | "cell_type": "code", 1031 | "execution_count": 21, 1032 | "metadata": { 1033 | "collapsed": false, 1034 | "deletable": true, 1035 | "editable": true 1036 | }, 1037 | "outputs": [], 1038 | "source": [ 1039 | "\"\"\"\n", 1040 | "DON'T MODIFY ANYTHING IN THIS CELL\n", 1041 | "\"\"\"\n", 1042 | "import tensorflow as tf\n", 1043 | "import numpy as np\n", 1044 | "import helper\n", 1045 | "import problem_unittests as tests\n", 1046 | "\n", 1047 | "_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()\n", 1048 | "seq_length, load_dir = helper.load_params()" 1049 | ] 1050 | }, 1051 | { 1052 | "cell_type": "markdown", 1053 | "metadata": { 1054 | "deletable": true, 1055 | "editable": true 1056 | }, 1057 | "source": [ 1058 | "## Implement Generate Functions\n", 1059 | "### Get Tensors\n", 1060 | "Get tensors from `loaded_graph` using the function [`get_tensor_by_name()`](https://www.tensorflow.org/api_docs/python/tf/Graph#get_tensor_by_name). Get the tensors using the following names:\n", 1061 | "- \"input:0\"\n", 1062 | "- \"initial_state:0\"\n", 1063 | "- \"final_state:0\"\n", 1064 | "- \"probs:0\"\n", 1065 | "\n", 1066 | "Return the tensors in the following tuple `(InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)` " 1067 | ] 1068 | }, 1069 | { 1070 | "cell_type": "code", 1071 | "execution_count": 22, 1072 | "metadata": { 1073 | "collapsed": false, 1074 | "deletable": true, 1075 | "editable": true 1076 | }, 1077 | "outputs": [ 1078 | { 1079 | "name": "stdout", 1080 | "output_type": "stream", 1081 | "text": [ 1082 | "Tests Passed\n" 1083 | ] 1084 | } 1085 | ], 1086 | "source": [ 1087 | "def get_tensors(loaded_graph):\n", 1088 | " \"\"\"\n", 1089 | " Get input, initial state, final state, and probabilities tensor from \n", 1090 | " :param loaded_graph: TensorFlow graph loaded from file\n", 1091 | " :return: Tuple (InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)\n", 1092 | " \"\"\"\n", 1093 | " # TODO: Implement Function\n", 1094 | " InputTensor = loaded_graph.get_tensor_by_name(\"input:0\")\n", 1095 | " InitialStateTensor = loaded_graph.get_tensor_by_name(\"initial_state:0\")\n", 1096 | " FinalStateTensor = loaded_graph.get_tensor_by_name(\"final_state:0\")\n", 1097 | " ProbsTensor = loaded_graph.get_tensor_by_name(\"probs:0\")\n", 1098 | " return InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor\n", 1099 | "\n", 1100 | "\n", 1101 | "\"\"\"\n", 1102 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1103 | "\"\"\"\n", 1104 | "tests.test_get_tensors(get_tensors)" 1105 | ] 1106 | }, 1107 | { 1108 | "cell_type": "markdown", 1109 | "metadata": { 1110 | "deletable": true, 1111 | "editable": true 1112 | }, 1113 | "source": [ 1114 | "### Choose Word\n", 1115 | "Implement the `pick_word()` function to select the next word using `probabilities`." 1116 | ] 1117 | }, 1118 | { 1119 | "cell_type": "code", 1120 | "execution_count": 23, 1121 | "metadata": { 1122 | "collapsed": false, 1123 | "deletable": true, 1124 | "editable": true 1125 | }, 1126 | "outputs": [ 1127 | { 1128 | "name": "stdout", 1129 | "output_type": "stream", 1130 | "text": [ 1131 | "Tests Passed\n" 1132 | ] 1133 | } 1134 | ], 1135 | "source": [ 1136 | "def pick_word(probabilities, int_to_vocab):\n", 1137 | " \"\"\"\n", 1138 | " Pick the next word in the generated text\n", 1139 | " :param probabilities: Probabilites of the next word\n", 1140 | " :param int_to_vocab: Dictionary of word ids as the keys and words as the values\n", 1141 | " :return: String of the predicted word\n", 1142 | " \"\"\"\n", 1143 | " # TODO: Implement Function\n", 1144 | " #use np.random.choice to generate next_word, p as probabilities\n", 1145 | " #turned int_to_vocab.values() into a list for 1D array\n", 1146 | " next_word = np.random.choice(list(int_to_vocab.values()), p=probabilities)\n", 1147 | " return next_word\n", 1148 | "\n", 1149 | "\n", 1150 | "\"\"\"\n", 1151 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1152 | "\"\"\"\n", 1153 | "tests.test_pick_word(pick_word)" 1154 | ] 1155 | }, 1156 | { 1157 | "cell_type": "markdown", 1158 | "metadata": { 1159 | "deletable": true, 1160 | "editable": true 1161 | }, 1162 | "source": [ 1163 | "## Generate TV Script\n", 1164 | "This will generate the TV script for you. Set `gen_length` to the length of TV script you want to generate." 1165 | ] 1166 | }, 1167 | { 1168 | "cell_type": "code", 1169 | "execution_count": 24, 1170 | "metadata": { 1171 | "collapsed": false, 1172 | "deletable": true, 1173 | "editable": true 1174 | }, 1175 | "outputs": [ 1176 | { 1177 | "name": "stdout", 1178 | "output_type": "stream", 1179 | "text": [ 1180 | "moe_szyslak: drinking will help us plan.\n", 1181 | "homer_simpson: this valentine's crap has to be a bar.(gets off) new_health_inspector: bar an idiot.\n", 1182 | "homer_simpson:(to self) sorry, i need i'm behind your foot.\n", 1183 | "moe_szyslak: but i suppose i got a two hundred and people all can use the kids.\n", 1184 | "homer_simpson: to be the best thing?\n", 1185 | "barney_gumble: 'cause only one i thought you said.\n", 1186 | "carl_carlson:(to self) someone's makes a little one, can i have a free? take this!(homer's sound)\n", 1187 | "homer_simpson: the one, but i did not going to find it out.\n", 1188 | "moe_szyslak:(sings) i just wanna tell my life till i'm on their go!\n", 1189 | "\n", 1190 | "\n", 1191 | "moe_szyslak: the guys are make around in the gentleman's of woman.\n", 1192 | "lenny_leonard: oh, you don't let me do being here? no, moe.\n", 1193 | "barney_gumble: you know, it's you, moe. the drinks are on you.\n", 1194 | "seymour_skinner:(sighs) isn't it eyes no more.\n", 1195 | "homer_simpson:(chuckles) all right.\n" 1196 | ] 1197 | } 1198 | ], 1199 | "source": [ 1200 | "gen_length = 200\n", 1201 | "# homer_simpson, moe_szyslak, or Barney_Gumble\n", 1202 | "prime_word = 'moe_szyslak'\n", 1203 | "\n", 1204 | "\"\"\"\n", 1205 | "DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE\n", 1206 | "\"\"\"\n", 1207 | "loaded_graph = tf.Graph()\n", 1208 | "with tf.Session(graph=loaded_graph) as sess:\n", 1209 | " # Load saved model\n", 1210 | " loader = tf.train.import_meta_graph(load_dir + '.meta')\n", 1211 | " loader.restore(sess, load_dir)\n", 1212 | "\n", 1213 | " # Get Tensors from loaded model\n", 1214 | " input_text, initial_state, final_state, probs = get_tensors(loaded_graph)\n", 1215 | "\n", 1216 | " # Sentences generation setup\n", 1217 | " gen_sentences = [prime_word + ':']\n", 1218 | " prev_state = sess.run(initial_state, {input_text: np.array([[1]])})\n", 1219 | "\n", 1220 | " # Generate sentences\n", 1221 | " for n in range(gen_length):\n", 1222 | " # Dynamic Input\n", 1223 | " dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]\n", 1224 | " dyn_seq_length = len(dyn_input[0])\n", 1225 | "\n", 1226 | " # Get Prediction\n", 1227 | " probabilities, prev_state = sess.run(\n", 1228 | " [probs, final_state],\n", 1229 | " {input_text: dyn_input, initial_state: prev_state})\n", 1230 | " \n", 1231 | " pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)\n", 1232 | "\n", 1233 | " gen_sentences.append(pred_word)\n", 1234 | " \n", 1235 | " # Remove tokens\n", 1236 | " tv_script = ' '.join(gen_sentences)\n", 1237 | " for key, token in token_dict.items():\n", 1238 | " ending = ' ' if key in ['\\n', '(', '\"'] else ''\n", 1239 | " tv_script = tv_script.replace(' ' + token.lower(), key)\n", 1240 | " tv_script = tv_script.replace('\\n ', '\\n')\n", 1241 | " tv_script = tv_script.replace('( ', '(')\n", 1242 | " \n", 1243 | " print(tv_script)" 1244 | ] 1245 | }, 1246 | { 1247 | "cell_type": "markdown", 1248 | "metadata": { 1249 | "deletable": true, 1250 | "editable": true 1251 | }, 1252 | "source": [ 1253 | "# The TV Script is Nonsensical\n", 1254 | "It's ok if the TV script doesn't make any sense. We trained on less than a megabyte of text. In order to get good results, you'll have to use a smaller vocabulary or get more data. Luckly there's more data! As we mentioned in the begging of this project, this is a subset of [another dataset](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data). We didn't have you train on all the data, because that would take too long. However, you are free to train your neural network on all the data. After you complete the project, of course.\n", 1255 | "# Submitting This Project\n", 1256 | "When submitting this project, make sure to run all the cells before saving the notebook. Save the notebook file as \"dlnd_tv_script_generation.ipynb\" and save it as a HTML file under \"File\" -> \"Download as\". Include the \"helper.py\" and \"problem_unittests.py\" files in your submission." 1257 | ] 1258 | } 1259 | ], 1260 | "metadata": { 1261 | "kernelspec": { 1262 | "display_name": "Python 3", 1263 | "language": "python", 1264 | "name": "python3" 1265 | }, 1266 | "language_info": { 1267 | "codemirror_mode": { 1268 | "name": "ipython", 1269 | "version": 3 1270 | }, 1271 | "file_extension": ".py", 1272 | "mimetype": "text/x-python", 1273 | "name": "python", 1274 | "nbconvert_exporter": "python", 1275 | "pygments_lexer": "ipython3", 1276 | "version": "3.6.0" 1277 | } 1278 | }, 1279 | "nbformat": 4, 1280 | "nbformat_minor": 0 1281 | } 1282 | --------------------------------------------------------------------------------