├── .gitignore ├── Appendix └── legacy.py ├── Chapter01 ├── keras_MINST_V1.py ├── keras_MINST_V2.py ├── keras_MINST_V3.py └── keras_MINST_V4.py ├── Chapter02 ├── cifar10_architecture.json ├── keras_Azure.py └── keras_VGG16_prebuilt.py ├── Chapter03 ├── keras_CIFAR10_V1.py ├── keras_CIFAR10_simple.py ├── keras_EvaluateCIFAR10.py ├── keras_LeNet.py └── keras_VGG16.py ├── Chapter04 ├── Chapter04.ipynb ├── cifar10 │ ├── Chapter04_cifar10.ipynb │ ├── cifar10_utils.py │ ├── example_gan_cifar10.py │ ├── example_gan_convolutional.py │ └── image_utils.py ├── dcgan.py ├── legacy.py └── wavnet │ ├── dataset.py │ ├── vctk │ └── download_vctk.sh │ ├── wavenet.py │ └── wavenet_utils.py ├── Chapter05 ├── data_download_glove.sh ├── data_download_text8.sh ├── finetune_glove_embeddings.py ├── finetune_glove_embeddings_tensorboard_embedding.py ├── finetune_word2vec_embeddings.py ├── finetune_word2vec_embeddings_tensorboard_embedding.py ├── keras_cbow.py ├── keras_skipgram.py ├── learn_embedding_from_scratch.py ├── learn_embedding_from_scratch_tensorboard_embedding.py ├── make_tensorboard.py ├── skipgram_example.py ├── transfer_glove_embeddings.py ├── transfer_glove_embeddings_tensorboard_embedding.py ├── transfer_word2vec_embeddings.py ├── transfer_word2vec_embeddings_tensorboard_embedding.py ├── word2vec_cbow.py ├── word2vec_gensim.py └── word2vec_skipgram.py ├── Chapter06 ├── alice_chargen_rnn.py ├── econs_data.py ├── econs_stateful.py ├── pos-tagging-explore.py ├── pos_tagging_data.py ├── pos_tagging_gru.py ├── treebank_data.py ├── treebank_pos_gru.py ├── treebank_pos_lstm.py └── umich_sentiment_lstm.py ├── Chapter07 ├── air-quality-regression.py ├── data_download_question_answer.sh ├── deep-dream.py ├── func-api-func.py ├── func-api-multi.py ├── func-api-seq.py ├── make_tensorboard.py ├── mem-network.py ├── sent-thoughts-parse.py ├── sent-thoughts-rnn.py ├── style-transfer.py └── tf-keras-func.py ├── Chapter08 ├── game.py ├── game_screenshots.py ├── rl-network-test.py ├── rl-network-train.py ├── rl-network-viz.py └── wrapped_game.py ├── Docker ├── Dockerfile ├── Dockerfile-gpu ├── Dockerfile-wavnet ├── Makefile └── README.md ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # DotEnv configuration 60 | .env 61 | 62 | # Database 63 | *.db 64 | *.rdb 65 | 66 | # Pycharm 67 | .idea 68 | 69 | # Jupyter NB Checkpoints 70 | .ipynb_checkpoints/ 71 | 72 | # exclude data from source control by default 73 | cover/* 74 | -------------------------------------------------------------------------------- /Appendix/legacy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to avoid warnings while testing both Keras 1 and 2. 3 | """ 4 | import keras 5 | 6 | keras_2 = int(keras.__version__.split(".")[0]) > 1 # Keras > 1 7 | 8 | 9 | def fit_generator(model, generator, epochs, steps_per_epoch): 10 | if keras_2: 11 | model.fit_generator(generator, epochs=epochs, steps_per_epoch=steps_per_epoch) 12 | else: 13 | model.fit_generator(generator, nb_epoch=epochs, samples_per_epoch=steps_per_epoch) 14 | 15 | 16 | def fit(model, x, y, nb_epoch=10, *args, **kwargs): 17 | if keras_2: 18 | return model.fit(x, y, *args, epochs=nb_epoch, **kwargs) 19 | else: 20 | return model.fit(x, y, *args, nb_epoch=nb_epoch, **kwargs) 21 | 22 | 23 | def l1l2(l1=0, l2=0): 24 | if keras_2: 25 | return keras.regularizers.L1L2(l1, l2) 26 | else: 27 | return keras.regularizers.l1l2(l1, l2) 28 | 29 | 30 | def Dense(units, W_regularizer=None, W_initializer='glorot_uniform', **kwargs): 31 | if keras_2: 32 | return keras.layers.Dense(units, kernel_regularizer=W_regularizer, kernel_initializer=W_initializer, **kwargs) 33 | else: 34 | return keras.layers.Dense(units, W_regularizer=W_regularizer, init=W_initializer, **kwargs) 35 | 36 | 37 | def BatchNormalization(mode=0, **kwargs): 38 | if keras_2: 39 | return keras.layers.BatchNormalization(**kwargs) 40 | else: 41 | return keras.layers.BatchNormalization(mode=mode, **kwargs) 42 | 43 | 44 | def Convolution2D(units, w, h, W_regularizer=None, W_initializer='glorot_uniform', border_mode='same', **kwargs): 45 | if keras_2: 46 | return keras.layers.Convolution2D(units, (w, h), padding=border_mode, kernel_regularizer=W_regularizer, 47 | kernel_initializer=W_initializer, 48 | **kwargs) 49 | else: 50 | return keras.layers.Convolution2D(units, w, h, border_mode=border_mode, W_regularizer=W_regularizer, 51 | init=W_initializer, 52 | **kwargs) 53 | 54 | 55 | def AveragePooling2D(pool_size, border_mode='valid', **kwargs): 56 | if keras_2: 57 | return keras.layers.AveragePooling2D(pool_size=pool_size, padding=border_mode, **kwargs) 58 | else: 59 | return keras.layers.AveragePooling2D(pool_size=pool_size, border_mode=border_mode, **kwargs) 60 | -------------------------------------------------------------------------------- /Chapter01/keras_MINST_V1.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from keras.datasets import mnist 4 | from keras.models import Sequential 5 | from keras.layers.core import Dense, Activation 6 | from keras.optimizers import SGD 7 | from keras.utils import np_utils 8 | 9 | np.random.seed(1671) # for reproducibility 10 | 11 | # network and training 12 | NB_EPOCH = 200 13 | BATCH_SIZE = 128 14 | VERBOSE = 1 15 | NB_CLASSES = 10 # number of outputs = number of digits 16 | OPTIMIZER = SGD() # SGD optimizer, explained later in this chapter 17 | N_HIDDEN = 128 18 | VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION 19 | 20 | # data: shuffled and split between train and test sets 21 | # 22 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 23 | 24 | #X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 25 | RESHAPED = 784 26 | # 27 | X_train = X_train.reshape(60000, RESHAPED) 28 | X_test = X_test.reshape(10000, RESHAPED) 29 | X_train = X_train.astype('float32') 30 | X_test = X_test.astype('float32') 31 | 32 | # normalize 33 | # 34 | X_train /= 255 35 | X_test /= 255 36 | print(X_train.shape[0], 'train samples') 37 | print(X_test.shape[0], 'test samples') 38 | 39 | # convert class vectors to binary class matrices 40 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 41 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 42 | 43 | # 10 outputs 44 | # final stage is softmax 45 | 46 | model = Sequential() 47 | model.add(Dense(NB_CLASSES, input_shape=(RESHAPED,))) 48 | model.add(Activation('softmax')) 49 | 50 | model.summary() 51 | 52 | model.compile(loss='categorical_crossentropy', 53 | optimizer=OPTIMIZER, 54 | metrics=['accuracy']) 55 | 56 | history = model.fit(X_train, Y_train, 57 | batch_size=BATCH_SIZE, epochs=NB_EPOCH, 58 | verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 59 | score = model.evaluate(X_test, Y_test, verbose=VERBOSE) 60 | print("\nTest score:", score[0]) 61 | print('Test accuracy:', score[1]) 62 | -------------------------------------------------------------------------------- /Chapter01/keras_MINST_V2.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from keras.datasets import mnist 4 | from keras.models import Sequential 5 | from keras.layers.core import Dense, Activation 6 | from keras.optimizers import SGD 7 | from keras.utils import np_utils 8 | 9 | np.random.seed(1671) # for reproducibility 10 | 11 | # network and training 12 | NB_EPOCH = 20 13 | BATCH_SIZE = 128 14 | VERBOSE = 1 15 | NB_CLASSES = 10 # number of outputs = number of digits 16 | OPTIMIZER = SGD() # optimizer, explained later in this chapter 17 | N_HIDDEN = 128 18 | VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION 19 | 20 | # data: shuffled and split between train and test sets 21 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 22 | 23 | #X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 24 | RESHAPED = 784 25 | # 26 | X_train = X_train.reshape(60000, RESHAPED) 27 | X_test = X_test.reshape(10000, RESHAPED) 28 | X_train = X_train.astype('float32') 29 | X_test = X_test.astype('float32') 30 | 31 | # normalize 32 | X_train /= 255 33 | X_test /= 255 34 | print(X_train.shape[0], 'train samples') 35 | print(X_test.shape[0], 'test samples') 36 | 37 | # convert class vectors to binary class matrices 38 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 39 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 40 | 41 | # M_HIDDEN hidden layers 42 | # 10 outputs 43 | # final stage is softmax 44 | 45 | model = Sequential() 46 | model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,))) 47 | model.add(Activation('relu')) 48 | model.add(Dense(N_HIDDEN)) 49 | model.add(Activation('relu')) 50 | model.add(Dense(NB_CLASSES)) 51 | model.add(Activation('softmax')) 52 | model.summary() 53 | 54 | model.compile(loss='categorical_crossentropy', 55 | optimizer=OPTIMIZER, 56 | metrics=['accuracy']) 57 | 58 | history = model.fit(X_train, Y_train, 59 | batch_size=BATCH_SIZE, epochs=NB_EPOCH, 60 | verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 61 | 62 | score = model.evaluate(X_test, Y_test, verbose=VERBOSE) 63 | print("\nTest score:", score[0]) 64 | print('Test accuracy:', score[1]) -------------------------------------------------------------------------------- /Chapter01/keras_MINST_V3.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from keras.datasets import mnist 4 | from keras.models import Sequential 5 | from keras.layers.core import Dense, Dropout, Activation 6 | from keras.optimizers import SGD 7 | from keras.utils import np_utils 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | np.random.seed(1671) # for reproducibility 12 | 13 | # network and training 14 | NB_EPOCH = 250 15 | BATCH_SIZE = 128 16 | VERBOSE = 1 17 | NB_CLASSES = 10 # number of outputs = number of digits 18 | OPTIMIZER = SGD() # optimizer, explained later in this chapter 19 | N_HIDDEN = 128 20 | VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION 21 | DROPOUT = 0.3 22 | 23 | # data: shuffled and split between train and test sets 24 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 25 | 26 | #X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 27 | RESHAPED = 784 28 | # 29 | X_train = X_train.reshape(60000, RESHAPED) 30 | X_test = X_test.reshape(10000, RESHAPED) 31 | X_train = X_train.astype('float32') 32 | X_test = X_test.astype('float32') 33 | 34 | # normalize 35 | X_train /= 255 36 | X_test /= 255 37 | print(X_train.shape[0], 'train samples') 38 | print(X_test.shape[0], 'test samples') 39 | 40 | # convert class vectors to binary class matrices 41 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 42 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 43 | 44 | # M_HIDDEN hidden layers 45 | # 10 outputs 46 | # final stage is softmax 47 | 48 | model = Sequential() 49 | model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,))) 50 | model.add(Activation('relu')) 51 | model.add(Dropout(DROPOUT)) 52 | model.add(Dense(N_HIDDEN)) 53 | model.add(Activation('relu')) 54 | model.add(Dropout(DROPOUT)) 55 | model.add(Dense(NB_CLASSES)) 56 | model.add(Activation('softmax')) 57 | model.summary() 58 | 59 | model.compile(loss='categorical_crossentropy', 60 | optimizer=OPTIMIZER, 61 | metrics=['accuracy']) 62 | 63 | history = model.fit(X_train, Y_train, 64 | batch_size=BATCH_SIZE, epochs=NB_EPOCH, 65 | verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 66 | 67 | score = model.evaluate(X_test, Y_test, verbose=VERBOSE) 68 | print("\nTest score:", score[0]) 69 | print('Test accuracy:', score[1]) 70 | 71 | # list all data in history 72 | print(history.history.keys()) 73 | # summarize history for accuracy 74 | plt.plot(history.history['acc']) 75 | plt.plot(history.history['val_acc']) 76 | plt.title('model accuracy') 77 | plt.ylabel('accuracy') 78 | plt.xlabel('epoch') 79 | plt.legend(['train', 'test'], loc='upper left') 80 | plt.show() 81 | # summarize history for loss 82 | plt.plot(history.history['loss']) 83 | plt.plot(history.history['val_loss']) 84 | plt.title('model loss') 85 | plt.ylabel('loss') 86 | plt.xlabel('epoch') 87 | plt.legend(['train', 'test'], loc='upper left') 88 | plt.show() -------------------------------------------------------------------------------- /Chapter01/keras_MINST_V4.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import numpy as np 3 | from keras.datasets import mnist 4 | from keras.models import Sequential 5 | from keras.layers.core import Dense, Dropout, Activation 6 | from keras.optimizers import RMSprop 7 | from keras.utils import np_utils 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | np.random.seed(1671) # for reproducibility 12 | 13 | # network and training 14 | NB_EPOCH = 20 15 | BATCH_SIZE = 128 16 | VERBOSE = 1 17 | NB_CLASSES = 10 # number of outputs = number of digits 18 | OPTIMIZER = RMSprop() # optimizer, explainedin this chapter 19 | N_HIDDEN = 128 20 | VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION 21 | DROPOUT = 0.3 22 | 23 | # data: shuffled and split between train and test sets 24 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 25 | 26 | #X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 27 | RESHAPED = 784 28 | # 29 | X_train = X_train.reshape(60000, RESHAPED) 30 | X_test = X_test.reshape(10000, RESHAPED) 31 | X_train = X_train.astype('float32') 32 | X_test = X_test.astype('float32') 33 | 34 | # normalize 35 | X_train /= 255 36 | X_test /= 255 37 | print(X_train.shape[0], 'train samples') 38 | print(X_test.shape[0], 'test samples') 39 | 40 | # convert class vectors to binary class matrices 41 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 42 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 43 | 44 | # M_HIDDEN hidden layers 45 | # 10 outputs 46 | # final stage is softmax 47 | 48 | model = Sequential() 49 | model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,))) 50 | model.add(Activation('relu')) 51 | model.add(Dropout(DROPOUT)) 52 | model.add(Dense(N_HIDDEN)) 53 | model.add(Activation('relu')) 54 | model.add(Dropout(DROPOUT)) 55 | model.add(Dense(NB_CLASSES)) 56 | model.add(Activation('softmax')) 57 | model.summary() 58 | 59 | model.compile(loss='categorical_crossentropy', 60 | optimizer=OPTIMIZER, 61 | metrics=['accuracy']) 62 | 63 | history = model.fit(X_train, Y_train, 64 | batch_size=BATCH_SIZE, epochs=NB_EPOCH, 65 | verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 66 | 67 | score = model.evaluate(X_test, Y_test, verbose=VERBOSE) 68 | print("\nTest score:", score[0]) 69 | print('Test accuracy:', score[1]) 70 | 71 | # list all data in history 72 | print(history.history.keys()) 73 | # summarize history for accuracy 74 | plt.plot(history.history['acc']) 75 | plt.plot(history.history['val_acc']) 76 | plt.title('model accuracy') 77 | plt.ylabel('accuracy') 78 | plt.xlabel('epoch') 79 | plt.legend(['train', 'test'], loc='upper left') 80 | plt.show() 81 | # summarize history for loss 82 | plt.plot(history.history['loss']) 83 | plt.plot(history.history['val_loss']) 84 | plt.title('model loss') 85 | plt.ylabel('loss') 86 | plt.xlabel('epoch') 87 | plt.legend(['train', 'test'], loc='upper left') 88 | plt.show() -------------------------------------------------------------------------------- /Chapter02/cifar10_architecture.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Sequential", "keras_version": "1.1.1", "config": [{"class_name": "Convolution2D", "config": {"b_regularizer": null, "W_constraint": null, "b_constraint": null, "name": "convolution2d_1", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "glorot_uniform", "bias": true, "nb_filter": 32, "input_dtype": "float32", "border_mode": "same", "batch_input_shape": [null, 3, 32, 32], "W_regularizer": null, "activation": "linear", "nb_row": 3}}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_1"}}, {"class_name": "Convolution2D", "config": {"W_constraint": null, "b_constraint": null, "name": "convolution2d_2", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "glorot_uniform", "bias": true, "nb_filter": 32, "border_mode": "same", "b_regularizer": null, "W_regularizer": null, "activation": "linear", "nb_row": 3}}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_2"}}, {"class_name": "MaxPooling2D", "config": {"name": "maxpooling2d_1", "trainable": true, "dim_ordering": "th", "pool_size": [2, 2], "strides": [2, 2], "border_mode": "valid"}}, {"class_name": "Dropout", "config": {"p": 0.25, "trainable": true, "name": "dropout_1"}}, {"class_name": "Convolution2D", "config": {"W_constraint": null, "b_constraint": null, "name": "convolution2d_3", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "glorot_uniform", "bias": true, "nb_filter": 64, "border_mode": "same", "b_regularizer": null, "W_regularizer": null, "activation": "linear", "nb_row": 3}}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_3"}}, {"class_name": "Convolution2D", "config": {"W_constraint": null, "b_constraint": null, "name": "convolution2d_4", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "glorot_uniform", "bias": true, "nb_filter": 64, "border_mode": "valid", "b_regularizer": null, "W_regularizer": null, "activation": "linear", "nb_row": 3}}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_4"}}, {"class_name": "MaxPooling2D", "config": {"name": "maxpooling2d_2", "trainable": true, "dim_ordering": "th", "pool_size": [2, 2], "strides": [2, 2], "border_mode": "valid"}}, {"class_name": "Dropout", "config": {"p": 0.25, "trainable": true, "name": "dropout_2"}}, {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1"}}, {"class_name": "Dense", "config": {"W_constraint": null, "b_constraint": null, "name": "dense_1", "activity_regularizer": null, "trainable": true, "init": "glorot_uniform", "bias": true, "input_dim": null, "b_regularizer": null, "W_regularizer": null, "activation": "linear", "output_dim": 512}}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_5"}}, {"class_name": "Dropout", "config": {"p": 0.5, "trainable": true, "name": "dropout_3"}}, {"class_name": "Dense", "config": {"W_constraint": null, "b_constraint": null, "name": "dense_2", "activity_regularizer": null, "trainable": true, "init": "glorot_uniform", "bias": true, "input_dim": null, "b_regularizer": null, "W_regularizer": null, "activation": "linear", "output_dim": 10}}, {"class_name": "Activation", "config": {"activation": "softmax", "trainable": true, "name": "activation_6"}}]} -------------------------------------------------------------------------------- /Chapter02/keras_Azure.py: -------------------------------------------------------------------------------- 1 | # The script MUST contain a function named azureml_main 2 | # which is the entry point for this module. 3 | 4 | # imports up here can be used to 5 | import pandas as pd 6 | import theano 7 | import theano.tensor as T 8 | from theano import function 9 | 10 | from keras.models import Sequential 11 | from keras.layers import Dense, Activation 12 | import numpy as np 13 | # The entry point function can contain up to two input arguments: 14 | # Param: a pandas.DataFrame 15 | # Param: a pandas.DataFrame 16 | def azureml_main(dataframe1 = None, dataframe2 = None): 17 | # Execution logic goes here 18 | # print('Input pandas.DataFrame #1:\r\n\r\n{0}'.format(dataframe1)) 19 | 20 | # If a zip file is connected to the third input port is connected, 21 | # it is unzipped under ".\Script Bundle". This directory is added 22 | # to sys.path. Therefore, if your zip file contains a Python file 23 | # mymodule.py you can import it using: 24 | # import mymodule 25 | model = Sequential() 26 | model.add(Dense(1, input_dim=784, activation="relu")) 27 | model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy']) 28 | data = np.random.random((1000,784)) 29 | labels = np.random.randint(2, size=(1000,1)) 30 | model.fit(data, labels, nb_epoch=10, batch_size=32) 31 | model.evaluate(data, labels) 32 | 33 | return dataframe1, 34 | -------------------------------------------------------------------------------- /Chapter02/keras_VGG16_prebuilt.py: -------------------------------------------------------------------------------- 1 | from keras.applications.vgg16 import VGG16 2 | from keras.models import Model 3 | from keras.preprocessing import image 4 | from keras.applications.vgg16 import preprocess_input 5 | import numpy as np 6 | 7 | 8 | # pre-built and pre-trained deep learning VGG16 model 9 | base_model = VGG16(weights='imagenet', include_top=True) 10 | for i, layer in enumerate(base_model.layers): 11 | print (i, layer.name, layer.output_shape) 12 | 13 | # extract features from block4_pool block 14 | model = Model(input=base_model.input, output=base_model.get_layer('block4_pool').output) 15 | 16 | img_path = 'cat.jpg' 17 | img = image.load_img(img_path, target_size=(224, 224)) 18 | x = image.img_to_array(img) 19 | x = np.expand_dims(x, axis=0) 20 | x = preprocess_input(x) 21 | 22 | # get the features from this block 23 | features = model.predict(x) 24 | 25 | print features 26 | -------------------------------------------------------------------------------- /Chapter03/keras_CIFAR10_V1.py: -------------------------------------------------------------------------------- 1 | from keras.datasets import cifar10 2 | from keras.utils import np_utils 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Dropout, Activation, Flatten 5 | from keras.layers.convolutional import Conv2D, MaxPooling2D 6 | from keras.optimizers import SGD, Adam, RMSprop 7 | 8 | from keras.preprocessing.image import ImageDataGenerator 9 | 10 | #from quiver_engine import server 11 | 12 | import matplotlib.pyplot as plt 13 | 14 | # CIFAR_10 is a set of 60K images 32x32 pixels on 3 channels 15 | IMG_CHANNELS = 3 16 | IMG_ROWS = 32 17 | IMG_COLS = 32 18 | 19 | #constant 20 | BATCH_SIZE = 128 21 | NB_EPOCH = 40 22 | NB_CLASSES = 10 23 | VERBOSE = 1 24 | VALIDATION_SPLIT = 0.2 25 | OPTIM = RMSprop() 26 | 27 | 28 | #load dataset 29 | (X_train, y_train), (X_test, y_test) = cifar10.load_data() 30 | print('X_train shape:', X_train.shape) 31 | print(X_train.shape[0], 'train samples') 32 | print(X_test.shape[0], 'test samples') 33 | 34 | # convert to categorical 35 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 36 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 37 | 38 | # float and normalization 39 | X_train = X_train.astype('float32') 40 | X_test = X_test.astype('float32') 41 | X_train /= 255 42 | X_test /= 255 43 | 44 | # network 45 | 46 | model = Sequential() 47 | 48 | model.add(Conv2D(32, kernel_size=3, padding='same', 49 | input_shape=(IMG_ROWS, IMG_COLS, IMG_CHANNELS))) 50 | model.add(Activation('relu')) 51 | model.add(Conv2D(32, kernel_size=3, padding='same')) 52 | model.add(Activation('relu')) 53 | model.add(MaxPooling2D(pool_size=(2, 2))) 54 | model.add(Dropout(0.25)) 55 | 56 | model.add(Conv2D(64, kernel_size=3, padding='same')) 57 | model.add(Activation('relu')) 58 | model.add(Conv2D(64, 3, 3)) 59 | model.add(Activation('relu')) 60 | model.add(MaxPooling2D(pool_size=(2, 2))) 61 | model.add(Dropout(0.25)) 62 | 63 | model.add(Flatten()) 64 | model.add(Dense(512)) 65 | model.add(Activation('relu')) 66 | model.add(Dropout(0.5)) 67 | model.add(Dense(NB_CLASSES)) 68 | model.add(Activation('softmax')) 69 | 70 | model.summary() 71 | 72 | model.compile(loss='categorical_crossentropy', optimizer=OPTIM, 73 | metrics=['accuracy']) 74 | 75 | 76 | datagen = ImageDataGenerator( 77 | featurewise_center=False, # set input mean to 0 over the dataset 78 | samplewise_center=False, # set each sample mean to 0 79 | featurewise_std_normalization=False, # divide inputs by std of the dataset 80 | samplewise_std_normalization=False, # divide each input by its std 81 | zca_whitening=False, # apply ZCA whitening 82 | rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) 83 | width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) 84 | height_shift_range=0.1, # randomly shift images vertically (fraction of total height) 85 | horizontal_flip=True, # randomly flip images 86 | vertical_flip=False) # randomly flip images 87 | 88 | datagen.fit(X_train) 89 | 90 | # train 91 | 92 | history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, 93 | epochs=NB_EPOCH, validation_split=VALIDATION_SPLIT, 94 | verbose=VERBOSE) 95 | 96 | #model.fit_generator(datagen.flow(X_train, Y_train, 97 | # batch_size=BATCH_SIZE), 98 | # samples_per_epoch=X_train.shape[0], 99 | # nb_epoch=NB_EPOCH, 100 | # verbose=VERBOSE) 101 | 102 | #server.launch(model) 103 | 104 | 105 | print('Testing...') 106 | score = model.evaluate(X_test, Y_test, 107 | batch_size=BATCH_SIZE, verbose=VERBOSE) 108 | print("\nTest score:", score[0]) 109 | print('Test accuracy:', score[1]) 110 | 111 | #save model 112 | model_json = model.to_json() 113 | open('cifar10_architecture.json', 'w').write(model_json) 114 | model.save_weights('cifar10_weights.h5', overwrite=True) 115 | 116 | # list all data in history 117 | print(history.history.keys()) 118 | # summarize history for accuracy 119 | plt.plot(history.history['acc']) 120 | plt.plot(history.history['val_acc']) 121 | plt.title('model accuracy') 122 | plt.ylabel('accuracy') 123 | plt.xlabel('epoch') 124 | plt.legend(['train', 'test'], loc='upper left') 125 | plt.show() 126 | # summarize history for loss 127 | plt.plot(history.history['loss']) 128 | plt.plot(history.history['val_loss']) 129 | plt.title('model loss') 130 | plt.ylabel('loss') 131 | plt.xlabel('epoch') 132 | plt.legend(['train', 'test'], loc='upper left') 133 | plt.show() 134 | -------------------------------------------------------------------------------- /Chapter03/keras_CIFAR10_simple.py: -------------------------------------------------------------------------------- 1 | from keras.datasets import cifar10 2 | from keras.utils import np_utils 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Dropout, Activation, Flatten 5 | from keras.layers.convolutional import Conv2D, MaxPooling2D 6 | from keras.optimizers import SGD, Adam, RMSprop 7 | 8 | import matplotlib.pyplot as plt 9 | 10 | #from quiver_engine import server 11 | # CIFAR_10 is a set of 60K images 32x32 pixels on 3 channels 12 | IMG_CHANNELS = 3 13 | IMG_ROWS = 32 14 | IMG_COLS = 32 15 | 16 | #constant 17 | BATCH_SIZE = 128 18 | NB_EPOCH = 20 19 | NB_CLASSES = 10 20 | VERBOSE = 1 21 | VALIDATION_SPLIT = 0.2 22 | OPTIM = RMSprop() 23 | 24 | 25 | #load dataset 26 | (X_train, y_train), (X_test, y_test) = cifar10.load_data() 27 | print('X_train shape:', X_train.shape) 28 | print(X_train.shape[0], 'train samples') 29 | print(X_test.shape[0], 'test samples') 30 | 31 | # convert to categorical 32 | Y_train = np_utils.to_categorical(y_train, NB_CLASSES) 33 | Y_test = np_utils.to_categorical(y_test, NB_CLASSES) 34 | 35 | # float and normalization 36 | X_train = X_train.astype('float32') 37 | X_test = X_test.astype('float32') 38 | X_train /= 255 39 | X_test /= 255 40 | 41 | # network 42 | 43 | model = Sequential() 44 | model.add(Conv2D(32, (3, 3), padding='same', 45 | input_shape=(IMG_ROWS, IMG_COLS, IMG_CHANNELS))) 46 | model.add(Activation('relu')) 47 | model.add(MaxPooling2D(pool_size=(2, 2))) 48 | model.add(Dropout(0.25)) 49 | 50 | model.add(Flatten()) 51 | model.add(Dense(512)) 52 | model.add(Activation('relu')) 53 | model.add(Dropout(0.5)) 54 | model.add(Dense(NB_CLASSES)) 55 | model.add(Activation('softmax')) 56 | 57 | model.summary() 58 | 59 | # train 60 | #optim = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) 61 | model.compile(loss='categorical_crossentropy', optimizer=OPTIM, 62 | metrics=['accuracy']) 63 | 64 | history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, 65 | epochs=NB_EPOCH, validation_split=VALIDATION_SPLIT, 66 | verbose=VERBOSE) 67 | 68 | print('Testing...') 69 | score = model.evaluate(X_test, Y_test, 70 | batch_size=BATCH_SIZE, verbose=VERBOSE) 71 | print("\nTest score:", score[0]) 72 | print('Test accuracy:', score[1]) 73 | 74 | #server.launch(model) 75 | 76 | 77 | #save model 78 | model_json = model.to_json() 79 | open('cifar10_architecture.json', 'w').write(model_json) 80 | model.save_weights('cifar10_weights.h5', overwrite=True) 81 | 82 | 83 | # list all data in history 84 | print(history.history.keys()) 85 | # summarize history for accuracy 86 | #plt.plot(mo) 87 | plt.plot(history.history['val_acc']) 88 | plt.title('model accuracy') 89 | plt.ylabel('accuracy') 90 | plt.xlabel('epoch') 91 | plt.legend(['train', 'test'], loc='upper left') 92 | plt.show() 93 | # summarize history for loss 94 | plt.plot(history.history['loss']) 95 | plt.plot(history.history['val_loss']) 96 | plt.title('model loss') 97 | plt.ylabel('loss') 98 | plt.xlabel('epoch') 99 | plt.legend(['train', 'test'], loc='upper left') 100 | plt.show() -------------------------------------------------------------------------------- /Chapter03/keras_EvaluateCIFAR10.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.misc 3 | from keras.models import model_from_json 4 | from keras.optimizers import SGD 5 | 6 | #load model 7 | model_architecture = 'cifar10_architecture.json' 8 | model_weights = 'cifar10_weights.h5' 9 | model = model_from_json(open(model_architecture).read()) 10 | model.load_weights(model_weights) 11 | 12 | #load images 13 | img_names = ['cat2.jpg', 'dog.jpg'] 14 | imgs = [np.transpose(scipy.misc.imresize(scipy.misc.imread(img_name), (32, 32)), 15 | (1, 0, 2)).astype('float32') 16 | for img_name in img_names] 17 | imgs = np.array(imgs) / 255 18 | 19 | # train 20 | optim = SGD() 21 | model.compile(loss='categorical_crossentropy', optimizer=optim, 22 | metrics=['accuracy']) 23 | 24 | predictions = model.predict_classes(imgs) 25 | print(predictions) 26 | 27 | -------------------------------------------------------------------------------- /Chapter03/keras_LeNet.py: -------------------------------------------------------------------------------- 1 | # import the necessary packages 2 | from keras import backend as K 3 | from keras.models import Sequential 4 | from keras.layers.convolutional import Conv2D 5 | from keras.layers.convolutional import MaxPooling2D 6 | from keras.layers.core import Activation 7 | from keras.layers.core import Flatten 8 | from keras.layers.core import Dense 9 | from keras.datasets import mnist 10 | from keras.utils import np_utils 11 | from keras.optimizers import SGD, RMSprop, Adam 12 | import numpy as np 13 | 14 | import matplotlib.pyplot as plt 15 | 16 | np.random.seed(1671) # for reproducibility 17 | 18 | #define the convnet 19 | class LeNet: 20 | @staticmethod 21 | def build(input_shape, classes): 22 | model = Sequential() 23 | # CONV => RELU => POOL 24 | model.add(Conv2D(20, kernel_size=5, padding="same", 25 | input_shape=input_shape)) 26 | model.add(Activation("relu")) 27 | model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) 28 | # CONV => RELU => POOL 29 | model.add(Conv2D(50, kernel_size=5, padding="same")) 30 | model.add(Activation("relu")) 31 | model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) 32 | # Flatten => RELU layers 33 | model.add(Flatten()) 34 | model.add(Dense(500)) 35 | model.add(Activation("relu")) 36 | 37 | # a softmax classifier 38 | model.add(Dense(classes)) 39 | model.add(Activation("softmax")) 40 | 41 | return model 42 | 43 | # network and training 44 | NB_EPOCH = 20 45 | BATCH_SIZE = 128 46 | VERBOSE = 1 47 | OPTIMIZER = Adam() 48 | VALIDATION_SPLIT=0.2 49 | 50 | IMG_ROWS, IMG_COLS = 28, 28 # input image dimensions 51 | NB_CLASSES = 10 # number of outputs = number of digits 52 | INPUT_SHAPE = (1, IMG_ROWS, IMG_COLS) 53 | 54 | # data: shuffled and split between train and test sets 55 | (X_train, y_train), (X_test, y_test) = mnist.load_data() 56 | K.set_image_dim_ordering("th") 57 | 58 | # consider them as float and normalize 59 | X_train = X_train.astype('float32') 60 | X_test = X_test.astype('float32') 61 | X_train /= 255 62 | X_test /= 255 63 | 64 | # we need a 60K x [1 x 28 x 28] shape as input to the CONVNET 65 | X_train = X_train[:, np.newaxis, :, :] 66 | X_test = X_test[:, np.newaxis, :, :] 67 | 68 | print(X_train.shape[0], 'train samples') 69 | print(X_test.shape[0], 'test samples') 70 | 71 | # convert class vectors to binary class matrices 72 | y_train = np_utils.to_categorical(y_train, NB_CLASSES) 73 | y_test = np_utils.to_categorical(y_test, NB_CLASSES) 74 | 75 | # initialize the optimizer and model 76 | model = LeNet.build(input_shape=INPUT_SHAPE, classes=NB_CLASSES) 77 | model.compile(loss="categorical_crossentropy", optimizer=OPTIMIZER, 78 | metrics=["accuracy"]) 79 | 80 | history = model.fit(X_train, y_train, 81 | batch_size=BATCH_SIZE, epochs=NB_EPOCH, 82 | verbose=VERBOSE, validation_split=VALIDATION_SPLIT) 83 | 84 | score = model.evaluate(X_test, y_test, verbose=VERBOSE) 85 | print("\nTest score:", score[0]) 86 | print('Test accuracy:', score[1]) 87 | 88 | # list all data in history 89 | print(history.history.keys()) 90 | # summarize history for accuracy 91 | plt.plot(history.history['acc']) 92 | plt.plot(history.history['val_acc']) 93 | plt.title('model accuracy') 94 | plt.ylabel('accuracy') 95 | plt.xlabel('epoch') 96 | plt.legend(['train', 'test'], loc='upper left') 97 | plt.show() 98 | # summarize history for loss 99 | plt.plot(history.history['loss']) 100 | plt.plot(history.history['val_loss']) 101 | plt.title('model loss') 102 | plt.ylabel('loss') 103 | plt.xlabel('epoch') 104 | plt.legend(['train', 'test'], loc='upper left') 105 | plt.show() 106 | -------------------------------------------------------------------------------- /Chapter03/keras_VGG16.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.models import Sequential 3 | from keras.layers.core import Flatten, Dense, Dropout 4 | from keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D 5 | from keras.optimizers import SGD 6 | import cv2, numpy as np 7 | 8 | # define a VGG16 network 9 | 10 | def VGG_16(weights_path=None): 11 | model = Sequential() 12 | model.add(ZeroPadding2D((1,1),input_shape=(3,224,224))) 13 | model.add(Conv2D(64, (3, 3), activation='relu')) 14 | model.add(ZeroPadding2D((1,1))) 15 | model.add(Conv2D(64, (3, 3), activation='relu')) 16 | model.add(MaxPooling2D((2,2), strides=(2,2))) 17 | 18 | model.add(ZeroPadding2D((1,1))) 19 | model.add(Conv2D(128, (3, 3), activation='relu')) 20 | model.add(ZeroPadding2D((1,1))) 21 | model.add(Conv2D(128, (3, 3), activation='relu')) 22 | model.add(MaxPooling2D((2,2), strides=(2,2))) 23 | 24 | model.add(ZeroPadding2D((1,1))) 25 | model.add(Conv2D(256, (3, 3), activation='relu')) 26 | model.add(ZeroPadding2D((1,1))) 27 | model.add(Conv2D(256, (3, 3), activation='relu')) 28 | model.add(ZeroPadding2D((1,1))) 29 | model.add(Conv2D(256, (3, 3), activation='relu')) 30 | model.add(MaxPooling2D((2,2), strides=(2,2))) 31 | 32 | model.add(ZeroPadding2D((1,1))) 33 | model.add(Conv2D(512, (3, 3), activation='relu')) 34 | model.add(ZeroPadding2D((1,1))) 35 | model.add(Conv2D(512, (3, 3), activation='relu')) 36 | model.add(ZeroPadding2D((1,1))) 37 | model.add(Conv2D(512, (3, 3), activation='relu')) 38 | model.add(MaxPooling2D((2,2), strides=(2,2))) 39 | 40 | model.add(ZeroPadding2D((1,1))) 41 | model.add(Conv2D(512, (3, 3), activation='relu')) 42 | model.add(ZeroPadding2D((1,1))) 43 | model.add(Conv2D(512, (3, 3), activation='relu')) 44 | model.add(ZeroPadding2D((1,1))) 45 | model.add(Conv2D(512, (3, 3), activation='relu')) 46 | model.add(MaxPooling2D((2,2), strides=(2,2))) 47 | 48 | model.add(Flatten()) 49 | 50 | #top layer of the VGG net 51 | model.add(Dense(4096, activation='relu')) 52 | model.add(Dropout(0.5)) 53 | model.add(Dense(4096, activation='relu')) 54 | model.add(Dropout(0.5)) 55 | model.add(Dense(1000, activation='softmax')) 56 | 57 | if weights_path: 58 | model.load_weights(weights_path) 59 | 60 | return model 61 | 62 | if __name__ == "__main__": 63 | im = cv2.resize(cv2.imread('cat.jpg'), (224, 224)).astype(np.float32) 64 | im = im.transpose((2,0,1)) 65 | im = np.expand_dims(im, axis=0) 66 | K.set_image_dim_ordering("th") 67 | 68 | # Test pretrained model 69 | model = VGG_16('/Users/gulli/Keras/codeBook/code/data/vgg16_weights.h5') 70 | optimizer = SGD() 71 | model.compile(optimizer=optimizer, loss='categorical_crossentropy') 72 | out = model.predict(im) 73 | print np.argmax(out) -------------------------------------------------------------------------------- /Chapter04/cifar10/cifar10_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.datasets import cifar10 3 | 4 | 5 | def cifar10_process(x): 6 | x = x.astype(np.float32) / 255.0 7 | return x 8 | 9 | 10 | def cifar10_data(): 11 | (xtrain, ytrain), (xtest, ytest) = cifar10.load_data() 12 | return cifar10_process(xtrain), cifar10_process(xtest) -------------------------------------------------------------------------------- /Chapter04/cifar10/example_gan_cifar10.py: -------------------------------------------------------------------------------- 1 | import matplotlib as mpl 2 | 3 | # This line allows mpl to run with no DISPLAY defined 4 | mpl.use('Agg') 5 | 6 | import pandas as pd 7 | import numpy as np 8 | import os 9 | from keras.layers import Reshape, Flatten, LeakyReLU, Activation, Dense, BatchNormalization 10 | from keras.layers.convolutional import Conv2D, UpSampling2D, MaxPooling2D, AveragePooling2D 11 | from keras.regularizers import L1L2 12 | from keras.models import Sequential 13 | from keras.optimizers import Adam 14 | from keras.callbacks import TensorBoard 15 | from keras_adversarial.image_grid_callback import ImageGridCallback 16 | 17 | from keras_adversarial import AdversarialModel, simple_gan, gan_targets 18 | from keras_adversarial import AdversarialOptimizerSimultaneous, normal_latent_sampling 19 | import keras.backend as K 20 | from cifar10_utils import cifar10_data 21 | from image_utils import dim_ordering_fix, dim_ordering_unfix, dim_ordering_shape 22 | 23 | 24 | def model_generator(): 25 | model = Sequential() 26 | nch = 256 27 | reg = lambda: L1L2(l1=1e-7, l2=1e-7) 28 | h = 5 29 | model.add(Dense(nch * 4 * 4, input_dim=100, kernel_regularizer=reg())) 30 | model.add(BatchNormalization()) 31 | model.add(Reshape(dim_ordering_shape((nch, 4, 4)))) 32 | model.add(Conv2D(int(nch / 2), (h, h), padding='same', kernel_regularizer=reg())) 33 | model.add(BatchNormalization(axis=1)) 34 | model.add(LeakyReLU(0.2)) 35 | model.add(UpSampling2D(size=(2, 2))) 36 | model.add(Conv2D(int(nch / 2), (h, h), padding='same', kernel_regularizer=reg())) 37 | model.add(BatchNormalization(axis=1)) 38 | model.add(LeakyReLU(0.2)) 39 | model.add(UpSampling2D(size=(2, 2))) 40 | model.add(Conv2D(int(nch / 4), (h, h), padding='same', kernel_regularizer=reg())) 41 | model.add(BatchNormalization(axis=1)) 42 | model.add(LeakyReLU(0.2)) 43 | model.add(UpSampling2D(size=(2, 2))) 44 | model.add(Conv2D(3, (h, h), padding='same', kernel_regularizer=reg())) 45 | model.add(Activation('sigmoid')) 46 | return model 47 | 48 | 49 | def model_discriminator(): 50 | nch = 256 51 | h = 5 52 | reg = lambda: L1L2(l1=1e-7, l2=1e-7) 53 | 54 | c1 = Conv2D(int(nch / 4), (h, h), padding='same', kernel_regularizer=reg(), 55 | input_shape=(32, 32, 3)) 56 | c2 = Conv2D(int(nch / 2), (h, h), padding='same', kernel_regularizer=reg()) 57 | c3 = Conv2D(nch, (h, h), padding='same', kernel_regularizer=reg()) 58 | c4 = Conv2D(1, (h, h), padding='same', kernel_regularizer=reg()) 59 | 60 | model = Sequential() 61 | model.add(c1) 62 | model.add(MaxPooling2D(pool_size=(2, 2))) 63 | model.add(LeakyReLU(0.2)) 64 | model.add(c2) 65 | model.add(MaxPooling2D(pool_size=(2, 2))) 66 | model.add(LeakyReLU(0.2)) 67 | model.add(c3) 68 | model.add(MaxPooling2D(pool_size=(2, 2))) 69 | model.add(LeakyReLU(0.2)) 70 | model.add(c4) 71 | model.add(AveragePooling2D(pool_size=(4, 4), padding='valid')) 72 | model.add(Flatten()) 73 | model.add(Activation('sigmoid')) 74 | return model 75 | 76 | 77 | def example_gan(adversarial_optimizer, path, opt_g, opt_d, nb_epoch, generator, discriminator, latent_dim, 78 | targets=gan_targets, loss='binary_crossentropy'): 79 | csvpath = os.path.join(path, "history.csv") 80 | if os.path.exists(csvpath): 81 | print("Already exists: {}".format(csvpath)) 82 | return 83 | 84 | print("Training: {}".format(csvpath)) 85 | # gan (x - > yfake, yreal), z is gaussian generated on GPU 86 | # can also experiment with uniform_latent_sampling 87 | generator.summary() 88 | discriminator.summary() 89 | gan = simple_gan(generator=generator, 90 | discriminator=discriminator, 91 | latent_sampling=normal_latent_sampling((latent_dim,))) 92 | 93 | # build adversarial model 94 | model = AdversarialModel(base_model=gan, 95 | player_params=[generator.trainable_weights, discriminator.trainable_weights], 96 | player_names=["generator", "discriminator"]) 97 | model.adversarial_compile(adversarial_optimizer=adversarial_optimizer, 98 | player_optimizers=[opt_g, opt_d], 99 | loss=loss) 100 | 101 | # create callback to generate images 102 | zsamples = np.random.normal(size=(10 * 10, latent_dim)) 103 | 104 | def generator_sampler(): 105 | xpred = dim_ordering_unfix(generator.predict(zsamples)).transpose((0, 2, 3, 1)) 106 | return xpred.reshape((10, 10) + xpred.shape[1:]) 107 | 108 | generator_cb = ImageGridCallback(os.path.join(path, "epoch-{:03d}.png"), generator_sampler, cmap=None) 109 | 110 | # train model 111 | xtrain, xtest = cifar10_data() 112 | y = targets(xtrain.shape[0]) 113 | ytest = targets(xtest.shape[0]) 114 | callbacks = [generator_cb] 115 | if K.backend() == "tensorflow": 116 | callbacks.append( 117 | TensorBoard(log_dir=os.path.join(path, 'logs'), histogram_freq=0, write_graph=True, write_images=True)) 118 | history = model.fit(x=xtrain, y=y, validation_data=(xtest, ytest), 119 | callbacks=callbacks, nb_epoch=nb_epoch, 120 | batch_size=32) 121 | 122 | # save history to CSV 123 | df = pd.DataFrame(history.history) 124 | df.to_csv(csvpath) 125 | 126 | # save models 127 | generator.save(os.path.join(path, "generator.h5")) 128 | discriminator.save(os.path.join(path, "discriminator.h5")) 129 | 130 | 131 | def main(): 132 | # z \in R^100 133 | latent_dim = 100 134 | # x \in R^{28x28} 135 | # generator (z -> x) 136 | generator = model_generator() 137 | # discriminator (x -> y) 138 | discriminator = model_discriminator() 139 | example_gan(AdversarialOptimizerSimultaneous(), "output/gan-cifar10", 140 | opt_g=Adam(1e-4, decay=1e-5), 141 | opt_d=Adam(1e-3, decay=1e-5), 142 | nb_epoch=100, generator=generator, discriminator=discriminator, 143 | latent_dim=latent_dim) 144 | 145 | 146 | if __name__ == "__main__": 147 | main() 148 | -------------------------------------------------------------------------------- /Chapter04/cifar10/example_gan_convolutional.py: -------------------------------------------------------------------------------- 1 | import matplotlib as mpl 2 | 3 | # This line allows mpl to run with no DISPLAY defined 4 | mpl.use('Agg') 5 | 6 | from keras.layers import Flatten, Dropout, LeakyReLU, Input, Activation 7 | from keras.models import Model 8 | from keras.layers.convolutional import UpSampling2D 9 | from keras.optimizers import Adam 10 | from keras.datasets import mnist 11 | import pandas as pd 12 | import numpy as np 13 | import keras.backend as K 14 | from keras_adversarial.legacy import Dense, BatchNormalization, Convolution2D 15 | from keras_adversarial.image_grid_callback import ImageGridCallback 16 | from keras_adversarial import AdversarialModel, simple_gan, gan_targets 17 | from keras_adversarial import AdversarialOptimizerSimultaneous, normal_latent_sampling 18 | from image_utils import dim_ordering_fix, dim_ordering_input, dim_ordering_reshape, dim_ordering_unfix 19 | 20 | 21 | def leaky_relu(x): 22 | return K.relu(x, 0.2) 23 | 24 | 25 | def model_generator(): 26 | nch = 256 27 | g_input = Input(shape=[100]) 28 | H = Dense(nch * 14 * 14)(g_input) 29 | H = BatchNormalization(mode=2)(H) 30 | H = Activation('relu')(H) 31 | H = dim_ordering_reshape(nch, 14)(H) 32 | H = UpSampling2D(size=(2, 2))(H) 33 | H = Convolution2D(int(nch / 2), 3, 3, border_mode='same')(H) 34 | H = BatchNormalization(mode=2, axis=1)(H) 35 | H = Activation('relu')(H) 36 | H = Convolution2D(int(nch / 4), 3, 3, border_mode='same')(H) 37 | H = BatchNormalization(mode=2, axis=1)(H) 38 | H = Activation('relu')(H) 39 | H = Convolution2D(1, 1, 1, border_mode='same')(H) 40 | g_V = Activation('sigmoid')(H) 41 | return Model(g_input, g_V) 42 | 43 | 44 | def model_discriminator(input_shape=(1, 28, 28), dropout_rate=0.5): 45 | d_input = dim_ordering_input(input_shape, name="input_x") 46 | nch = 512 47 | # nch = 128 48 | H = Convolution2D(int(nch / 2), 5, 5, subsample=(2, 2), border_mode='same', activation='relu')(d_input) 49 | H = LeakyReLU(0.2)(H) 50 | H = Dropout(dropout_rate)(H) 51 | H = Convolution2D(nch, 5, 5, subsample=(2, 2), border_mode='same', activation='relu')(H) 52 | H = LeakyReLU(0.2)(H) 53 | H = Dropout(dropout_rate)(H) 54 | H = Flatten()(H) 55 | H = Dense(int(nch / 2))(H) 56 | H = LeakyReLU(0.2)(H) 57 | H = Dropout(dropout_rate)(H) 58 | d_V = Dense(1, activation='sigmoid')(H) 59 | return Model(d_input, d_V) 60 | 61 | 62 | def mnist_process(x): 63 | x = x.astype(np.float32) / 255.0 64 | return x 65 | 66 | 67 | def mnist_data(): 68 | (xtrain, ytrain), (xtest, ytest) = mnist.load_data() 69 | return mnist_process(xtrain), mnist_process(xtest) 70 | 71 | 72 | def generator_sampler(latent_dim, generator): 73 | def fun(): 74 | zsamples = np.random.normal(size=(10 * 10, latent_dim)) 75 | gen = dim_ordering_unfix(generator.predict(zsamples)) 76 | return gen.reshape((10, 10, 28, 28)) 77 | 78 | return fun 79 | 80 | 81 | if __name__ == "__main__": 82 | # z \in R^100 83 | latent_dim = 100 84 | # x \in R^{28x28} 85 | input_shape = (1, 28, 28) 86 | 87 | # generator (z -> x) 88 | generator = model_generator() 89 | # discriminator (x -> y) 90 | discriminator = model_discriminator(input_shape=input_shape) 91 | # gan (x - > yfake, yreal), z generated on GPU 92 | gan = simple_gan(generator, discriminator, normal_latent_sampling((latent_dim,))) 93 | 94 | # print summary of models 95 | generator.summary() 96 | discriminator.summary() 97 | gan.summary() 98 | 99 | # build adversarial model 100 | model = AdversarialModel(base_model=gan, 101 | player_params=[generator.trainable_weights, discriminator.trainable_weights], 102 | player_names=["generator", "discriminator"]) 103 | model.adversarial_compile(adversarial_optimizer=AdversarialOptimizerSimultaneous(), 104 | player_optimizers=[Adam(1e-4, decay=1e-4), Adam(1e-3, decay=1e-4)], 105 | loss='binary_crossentropy') 106 | 107 | # train model 108 | generator_cb = ImageGridCallback("output/gan_convolutional/epoch-{:03d}.png", 109 | generator_sampler(latent_dim, generator)) 110 | 111 | xtrain, xtest = mnist_data() 112 | xtrain = dim_ordering_fix(xtrain.reshape((-1, 1, 28, 28))) 113 | xtest = dim_ordering_fix(xtest.reshape((-1, 1, 28, 28))) 114 | y = gan_targets(xtrain.shape[0]) 115 | ytest = gan_targets(xtest.shape[0]) 116 | history = model.fit(x=xtrain, y=y, validation_data=(xtest, ytest), callbacks=[generator_cb], nb_epoch=100, 117 | batch_size=32) 118 | df = pd.DataFrame(history.history) 119 | df.to_csv("output/gan_convolutional/history.csv") 120 | 121 | generator.save("output/gan_convolutional/generator.h5") 122 | discriminator.save("output/gan_convolutional/discriminator.h5") 123 | -------------------------------------------------------------------------------- /Chapter04/cifar10/image_utils.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | import numpy as np 3 | from keras.layers import Input, Reshape 4 | 5 | 6 | def dim_ordering_fix(x): 7 | if K.image_dim_ordering() == 'th': 8 | return x 9 | else: 10 | return np.transpose(x, (0, 2, 3, 1)) 11 | 12 | 13 | def dim_ordering_unfix(x): 14 | if K.image_dim_ordering() == 'th': 15 | return x 16 | else: 17 | return np.transpose(x, (0, 3, 1, 2)) 18 | 19 | 20 | def dim_ordering_shape(input_shape): 21 | if K.image_dim_ordering() == 'th': 22 | return input_shape 23 | else: 24 | return (input_shape[1], input_shape[2], input_shape[0]) 25 | 26 | 27 | def dim_ordering_input(input_shape, name): 28 | if K.image_dim_ordering() == 'th': 29 | return Input(input_shape, name=name) 30 | else: 31 | return Input((input_shape[1], input_shape[2], input_shape[0]), name=name) 32 | 33 | 34 | def dim_ordering_reshape(k, w, **kwargs): 35 | if K.image_dim_ordering() == 'th': 36 | return Reshape((k, w, w), **kwargs) 37 | else: 38 | return Reshape((w, w, k), **kwargs) 39 | 40 | 41 | def channel_axis(): 42 | if K.image_dim_ordering() == 'th': 43 | return 1 44 | else: 45 | return 3 -------------------------------------------------------------------------------- /Chapter04/legacy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions to avoid warnings while testing both Keras 1 and 2. 3 | """ 4 | import keras 5 | 6 | keras_2 = int(keras.__version__.split(".")[0]) > 1 # Keras > 1 7 | 8 | 9 | def fit_generator(model, generator, epochs, steps_per_epoch): 10 | if keras_2: 11 | model.fit_generator(generator, epochs=epochs, steps_per_epoch=steps_per_epoch) 12 | else: 13 | model.fit_generator(generator, nb_epoch=epochs, samples_per_epoch=steps_per_epoch) 14 | 15 | 16 | def fit(model, x, y, nb_epoch=10, *args, **kwargs): 17 | if keras_2: 18 | return model.fit(x, y, *args, epochs=nb_epoch, **kwargs) 19 | else: 20 | return model.fit(x, y, *args, nb_epoch=nb_epoch, **kwargs) 21 | 22 | 23 | def l1l2(l1=0, l2=0): 24 | if keras_2: 25 | return keras.regularizers.L1L2(l1, l2) 26 | else: 27 | return keras.regularizers.l1l2(l1, l2) 28 | 29 | 30 | def Dense(units, W_regularizer=None, W_initializer='glorot_uniform', **kwargs): 31 | if keras_2: 32 | return keras.layers.Dense(units, kernel_regularizer=W_regularizer, kernel_initializer=W_initializer, **kwargs) 33 | else: 34 | return keras.layers.Dense(units, W_regularizer=W_regularizer, init=W_initializer, **kwargs) 35 | 36 | 37 | def BatchNormalization(mode=0, **kwargs): 38 | if keras_2: 39 | return keras.layers.BatchNormalization(**kwargs) 40 | else: 41 | return keras.layers.BatchNormalization(mode=mode, **kwargs) 42 | 43 | 44 | def Convolution2D(units, w, h, W_regularizer=None, W_initializer='glorot_uniform', border_mode='same', **kwargs): 45 | if keras_2: 46 | return keras.layers.Convolution2D(units, (w, h), padding=border_mode, kernel_regularizer=W_regularizer, 47 | kernel_initializer=W_initializer, 48 | **kwargs) 49 | else: 50 | return keras.layers.Convolution2D(units, w, h, border_mode=border_mode, W_regularizer=W_regularizer, 51 | init=W_initializer, 52 | **kwargs) 53 | 54 | 55 | def AveragePooling2D(pool_size, border_mode='valid', **kwargs): 56 | if keras_2: 57 | return keras.layers.AveragePooling2D(pool_size=pool_size, padding=border_mode, **kwargs) 58 | else: 59 | return keras.layers.AveragePooling2D(pool_size=pool_size, border_mode=border_mode, **kwargs) 60 | -------------------------------------------------------------------------------- /Chapter04/wavnet/vctk/download_vctk.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Available diskspace in $(pwd):" 3 | df -h . 4 | read -p "This will download the VCTK corputs 11GB and extract it 14.9GB are you sure (y/n): " -n 1 -r 5 | echo 6 | if [[ $REPLY =~ ^[Yy]$ ]] 7 | then 8 | wget http://homepages.inf.ed.ac.uk/jyamagis/release/VCTK-Corpus.tar.gz 9 | if [[ "$OSTYPE" == "darwin"* ]]; then 10 | open VCTK.Corpus.tar.gz 11 | else 12 | tar -xvfz VCTK.Corpus.tar.gz 13 | fi 14 | fi -------------------------------------------------------------------------------- /Chapter04/wavnet/wavenet_utils.py: -------------------------------------------------------------------------------- 1 | import keras.backend as K 2 | from keras.layers import AtrousConvolution1D 3 | from keras.utils.np_utils import conv_output_length 4 | 5 | 6 | def categorical_mean_squared_error(y_true, y_pred): 7 | """MSE for categorical variables.""" 8 | return K.mean(K.square(K.argmax(y_true, axis=-1) - 9 | K.argmax(y_pred, axis=-1))) 10 | 11 | class CausalAtrousConvolution1D(AtrousConvolution1D): 12 | def __init__(self, nb_filter, filter_length, init='glorot_uniform', activation=None, weights=None, 13 | border_mode='valid', subsample_length=1, atrous_rate=1, W_regularizer=None, b_regularizer=None, 14 | activity_regularizer=None, W_constraint=None, b_constraint=None, bias=True, causal=False, **kwargs): 15 | super(CausalAtrousConvolution1D, self).__init__(nb_filter, filter_length, init, activation, weights, 16 | border_mode, subsample_length, atrous_rate, W_regularizer, 17 | b_regularizer, activity_regularizer, W_constraint, b_constraint, 18 | bias, **kwargs) 19 | self.causal = causal 20 | if self.causal and border_mode != 'valid': 21 | raise ValueError("Causal mode dictates border_mode=valid.") 22 | 23 | def get_output_shape_for(self, input_shape): 24 | input_length = input_shape[1] 25 | 26 | if self.causal: 27 | input_length += self.atrous_rate * (self.filter_length - 1) 28 | 29 | length = conv_output_length(input_length, 30 | self.filter_length, 31 | self.border_mode, 32 | self.subsample[0], 33 | dilation=self.atrous_rate) 34 | 35 | return (input_shape[0], length, self.nb_filter) 36 | 37 | def call(self, x, mask=None): 38 | if self.causal: 39 | x = K.asymmetric_temporal_padding(x, self.atrous_rate * (self.filter_length - 1), 0) 40 | return super(CausalAtrousConvolution1D, self).call(x, mask) 41 | -------------------------------------------------------------------------------- /Chapter05/data_download_glove.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Available diskspace in $(pwd):" 3 | df -h . 4 | read -p "This will download the Glove corputs 822 MB and extract it MB are you sure (y/n): " -n 1 -r 5 | echo 6 | if [[ $REPLY =~ ^[Yy]$ ]] 7 | then 8 | wget wget http://nlp.stanford.edu/data/glove.6B.zip 9 | unzip glove.6B.zip 10 | fi 11 | -------------------------------------------------------------------------------- /Chapter05/data_download_text8.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Available diskspace in $(pwd):" 3 | df -h . 4 | read -p "This will download the Text8 corputs 31.3 MB and extract it 100 MB are you sure (y/n): " -n 1 -r 5 | echo 6 | if [[ $REPLY =~ ^[Yy]$ ]] 7 | then 8 | wget http://mattmahoney.net/dc/text8.zip 9 | unzip text8.zip 10 | fi 11 | -------------------------------------------------------------------------------- /Chapter05/finetune_glove_embeddings.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, SpatialDropout1D 2 | from keras.layers.convolutional import Conv1D 3 | from keras.layers.embeddings import Embedding 4 | from keras.layers.pooling import GlobalMaxPooling1D 5 | from keras.models import Sequential 6 | from keras.preprocessing.sequence import pad_sequences 7 | from keras.utils import np_utils 8 | from sklearn.model_selection import train_test_split 9 | import collections 10 | import nltk 11 | import numpy as np 12 | from make_tensorboard import make_tensorboard 13 | import codecs 14 | 15 | 16 | np.random.seed(42) 17 | 18 | INPUT_FILE = "data/umich-sentiment-train.txt" 19 | GLOVE_MODEL = "data/glove.6B.300d.txt" 20 | VOCAB_SIZE = 5000 21 | EMBED_SIZE = 300 22 | NUM_FILTERS = 256 23 | NUM_WORDS = 3 24 | BATCH_SIZE = 64 25 | NUM_EPOCHS = 10 26 | 27 | counter = collections.Counter() 28 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 29 | maxlen = 0 30 | for line in fin: 31 | _, sent = line.strip().split("\t") 32 | words = [x.lower() for x in nltk.word_tokenize(sent)] 33 | if len(words) > maxlen: 34 | maxlen = len(words) 35 | for word in words: 36 | counter[word] += 1 37 | fin.close() 38 | 39 | word2index = collections.defaultdict(int) 40 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 41 | word2index[word[0]] = wid + 1 42 | vocab_sz = len(word2index) + 1 43 | index2word = {v: k for k, v in word2index.items()} 44 | 45 | xs, ys = [], [] 46 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 47 | for line in fin: 48 | label, sent = line.strip().split("\t") 49 | ys.append(int(label)) 50 | words = [x.lower() for x in nltk.word_tokenize(sent)] 51 | wids = [word2index[word] for word in words] 52 | xs.append(wids) 53 | fin.close() 54 | X = pad_sequences(xs, maxlen=maxlen) 55 | Y = np_utils.to_categorical(ys) 56 | 57 | Xtrain, Xtest, Ytrain, Ytest = \ 58 | train_test_split(X, Y, test_size=0.3, random_state=42) 59 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 60 | 61 | # load GloVe vectors 62 | word2emb = {} 63 | fglove = open(GLOVE_MODEL, "rb") 64 | for line in fglove: 65 | cols = line.strip().split() 66 | word = cols[0].decode('utf-8') 67 | embedding = np.array(cols[1:], dtype="float32") 68 | word2emb[word] = embedding 69 | fglove.close() 70 | embedding_weights = np.zeros((vocab_sz, EMBED_SIZE)) 71 | for word, index in word2index.items(): 72 | try: 73 | embedding_weights[index, :] = word2emb[word] 74 | except KeyError: 75 | pass 76 | 77 | model = Sequential() 78 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen, 79 | weights=[embedding_weights], 80 | trainable=True)) 81 | model.add(SpatialDropout1D(0.2)) 82 | model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, 83 | activation="relu")) 84 | model.add(GlobalMaxPooling1D()) 85 | model.add(Dense(2, activation="softmax")) 86 | 87 | model.compile(optimizer="adam", loss="categorical_crossentropy", 88 | metrics=["accuracy"]) 89 | 90 | tensorboard, _ = make_tensorboard( 91 | set_dir_name='keras_finetune_glove_embeddings') 92 | 93 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 94 | epochs=NUM_EPOCHS, 95 | callbacks=[tensorboard], 96 | validation_data=(Xtest, Ytest)) 97 | 98 | 99 | # evaluate model 100 | score = model.evaluate(Xtest, Ytest, verbose=1) 101 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 102 | -------------------------------------------------------------------------------- /Chapter05/finetune_glove_embeddings_tensorboard_embedding.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, SpatialDropout1D 2 | from keras.layers.convolutional import Conv1D 3 | from keras.layers.embeddings import Embedding 4 | from keras.layers.pooling import GlobalMaxPooling1D 5 | from keras.models import Sequential 6 | from keras.preprocessing.sequence import pad_sequences 7 | from keras.utils import np_utils 8 | from sklearn.model_selection import train_test_split 9 | import collections 10 | import nltk 11 | import numpy as np 12 | from make_tensorboard import make_tensorboard 13 | import os 14 | import codecs 15 | 16 | 17 | np.random.seed(42) 18 | 19 | INPUT_FILE = "data/umich-sentiment-train.txt" 20 | GLOVE_MODEL = "data/glove.6B.300d.txt" 21 | VOCAB_SIZE = 5000 22 | EMBED_SIZE = 300 23 | NUM_FILTERS = 256 24 | NUM_WORDS = 3 25 | BATCH_SIZE = 64 26 | NUM_EPOCHS = 10 27 | 28 | counter = collections.Counter() 29 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 30 | maxlen = 0 31 | for line in fin: 32 | _, sent = line.strip().split("\t") 33 | words = [x.lower() for x in nltk.word_tokenize(sent)] 34 | if len(words) > maxlen: 35 | maxlen = len(words) 36 | for word in words: 37 | counter[word] += 1 38 | fin.close() 39 | 40 | word2index = collections.defaultdict(int) 41 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 42 | word2index[word[0]] = wid + 1 43 | vocab_sz = len(word2index) + 1 44 | index2word = {v: k for k, v in word2index.items()} 45 | 46 | xs, ys = [], [] 47 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 48 | for line in fin: 49 | label, sent = line.strip().split("\t") 50 | ys.append(int(label)) 51 | words = [x.lower() for x in nltk.word_tokenize(sent)] 52 | wids = [word2index[word] for word in words] 53 | xs.append(wids) 54 | fin.close() 55 | X = pad_sequences(xs, maxlen=maxlen) 56 | Y = np_utils.to_categorical(ys) 57 | 58 | Xtrain, Xtest, Ytrain, Ytest = \ 59 | train_test_split(X, Y, test_size=0.3, random_state=42) 60 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 61 | 62 | # load GloVe vectors 63 | word2emb = {} 64 | fglove = open(GLOVE_MODEL, "rb") 65 | for line in fglove: 66 | cols = line.strip().split() 67 | word = cols[0] 68 | embedding = np.array(cols[1:], dtype="float32") 69 | word2emb[word] = embedding 70 | fglove.close() 71 | embedding_weights = np.zeros((vocab_sz, EMBED_SIZE)) 72 | for word, index in word2index.items(): 73 | try: 74 | embedding_weights[index, :] = word2emb[word] 75 | except KeyError: 76 | pass 77 | 78 | model = Sequential() 79 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen, 80 | weights=[embedding_weights], 81 | trainable=True)) 82 | model.add(SpatialDropout1D(0.2)) 83 | model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, 84 | activation="relu")) 85 | model.add(GlobalMaxPooling1D()) 86 | model.add(Dense(2, activation="softmax")) 87 | 88 | model.compile(optimizer="adam", loss="categorical_crossentropy", 89 | metrics=["accuracy"]) 90 | 91 | tensorboard, log_dir = make_tensorboard( 92 | set_dir_name='keras_finetune_glove_embeddings', 93 | embeddings_freq=1, 94 | embeddings_metadata='embedding/metadata.tsv' 95 | ) 96 | 97 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 98 | epochs=NUM_EPOCHS, 99 | callbacks=[tensorboard], 100 | validation_data=(Xtest, Ytest)) 101 | 102 | 103 | # evaluate model 104 | score = model.evaluate(Xtest, Ytest, verbose=1) 105 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 106 | 107 | word_list = [] 108 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 109 | word_list.append(word[0]) 110 | log_dir = log_dir + "/embedding/" 111 | if os.path.exists(log_dir) is not True: 112 | os.mkdir(log_dir) 113 | meta_data_file = log_dir + 'metadata.tsv' 114 | with open(meta_data_file, 'w', encoding='utf8') as f: 115 | for name in word_list: 116 | f.write('%s\n' % str(name)) 117 | f.write('__UNK__\n') 118 | -------------------------------------------------------------------------------- /Chapter05/finetune_word2vec_embeddings.py: -------------------------------------------------------------------------------- 1 | from gensim.models import KeyedVectors 2 | from keras.layers.core import Dense, SpatialDropout1D 3 | from keras.layers.convolutional import Conv1D 4 | from keras.layers.embeddings import Embedding 5 | from keras.layers.pooling import GlobalMaxPooling1D 6 | from keras.models import Sequential 7 | from keras.preprocessing.sequence import pad_sequences 8 | from keras.utils import np_utils 9 | from sklearn.model_selection import train_test_split 10 | import collections 11 | from make_tensorboard import make_tensorboard 12 | import nltk 13 | import numpy as np 14 | import codecs 15 | 16 | np.random.seed(42) 17 | 18 | INPUT_FILE = "data/umich-sentiment-train.txt" 19 | # You have to get the model 20 | # https://drive.google.com/uc?id=0B7XkCwpI5KDYNlNUTTlSS21pQmM&export=download 21 | WORD2VEC_MODEL = "data/GoogleNews-vectors-negative300.bin.gz" 22 | VOCAB_SIZE = 5000 23 | EMBED_SIZE = 300 24 | NUM_FILTERS = 256 25 | NUM_WORDS = 3 26 | BATCH_SIZE = 64 27 | NUM_EPOCHS = 10 28 | 29 | counter = collections.Counter() 30 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 31 | maxlen = 0 32 | for line in fin: 33 | _, sent = line.strip().split("\t") 34 | words = [x.lower() for x in nltk.word_tokenize(sent)] 35 | if len(words) > maxlen: 36 | maxlen = len(words) 37 | for word in words: 38 | counter[word] += 1 39 | fin.close() 40 | 41 | word2index = collections.defaultdict(int) 42 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 43 | word2index[word[0]] = wid + 1 44 | vocab_sz = len(word2index) + 1 45 | index2word = {v: k for k, v in word2index.items()} 46 | 47 | xs, ys = [], [] 48 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 49 | for line in fin: 50 | label, sent = line.strip().split("\t") 51 | ys.append(int(label)) 52 | words = [x.lower() for x in nltk.word_tokenize(sent)] 53 | wids = [word2index[word] for word in words] 54 | xs.append(wids) 55 | fin.close() 56 | X = pad_sequences(xs, maxlen=maxlen) 57 | Y = np_utils.to_categorical(ys) 58 | 59 | Xtrain, Xtest, Ytrain, Ytest = \ 60 | train_test_split(X, Y, test_size=0.3, random_state=42) 61 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 62 | 63 | # load word2vec model 64 | word2vec = KeyedVectors.load_word2vec_format(WORD2VEC_MODEL, binary=True) 65 | embedding_weights = np.zeros((vocab_sz, EMBED_SIZE)) 66 | for word, index in word2index.items(): 67 | try: 68 | embedding_weights[index, :] = word2vec[word] 69 | except KeyError: 70 | pass 71 | 72 | model = Sequential() 73 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen, 74 | weights=[embedding_weights], 75 | trainable=True)) 76 | model.add(SpatialDropout1D(0.2)) 77 | model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, 78 | activation="relu")) 79 | model.add(GlobalMaxPooling1D()) 80 | model.add(Dense(2, activation="softmax")) 81 | 82 | model.compile(optimizer="adam", loss="categorical_crossentropy", 83 | metrics=["accuracy"]) 84 | tensorboard, log_dir = make_tensorboard( 85 | set_dir_name='keras_finetune_word2vec_embeddings') 86 | 87 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 88 | epochs=NUM_EPOCHS, 89 | callbacks=[tensorboard], 90 | validation_data=(Xtest, Ytest)) 91 | 92 | # evaluate model 93 | score = model.evaluate(Xtest, Ytest, verbose=1) 94 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 95 | -------------------------------------------------------------------------------- /Chapter05/finetune_word2vec_embeddings_tensorboard_embedding.py: -------------------------------------------------------------------------------- 1 | from gensim.models import KeyedVectors 2 | from keras.layers.core import Dense, SpatialDropout1D 3 | from keras.layers.convolutional import Conv1D 4 | from keras.layers.embeddings import Embedding 5 | from keras.layers.pooling import GlobalMaxPooling1D 6 | from keras.models import Sequential 7 | from keras.preprocessing.sequence import pad_sequences 8 | from keras.utils import np_utils 9 | from sklearn.model_selection import train_test_split 10 | import collections 11 | from make_tensorboard import make_tensorboard 12 | import nltk 13 | import numpy as np 14 | import os 15 | import codecs 16 | 17 | np.random.seed(42) 18 | 19 | INPUT_FILE = "data/umich-sentiment-train.txt" 20 | # You have to get the model 21 | # https://drive.google.com/uc?id=0B7XkCwpI5KDYNlNUTTlSS21pQmM&export=download 22 | WORD2VEC_MODEL = "data/GoogleNews-vectors-negative300.bin.gz" 23 | VOCAB_SIZE = 5000 24 | EMBED_SIZE = 300 25 | NUM_FILTERS = 256 26 | NUM_WORDS = 3 27 | BATCH_SIZE = 64 28 | NUM_EPOCHS = 20 29 | 30 | counter = collections.Counter() 31 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 32 | maxlen = 0 33 | for line in fin: 34 | _, sent = line.strip().split("\t") 35 | words = [x.lower() for x in nltk.word_tokenize(sent)] 36 | if len(words) > maxlen: 37 | maxlen = len(words) 38 | for word in words: 39 | counter[word] += 1 40 | fin.close() 41 | 42 | word2index = collections.defaultdict(int) 43 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 44 | word2index[word[0]] = wid + 1 45 | vocab_sz = len(word2index) + 1 46 | index2word = {v: k for k, v in word2index.items()} 47 | 48 | xs, ys = [], [] 49 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 50 | for line in fin: 51 | label, sent = line.strip().split("\t") 52 | ys.append(int(label)) 53 | words = [x.lower() for x in nltk.word_tokenize(sent)] 54 | wids = [word2index[word] for word in words] 55 | xs.append(wids) 56 | fin.close() 57 | X = pad_sequences(xs, maxlen=maxlen) 58 | Y = np_utils.to_categorical(ys) 59 | 60 | Xtrain, Xtest, Ytrain, Ytest = \ 61 | train_test_split(X, Y, test_size=0.3, random_state=42) 62 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 63 | 64 | # load word2vec model 65 | word2vec = KeyedVectors.load_word2vec_format(WORD2VEC_MODEL, binary=True) 66 | embedding_weights = np.zeros((vocab_sz, EMBED_SIZE)) 67 | for word, index in word2index.items(): 68 | try: 69 | embedding_weights[index, :] = word2vec[word] 70 | except KeyError: 71 | pass 72 | 73 | model = Sequential() 74 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen, 75 | weights=[embedding_weights], 76 | trainable=True)) 77 | model.add(SpatialDropout1D(0.2)) 78 | model.add(Conv1D(filters=NUM_FILTERS, kernel_size=NUM_WORDS, 79 | activation="relu")) 80 | model.add(GlobalMaxPooling1D()) 81 | model.add(Dense(2, activation="softmax")) 82 | 83 | model.compile(optimizer="adam", loss="categorical_crossentropy", 84 | metrics=["accuracy"]) 85 | tensorboard, log_dir = make_tensorboard( 86 | set_dir_name='keras_finetune_word2vec_embeddings', 87 | embeddings_freq=1, 88 | embeddings_metadata='embedding/metadata.tsv' 89 | ) 90 | 91 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 92 | epochs=NUM_EPOCHS, 93 | callbacks=[tensorboard], 94 | validation_data=(Xtest, Ytest)) 95 | 96 | # evaluate model 97 | score = model.evaluate(Xtest, Ytest, verbose=1) 98 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 99 | 100 | word_list = [] 101 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 102 | word_list.append(word[0]) 103 | log_dir = log_dir + "/embedding/" 104 | if os.path.exists(log_dir) is not True: 105 | os.mkdir(log_dir) 106 | meta_data_file = log_dir + 'metadata.tsv' 107 | with open(meta_data_file, 'w', encoding='utf8') as f: 108 | for name in word_list: 109 | f.write('%s\n' % str(name)) 110 | f.write('__UNK__\n') 111 | -------------------------------------------------------------------------------- /Chapter05/keras_cbow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Lambda 5 | from keras.layers.embeddings import Embedding 6 | import keras.backend as K 7 | 8 | vocab_size = 5000 9 | embed_size = 300 10 | window_size = 1 11 | 12 | model = Sequential() 13 | model.add(Embedding(input_dim=vocab_size, output_dim=embed_size, 14 | embeddings_initializer='glorot_uniform', 15 | input_length=window_size*2)) 16 | model.add(Lambda(lambda x: K.mean(x, axis=1), output_shape=(embed_size,))) 17 | model.add(Dense(vocab_size, kernel_initializer='glorot_uniform', 18 | activation='softmax')) 19 | 20 | model.compile(loss='categorical_crossentropy', optimizer="adadelta") 21 | 22 | # get weights 23 | weights = model.layers[0].get_weights()[0] 24 | -------------------------------------------------------------------------------- /Chapter05/keras_skipgram.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.layers import Merge 4 | from keras.layers.core import Dense, Reshape 5 | from keras.layers.embeddings import Embedding 6 | from keras.models import Sequential 7 | 8 | vocab_size = 5000 9 | embed_size = 300 10 | 11 | word_model = Sequential() 12 | word_model.add(Embedding(vocab_size, embed_size, 13 | embeddings_initializer="glorot_uniform", 14 | input_length=1)) 15 | word_model.add(Reshape((embed_size,))) 16 | 17 | context_model = Sequential() 18 | context_model.add(Embedding(vocab_size, embed_size, 19 | embeddings_initializer="glorot_uniform", 20 | input_length=1)) 21 | context_model.add(Reshape((embed_size,))) 22 | 23 | model = Sequential() 24 | model.add(Merge([word_model, context_model], mode="dot", dot_axes=0)) 25 | model.add(Dense(1, kernel_initializer="glorot_uniform", activation="sigmoid")) 26 | 27 | model.compile(loss="mean_squared_error", optimizer="adam") 28 | 29 | merge_layer = model.layers[0] 30 | word_model = merge_layer.layers[0] 31 | word_embed_layer = word_model.layers[0] 32 | weights = word_embed_layer.get_weights()[0] 33 | -------------------------------------------------------------------------------- /Chapter05/learn_embedding_from_scratch.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, SpatialDropout1D 2 | from keras.layers.convolutional import Conv1D 3 | from keras.layers.embeddings import Embedding 4 | from keras.layers.pooling import GlobalMaxPooling1D 5 | from keras.models import Sequential 6 | from keras.preprocessing.sequence import pad_sequences 7 | from keras.utils import np_utils 8 | from sklearn.model_selection import train_test_split 9 | import collections 10 | import nltk 11 | import numpy as np 12 | from make_tensorboard import make_tensorboard 13 | import codecs 14 | 15 | 16 | np.random.seed(42) 17 | 18 | INPUT_FILE = "data/umich-sentiment-train.txt" 19 | VOCAB_SIZE = 5000 20 | EMBED_SIZE = 100 21 | NUM_FILTERS = 256 22 | NUM_WORDS = 3 23 | BATCH_SIZE = 64 24 | NUM_EPOCHS = 20 25 | 26 | counter = collections.Counter() 27 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 28 | maxlen = 0 29 | # you have to download the nltk data 30 | # You will see the below command 31 | # import nltk 32 | # nltk.download() 33 | # NLTK Downloader 34 | # --------------------------------------------------------------------------- 35 | # d) Download l) List u) Update c) Config h) Help q) Quit 36 | # --------------------------------------------------------------------------- 37 | # Downloader> d 38 | # 39 | # Download which package (l=list; x=cancel)? 40 | # Identifier> all 41 | # 42 | 43 | for line in fin: 44 | _, sent = line.strip().split("\t") 45 | words = [x.lower() for x in nltk.word_tokenize(sent)] 46 | if len(words) > maxlen: 47 | maxlen = len(words) 48 | for word in words: 49 | counter[word] += 1 50 | fin.close() 51 | 52 | word2index = collections.defaultdict(int) 53 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 54 | word2index[word[0]] = wid + 1 55 | # Adding one because UNK. 56 | # It means representing words that are not seen in the vocubulary 57 | vocab_sz = len(word2index) + 1 58 | index2word = {v: k for k, v in word2index.items()} 59 | 60 | xs, ys = [], [] 61 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 62 | for line in fin: 63 | label, sent = line.strip().split("\t") 64 | ys.append(int(label)) 65 | words = [x.lower() for x in nltk.word_tokenize(sent)] 66 | wids = [word2index[word] for word in words] 67 | xs.append(wids) 68 | fin.close() 69 | X = pad_sequences(xs, maxlen=maxlen) 70 | Y = np_utils.to_categorical(ys) 71 | 72 | Xtrain, Xtest, Ytrain, Ytest = \ 73 | train_test_split(X, Y, test_size=0.3, random_state=42) 74 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 75 | 76 | model = Sequential() 77 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen)) 78 | model.add(SpatialDropout1D(0.2)) 79 | model.add(Conv1D(filters=NUM_FILTERS, 80 | kernel_size=NUM_WORDS, 81 | activation="relu")) 82 | model.add(GlobalMaxPooling1D()) 83 | model.add(Dense(2, activation="softmax")) 84 | 85 | model.compile(optimizer="adam", loss="categorical_crossentropy", 86 | metrics=["accuracy"]) 87 | 88 | tensorboard, _ = make_tensorboard( 89 | set_dir_name='keras_learn_embedding_from_scratch') 90 | 91 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 92 | epochs=NUM_EPOCHS, 93 | callbacks=[tensorboard], 94 | validation_data=(Xtest, Ytest)) 95 | 96 | # evaluate model 97 | score = model.evaluate(Xtest, Ytest, verbose=1) 98 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 99 | -------------------------------------------------------------------------------- /Chapter05/learn_embedding_from_scratch_tensorboard_embedding.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, SpatialDropout1D 2 | from keras.layers.convolutional import Conv1D 3 | from keras.layers.embeddings import Embedding 4 | from keras.layers.pooling import GlobalMaxPooling1D 5 | from keras.models import Sequential 6 | from keras.preprocessing.sequence import pad_sequences 7 | from keras.utils import np_utils 8 | from sklearn.model_selection import train_test_split 9 | import collections 10 | import nltk 11 | import numpy as np 12 | from make_tensorboard import make_tensorboard 13 | import codecs 14 | import os 15 | 16 | 17 | np.random.seed(42) 18 | 19 | INPUT_FILE = "data/umich-sentiment-train.txt" 20 | VOCAB_SIZE = 5000 21 | EMBED_SIZE = 100 22 | NUM_FILTERS = 256 23 | NUM_WORDS = 3 24 | BATCH_SIZE = 64 25 | NUM_EPOCHS = 20 26 | 27 | counter = collections.Counter() 28 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 29 | maxlen = 0 30 | # you have to download the nltk data 31 | # You will see the below command 32 | # import nltk 33 | # nltk.download() 34 | # NLTK Downloader 35 | # --------------------------------------------------------------------------- 36 | # d) Download l) List u) Update c) Config h) Help q) Quit 37 | # --------------------------------------------------------------------------- 38 | # Downloader> d 39 | # 40 | # Download which package (l=list; x=cancel)? 41 | # Identifier> all 42 | # 43 | 44 | for line in fin: 45 | _, sent = line.strip().split("\t") 46 | words = [x.lower() for x in nltk.word_tokenize(sent)] 47 | if len(words) > maxlen: 48 | maxlen = len(words) 49 | for word in words: 50 | counter[word] += 1 51 | fin.close() 52 | 53 | word2index = collections.defaultdict(int) 54 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 55 | word2index[word[0]] = wid + 1 56 | # Adding one because UNK. 57 | # It means representing words that are not seen in the vocubulary 58 | vocab_sz = len(word2index) + 1 59 | index2word = {v: k for k, v in word2index.items()} 60 | 61 | xs, ys = [], [] 62 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 63 | for line in fin: 64 | label, sent = line.strip().split("\t") 65 | ys.append(int(label)) 66 | words = [x.lower() for x in nltk.word_tokenize(sent)] 67 | wids = [word2index[word] for word in words] 68 | xs.append(wids) 69 | fin.close() 70 | X = pad_sequences(xs, maxlen=maxlen) 71 | Y = np_utils.to_categorical(ys) 72 | 73 | Xtrain, Xtest, Ytrain, Ytest = \ 74 | train_test_split(X, Y, test_size=0.3, random_state=42) 75 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 76 | 77 | model = Sequential() 78 | model.add(Embedding(vocab_sz, EMBED_SIZE, input_length=maxlen)) 79 | model.add(SpatialDropout1D(0.2)) 80 | model.add(Conv1D(filters=NUM_FILTERS, 81 | kernel_size=NUM_WORDS, 82 | activation="relu")) 83 | model.add(GlobalMaxPooling1D()) 84 | model.add(Dense(2, activation="softmax")) 85 | 86 | model.compile(optimizer="adam", loss="categorical_crossentropy", 87 | metrics=["accuracy"]) 88 | 89 | tensorboard, log_dir = make_tensorboard( 90 | set_dir_name='keras_learn_embedding_from_scratch', 91 | embeddings_freq=1, 92 | embeddings_metadata='embedding/metadata.tsv' 93 | ) 94 | 95 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 96 | epochs=NUM_EPOCHS, 97 | callbacks=[tensorboard], 98 | validation_data=(Xtest, Ytest)) 99 | 100 | # evaluate model 101 | score = model.evaluate(Xtest, Ytest, verbose=1) 102 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 103 | word_list = [] 104 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 105 | word_list.append(word[0]) 106 | 107 | log_dir = log_dir + "/embedding/" 108 | if os.path.exists(log_dir) is not True: 109 | os.mkdir(log_dir) 110 | meta_data_file = log_dir + 'metadata.tsv' 111 | with open(meta_data_file, 'w', encoding='utf8') as f: 112 | for name in word_list: 113 | f.write('%s\n' % str(name)) 114 | f.write('__UNK__\n') 115 | -------------------------------------------------------------------------------- /Chapter05/make_tensorboard.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | from __future__ import unicode_literals 4 | from time import gmtime, strftime 5 | from keras.callbacks import TensorBoard 6 | import os 7 | 8 | 9 | def make_tensorboard(set_dir_name='', 10 | embeddings_freq=0, 11 | embeddings_layer_names=None, 12 | embeddings_metadata=None): 13 | tictoc = strftime("%a_%d_%b_%Y_%H_%M_%S", gmtime()) 14 | directory_name = tictoc 15 | log_dir = set_dir_name + '_' + directory_name 16 | os.mkdir(log_dir) 17 | tensorboard = TensorBoard(log_dir=log_dir, 18 | embeddings_freq=embeddings_freq, 19 | embeddings_layer_names=embeddings_layer_names, 20 | embeddings_metadata=embeddings_metadata) 21 | return tensorboard, log_dir 22 | -------------------------------------------------------------------------------- /Chapter05/skipgram_example.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.preprocessing.text import Tokenizer, text_to_word_sequence 3 | from keras.preprocessing.sequence import skipgrams 4 | 5 | text = "I love green eggs and ham ." 6 | 7 | tokenizer = Tokenizer() 8 | tokenizer.fit_on_texts([text]) 9 | 10 | word2id = tokenizer.word_index 11 | id2word = {v: k for k, v in word2id.items()} 12 | 13 | wids = [word2id[w] for w in text_to_word_sequence(text)] 14 | pairs, labels = skipgrams(wids, len(word2id)) 15 | print(len(pairs), len(labels)) 16 | for i in range(10): 17 | print("({:s} ({:d}), {:s} ({:d})) -> {:d}".format( 18 | id2word[pairs[i][0]], pairs[i][0], 19 | id2word[pairs[i][1]], pairs[i][1], 20 | labels[i])) 21 | -------------------------------------------------------------------------------- /Chapter05/transfer_glove_embeddings.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, Dropout 2 | from keras.models import Sequential 3 | from keras.preprocessing.sequence import pad_sequences 4 | from keras.utils import np_utils 5 | from sklearn.model_selection import train_test_split 6 | import collections 7 | import nltk 8 | import numpy as np 9 | from make_tensorboard import make_tensorboard 10 | import codecs 11 | 12 | np.random.seed(42) 13 | 14 | INPUT_FILE = "data/umich-sentiment-train.txt" 15 | GLOVE_MODEL = "data/glove.6B.100d.txt" 16 | VOCAB_SIZE = 5000 17 | EMBED_SIZE = 100 18 | BATCH_SIZE = 64 19 | NUM_EPOCHS = 10 20 | 21 | print("reading data...") 22 | counter = collections.Counter() 23 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 24 | maxlen = 0 25 | for line in fin: 26 | _, sent = line.strip().split("\t") 27 | words = [x.lower() for x in nltk.word_tokenize(sent)] 28 | if len(words) > maxlen: 29 | maxlen = len(words) 30 | for word in words: 31 | counter[word] += 1 32 | fin.close() 33 | 34 | print("creating vocabulary...") 35 | word2index = collections.defaultdict(int) 36 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 37 | word2index[word[0]] = wid + 1 38 | vocab_sz = len(word2index) + 1 39 | index2word = {v: k for k, v in word2index.items()} 40 | index2word[0] = "_UNK_" 41 | 42 | print("creating word sequences...") 43 | ws, ys = [], [] 44 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 45 | for line in fin: 46 | label, sent = line.strip().split("\t") 47 | ys.append(int(label)) 48 | words = [x.lower() for x in nltk.word_tokenize(sent)] 49 | wids = [word2index[word] for word in words] 50 | ws.append(wids) 51 | fin.close() 52 | W = pad_sequences(ws, maxlen=maxlen) 53 | Y = np_utils.to_categorical(ys) 54 | 55 | # load GloVe vectors 56 | print("loading GloVe vectors...") 57 | word2emb = collections.defaultdict(int) 58 | fglove = open(GLOVE_MODEL, "rb") 59 | for line in fglove: 60 | cols = line.strip().split() 61 | word = cols[0].decode('utf-8') 62 | embedding = np.array(cols[1:], dtype="float32") 63 | word2emb[word] = embedding 64 | fglove.close() 65 | 66 | print("transferring embeddings...") 67 | X = np.zeros((W.shape[0], EMBED_SIZE)) 68 | for i in range(W.shape[0]): 69 | E = np.zeros((EMBED_SIZE, maxlen)) 70 | words = [index2word[wid] for wid in W[i].tolist()] 71 | for j in range(maxlen): 72 | E[:, j] = word2emb[words[j]] 73 | X[i, :] = np.sum(E, axis=1) 74 | 75 | Xtrain, Xtest, Ytrain, Ytest = \ 76 | train_test_split(X, Y, test_size=0.3, random_state=42) 77 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 78 | 79 | model = Sequential() 80 | model.add(Dense(32, input_dim=EMBED_SIZE, activation="relu")) 81 | model.add(Dropout(0.2)) 82 | model.add(Dense(2, activation="softmax")) 83 | 84 | model.compile(optimizer="adam", loss="categorical_crossentropy", 85 | metrics=["accuracy"]) 86 | 87 | tensorboard, log_dir = make_tensorboard( 88 | set_dir_name='keras_transfer_glove_embeddings') 89 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 90 | epochs=NUM_EPOCHS, 91 | callbacks=[tensorboard], 92 | validation_data=(Xtest, Ytest)) 93 | 94 | # evaluate model 95 | score = model.evaluate(Xtest, Ytest, verbose=1) 96 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 97 | -------------------------------------------------------------------------------- /Chapter05/transfer_glove_embeddings_tensorboard_embedding.py: -------------------------------------------------------------------------------- 1 | from keras.layers.core import Dense, Dropout 2 | from keras.models import Sequential 3 | from keras.preprocessing.sequence import pad_sequences 4 | from keras.utils import np_utils 5 | from sklearn.model_selection import train_test_split 6 | import collections 7 | import nltk 8 | import numpy as np 9 | from make_tensorboard import make_tensorboard 10 | import codecs 11 | 12 | np.random.seed(42) 13 | 14 | INPUT_FILE = "data/umich-sentiment-train.txt" 15 | GLOVE_MODEL = "data/glove.6B.100d.txt" 16 | VOCAB_SIZE = 5000 17 | EMBED_SIZE = 100 18 | BATCH_SIZE = 64 19 | NUM_EPOCHS = 10 20 | 21 | print("reading data...") 22 | counter = collections.Counter() 23 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 24 | maxlen = 0 25 | for line in fin: 26 | _, sent = line.strip().split("\t") 27 | words = [x.lower() for x in nltk.word_tokenize(sent)] 28 | if len(words) > maxlen: 29 | maxlen = len(words) 30 | for word in words: 31 | counter[word] += 1 32 | fin.close() 33 | 34 | print("creating vocabulary...") 35 | word2index = collections.defaultdict(int) 36 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 37 | word2index[word[0]] = wid + 1 38 | vocab_sz = len(word2index) + 1 39 | index2word = {v: k for k, v in word2index.items()} 40 | index2word[0] = "_UNK_" 41 | 42 | print("creating word sequences...") 43 | ws, ys = [], [] 44 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 45 | for line in fin: 46 | label, sent = line.strip().split("\t") 47 | ys.append(int(label)) 48 | words = [x.lower() for x in nltk.word_tokenize(sent)] 49 | wids = [word2index[word] for word in words] 50 | ws.append(wids) 51 | fin.close() 52 | W = pad_sequences(ws, maxlen=maxlen) 53 | Y = np_utils.to_categorical(ys) 54 | 55 | # load GloVe vectors 56 | print("loading GloVe vectors...") 57 | word2emb = collections.defaultdict(int) 58 | fglove = open(GLOVE_MODEL, "rb") 59 | for line in fglove: 60 | cols = line.strip().split() 61 | word = cols[0].decode('utf-8') 62 | embedding = np.array(cols[1:], dtype="float32") 63 | word2emb[word] = embedding 64 | fglove.close() 65 | 66 | print("transferring embeddings...") 67 | X = np.zeros((W.shape[0], EMBED_SIZE)) 68 | for i in range(W.shape[0]): 69 | E = np.zeros((EMBED_SIZE, maxlen)) 70 | words = [index2word[wid] for wid in W[i].tolist()] 71 | for j in range(maxlen): 72 | E[:, j] = word2emb[words[j]] 73 | X[i, :] = np.sum(E, axis=1) 74 | 75 | Xtrain, Xtest, Ytrain, Ytest = \ 76 | train_test_split(X, Y, test_size=0.3, random_state=42) 77 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 78 | 79 | model = Sequential() 80 | model.add(Dense(32, input_dim=EMBED_SIZE, activation="relu")) 81 | model.add(Dropout(0.2)) 82 | model.add(Dense(2, activation="softmax")) 83 | 84 | model.compile(optimizer="adam", loss="categorical_crossentropy", 85 | metrics=["accuracy"]) 86 | 87 | tensorboard, log_dir = make_tensorboard( 88 | set_dir_name='keras_transfer_glove_embeddings', 89 | embeddings_freq=1, 90 | embeddings_layer_names='dense_1', 91 | ) 92 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 93 | epochs=NUM_EPOCHS, 94 | callbacks=[tensorboard], 95 | validation_data=(Xtest, Ytest)) 96 | 97 | # evaluate model 98 | score = model.evaluate(Xtest, Ytest, verbose=1) 99 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 100 | -------------------------------------------------------------------------------- /Chapter05/transfer_word2vec_embeddings.py: -------------------------------------------------------------------------------- 1 | from gensim.models import KeyedVectors 2 | from keras.layers.core import Dense, Dropout 3 | from keras.models import Sequential 4 | from keras.preprocessing.sequence import pad_sequences 5 | from keras.utils import np_utils 6 | from sklearn.model_selection import train_test_split 7 | import collections 8 | import nltk 9 | import numpy as np 10 | from make_tensorboard import make_tensorboard 11 | import codecs 12 | 13 | np.random.seed(42) 14 | 15 | INPUT_FILE = "data/umich-sentiment-train.txt" 16 | WORD2VEC_MODEL = "data/GoogleNews-vectors-negative300.bin.gz" 17 | VOCAB_SIZE = 5000 18 | EMBED_SIZE = 300 19 | BATCH_SIZE = 64 20 | NUM_EPOCHS = 10 21 | 22 | print("reading data...") 23 | counter = collections.Counter() 24 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 25 | maxlen = 0 26 | for line in fin: 27 | _, sent = line.strip().split("\t") 28 | words = [x.lower() for x in nltk.word_tokenize(sent)] 29 | if len(words) > maxlen: 30 | maxlen = len(words) 31 | for word in words: 32 | counter[word] += 1 33 | fin.close() 34 | 35 | print("creating vocabulary...") 36 | word2index = collections.defaultdict(int) 37 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 38 | word2index[word[0]] = wid + 1 39 | vocab_sz = len(word2index) + 1 40 | index2word = {v: k for k, v in word2index.items()} 41 | index2word[0] = "_UNK_" 42 | 43 | print("creating word sequences...") 44 | ws, ys = [], [] 45 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 46 | for line in fin: 47 | label, sent = line.strip().split("\t") 48 | ys.append(int(label)) 49 | words = [x.lower() for x in nltk.word_tokenize(sent)] 50 | wids = [word2index[word] for word in words] 51 | ws.append(wids) 52 | fin.close() 53 | W = pad_sequences(ws, maxlen=maxlen) 54 | Y = np_utils.to_categorical(ys) 55 | 56 | # load GloVe vectors 57 | print("loading word2vec vectors...") 58 | word2vec = KeyedVectors.load_word2vec_format(WORD2VEC_MODEL, binary=True) 59 | 60 | print("transferring embeddings...") 61 | X = np.zeros((W.shape[0], EMBED_SIZE)) 62 | for i in range(W.shape[0]): 63 | E = np.zeros((EMBED_SIZE, maxlen)) 64 | words = [index2word[wid] for wid in W[i].tolist()] 65 | for j in range(maxlen): 66 | try: 67 | E[:, j] = word2vec[words[j]] 68 | except KeyError: 69 | pass 70 | X[i, :] = np.sum(E, axis=1) 71 | 72 | Xtrain, Xtest, Ytrain, Ytest = \ 73 | train_test_split(X, Y, test_size=0.3, random_state=42) 74 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 75 | 76 | model = Sequential() 77 | model.add(Dense(32, input_dim=EMBED_SIZE, activation="relu")) 78 | model.add(Dropout(0.2)) 79 | model.add(Dense(2, activation="softmax")) 80 | 81 | model.compile(optimizer="adam", loss="categorical_crossentropy", 82 | metrics=["accuracy"]) 83 | callbacks, log_dir = make_tensorboard( 84 | set_dir_name='keras_transfer_word2vec_embeddings') 85 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 86 | epochs=NUM_EPOCHS, 87 | callbacks=[callbacks], 88 | validation_data=(Xtest, Ytest)) 89 | 90 | 91 | # evaluate model 92 | score = model.evaluate(Xtest, Ytest, verbose=1) 93 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 94 | -------------------------------------------------------------------------------- /Chapter05/transfer_word2vec_embeddings_tensorboard_embedding.py: -------------------------------------------------------------------------------- 1 | from gensim.models import KeyedVectors 2 | from keras.layers.core import Dense, Dropout 3 | from keras.models import Sequential 4 | from keras.preprocessing.sequence import pad_sequences 5 | from keras.utils import np_utils 6 | from sklearn.model_selection import train_test_split 7 | import collections 8 | import nltk 9 | import numpy as np 10 | from make_tensorboard import make_tensorboard 11 | import codecs 12 | 13 | np.random.seed(42) 14 | 15 | INPUT_FILE = "data/umich-sentiment-train.txt" 16 | WORD2VEC_MODEL = "data/GoogleNews-vectors-negative300.bin.gz" 17 | VOCAB_SIZE = 5000 18 | EMBED_SIZE = 300 19 | BATCH_SIZE = 64 20 | NUM_EPOCHS = 10 21 | 22 | print("reading data...") 23 | counter = collections.Counter() 24 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 25 | maxlen = 0 26 | for line in fin: 27 | _, sent = line.strip().split("\t") 28 | words = [x.lower() for x in nltk.word_tokenize(sent)] 29 | if len(words) > maxlen: 30 | maxlen = len(words) 31 | for word in words: 32 | counter[word] += 1 33 | fin.close() 34 | 35 | print("creating vocabulary...") 36 | word2index = collections.defaultdict(int) 37 | for wid, word in enumerate(counter.most_common(VOCAB_SIZE)): 38 | word2index[word[0]] = wid + 1 39 | vocab_sz = len(word2index) + 1 40 | index2word = {v: k for k, v in word2index.items()} 41 | index2word[0] = "_UNK_" 42 | 43 | print("creating word sequences...") 44 | ws, ys = [], [] 45 | fin = codecs.open(INPUT_FILE, "r", encoding='utf-8') 46 | for line in fin: 47 | label, sent = line.strip().split("\t") 48 | ys.append(int(label)) 49 | words = [x.lower() for x in nltk.word_tokenize(sent)] 50 | wids = [word2index[word] for word in words] 51 | ws.append(wids) 52 | fin.close() 53 | W = pad_sequences(ws, maxlen=maxlen) 54 | Y = np_utils.to_categorical(ys) 55 | 56 | # load GloVe vectors 57 | print("loading word2vec vectors...") 58 | word2vec = KeyedVectors.load_word2vec_format(WORD2VEC_MODEL, binary=True) 59 | 60 | print("transferring embeddings...") 61 | X = np.zeros((W.shape[0], EMBED_SIZE)) 62 | for i in range(W.shape[0]): 63 | E = np.zeros((EMBED_SIZE, maxlen)) 64 | words = [index2word[wid] for wid in W[i].tolist()] 65 | for j in range(maxlen): 66 | try: 67 | E[:, j] = word2vec[words[j]] 68 | except KeyError: 69 | pass 70 | X[i, :] = np.sum(E, axis=1) 71 | 72 | Xtrain, Xtest, Ytrain, Ytest = \ 73 | train_test_split(X, Y, test_size=0.3, random_state=42) 74 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 75 | 76 | model = Sequential() 77 | model.add(Dense(32, input_dim=EMBED_SIZE, activation="relu")) 78 | model.add(Dropout(0.2)) 79 | model.add(Dense(2, activation="softmax")) 80 | 81 | model.compile(optimizer="adam", loss="categorical_crossentropy", 82 | metrics=["accuracy"]) 83 | callbacks, log_dir = make_tensorboard( 84 | set_dir_name='keras_transfer_word2vec_embeddings', 85 | embeddings_freq=1, 86 | embeddings_layer_names='dense_1', 87 | ) 88 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 89 | epochs=NUM_EPOCHS, 90 | callbacks=[callbacks], 91 | validation_data=(Xtest, Ytest)) 92 | 93 | 94 | # evaluate model 95 | score = model.evaluate(Xtest, Ytest, verbose=1) 96 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 97 | -------------------------------------------------------------------------------- /Chapter05/word2vec_cbow.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Dropout, Activation 4 | from keras.preprocessing.text import Tokenizer, one_hot 5 | from sklearn.preprocessing import OneHotEncoder 6 | from sklearn.model_selection import train_test_split 7 | from sklearn.metrics.pairwise import cosine_distances 8 | import matplotlib.pyplot as plt 9 | import nltk 10 | import numpy as np 11 | import operator 12 | 13 | np.random.seed(42) 14 | 15 | BATCH_SIZE = 128 16 | NUM_EPOCHS = 20 17 | 18 | lines = [] 19 | fin = open("../data/alice_in_wonderland.txt", "rb") 20 | for line in fin: 21 | line = line.strip().decode("ascii", "ignore").encode("utf-8") 22 | if len(line) == 0: 23 | continue 24 | lines.append(line) 25 | fin.close() 26 | 27 | sents = nltk.sent_tokenize(" ".join(lines)) 28 | 29 | tokenizer = Tokenizer(5000) # use top 5000 words only 30 | tokens = tokenizer.fit_on_texts(sents) 31 | vocab_size = len(tokenizer.word_index) + 1 32 | 33 | w_lefts, w_centers, w_rights = [], [], [] 34 | for sent in sents: 35 | embedding = one_hot(sent, vocab_size) 36 | triples = list(nltk.trigrams(embedding)) 37 | w_lefts.extend([x[0] for x in triples]) 38 | w_centers.extend([x[1] for x in triples]) 39 | w_rights.extend([x[2] for x in triples]) 40 | 41 | ohe = OneHotEncoder(n_values=vocab_size) 42 | Xleft = ohe.fit_transform(np.array(w_lefts).reshape(-1, 1)).todense() 43 | Xright = ohe.fit_transform(np.array(w_rights).reshape(-1, 1)).todense() 44 | X = (Xleft + Xright) / 2.0 45 | Y = ohe.fit_transform(np.array(w_centers).reshape(-1, 1)).todense() 46 | Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.3, 47 | random_state=42) 48 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 49 | 50 | model = Sequential() 51 | model.add(Dense(300, input_shape=(Xtrain.shape[1],))) 52 | model.add(Activation("relu")) 53 | model.add(Dropout(0.5)) 54 | model.add(Dense(Ytrain.shape[1])) 55 | model.add(Activation("softmax")) 56 | 57 | model.compile(optimizer="rmsprop", loss="categorical_crossentropy", 58 | metrics=["accuracy"]) 59 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 60 | epochs=NUM_EPOCHS, verbose=1, 61 | validation_data=(Xtest, Ytest)) 62 | 63 | # plot loss function 64 | plt.subplot(211) 65 | plt.title("accuracy") 66 | plt.plot(history.history["acc"], color="r", label="train") 67 | plt.plot(history.history["val_acc"], color="b", label="validation") 68 | plt.legend(loc="best") 69 | 70 | plt.subplot(212) 71 | plt.title("loss") 72 | plt.plot(history.history["loss"], color="r", label="train") 73 | plt.plot(history.history["val_loss"], color="b", label="validation") 74 | plt.legend(loc="best") 75 | 76 | plt.tight_layout() 77 | plt.show() 78 | 79 | # evaluate model 80 | score = model.evaluate(Xtest, Ytest, verbose=1) 81 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 82 | 83 | # using the word2vec model 84 | word2idx = tokenizer.word_index 85 | idx2word = {v: k for k, v in word2idx.items()} 86 | 87 | # retrieve the weights from the first dense layer. This will convert 88 | # the input vector from a one-hot sum of two words to a dense 300 89 | # dimensional representation 90 | W, b = model.layers[0].get_weights() 91 | 92 | idx2emb = {} 93 | for word in word2idx.keys(): 94 | wid = word2idx[word] 95 | vec_in = ohe.fit_transform(np.array(wid)).todense() 96 | vec_emb = np.dot(vec_in, W) 97 | idx2emb[wid] = vec_emb 98 | 99 | for word in ["stupid", "alice", "succeeded"]: 100 | wid = word2idx[word] 101 | source_emb = idx2emb[wid] 102 | distances = [] 103 | for i in range(1, vocab_size): 104 | if i == wid: 105 | continue 106 | target_emb = idx2emb[i] 107 | distances.append(((wid, i), 108 | cosine_distances(source_emb, target_emb))) 109 | sorted_distances = sorted(distances, key=operator.itemgetter(1))[0:10] 110 | predictions = [idx2word[x[0][1]] for x in sorted_distances] 111 | print("{:s} => {:s}".format(word, ", ".join(predictions))) 112 | -------------------------------------------------------------------------------- /Chapter05/word2vec_gensim.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from gensim.models import word2vec 3 | import os 4 | import logging 5 | from pathlib import Path 6 | 7 | 8 | class Text8Sentences(object): 9 | def __init__(self, fname, maxlen): 10 | self.fname = fname 11 | self.maxlen = maxlen 12 | 13 | def __iter__(self): 14 | with open(os.path.join(DATA_DIR, "text8"), "r") as ftext: 15 | text = ftext.read().split(" ") 16 | words = [] 17 | for word in text: 18 | if len(words) >= self.maxlen: 19 | yield words 20 | words = [] 21 | words.append(word) 22 | yield words 23 | 24 | 25 | logging.basicConfig( 26 | format='%(asctime)s : %(levelname)s : %(message)s', 27 | level=logging.INFO) 28 | 29 | DATA_DIR = "data/" 30 | MODEL_NAME = "word2vec_text8" 31 | model_file = Path(MODEL_NAME) 32 | sentences = Text8Sentences(os.path.join(DATA_DIR, "text8"), 50) 33 | if model_file.is_file(): 34 | model = word2vec.Word2Vec.load(MODEL_NAME) 35 | else: 36 | model = word2vec.Word2Vec(sentences, size=300, min_count=30) 37 | 38 | print("""model.most_similar("woman")""") 39 | print(model.most_similar("woman")) 40 | # [('child', 0.7057571411132812), 41 | # ('girl', 0.702182412147522), 42 | # ('man', 0.6846336126327515), 43 | # ('herself', 0.6292711496353149), 44 | # ('lady', 0.6229539513587952), 45 | # ('person', 0.6190367937088013), 46 | # ('lover', 0.6062309741973877), 47 | # ('baby', 0.5993420481681824), 48 | # ('mother', 0.5954475402832031), 49 | # ('daughter', 0.5871444940567017)] 50 | 51 | print("""model.most_similar(positive=["woman", "king"], 52 | negative=["man"], topn=10)""") 53 | print(model.most_similar(positive=['woman', 'king'], 54 | negative=['man'], 55 | topn=10)) 56 | # [('queen', 0.6237582564353943), 57 | # ('prince', 0.5638638734817505), 58 | # ('elizabeth', 0.5557916164398193), 59 | # ('princess', 0.5456407070159912), 60 | # ('throne', 0.5439794063568115), 61 | # ('daughter', 0.5364126563072205), 62 | # ('empress', 0.5354889631271362), 63 | # ('isabella', 0.5233952403068542), 64 | # ('regent', 0.520746111869812), 65 | # ('matilda', 0.5167444944381714)] 66 | 67 | print("""model.similarity("girl", "woman")""") 68 | print(model.similarity("girl", "woman")) 69 | print("""model.similarity("girl", "man")""") 70 | print(model.similarity("girl", "man")) 71 | print("""model.similarity("girl", "car")""") 72 | print(model.similarity("girl", "car")) 73 | print("""model.similarity("bus", "car")""") 74 | print(model.similarity("bus", "car")) 75 | # model.similarity("girl", "woman") 76 | # 0.702182479574 77 | # model.similarity("girl", "man") 78 | # 0.574259909834 79 | # model.similarity("girl", "car") 80 | # 0.289332921793 81 | # model.similarity("bus", "car") 82 | # 0.483853497748 83 | 84 | model.save(MODEL_NAME) 85 | -------------------------------------------------------------------------------- /Chapter05/word2vec_skipgram.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Dropout, Activation 4 | from keras.preprocessing.text import Tokenizer, one_hot 5 | from sklearn.metrics.pairwise import cosine_distances 6 | from sklearn.model_selection import train_test_split 7 | from sklearn.preprocessing import OneHotEncoder 8 | import matplotlib.pyplot as plt 9 | import nltk 10 | import numpy as np 11 | import operator 12 | 13 | np.random.seed(42) 14 | 15 | BATCH_SIZE = 128 16 | NUM_EPOCHS = 20 17 | 18 | lines = [] 19 | fin = open("../data/alice_in_wonderland.txt", "rb") 20 | for line in fin: 21 | line = line.strip().decode("ascii", "ignore").encode("utf-8") 22 | if len(line) == 0: 23 | continue 24 | lines.append(line) 25 | fin.close() 26 | 27 | sents = nltk.sent_tokenize(" ".join(lines)) 28 | 29 | tokenizer = Tokenizer(5000) # use top 5000 words only 30 | tokens = tokenizer.fit_on_texts(sents) 31 | vocab_size = len(tokenizer.word_counts) + 1 32 | 33 | xs = [] 34 | ys = [] 35 | for sent in sents: 36 | embedding = one_hot(sent, vocab_size) 37 | triples = list(nltk.trigrams(embedding)) 38 | w_lefts = [x[0] for x in triples] 39 | w_centers = [x[1] for x in triples] 40 | w_rights = [x[2] for x in triples] 41 | xs.extend(w_centers) 42 | ys.extend(w_lefts) 43 | xs.extend(w_centers) 44 | ys.extend(w_rights) 45 | 46 | ohe = OneHotEncoder(n_values=vocab_size) 47 | X = ohe.fit_transform(np.array(xs).reshape(-1, 1)).todense() 48 | Y = ohe.fit_transform(np.array(ys).reshape(-1, 1)).todense() 49 | Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.3, 50 | random_state=42) 51 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 52 | 53 | model = Sequential() 54 | model.add(Dense(300, input_shape=(Xtrain.shape[1],))) 55 | model.add(Activation("relu")) 56 | model.add(Dropout(0.5)) 57 | model.add(Dense(Ytrain.shape[1])) 58 | model.add(Activation("softmax")) 59 | 60 | model.compile(optimizer="rmsprop", loss="categorical_crossentropy", 61 | metrics=["accuracy"]) 62 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, 63 | epochs=NUM_EPOCHS, verbose=1, 64 | validation_data=(Xtest, Ytest)) 65 | 66 | # plot loss function 67 | plt.subplot(211) 68 | plt.title("accuracy") 69 | plt.plot(history.history["acc"], color="r", label="train") 70 | plt.plot(history.history["val_acc"], color="b", label="validation") 71 | plt.legend(loc="best") 72 | 73 | plt.subplot(212) 74 | plt.title("loss") 75 | plt.plot(history.history["loss"], color="r", label="train") 76 | plt.plot(history.history["val_loss"], color="b", label="validation") 77 | plt.legend(loc="best") 78 | 79 | plt.tight_layout() 80 | plt.show() 81 | 82 | # evaluate model 83 | score = model.evaluate(Xtest, Ytest, verbose=1) 84 | print("Test score: {:.3f}, accuracy: {:.3f}".format(score[0], score[1])) 85 | 86 | # using the word2vec model 87 | word2idx = tokenizer.word_index 88 | idx2word = {v: k for k, v in word2idx.items()} 89 | 90 | # retrieve the weights from the first dense layer. This will convert 91 | # the input vector from a one-hot sum of two words to a dense 300 92 | # dimensional representation 93 | W, b = model.layers[0].get_weights() 94 | 95 | idx2emb = {} 96 | for word in word2idx.keys(): 97 | wid = word2idx[word] 98 | vec_in = ohe.fit_transform(np.array(wid)).todense() 99 | vec_emb = np.dot(vec_in, W) 100 | idx2emb[wid] = vec_emb 101 | 102 | for word in ["stupid", "alice", "succeeded"]: 103 | wid = word2idx[word] 104 | source_emb = idx2emb[wid] 105 | distances = [] 106 | for i in range(1, vocab_size): 107 | if i == wid: 108 | continue 109 | target_emb = idx2emb[i] 110 | distances.append(((wid, i), 111 | cosine_distances(source_emb, target_emb))) 112 | sorted_distances = sorted(distances, key=operator.itemgetter(1))[0:10] 113 | predictions = [idx2word[x[0][1]] for x in sorted_distances] 114 | print("{:s} => {:s}".format(word, ", ".join(predictions))) 115 | -------------------------------------------------------------------------------- /Chapter06/alice_chargen_rnn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Adapted from lstm_text_generation.py in keras/examples 3 | from __future__ import print_function 4 | from keras.layers.recurrent import SimpleRNN 5 | from keras.models import Sequential 6 | from keras.layers import Dense, Activation 7 | import numpy as np 8 | 9 | INPUT_FILE = "../data/alice_in_wonderland.txt" 10 | 11 | # extract the input as a stream of characters 12 | print("Extracting text from input...") 13 | fin = open(INPUT_FILE, 'rb') 14 | lines = [] 15 | for line in fin: 16 | line = line.strip().lower() 17 | line = line.decode("ascii", "ignore") 18 | if len(line) == 0: 19 | continue 20 | lines.append(line) 21 | fin.close() 22 | text = " ".join(lines) 23 | 24 | # creating lookup tables 25 | # Here chars is the number of features in our character "vocabulary" 26 | chars = set([c for c in text]) 27 | nb_chars = len(chars) 28 | char2index = dict((c, i) for i, c in enumerate(chars)) 29 | index2char = dict((i, c) for i, c in enumerate(chars)) 30 | 31 | # create inputs and labels from the text. We do this by stepping 32 | # through the text ${step} character at a time, and extracting a 33 | # sequence of size ${seqlen} and the next output char. For example, 34 | # assuming an input text "The sky was falling", we would get the 35 | # following sequence of input_chars and label_chars (first 5 only) 36 | # The sky wa -> s 37 | # he sky was -> 38 | # e sky was -> f 39 | # sky was f -> a 40 | # sky was fa -> l 41 | print("Creating input and label text...") 42 | SEQLEN = 10 43 | STEP = 1 44 | 45 | input_chars = [] 46 | label_chars = [] 47 | for i in range(0, len(text) - SEQLEN, STEP): 48 | input_chars.append(text[i:i + SEQLEN]) 49 | label_chars.append(text[i + SEQLEN]) 50 | 51 | # vectorize the input and label chars 52 | # Each row of the input is represented by seqlen characters, each 53 | # represented as a 1-hot encoding of size len(char). There are 54 | # len(input_chars) such rows, so shape(X) is (len(input_chars), 55 | # seqlen, nb_chars). 56 | # Each row of output is a single character, also represented as a 57 | # dense encoding of size len(char). Hence shape(y) is (len(input_chars), 58 | # nb_chars). 59 | print("Vectorizing input and label text...") 60 | X = np.zeros((len(input_chars), SEQLEN, nb_chars), dtype=np.bool) 61 | y = np.zeros((len(input_chars), nb_chars), dtype=np.bool) 62 | for i, input_char in enumerate(input_chars): 63 | for j, ch in enumerate(input_char): 64 | X[i, j, char2index[ch]] = 1 65 | y[i, char2index[label_chars[i]]] = 1 66 | 67 | # Build the model. We use a single RNN with a fully connected layer 68 | # to compute the most likely predicted output char 69 | HIDDEN_SIZE = 128 70 | BATCH_SIZE = 128 71 | NUM_ITERATIONS = 25 72 | NUM_EPOCHS_PER_ITERATION = 1 73 | NUM_PREDS_PER_EPOCH = 100 74 | 75 | model = Sequential() 76 | model.add(SimpleRNN(HIDDEN_SIZE, return_sequences=False, 77 | input_shape=(SEQLEN, nb_chars), 78 | unroll=True)) 79 | model.add(Dense(nb_chars)) 80 | model.add(Activation("softmax")) 81 | 82 | model.compile(loss="categorical_crossentropy", optimizer="rmsprop") 83 | 84 | # We train the model in batches and test output generated at each step 85 | for iteration in range(NUM_ITERATIONS): 86 | print("=" * 50) 87 | print("Iteration #: %d" % (iteration)) 88 | model.fit(X, y, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS_PER_ITERATION) 89 | 90 | # testing model 91 | # randomly choose a row from input_chars, then use it to 92 | # generate text from model for next 100 chars 93 | test_idx = np.random.randint(len(input_chars)) 94 | test_chars = input_chars[test_idx] 95 | print("Generating from seed: %s" % (test_chars)) 96 | print(test_chars, end="") 97 | for i in range(NUM_PREDS_PER_EPOCH): 98 | Xtest = np.zeros((1, SEQLEN, nb_chars)) 99 | for i, ch in enumerate(test_chars): 100 | Xtest[0, i, char2index[ch]] = 1 101 | pred = model.predict(Xtest, verbose=0)[0] 102 | ypred = index2char[np.argmax(pred)] 103 | print(ypred, end="") 104 | # move forward with test_chars + ypred 105 | test_chars = test_chars[1:] + ypred 106 | print() 107 | -------------------------------------------------------------------------------- /Chapter06/econs_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import os 6 | import re 7 | 8 | DATA_DIR = "../data" 9 | 10 | fld = open(os.path.join(DATA_DIR, "LD2011_2014.txt"), "rb") 11 | data = [] 12 | line_num = 0 13 | #cid = np.random.randint(0, 370, 1) 14 | cid = 250 15 | for line in fld: 16 | if line.startswith("\"\";"): 17 | continue 18 | if line_num % 100 == 0: 19 | print("{:d} lines read".format(line_num)) 20 | cols = [float(re.sub(",", ".", x)) for x in 21 | line.strip().split(";")[1:]] 22 | data.append(cols[cid]) 23 | line_num += 1 24 | fld.close() 25 | 26 | NUM_ENTRIES = 1000 27 | plt.plot(range(NUM_ENTRIES), data[0:NUM_ENTRIES]) 28 | plt.ylabel("electricity consumption") 29 | plt.xlabel("time (1pt = 15 mins)") 30 | plt.show() 31 | 32 | np.save(os.path.join(DATA_DIR, "LD_250.npy"), np.array(data)) 33 | -------------------------------------------------------------------------------- /Chapter06/econs_stateful.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.layers.core import Dense 4 | from keras.layers.recurrent import LSTM 5 | from keras.models import Sequential 6 | from sklearn.preprocessing import MinMaxScaler 7 | import numpy as np 8 | import math 9 | import os 10 | 11 | DATA_DIR = "../data" 12 | 13 | data = np.load(os.path.join(DATA_DIR, "LD_250.npy")) 14 | 15 | STATELESS = False 16 | 17 | NUM_TIMESTEPS = 20 18 | HIDDEN_SIZE = 10 19 | BATCH_SIZE = 96 # 24 hours (15 min intervals) 20 | NUM_EPOCHS = 5 21 | 22 | # scale the data to be in the range (0, 1) 23 | data = data.reshape(-1, 1) 24 | scaler = MinMaxScaler(feature_range=(0, 1), copy=False) 25 | data = scaler.fit_transform(data) 26 | 27 | # transform to 4 inputs -> 1 label format 28 | X = np.zeros((data.shape[0], NUM_TIMESTEPS)) 29 | Y = np.zeros((data.shape[0], 1)) 30 | for i in range(len(data) - NUM_TIMESTEPS - 1): 31 | X[i] = data[i:i + NUM_TIMESTEPS].T 32 | Y[i] = data[i + NUM_TIMESTEPS + 1] 33 | 34 | # reshape X to three dimensions (samples, timesteps, features) 35 | X = np.expand_dims(X, axis=2) 36 | 37 | # split into training and test sets (add the extra offsets so 38 | # we can use batch size of 5) 39 | sp = int(0.7 * len(data)) 40 | Xtrain, Xtest, Ytrain, Ytest = X[0:sp], X[sp:], Y[0:sp], Y[sp:] 41 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 42 | 43 | if STATELESS: 44 | # stateless 45 | model = Sequential() 46 | model.add(LSTM(HIDDEN_SIZE, input_shape=(NUM_TIMESTEPS, 1), 47 | return_sequences=False)) 48 | model.add(Dense(1)) 49 | else: 50 | # stateful 51 | model = Sequential() 52 | model.add(LSTM(HIDDEN_SIZE, stateful=True, 53 | batch_input_shape=(BATCH_SIZE, NUM_TIMESTEPS, 1), 54 | return_sequences=False)) 55 | model.add(Dense(1)) 56 | 57 | model.compile(loss="mean_squared_error", optimizer="adam", 58 | metrics=["mean_squared_error"]) 59 | 60 | if STATELESS: 61 | # stateless 62 | model.fit(Xtrain, Ytrain, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE, 63 | validation_data=(Xtest, Ytest), 64 | shuffle=False) 65 | else: 66 | # stateful 67 | # need to make training and test data to multiple of BATCH_SIZE 68 | train_size = (Xtrain.shape[0] // BATCH_SIZE) * BATCH_SIZE 69 | test_size = (Xtest.shape[0] // BATCH_SIZE) * BATCH_SIZE 70 | Xtrain, Ytrain = Xtrain[0:train_size], Ytrain[0:train_size] 71 | Xtest, Ytest = Xtest[0:test_size], Ytest[0:test_size] 72 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 73 | for i in range(NUM_EPOCHS): 74 | print("Epoch {:d}/{:d}".format(i+1, NUM_EPOCHS)) 75 | model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=1, 76 | validation_data=(Xtest, Ytest), 77 | shuffle=False) 78 | model.reset_states() 79 | 80 | score, _ = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE) 81 | rmse = math.sqrt(score) 82 | print("\nMSE: {:.3f}, RMSE: {:.3f}".format(score, rmse)) 83 | -------------------------------------------------------------------------------- /Chapter06/pos-tagging-explore.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.layers.core import Activation, Dense, Dropout, RepeatVector, SpatialDropout1D 4 | from keras.layers.embeddings import Embedding 5 | from keras.layers.recurrent import GRU, LSTM 6 | from keras.layers.wrappers import TimeDistributed, Bidirectional 7 | from keras.models import Sequential 8 | from keras.preprocessing import sequence 9 | from keras.utils import np_utils 10 | import collections 11 | import matplotlib.pyplot as plt 12 | import numpy as np 13 | import os 14 | 15 | def explore_data(datadir, datafiles): 16 | counter = collections.Counter() 17 | maxlen = 0 18 | for datafile in datafiles: 19 | fdata = open(os.path.join(datadir, datafile), "rb") 20 | for line in fdata: 21 | words = line.strip().split() 22 | if len(words) > maxlen: 23 | maxlen = len(words) 24 | for word in words: 25 | counter[word] += 1 26 | fdata.close() 27 | return maxlen, counter 28 | 29 | def build_tensor(filename, numrecs, word2index, maxlen, 30 | make_categorical=False): 31 | data = np.empty((numrecs, ), dtype=list) 32 | fin = open(filename, "rb") 33 | i = 0 34 | for line in fin: 35 | wids = [] 36 | for word in line.strip().split(): 37 | if word2index.has_key(word): 38 | wids.append(word2index[word]) 39 | else: 40 | wids.append(word2index["UNK"]) 41 | if make_categorical: 42 | data[i] = np_utils.to_categorical( 43 | wids, num_classes=len(word2index)) 44 | else: 45 | data[i] = wids 46 | i += 1 47 | fin.close() 48 | pdata = sequence.pad_sequences(data, maxlen=maxlen) 49 | return pdata 50 | 51 | def evaluate_model(model, Xtest, Ytest, batch_size): 52 | pass 53 | 54 | DATA_DIR = "../data" 55 | 56 | s_maxlen, s_counter = explore_data(DATA_DIR, ["babi-sent-train.txt", 57 | "babi-sent-test.txt"]) 58 | t_maxlen, t_counter = explore_data(DATA_DIR, ["babi-pos-train.txt", 59 | "babi-pos-test.txt"]) 60 | 61 | print(s_maxlen, len(s_counter), t_maxlen, len(t_counter)) 62 | # 7 21 7 9 63 | # maxlen: 7 64 | # size of source vocab: 21 65 | # size of target vocab: 9 66 | 67 | # lookup tables 68 | s_word2id = {k:v+1 for v, (k, _) in enumerate(s_counter.most_common())} 69 | s_word2id["PAD"] = 0 70 | s_id2word = {v:k for k, v in s_word2id.items()} 71 | t_pos2id = {k:v+1 for v, (k, _) in enumerate(t_counter.most_common())} 72 | t_pos2id["PAD"] = 0 73 | t_id2pos = {v:k for k, v in t_pos2id.items()} 74 | 75 | # vectorize data 76 | MAX_SEQLEN = 10 77 | 78 | Xtrain = build_tensor(os.path.join(DATA_DIR, "babi-sent-train.txt"), 79 | 30000, s_word2id, MAX_SEQLEN) 80 | Xtest = build_tensor(os.path.join(DATA_DIR, "babi-sent-test.txt"), 81 | 3000, s_word2id, MAX_SEQLEN) 82 | Ytrain = build_tensor(os.path.join(DATA_DIR, "babi-pos-train.txt"), 83 | 30000, t_pos2id, MAX_SEQLEN, make_categorical=True) 84 | Ytest = build_tensor(os.path.join(DATA_DIR, "babi-pos-test.txt"), 85 | 3000, t_pos2id, MAX_SEQLEN, make_categorical=True) 86 | print(Xtrain.shape, Xtest.shape, Ytrain.shape, Ytest.shape) 87 | 88 | # define network 89 | EMBED_SIZE = 32 90 | HIDDEN_SIZE = 32 91 | 92 | BATCH_SIZE = 32 93 | NUM_EPOCHS = 5 94 | 95 | model = Sequential() 96 | model.add(Embedding(len(s_word2id), EMBED_SIZE, 97 | input_length=MAX_SEQLEN)) 98 | model.add(SpatialDropout1D(Dropout(0.2))) 99 | model.add(LSTM(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2)) 100 | #model.add(GRU(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2)) 101 | #model.add(Bidirectional(LSTM(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2))) 102 | model.add(RepeatVector(MAX_SEQLEN)) 103 | model.add(LSTM(HIDDEN_SIZE, return_sequences=True)) 104 | #model.add(GRU(HIDDEN_SIZE, return_sequences=True)) 105 | #model.add(Bidirectional(LSTM(HIDDEN_SIZE, return_sequences=True))) 106 | model.add(TimeDistributed(Dense(len(t_pos2id)))) 107 | model.add(Activation("softmax")) 108 | 109 | model.compile(loss="categorical_crossentropy", optimizer="adam", 110 | metrics=["accuracy"]) 111 | 112 | history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, 113 | validation_data=[Xtest, Ytest]) 114 | 115 | # plot loss and accuracy 116 | plt.subplot(211) 117 | plt.title("Accuracy") 118 | plt.plot(history.history["acc"], color="g", label="Train") 119 | plt.plot(history.history["val_acc"], color="b", label="Validation") 120 | plt.legend(loc="best") 121 | 122 | plt.subplot(212) 123 | plt.title("Loss") 124 | plt.plot(history.history["loss"], color="g", label="Train") 125 | plt.plot(history.history["val_loss"], color="b", label="Validation") 126 | plt.legend(loc="best") 127 | 128 | plt.tight_layout() 129 | plt.show() 130 | 131 | # evaluate model 132 | score, acc = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE) 133 | print("Test score: %.3f, accuracy: %.3f" % (score, acc)) 134 | 135 | # custom evaluate 136 | hit_rates = [] 137 | num_iters = Xtest.shape[0] // BATCH_SIZE 138 | for i in range(num_iters - 1): 139 | xtest = Xtest[i * BATCH_SIZE : (i + 1) * BATCH_SIZE] 140 | ytest = np.argmax(Ytest[i * BATCH_SIZE : (i + 1) * BATCH_SIZE], axis=2) 141 | ytest_ = np.argmax(model.predict(xtest), axis=2) 142 | # print(ytest.shape, ytest_.shape) 143 | for j in range(BATCH_SIZE): 144 | # print("sentence: " + " ".join([s_id2word[x] for x in xtest[j].tolist()])) 145 | # print("predicted: " + " ".join([t_id2pos[y] for y in ytest_[j].tolist()])) 146 | # print("label: " + " ".join([t_id2pos[y] for y in ytest[j].tolist()])) 147 | word_indices = np.nonzero(xtest[j]) 148 | pos_labels = ytest[j][word_indices] 149 | pos_pred = ytest_[j][word_indices] 150 | hit_rates.append(np.sum(pos_labels == pos_pred) / len(pos_pred)) 151 | break 152 | 153 | accuracy = sum(hit_rates) / len(hit_rates) 154 | print("accuracy: {:.3f}".format(accuracy)) 155 | 156 | # prediction 157 | pred_ids = np.random.randint(0, 3000, 5) 158 | for pred_id in pred_ids: 159 | xtest = Xtest[pred_id].reshape(1, 10) 160 | ytest_ = np.argmax(model.predict(xtest), axis=2) 161 | ytest = np.argmax(Ytest[pred_id], axis=1) 162 | print("sentence: " + " ".join([s_id2word[x] for x in xtest[0].tolist()])) 163 | print("predicted: " + " ".join([t_id2pos[y] for y in ytest_[0].tolist()])) 164 | print("label: " + " ".join([t_id2pos[y] for y in ytest.tolist()])) 165 | word_indices = np.nonzero(xtest)[1] 166 | ypred_tags = ytest_[0][word_indices] 167 | ytrue_tags = ytest[word_indices] 168 | hit_rate = np.sum(ypred_tags == ytrue_tags) / len(ypred_tags) 169 | print("hit rate: {:.3f}".format(hit_rate)) 170 | print() 171 | -------------------------------------------------------------------------------- /Chapter06/pos_tagging_data.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copied from: Out of core classification of Text Documents 3 | # from the scikit-learn documentation. 4 | # http://scikit-learn.org/stable/auto_examples/applications/plot_out_of_core_classification.html 5 | # 6 | from __future__ import division, print_function 7 | from sklearn.externals.six.moves import html_parser 8 | from glob import glob 9 | import collections 10 | import nltk 11 | import os 12 | import re 13 | 14 | class ReutersParser(html_parser.HTMLParser): 15 | """ Utility class to parse a SGML file and yield documents one at 16 | a time. 17 | """ 18 | def __init__(self, encoding='latin-1'): 19 | html_parser.HTMLParser.__init__(self) 20 | self._reset() 21 | self.encoding = encoding 22 | 23 | def handle_starttag(self, tag, attrs): 24 | method = 'start_' + tag 25 | getattr(self, method, lambda x: None)(attrs) 26 | 27 | def handle_endtag(self, tag): 28 | method = 'end_' + tag 29 | getattr(self, method, lambda: None)() 30 | 31 | def _reset(self): 32 | self.in_title = 0 33 | self.in_body = 0 34 | self.in_topics = 0 35 | self.in_topic_d = 0 36 | self.title = "" 37 | self.body = "" 38 | self.topics = [] 39 | self.topic_d = "" 40 | 41 | def parse(self, fd): 42 | self.docs = [] 43 | for chunk in fd: 44 | self.feed(chunk.decode(self.encoding)) 45 | for doc in self.docs: 46 | yield doc 47 | self.docs = [] 48 | self.close() 49 | 50 | def handle_data(self, data): 51 | if self.in_body: 52 | self.body += data 53 | elif self.in_title: 54 | self.title += data 55 | elif self.in_topic_d: 56 | self.topic_d += data 57 | 58 | def start_reuters(self, attributes): 59 | pass 60 | 61 | def end_reuters(self): 62 | self.body = re.sub(r'\s+', r' ', self.body) 63 | self.docs.append({'title': self.title, 64 | 'body': self.body, 65 | 'topics': self.topics}) 66 | self._reset() 67 | 68 | def start_title(self, attributes): 69 | self.in_title = 1 70 | 71 | def end_title(self): 72 | self.in_title = 0 73 | 74 | def start_body(self, attributes): 75 | self.in_body = 1 76 | 77 | def end_body(self): 78 | self.in_body = 0 79 | 80 | def start_topics(self, attributes): 81 | self.in_topics = 1 82 | 83 | def end_topics(self): 84 | self.in_topics = 0 85 | 86 | def start_d(self, attributes): 87 | self.in_topic_d = 1 88 | 89 | def end_d(self): 90 | self.in_topic_d = 0 91 | self.topics.append(self.topic_d) 92 | self.topic_d = "" 93 | 94 | 95 | def stream_reuters_documents(reuters_dir): 96 | """ Iterate over documents of the Reuters dataset. 97 | 98 | The Reuters archive will automatically be downloaded and uncompressed if 99 | the `data_path` directory does not exist. 100 | 101 | Documents are represented as dictionaries with 'body' (str), 102 | 'title' (str), 'topics' (list(str)) keys. 103 | 104 | """ 105 | parser = ReutersParser() 106 | for filename in glob(os.path.join(reuters_dir, "*.sgm")): 107 | for doc in parser.parse(open(filename, 'rb')): 108 | yield doc 109 | 110 | 111 | ##################### main ###################### 112 | 113 | DATA_DIR = "../data" 114 | REUTERS_DIR = os.path.join(DATA_DIR, "reuters-21578") 115 | 116 | num_read = 0 117 | num_sents = 0 118 | 119 | fsent = open(os.path.join(DATA_DIR, "reuters-sent.txt"), "wb") 120 | fpos = open(os.path.join(DATA_DIR, "reuters-pos.txt"), "wb") 121 | tagger = nltk.tag.PerceptronTagger() 122 | 123 | for doc in stream_reuters_documents(REUTERS_DIR): 124 | # skip docs without specified topic 125 | topics = doc["topics"] 126 | if len(topics) == 0: 127 | continue 128 | title = doc["title"] 129 | body = doc["body"] 130 | sents = nltk.sent_tokenize(body) 131 | for sent in sents: 132 | if num_sents % 100 == 0: 133 | print("{:d} sentences written".format(num_sents)) 134 | if len(sent) <= 20: 135 | continue 136 | sent = sent.encode("utf8").decode("ascii", "ignore") 137 | words = nltk.word_tokenize(sent) 138 | fsent.write("{:s}\n".format(" ".join(words))) 139 | tokentags = nltk.tag._pos_tag(words, None, tagger) 140 | fpos.write("{:s}\n".format(" ".join([x[1] for x in tokentags]))) 141 | num_sents += 1 142 | 143 | fsent.close() 144 | fpos.close() 145 | print("{:d} sentences written, COMPLETE".format(num_sents)) 146 | -------------------------------------------------------------------------------- /Chapter06/treebank_data.py: -------------------------------------------------------------------------------- 1 | import nltk 2 | import os 3 | 4 | DATA_DIR = "../data" 5 | 6 | fedata = open(os.path.join(DATA_DIR, "treebank_sents.txt"), "wb") 7 | ffdata = open(os.path.join(DATA_DIR, "treebank_poss.txt"), "wb") 8 | 9 | sents = nltk.corpus.treebank.tagged_sents() 10 | for sent in sents: 11 | words, poss = [], [] 12 | for word, pos in sent: 13 | if pos == "-NONE-": 14 | continue 15 | words.append(word) 16 | poss.append(pos) 17 | fedata.write("{:s}\n".format(" ".join(words))) 18 | ffdata.write("{:s}\n".format(" ".join(poss))) 19 | 20 | fedata.close() 21 | ffdata.close() 22 | -------------------------------------------------------------------------------- /Chapter06/treebank_pos_gru.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | np.random.seed(42) # setting seed before importing from keras 3 | from keras.layers.core import Activation, Dense, Dropout, RepeatVector, SpatialDropout1D 4 | from keras.layers.embeddings import Embedding 5 | from keras.layers.recurrent import GRU 6 | from keras.layers.wrappers import TimeDistributed 7 | from keras.models import Sequential 8 | from keras.preprocessing import sequence 9 | from keras.utils import np_utils 10 | from sklearn.model_selection import train_test_split 11 | import collections 12 | import os 13 | 14 | def parse_sentences(filename): 15 | word_freqs = collections.Counter() 16 | num_recs, maxlen = 0, 0 17 | fin = open(filename, "rb") 18 | for line in fin: 19 | words = line.strip().lower().split() 20 | for word in words: 21 | word_freqs[word] += 1 22 | if len(words) > maxlen: 23 | maxlen = len(words) 24 | num_recs += 1 25 | fin.close() 26 | return word_freqs, maxlen, num_recs 27 | 28 | DATA_DIR = "../data" 29 | 30 | s_wordfreqs, s_maxlen, s_numrecs = parse_sentences(os.path.join(DATA_DIR, "treebank_sents.txt")) 31 | t_wordfreqs, t_maxlen, t_numrecs = parse_sentences(os.path.join(DATA_DIR, "treebank_poss.txt")) 32 | print(len(s_wordfreqs), s_maxlen, s_numrecs, len(t_wordfreqs), t_maxlen, t_numrecs) 33 | 34 | MAX_SEQLEN = 250 35 | S_MAX_FEATURES = 5000 36 | T_MAX_FEATURES = 45 37 | 38 | s_vocabsize = min(len(s_wordfreqs), S_MAX_FEATURES) + 2 39 | s_word2index = {x[0]:i+2 for i, x in enumerate(s_wordfreqs.most_common(S_MAX_FEATURES))} 40 | s_word2index["PAD"] = 0 41 | s_word2index["UNK"] = 1 42 | s_index2word = {v:k for k, v in s_word2index.items()} 43 | 44 | t_vocabsize = len(t_wordfreqs) + 1 45 | t_word2index = {x[0]:i for i, x in enumerate(t_wordfreqs.most_common(T_MAX_FEATURES))} 46 | t_word2index["PAD"] = 0 47 | t_index2word = {v:k for k, v in t_word2index.items()} 48 | 49 | def build_tensor(filename, numrecs, word2index, maxlen, make_categorical = False, num_classes = 0): 50 | data = np.empty((numrecs, ), dtype=list) 51 | fin = open(filename, "rb") 52 | i = 0 53 | for line in fin: 54 | wids = [] 55 | for word in line.strip().lower().split(): 56 | if word2index.has_key(word): 57 | wids.append(word2index[word]) 58 | else: 59 | wids.append(word2index["UNK"]) 60 | if make_categorical: 61 | data[i] = np_utils.to_categorical(wids, num_classes=num_classes) 62 | else: 63 | data[i] = wids 64 | i += 1 65 | fin.close() 66 | pdata = sequence.pad_sequences(data, maxlen=maxlen) 67 | return pdata 68 | 69 | X = build_tensor(os.path.join(DATA_DIR, "treebank_sents.txt"), s_numrecs, s_word2index, MAX_SEQLEN) 70 | Y = build_tensor(os.path.join(DATA_DIR, "treebank_poss.txt"), t_numrecs, t_word2index, MAX_SEQLEN, True, t_vocabsize) 71 | 72 | Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.2, random_state=42) 73 | 74 | EMBED_SIZE = 128 75 | HIDDEN_SIZE = 64 76 | BATCH_SIZE = 32 77 | NUM_EPOCHS = 1 78 | 79 | model = Sequential() 80 | model.add(Embedding(s_vocabsize, EMBED_SIZE, input_length=MAX_SEQLEN)) 81 | model.add(SpatialDropout1D(Dropout(0.2))) 82 | model.add(GRU(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2)) 83 | model.add(RepeatVector(MAX_SEQLEN)) 84 | model.add(GRU(HIDDEN_SIZE, return_sequences=True)) 85 | model.add(TimeDistributed(Dense(t_vocabsize))) 86 | model.add(Activation("softmax")) 87 | 88 | model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) 89 | 90 | model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_data=[Xtest, Ytest]) 91 | 92 | score, acc = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE) 93 | print("Test score: %.3f, accuracy: %.3f" % (score, acc)) 94 | -------------------------------------------------------------------------------- /Chapter06/treebank_pos_lstm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | np.random.seed(42) # setting seed before importing from Keras 3 | from keras.layers.core import Activation, Dense, Dropout, RepeatVector, SpatialDropout1D 4 | from keras.layers.embeddings import Embedding 5 | from keras.layers.recurrent import LSTM 6 | from keras.layers.wrappers import TimeDistributed 7 | from keras.models import Sequential 8 | from keras.preprocessing import sequence 9 | from keras.utils import np_utils 10 | from sklearn.model_selection import train_test_split 11 | import collections 12 | import os 13 | 14 | def parse_sentences(filename): 15 | word_freqs = collections.Counter() 16 | num_recs, maxlen = 0, 0 17 | fin = open(filename, "rb") 18 | for line in fin: 19 | words = line.strip().lower().split() 20 | for word in words: 21 | word_freqs[word] += 1 22 | if len(words) > maxlen: 23 | maxlen = len(words) 24 | num_recs += 1 25 | fin.close() 26 | return word_freqs, maxlen, num_recs 27 | 28 | DATA_DIR = "../data" 29 | 30 | s_wordfreqs, s_maxlen, s_numrecs = parse_sentences(os.path.join(DATA_DIR, "treebank_sents.txt")) 31 | t_wordfreqs, t_maxlen, t_numrecs = parse_sentences(os.path.join(DATA_DIR, "treebank_poss.txt")) 32 | print(len(s_wordfreqs), s_maxlen, s_numrecs, len(t_wordfreqs), t_maxlen, t_numrecs) 33 | 34 | MAX_SEQLEN = 250 35 | S_MAX_FEATURES = 5000 36 | T_MAX_FEATURES = 45 37 | 38 | s_vocabsize = min(len(s_wordfreqs), S_MAX_FEATURES) + 2 39 | s_word2index = {x[0]:i+2 for i, x in enumerate(s_wordfreqs.most_common(S_MAX_FEATURES))} 40 | s_word2index["PAD"] = 0 41 | s_word2index["UNK"] = 1 42 | s_index2word = {v:k for k, v in s_word2index.items()} 43 | 44 | t_vocabsize = len(t_wordfreqs) + 1 45 | t_word2index = {x[0]:i for i, x in enumerate(t_wordfreqs.most_common(T_MAX_FEATURES))} 46 | t_word2index["PAD"] = 0 47 | t_index2word = {v:k for k, v in t_word2index.items()} 48 | 49 | def build_tensor(filename, numrecs, word2index, maxlen, make_categorical = False, num_classes = 0): 50 | data = np.empty((numrecs, ), dtype=list) 51 | fin = open(filename, "rb") 52 | i = 0 53 | for line in fin: 54 | wids = [] 55 | for word in line.strip().lower().split(): 56 | if word2index.has_key(word): 57 | wids.append(word2index[word]) 58 | else: 59 | wids.append(word2index["UNK"]) 60 | if make_categorical: 61 | data[i] = np_utils.to_categorical(wids, num_classes=num_classes) 62 | else: 63 | data[i] = wids 64 | i += 1 65 | fin.close() 66 | pdata = sequence.pad_sequences(data, maxlen=maxlen) 67 | return pdata 68 | 69 | X = build_tensor(os.path.join(DATA_DIR, "treebank_sents.txt"), s_numrecs, s_word2index, MAX_SEQLEN) 70 | Y = build_tensor(os.path.join(DATA_DIR, "treebank_poss.txt"), t_numrecs, t_word2index, MAX_SEQLEN, True, t_vocabsize) 71 | 72 | Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.2, random_state=42) 73 | 74 | EMBED_SIZE = 128 75 | HIDDEN_SIZE = 64 76 | BATCH_SIZE = 32 77 | NUM_EPOCHS = 1 78 | 79 | model = Sequential() 80 | model.add(Embedding(s_vocabsize, EMBED_SIZE, input_length=MAX_SEQLEN)) 81 | model.add(SpatialDropout1D(Dropout(0.2))) 82 | model.add(LSTM(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2)) 83 | model.add(RepeatVector(MAX_SEQLEN)) 84 | model.add(LSTM(HIDDEN_SIZE, return_sequences=True)) 85 | model.add(TimeDistributed(Dense(t_vocabsize))) 86 | model.add(Activation("softmax")) 87 | 88 | model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) 89 | 90 | model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_data=[Xtest, Ytest]) 91 | 92 | score, acc = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE) 93 | print("Test score: %.3f, accuracy: %.3f" % (score, acc)) 94 | -------------------------------------------------------------------------------- /Chapter06/umich_sentiment_lstm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.layers.core import Activation, Dense, Dropout, SpatialDropout1D 3 | from keras.layers.embeddings import Embedding 4 | from keras.layers.recurrent import LSTM 5 | from keras.models import Sequential 6 | from keras.preprocessing import sequence 7 | from sklearn.model_selection import train_test_split 8 | import collections 9 | import matplotlib.pyplot as plt 10 | import nltk 11 | import numpy as np 12 | import os 13 | 14 | DATA_DIR = "../data" 15 | 16 | MAX_FEATURES = 2000 17 | MAX_SENTENCE_LENGTH = 40 18 | 19 | EMBEDDING_SIZE = 128 20 | HIDDEN_LAYER_SIZE = 64 21 | BATCH_SIZE = 32 22 | NUM_EPOCHS = 10 23 | 24 | # Read training data and generate vocabulary 25 | maxlen = 0 26 | word_freqs = collections.Counter() 27 | num_recs = 0 28 | ftrain = open(os.path.join(DATA_DIR, "umich-sentiment-train.txt"), 'rb') 29 | for line in ftrain: 30 | label, sentence = line.strip().split("\t") 31 | words = nltk.word_tokenize(sentence.decode("ascii", "ignore").lower()) 32 | if len(words) > maxlen: 33 | maxlen = len(words) 34 | for word in words: 35 | word_freqs[word] += 1 36 | num_recs += 1 37 | ftrain.close() 38 | 39 | ## Get some information about our corpus 40 | #print maxlen # 42 41 | #print len(word_freqs) # 2313 42 | 43 | # 1 is UNK, 0 is PAD 44 | # We take MAX_FEATURES-1 featurs to accound for PAD 45 | vocab_size = min(MAX_FEATURES, len(word_freqs)) + 2 46 | word2index = {x[0]: i+2 for i, x in 47 | enumerate(word_freqs.most_common(MAX_FEATURES))} 48 | word2index["PAD"] = 0 49 | word2index["UNK"] = 1 50 | index2word = {v:k for k, v in word2index.items()} 51 | 52 | # convert sentences to sequences 53 | X = np.empty((num_recs, ), dtype=list) 54 | y = np.zeros((num_recs, )) 55 | i = 0 56 | ftrain = open(os.path.join(DATA_DIR, "umich-sentiment-train.txt"), 'rb') 57 | for line in ftrain: 58 | label, sentence = line.strip().split("\t") 59 | words = nltk.word_tokenize(sentence.decode("ascii", "ignore").lower()) 60 | seqs = [] 61 | for word in words: 62 | if word2index.has_key(word): 63 | seqs.append(word2index[word]) 64 | else: 65 | seqs.append(word2index["UNK"]) 66 | X[i] = seqs 67 | y[i] = int(label) 68 | i += 1 69 | ftrain.close() 70 | 71 | # Pad the sequences (left padded with zeros) 72 | X = sequence.pad_sequences(X, maxlen=MAX_SENTENCE_LENGTH) 73 | 74 | # Split input into training and test 75 | Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.2, 76 | random_state=42) 77 | print(Xtrain.shape, Xtest.shape, ytrain.shape, ytest.shape) 78 | 79 | # Build model 80 | model = Sequential() 81 | model.add(Embedding(vocab_size, EMBEDDING_SIZE, 82 | input_length=MAX_SENTENCE_LENGTH)) 83 | model.add(SpatialDropout1D(Dropout(0.2))) 84 | model.add(LSTM(HIDDEN_LAYER_SIZE, dropout=0.2, recurrent_dropout=0.2)) 85 | model.add(Dense(1)) 86 | model.add(Activation("sigmoid")) 87 | 88 | model.compile(loss="binary_crossentropy", optimizer="adam", 89 | metrics=["accuracy"]) 90 | 91 | history = model.fit(Xtrain, ytrain, batch_size=BATCH_SIZE, 92 | epochs=NUM_EPOCHS, 93 | validation_data=(Xtest, ytest)) 94 | 95 | # plot loss and accuracy 96 | plt.subplot(211) 97 | plt.title("Accuracy") 98 | plt.plot(history.history["acc"], color="g", label="Train") 99 | plt.plot(history.history["val_acc"], color="b", label="Validation") 100 | plt.legend(loc="best") 101 | 102 | plt.subplot(212) 103 | plt.title("Loss") 104 | plt.plot(history.history["loss"], color="g", label="Train") 105 | plt.plot(history.history["val_loss"], color="b", label="Validation") 106 | plt.legend(loc="best") 107 | 108 | plt.tight_layout() 109 | plt.show() 110 | 111 | # evaluate 112 | score, acc = model.evaluate(Xtest, ytest, batch_size=BATCH_SIZE) 113 | print("Test score: %.3f, accuracy: %.3f" % (score, acc)) 114 | 115 | for i in range(5): 116 | idx = np.random.randint(len(Xtest)) 117 | xtest = Xtest[idx].reshape(1,40) 118 | ylabel = ytest[idx] 119 | ypred = model.predict(xtest)[0][0] 120 | sent = " ".join([index2word[x] for x in xtest[0].tolist() if x != 0]) 121 | print("%.0f\t%d\t%s" % (ypred, ylabel, sent)) 122 | -------------------------------------------------------------------------------- /Chapter07/air-quality-regression.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.layers import Input 4 | from keras.layers.core import Dense 5 | from keras.models import Model 6 | from sklearn.preprocessing import StandardScaler 7 | import numpy as np 8 | import os 9 | import pandas as pd 10 | import matplotlib.pyplot as plt 11 | 12 | DATA_DIR = "data" 13 | 14 | AIRQUALITY_FILE = os.path.join(DATA_DIR, "AirQualityUCI.csv") 15 | 16 | aqdf = pd.read_csv(AIRQUALITY_FILE, sep=";", decimal=",", header=0) 17 | # remove first and last 2 cols 18 | del aqdf["Date"] 19 | del aqdf["Time"] 20 | del aqdf["Unnamed: 15"] 21 | del aqdf["Unnamed: 16"] 22 | # fill NaNs in each column with the mean value 23 | aqdf = aqdf.fillna(aqdf.mean()) 24 | # aqdf.head() 25 | 26 | Xorig = aqdf.as_matrix() 27 | Xorig.shape 28 | 29 | scaler = StandardScaler() 30 | Xscaled = scaler.fit_transform(Xorig) 31 | # store these off for predictions with unseen data 32 | Xmeans = scaler.mean_ 33 | Xstds = scaler.scale_ 34 | 35 | y = Xscaled[:, 3] 36 | X = np.delete(Xscaled, 3, axis=1) 37 | 38 | print(X.shape, y.shape, Xmeans.shape, Xstds.shape) 39 | 40 | train_size = int(0.7 * X.shape[0]) 41 | Xtrain, Xtest, ytrain, ytest = \ 42 | X[0:train_size], X[train_size:], y[0:train_size], y[train_size:] 43 | print(Xtrain.shape, Xtest.shape, ytrain.shape, ytest.shape) 44 | 45 | # define the network 46 | readings = Input(shape=(12,)) 47 | x = Dense(8, activation="relu", kernel_initializer="glorot_uniform")(readings) 48 | benzene = Dense(1, kernel_initializer="glorot_uniform")(x) 49 | 50 | model = Model(inputs=[readings], outputs=[benzene]) 51 | model.compile(loss="mse", optimizer="adam") 52 | 53 | # train network 54 | NUM_EPOCHS = 20 55 | BATCH_SIZE = 10 56 | 57 | history = model.fit(Xtrain, ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, 58 | validation_split=0.2) 59 | 60 | # show some predictions 61 | ytest_ = model.predict(Xtest).flatten() 62 | for i in range(10): 63 | label = (ytest[i] * Xstds[3]) + Xmeans[3] 64 | prediction = (ytest_[i] * Xstds[3]) + Xmeans[3] 65 | print("Benzene Conc. expected: {:.3f}, " 66 | "predicted: {:.3f}".format(label, prediction)) 67 | 68 | # plot all predictions 69 | plt.plot(np.arange(ytest.shape[0]), (ytest * Xstds[3]) / Xmeans[3], 70 | color="b", label="actual") 71 | plt.plot(np.arange(ytest_.shape[0]), (ytest_ * Xstds[3]) / Xmeans[3], 72 | color="r", alpha=0.5, label="predicted") 73 | plt.xlabel("time") 74 | plt.ylabel("C6H6 concentrations") 75 | plt.legend(loc="best") 76 | plt.show() 77 | -------------------------------------------------------------------------------- /Chapter07/data_download_question_answer.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Available diskspace in $(pwd):" 3 | df -h . 4 | read -p "This will download the Text8 corputs 15 MB and extract it 149 MB are you sure (y/n): " -n 1 -r 5 | echo 6 | if [[ $REPLY =~ ^[Yy]$ ]] 7 | then 8 | wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz 9 | tar -xvzf tasks_1-20_v1-2.tar.gz 10 | fi -------------------------------------------------------------------------------- /Chapter07/deep-dream.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras import backend as K 4 | from keras.applications import vgg16 5 | from keras.layers import Input 6 | import numpy as np 7 | import os 8 | import matplotlib.pyplot as plt 9 | 10 | 11 | def preprocess(img): 12 | img4d = img.copy() 13 | img4d = img4d.astype("float64") 14 | if K.image_dim_ordering() == "th": 15 | # (H, W, C) -> (C, H, W) 16 | img4d = img4d.transpose((2, 0, 1)) 17 | img4d = np.expand_dims(img4d, axis=0) 18 | img4d = vgg16.preprocess_input(img4d) 19 | return img4d 20 | 21 | 22 | def deprocess(img4d): 23 | img = img4d.copy() 24 | if K.image_dim_ordering() == "th": 25 | # (B, C, H, W) 26 | img = img.reshape((img4d.shape[1], img4d.shape[2], img4d.shape[3])) 27 | # (C, H, W) -> (H, W, C) 28 | img = img.transpose((1, 2, 0)) 29 | else: 30 | # (B, H, W, C) 31 | img = img.reshape((img4d.shape[1], img4d.shape[2], img4d.shape[3])) 32 | img[:, :, 0] += 103.939 33 | img[:, :, 1] += 116.779 34 | img[:, :, 2] += 123.68 35 | # BGR -> RGB 36 | img = img[:, :, ::-1] 37 | img = np.clip(img, 0, 255).astype("uint8") 38 | return img 39 | 40 | 41 | # ########################## main ########################### 42 | 43 | DATA_DIR = "data" 44 | 45 | IMAGE_FILE = os.path.join(DATA_DIR, "cat.jpg") 46 | 47 | img = plt.imread(IMAGE_FILE) 48 | plt.imshow(img) 49 | img_copy = img.copy() 50 | print("Original image shape:", img.shape) 51 | image_shape_widh = img.shape[0] 52 | image_shape_height = img.shape[1] 53 | p_img = preprocess(img_copy) 54 | print("After preprocess:", p_img.shape) 55 | d_img = deprocess(p_img) 56 | print("After deprocess:", d_img.shape) 57 | plt.imshow(d_img) 58 | plt.show() 59 | 60 | # load pretrained VGG-16 61 | batch_shape = p_img.shape 62 | dream = Input(batch_shape=batch_shape) 63 | model = vgg16.VGG16(input_tensor=dream, weights="imagenet", include_top=False) 64 | 65 | # create layer name to layer dictionary 66 | layer_dict = {layer.name: layer for layer in model.layers} 67 | print(layer_dict) 68 | # layer_dict 69 | 70 | # visualize gradients at pooling layers 71 | num_pool_layers = 5 72 | lr = 0.01 73 | fig, axes = plt.subplots(1, num_pool_layers, figsize=(20, 10)) 74 | for i in range(num_pool_layers): 75 | layer_name = "block{:d}_pool".format(i + 1) 76 | layer_output = layer_dict[layer_name].output 77 | loss = K.mean(layer_output) 78 | grads = K.gradients(loss, dream)[0] 79 | grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) * lr 80 | f = K.function([dream], [loss, grads]) 81 | img_value = p_img.copy() 82 | loss_value, grads_value = f([img_value]) 83 | axes[i].set_title(layer_name) 84 | axes[i].imshow(deprocess(grads_value)) 85 | 86 | plt.tight_layout() 87 | plt.show() 88 | 89 | # deep dreaming 90 | first_layer = model.layers[-1] 91 | input_img = first_layer.input 92 | print(first_layer.name, first_layer.output_shape) 93 | 94 | num_pool_layers = 5 95 | num_iters_per_layer = 3 96 | step = 2000 97 | 98 | for i in range(num_pool_layers): 99 | layer_name = "block{:d}_pool".format(i+1) 100 | print("Pooling Layer: {:s}".format(layer_name)) 101 | layer_output = layer_dict[layer_name].output 102 | # loss function 103 | loss = K.mean(layer_output) 104 | # gradient 105 | grads = K.gradients(loss, dream)[0] 106 | grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) 107 | # optimizer 108 | f = K.function([dream], [loss, grads]) 109 | img_value = p_img.copy() 110 | fig, axes = plt.subplots(1, num_iters_per_layer, figsize=(20, 10)) 111 | for it in range(num_iters_per_layer): 112 | loss_value, grads_value = f([img_value]) 113 | img_value += grads_value * step 114 | axes[it].imshow(deprocess(img_value)) 115 | plt.show() 116 | 117 | # try to dream structure out of random noise 118 | img_noise = np.random.randint(100, 150, 119 | size=(image_shape_widh, image_shape_height, 3), 120 | dtype=np.uint8) 121 | print(img_noise.shape) 122 | plt.imshow(img_noise) 123 | plt.show() 124 | 125 | p_img = img_noise.copy() 126 | p_img = preprocess(p_img) 127 | 128 | num_pool_layers = 5 129 | num_iters_per_layer = 3 130 | step = 2000 131 | 132 | for i in range(num_pool_layers): 133 | layer_name = "block{:d}_pool".format(i+1) 134 | print("Pooling Layer: {:s}".format(layer_name)) 135 | layer_output = layer_dict[layer_name].output 136 | # loss function 137 | loss = K.mean(layer_output) 138 | # loss = layer_output[:,:,:,24] 139 | # gradient 140 | grads = K.gradients(loss, dream)[0] 141 | grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) 142 | # optimizer 143 | f = K.function([dream], [loss, grads]) 144 | img_value = p_img.copy() 145 | fig, axes = plt.subplots(1, num_iters_per_layer, figsize=(20, 10)) 146 | for it in range(num_iters_per_layer): 147 | loss_value, grads_value = f([img_value]) 148 | img_value += grads_value * step 149 | axes[it].imshow(deprocess(img_value)) 150 | plt.show() 151 | 152 | # random noise with specific objective. Only do gradient ascent on 153 | # specific label and see that this pattern shows up 154 | num_pool_layers = 5 155 | num_iters_per_layer = 3 156 | step = 2000 157 | 158 | for i in range(num_pool_layers): 159 | layer_name = "block{:d}_pool".format(i+1) 160 | print("Pooling Layer: {:s}".format(layer_name)) 161 | layer_output = layer_dict[layer_name].output 162 | # loss function 163 | loss = layer_output[:, :, :, 24] 164 | # gradient 165 | grads = K.gradients(loss, dream)[0] 166 | grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) 167 | # optimizer 168 | f = K.function([dream], [loss, grads]) 169 | img_value = p_img.copy() 170 | fig, axes = plt.subplots(1, num_iters_per_layer, figsize=(20, 10)) 171 | for it in range(num_iters_per_layer): 172 | loss_value, grads_value = f([img_value]) 173 | img_value += grads_value * step 174 | axes[it].imshow(deprocess(img_value)) 175 | plt.show() 176 | -------------------------------------------------------------------------------- /Chapter07/func-api-func.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.layers import Input 3 | from keras.layers.core import Dense 4 | from keras.models import Model 5 | from keras.layers.core import Activation 6 | 7 | inputs = Input(shape=(784,)) 8 | 9 | x = Dense(32)(inputs) 10 | x = Activation("sigmoid")(x) 11 | x = Dense(10)(x) 12 | predictions = Activation("softmax")(x) 13 | 14 | model = Model(inputs=inputs, outputs=predictions) 15 | 16 | model.compile(loss="categorical_crossentropy", optimizer="adam") 17 | -------------------------------------------------------------------------------- /Chapter07/func-api-multi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.layers import Input 3 | from keras.layers.core import Dense 4 | from keras.models import Model 5 | from keras.layers.core import Activation 6 | 7 | input1 = Input(shape=(784,)) 8 | input2 = Input(shape=(784,)) 9 | 10 | x = Dense(32)(input1) 11 | x = Activation("sigmoid")(x) 12 | x = Dense(10)(x) 13 | output1 = Activation("softmax")(x) 14 | 15 | x = Dense(32)(input2) 16 | x = Activation("sigmoid")(x) 17 | x = Dense(10)(x) 18 | output2 = Activation("softmax")(x) 19 | 20 | model = Model(inputs=[input1, input2], outputs=[output1, output2]) 21 | 22 | model.compile(loss="categorical_crossentropy", optimizer="adam") 23 | -------------------------------------------------------------------------------- /Chapter07/func-api-seq.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | 5 | model = Sequential([ 6 | Dense(32, input_dim=784), 7 | Activation("sigmoid"), 8 | Dense(10), 9 | Activation("softmax"), 10 | ]) 11 | model.compile(loss="categorical_crossentropy", optimizer="adam") 12 | -------------------------------------------------------------------------------- /Chapter07/make_tensorboard.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import absolute_import 3 | from __future__ import unicode_literals 4 | from time import gmtime, strftime 5 | from keras.callbacks import TensorBoard 6 | import os 7 | 8 | 9 | def make_tensorboard(set_dir_name=''): 10 | tictoc = strftime("%a_%d_%b_%Y_%H_%M_%S", gmtime()) 11 | directory_name = tictoc 12 | log_dir = set_dir_name + '_' + directory_name 13 | os.mkdir(log_dir) 14 | tensorboard = TensorBoard(log_dir=log_dir) 15 | return tensorboard 16 | -------------------------------------------------------------------------------- /Chapter07/mem-network.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.layers import Input 4 | from keras.layers.core import Activation, Dense, Dropout, Permute 5 | from keras.layers.embeddings import Embedding 6 | from keras.layers.merge import add, concatenate, dot 7 | from keras.layers.recurrent import LSTM 8 | from keras.models import Model 9 | from keras.preprocessing.sequence import pad_sequences 10 | from keras.utils import np_utils 11 | import collections 12 | import itertools 13 | import nltk 14 | import numpy as np 15 | import os 16 | from make_tensorboard import make_tensorboard 17 | 18 | 19 | def get_data(infile): 20 | stories, questions, answers = [], [], [] 21 | story_text = [] 22 | fin = open(TRAIN_FILE, "rb") 23 | for line in fin: 24 | line = line.decode("utf-8").strip() 25 | lno, text = line.split(" ", 1) 26 | if "\t" in text: 27 | question, answer, _ = text.split("\t") 28 | stories.append(story_text) 29 | questions.append(question) 30 | answers.append(answer) 31 | story_text = [] 32 | else: 33 | story_text.append(text) 34 | fin.close() 35 | return stories, questions, answers 36 | 37 | 38 | def build_vocab(train_data, test_data): 39 | counter = collections.Counter() 40 | for stories, questions, answers in [train_data, test_data]: 41 | for story in stories: 42 | for sent in story: 43 | for word in nltk.word_tokenize(sent): 44 | counter[word.lower()] += 1 45 | for question in questions: 46 | for word in nltk.word_tokenize(question): 47 | counter[word.lower()] += 1 48 | for answer in answers: 49 | for word in nltk.word_tokenize(answer): 50 | counter[word.lower()] += 1 51 | # no OOV here because there are not too many words in dataset 52 | word2idx = {w: (i+1) for i, (w, _) in enumerate(counter.most_common())} 53 | word2idx["PAD"] = 0 54 | idx2word = {v: k for k, v in word2idx.items()} 55 | return word2idx, idx2word 56 | 57 | 58 | def get_maxlens(train_data, test_data): 59 | story_maxlen, question_maxlen = 0, 0 60 | for stories, questions, _ in [train_data, test_data]: 61 | for story in stories: 62 | story_len = 0 63 | for sent in story: 64 | swords = nltk.word_tokenize(sent) 65 | story_len += len(swords) 66 | if story_len > story_maxlen: 67 | story_maxlen = story_len 68 | for question in questions: 69 | question_len = len(nltk.word_tokenize(question)) 70 | if question_len > question_maxlen: 71 | question_maxlen = question_len 72 | return story_maxlen, question_maxlen 73 | 74 | 75 | def vectorize(data, word2idx, story_maxlen, question_maxlen): 76 | Xs, Xq, Y = [], [], [] 77 | stories, questions, answers = data 78 | for story, question, answer in zip(stories, questions, answers): 79 | xs = [[word2idx[w.lower()] for w in nltk.word_tokenize(s)] 80 | for s in story] 81 | xs = list(itertools.chain.from_iterable(xs)) 82 | xq = [word2idx[w.lower()] for w in nltk.word_tokenize(question)] 83 | Xs.append(xs) 84 | Xq.append(xq) 85 | Y.append(word2idx[answer.lower()]) 86 | pad_sequences_Xs = pad_sequences(Xs, maxlen=story_maxlen) 87 | pad_sequences_Xq = pad_sequences(Xq, maxlen=question_maxlen) 88 | categorical_Y = np_utils.to_categorical(Y, num_classes=len(word2idx)) 89 | 90 | return pad_sequences_Xs, pad_sequences_Xq, categorical_Y 91 | 92 | 93 | DATA_DIR = "data/tasks_1-20_v1-2/en" 94 | 95 | TRAIN_FILE = os.path.join(DATA_DIR, "qa1_single-supporting-fact_train.txt") 96 | TEST_FILE = os.path.join(DATA_DIR, "qa1_single-supporting-fact_test.txt") 97 | 98 | # get the data 99 | data_train = get_data(TRAIN_FILE) 100 | data_test = get_data(TEST_FILE) 101 | 102 | print(len(data_train[0]), len(data_test[0])) 103 | 104 | # build vocabulary from all the data 105 | word2idx, idx2word = build_vocab(data_train, data_test) 106 | 107 | vocab_size = len(word2idx) 108 | print("vocab size: {:d}".format(len(word2idx))) 109 | 110 | # compute max sequence length for each entity 111 | story_maxlen, question_maxlen = get_maxlens(data_train, data_test) 112 | print("story maxlen: {:d}, " 113 | "question maxlen: {:d}".format(story_maxlen, question_maxlen)) 114 | 115 | # vectorize the data 116 | Xstrain, Xqtrain, Ytrain = \ 117 | vectorize(data_train, word2idx, story_maxlen, question_maxlen) 118 | Xstest, Xqtest, Ytest = \ 119 | vectorize(data_test, word2idx, story_maxlen, question_maxlen) 120 | 121 | print(Xstrain.shape, Xqtrain.shape, Ytrain.shape, 122 | Xstest.shape, Xqtest.shape, Ytest.shape) 123 | 124 | # define network 125 | EMBEDDING_SIZE = 64 126 | LATENT_SIZE = 32 127 | BATCH_SIZE = 32 128 | NUM_EPOCHS = 400 129 | 130 | # inputs 131 | story_input = Input(shape=(story_maxlen,)) 132 | question_input = Input(shape=(question_maxlen,)) 133 | 134 | # story encoder memory 135 | story_encoder = Embedding(input_dim=vocab_size, 136 | output_dim=EMBEDDING_SIZE, 137 | input_length=story_maxlen)(story_input) 138 | story_encoder = Dropout(0.3)(story_encoder) 139 | 140 | # question encoder 141 | question_encoder = Embedding(input_dim=vocab_size, 142 | output_dim=EMBEDDING_SIZE, 143 | input_length=question_maxlen)(question_input) 144 | question_encoder = Dropout(0.3)(question_encoder) 145 | 146 | # match between story and question 147 | match = dot([story_encoder, question_encoder], axes=[2, 2]) 148 | 149 | # encode story into vector space of question 150 | story_encoder_c = Embedding(input_dim=vocab_size, 151 | output_dim=question_maxlen, 152 | input_length=story_maxlen)(story_input) 153 | story_encoder_c = Dropout(0.3)(story_encoder_c) 154 | 155 | # combine match and story vectors 156 | response = add([match, story_encoder_c]) 157 | response = Permute((2, 1))(response) 158 | 159 | # combine response and question vectors 160 | answer = concatenate([response, question_encoder], axis=-1) 161 | answer = LSTM(LATENT_SIZE)(answer) 162 | answer = Dropout(0.3)(answer) 163 | answer = Dense(vocab_size)(answer) 164 | output = Activation("softmax")(answer) 165 | 166 | model = Model(inputs=[story_input, question_input], outputs=output) 167 | model.compile(optimizer="rmsprop", loss="categorical_crossentropy", 168 | metrics=["accuracy"]) 169 | 170 | tensorboard = make_tensorboard(set_dir_name='mem-network') 171 | 172 | # train model 173 | history = model.fit([Xstrain, Xqtrain], [Ytrain], batch_size=BATCH_SIZE, 174 | epochs=NUM_EPOCHS, 175 | callbacks=[tensorboard], 176 | validation_data=([Xstest, Xqtest], [Ytest])) 177 | 178 | # labels 179 | ytest = np.argmax(Ytest, axis=1) 180 | 181 | # get predictions 182 | Ytest_ = model.predict([Xstest, Xqtest]) 183 | ytest_ = np.argmax(Ytest_, axis=1) 184 | 185 | NUM_DISPLAY = 10 186 | 187 | for i in range(NUM_DISPLAY): 188 | story = " ".join([idx2word[x] for x in Xstest[i].tolist() if x != 0]) 189 | question = " ".join([idx2word[x] for x in Xqtest[i].tolist()]) 190 | label = idx2word[ytest[i]] 191 | prediction = idx2word[ytest_[i]] 192 | print(story, question, label, prediction) 193 | -------------------------------------------------------------------------------- /Chapter07/sent-thoughts-parse.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copied from: Out of core classification of Text Documents 3 | # from the scikit-learn documentation. 4 | # http://scikit-learn.org/stable/auto_examples/applications/plot_out_of_core_classification.html 5 | # 6 | from __future__ import division, print_function 7 | from sklearn.externals.six.moves import html_parser 8 | from glob import glob 9 | import collections 10 | import nltk 11 | import os 12 | import re 13 | 14 | 15 | class ReutersParser(html_parser.HTMLParser): 16 | """ Utility class to parse a SGML file and yield documents one at 17 | a time. 18 | """ 19 | def __init__(self, encoding='latin-1'): 20 | html_parser.HTMLParser.__init__(self) 21 | self._reset() 22 | self.encoding = encoding 23 | 24 | def handle_starttag(self, tag, attrs): 25 | method = 'start_' + tag 26 | getattr(self, method, lambda x: None)(attrs) 27 | 28 | def handle_endtag(self, tag): 29 | method = 'end_' + tag 30 | getattr(self, method, lambda: None)() 31 | 32 | def _reset(self): 33 | self.in_title = 0 34 | self.in_body = 0 35 | self.in_topics = 0 36 | self.in_topic_d = 0 37 | self.title = "" 38 | self.body = "" 39 | self.topics = [] 40 | self.topic_d = "" 41 | 42 | def parse(self, fd): 43 | self.docs = [] 44 | for chunk in fd: 45 | self.feed(chunk.decode(self.encoding)) 46 | for doc in self.docs: 47 | yield doc 48 | self.docs = [] 49 | self.close() 50 | 51 | def handle_data(self, data): 52 | if self.in_body: 53 | self.body += data 54 | elif self.in_title: 55 | self.title += data 56 | elif self.in_topic_d: 57 | self.topic_d += data 58 | 59 | def start_reuters(self, attributes): 60 | pass 61 | 62 | def end_reuters(self): 63 | self.body = re.sub(r'\s+', r' ', self.body) 64 | self.docs.append({'title': self.title, 65 | 'body': self.body, 66 | 'topics': self.topics}) 67 | self._reset() 68 | 69 | def start_title(self, attributes): 70 | self.in_title = 1 71 | 72 | def end_title(self): 73 | self.in_title = 0 74 | 75 | def start_body(self, attributes): 76 | self.in_body = 1 77 | 78 | def end_body(self): 79 | self.in_body = 0 80 | 81 | def start_topics(self, attributes): 82 | self.in_topics = 1 83 | 84 | def end_topics(self): 85 | self.in_topics = 0 86 | 87 | def start_d(self, attributes): 88 | self.in_topic_d = 1 89 | 90 | def end_d(self): 91 | self.in_topic_d = 0 92 | self.topics.append(self.topic_d) 93 | self.topic_d = "" 94 | 95 | 96 | def stream_reuters_documents(reuters_dir): 97 | """ Iterate over documents of the Reuters dataset. 98 | 99 | The Reuters archive will automatically be downloaded and uncompressed if 100 | the `data_path` directory does not exist. 101 | 102 | Documents are represented as dictionaries with 'body' (str), 103 | 'title' (str), 'topics' (list(str)) keys. 104 | 105 | """ 106 | parser = ReutersParser() 107 | for filename in glob(os.path.join(reuters_dir, "*.sgm")): 108 | for doc in parser.parse(open(filename, 'rb')): 109 | yield doc 110 | 111 | 112 | def maybe_build_vocab(reuters_dir, vocab_file): 113 | vocab = collections.defaultdict(int) 114 | if os.path.exists(vocab_file): 115 | fvoc = open(vocab_file, "rb") 116 | for line in fvoc: 117 | word, idx = line.strip().split("\t") 118 | vocab[word] = int(idx) 119 | fvoc.close() 120 | else: 121 | counter = collections.Counter() 122 | num_docs_read = 0 123 | for doc in stream_reuters_documents(reuters_dir): 124 | if num_docs_read % 100 == 0: 125 | print("building vocab from {:d} " 126 | "docs".format(num_docs_read)) 127 | topics = doc["topics"] 128 | if len(topics) == 0: 129 | continue 130 | title = doc["title"] 131 | body = doc["body"] 132 | title_body = ". ".join([title, body]).lower() 133 | for sent in nltk.sent_tokenize(title_body): 134 | for word in nltk.word_tokenize(sent): 135 | counter[word] += 1 136 | for i, c in enumerate(counter.most_common(VOCAB_SIZE)): 137 | vocab[c[0]] = i + 1 138 | num_docs_read += 1 139 | print("vocab built from {:d} " 140 | "docs, complete".format(num_docs_read)) 141 | fvoc = open(vocab_file, "w") 142 | for k in vocab.keys(): 143 | fvoc.write("{:s}\t{:d}\n".format(k, vocab[k])) 144 | fvoc.close() 145 | return vocab 146 | 147 | 148 | def build_numeric_text(vocab, text): 149 | wids = [] 150 | for sent in nltk.sent_tokenize(text): 151 | for word in nltk.word_tokenize(sent): 152 | wids.append(vocab[word]) 153 | return ",".join([str(x) for x in wids]) 154 | 155 | 156 | # #################### main ###################### 157 | 158 | DATA_DIR = "data" 159 | REUTERS_DIR = os.path.join(DATA_DIR, "reuters-21578") 160 | VOCAB_FILE = os.path.join(DATA_DIR, "vocab.txt") 161 | VOCAB_SIZE = 5000 162 | 163 | vocab = maybe_build_vocab(REUTERS_DIR, VOCAB_FILE) 164 | 165 | ftext = open(os.path.join(DATA_DIR, "text.tsv"), "w") 166 | ftags = open(os.path.join(DATA_DIR, "tags.tsv"), "w") 167 | num_read = 0 168 | for doc in stream_reuters_documents(REUTERS_DIR): 169 | # periodic heartbeat report 170 | if num_read % 100 == 0: 171 | print("building features from {:d} docs".format(num_read)) 172 | # skip docs without specified topic 173 | topics = doc["topics"] 174 | if len(topics) == 0: 175 | continue 176 | title = doc["title"] 177 | body = doc["body"] 178 | num_read += 1 179 | # concatenate title and body and convert to list of word indexes 180 | title_body = ". ".join([title, body]).lower() 181 | title_body = re.sub("\n", "", title_body) 182 | title_body = title_body.encode("utf8").decode("ascii", "ignore") 183 | ftext.write("{:d}\t{:s}\n".format(num_read, title_body)) 184 | ftags.write("{:d}\t{:s}\n".format(num_read, ",".join(topics))) 185 | 186 | print("features built from {:d} docs, complete".format(num_read)) 187 | ftext.close() 188 | ftags.close() 189 | -------------------------------------------------------------------------------- /Chapter07/sent-thoughts-rnn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from sklearn.model_selection import train_test_split 4 | from keras.callbacks import ModelCheckpoint 5 | from keras.layers import Input 6 | from keras.layers.core import RepeatVector 7 | from keras.layers.recurrent import LSTM 8 | from keras.layers.wrappers import Bidirectional 9 | from keras.models import Model 10 | from keras.preprocessing import sequence 11 | import collections 12 | from make_tensorboard import make_tensorboard 13 | import nltk 14 | import numpy as np 15 | import os 16 | import matplotlib.pyplot as plt 17 | 18 | 19 | def lookup_word2id(word): 20 | try: 21 | return word2id[word] 22 | except KeyError: 23 | return word2id["UNK"] 24 | 25 | 26 | def load_glove_vectors(glove_file, word2id, embed_size): 27 | embedding = np.zeros((len(word2id), embed_size)) 28 | fglove = open(glove_file, "rb") 29 | for line in fglove: 30 | cols = line.strip().split() 31 | word = cols[0].decode('utf-8') 32 | if embed_size == 0: 33 | embed_size = len(cols) - 1 34 | if word in word2id: 35 | vec = np.array([float(v) for v in cols[1:]]) 36 | embedding[lookup_word2id(word)] = vec 37 | embedding[word2id["PAD"]] = np.zeros((embed_size)) 38 | embedding[word2id["UNK"]] = np.random.uniform(-1, 1, embed_size) 39 | return embedding 40 | 41 | 42 | def sentence_generator(X, embeddings, batch_size): 43 | while True: 44 | # loop once per epoch 45 | num_recs = X.shape[0] 46 | indices = np.random.permutation(np.arange(num_recs)) 47 | num_batches = num_recs // batch_size 48 | for bid in range(num_batches): 49 | sids = indices[bid * batch_size: (bid + 1) * batch_size] 50 | Xbatch = embeddings[X[sids, :]] 51 | yield Xbatch, Xbatch 52 | 53 | 54 | def compute_cosine_similarity(x, y): 55 | return np.dot(x, y) / (np.linalg.norm(x, 2) * np.linalg.norm(y, 2)) 56 | 57 | 58 | # ############################## msin ############################### 59 | 60 | DATA_DIR = "data" 61 | 62 | # parsing sentences and building vocabulary 63 | word_freqs = collections.Counter() 64 | ftext = open(os.path.join(DATA_DIR, "text.tsv"), "r") 65 | sents = [] 66 | for line in ftext: 67 | docid, text = line.strip().split("\t") 68 | for sent in nltk.sent_tokenize(text): 69 | for word in nltk.word_tokenize(sent): 70 | word = word.lower() 71 | word_freqs[word] += 1 72 | sents.append(sent) 73 | ftext.close() 74 | 75 | VOCAB_SIZE = 5000 76 | EMBED_SIZE = 100 77 | LATENT_SIZE = 512 78 | SEQUENCE_LEN = 50 79 | 80 | BATCH_SIZE = 64 81 | NUM_EPOCHS = 10 82 | 83 | # word2id = collections.defaultdict(lambda: 1) 84 | word2id = {} 85 | word2id["PAD"] = 0 86 | word2id["UNK"] = 1 87 | for v, (k, _) in enumerate(word_freqs.most_common(VOCAB_SIZE - 2)): 88 | word2id[k] = v + 2 89 | id2word = {v: k for k, v in word2id.items()} 90 | 91 | print("vocabulary sizes:", len(word2id), len(id2word)) 92 | 93 | sent_wids = [[lookup_word2id(w) for w in s.split()] for s in sents] 94 | sent_wids = sequence.pad_sequences(sent_wids, SEQUENCE_LEN) 95 | 96 | # load glove vectors into weight matrix 97 | embeddings = load_glove_vectors(os.path.join( 98 | DATA_DIR, "glove.6B.{:d}d.txt".format(EMBED_SIZE)), word2id, EMBED_SIZE) 99 | print(embeddings.shape) 100 | 101 | # split sentences into training and test 102 | train_size = 0.7 103 | Xtrain, Xtest = train_test_split(sent_wids, train_size=train_size) 104 | print("number of sentences: ", len(sent_wids)) 105 | print(Xtrain.shape, Xtest.shape) 106 | 107 | # define training and test generators 108 | train_gen = sentence_generator(Xtrain, embeddings, BATCH_SIZE) 109 | test_gen = sentence_generator(Xtest, embeddings, BATCH_SIZE) 110 | 111 | # define autoencoder network 112 | inputs = Input(shape=(SEQUENCE_LEN, EMBED_SIZE), name="input") 113 | encoded = Bidirectional(LSTM(LATENT_SIZE), merge_mode="sum", 114 | name="encoder_lstm")(inputs) 115 | decoded = RepeatVector(SEQUENCE_LEN, name="repeater")(encoded) 116 | decoded = Bidirectional(LSTM(EMBED_SIZE, return_sequences=True), 117 | merge_mode="sum", 118 | name="decoder_lstm")(decoded) 119 | 120 | autoencoder = Model(inputs, decoded) 121 | 122 | tensorboard = make_tensorboard(set_dir_name='rnn') 123 | 124 | autoencoder.compile(optimizer="sgd", loss="mse") 125 | 126 | # train 127 | num_train_steps = len(Xtrain) // BATCH_SIZE 128 | num_test_steps = len(Xtest) // BATCH_SIZE 129 | checkpoint = ModelCheckpoint( 130 | filepath=os.path.join(DATA_DIR, "sent-thoughts-autoencoder.h5"), 131 | save_best_only=True) 132 | history = autoencoder.fit_generator(train_gen, 133 | steps_per_epoch=num_train_steps, 134 | epochs=NUM_EPOCHS, 135 | validation_data=test_gen, 136 | validation_steps=num_test_steps, 137 | callbacks=[checkpoint, tensorboard]) 138 | 139 | # collect autoencoder predictions for test set 140 | test_inputs, test_labels = next(test_gen) 141 | preds = autoencoder.predict(test_inputs) 142 | 143 | # extract encoder part from autoencoder 144 | encoder = Model(autoencoder.input, 145 | autoencoder.get_layer("encoder_lstm").output) 146 | # encoder.summary() 147 | 148 | # compute difference between vector produced by original and autoencoded 149 | k = 500 150 | cosims = np.zeros((k)) 151 | i = 0 152 | for bid in range(num_test_steps): 153 | xtest, ytest = next(test_gen) 154 | ytest_ = autoencoder.predict(xtest) 155 | Xvec = encoder.predict(xtest) 156 | Yvec = encoder.predict(ytest_) 157 | for rid in range(Xvec.shape[0]): 158 | if i >= k: 159 | break 160 | cosims[i] = compute_cosine_similarity(Xvec[rid], Yvec[rid]) 161 | if i <= 10: 162 | print(cosims[i]) 163 | i += 1 164 | if i >= k: 165 | break 166 | 167 | plt.hist(cosims, bins=10, normed=True) 168 | plt.xlabel("cosine similarity") 169 | plt.ylabel("frequency") 170 | plt.show() 171 | -------------------------------------------------------------------------------- /Chapter07/style-transfer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.applications import vgg16 4 | from keras import backend as K 5 | from scipy.misc import imresize 6 | import numpy as np 7 | import os 8 | import matplotlib.pyplot as plt # noqa 9 | 10 | 11 | def preprocess(img): 12 | img4d = img.copy() 13 | img4d = img4d.astype("float64") 14 | if K.image_dim_ordering() == "th": 15 | # (H, W, C) -> (C, H, W) 16 | img4d = img4d.transpose((2, 0, 1)) 17 | img4d = np.expand_dims(img4d, axis=0) 18 | img4d = vgg16.preprocess_input(img4d) 19 | return img4d 20 | 21 | 22 | def deprocess(img4d): 23 | img = img4d.copy() 24 | if K.image_dim_ordering() == "th": 25 | # (B, C, H, W) 26 | img = img.reshape((img4d.shape[1], img4d.shape[2], img4d.shape[3])) 27 | # (C, H, W) -> (H, W, C) 28 | img = img.transpose((1, 2, 0)) 29 | else: 30 | # (B, H, W, C) 31 | img = img.reshape((img4d.shape[1], img4d.shape[2], img4d.shape[3])) 32 | img[:, :, 0] += 103.939 33 | img[:, :, 1] += 116.779 34 | img[:, :, 2] += 123.68 35 | # BGR -> RGB 36 | img = img[:, :, ::-1] 37 | img = np.clip(img, 0, 255).astype("uint8") 38 | return img 39 | 40 | 41 | def content_loss(content, comb): 42 | return K.sum(K.square(comb - content)) 43 | 44 | 45 | def gram_matrix(x): 46 | if K.image_dim_ordering() == "th": 47 | features = K.batch_flatten(x) 48 | else: 49 | features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1))) 50 | gram = K.dot(features, K.transpose(features)) 51 | return gram 52 | 53 | 54 | def style_loss_per_layer(style, comb): 55 | S = gram_matrix(style) 56 | C = gram_matrix(comb) 57 | channels = 3 58 | size = RESIZED_WH * RESIZED_WH 59 | return K.sum(K.square(S - C)) / (4 * (channels ** 2) * (size ** 2)) 60 | 61 | 62 | def style_loss(): 63 | stl_loss = 0.0 64 | for i in range(NUM_LAYERS): 65 | layer_name = "block{:d}_conv1".format(i+1) 66 | layer_features = layer_dict[layer_name] 67 | style_features = layer_features[1, :, :, :] 68 | comb_features = layer_features[2, :, :, :] 69 | stl_loss += style_loss_per_layer(style_features, comb_features) 70 | return stl_loss / NUM_LAYERS 71 | 72 | 73 | def variation_loss(comb): 74 | if K.image_dim_ordering() == "th": 75 | dx = K.square(comb[:, :, :RESIZED_WH-1, :RESIZED_WH-1] - 76 | comb[:, :, 1:, :RESIZED_WH-1]) 77 | dy = K.square(comb[:, :, :RESIZED_WH-1, :RESIZED_WH-1] - 78 | comb[:, :, :RESIZED_WH-1, 1:]) 79 | else: 80 | dx = K.square(comb[:, :RESIZED_WH-1, :RESIZED_WH-1, :] - 81 | comb[:, 1:, :RESIZED_WH-1, :]) 82 | dy = K.square(comb[:, :RESIZED_WH-1, :RESIZED_WH-1, :] - 83 | comb[:, :RESIZED_WH-1, 1:, :]) 84 | return K.sum(K.pow(dx + dy, 1.25)) 85 | 86 | # ########################### main ############################ 87 | 88 | 89 | DATA_DIR = "data" 90 | 91 | CONTENT_IMAGE_FILE = os.path.join(DATA_DIR, "cat.jpg") 92 | STYLE_IMAGE_FILE = os.path.join(DATA_DIR, "JapaneseBridgeMomentCopy.jpg") 93 | 94 | 95 | RESIZED_WH = 400 96 | 97 | # verify that the content and style images are readable 98 | content_img_value = imresize(plt.imread(CONTENT_IMAGE_FILE), 99 | (RESIZED_WH, RESIZED_WH)) 100 | style_img_value = imresize(plt.imread(STYLE_IMAGE_FILE), 101 | (RESIZED_WH, RESIZED_WH)) 102 | 103 | plt.subplot(121) 104 | plt.title("content") 105 | plt.imshow(content_img_value) 106 | 107 | plt.subplot(122) 108 | plt.title("style") 109 | plt.imshow(style_img_value) 110 | 111 | plt.show() 112 | 113 | # define content and style tensors 114 | content_img = K.variable(preprocess(content_img_value)) 115 | style_img = K.variable(preprocess(style_img_value)) 116 | if K.image_dim_ordering() == "th": 117 | comb_img = K.placeholder((1, 3, RESIZED_WH, RESIZED_WH)) 118 | else: 119 | comb_img = K.placeholder((1, RESIZED_WH, RESIZED_WH, 3)) 120 | 121 | # concatenate images into single input 122 | input_tensor = K.concatenate([content_img, style_img, comb_img], axis=0) 123 | input_tensor.get_shape() 124 | 125 | # download VGG16 model 126 | model = vgg16.VGG16(input_tensor=input_tensor, 127 | weights="imagenet", include_top=False) 128 | 129 | layer_dict = {layer.name: layer.output for layer in model.layers} 130 | 131 | CONTENT_WEIGHT = 0.1 132 | STYLE_WEIGHT = 5.0 133 | VAR_WEIGHT = 0.01 134 | 135 | NUM_LAYERS = 5 136 | 137 | # define loss function as linear combination of content, style and variational 138 | # losses (defined above) 139 | c_loss = content_loss(content_img, comb_img) 140 | s_loss = style_loss() 141 | v_loss = variation_loss(comb_img) 142 | loss = (CONTENT_WEIGHT * c_loss) + (STYLE_WEIGHT * s_loss) + \ 143 | (VAR_WEIGHT * v_loss) 144 | 145 | grads = K.gradients(loss, comb_img)[0] 146 | f = K.function([comb_img], [loss, grads]) 147 | 148 | NUM_ITERATIONS = 5 149 | LEARNING_RATE = 0.00092 150 | 151 | content_img4d = preprocess(content_img_value) 152 | for i in range(NUM_ITERATIONS): 153 | print("Epoch {:d}/{:d}".format(i+1, NUM_ITERATIONS)) 154 | loss_value, grads_value = f([content_img4d]) 155 | content_img4d += grads_value * LEARNING_RATE 156 | plt.imshow(deprocess(content_img4d)) 157 | plt.show() 158 | -------------------------------------------------------------------------------- /Chapter07/tf-keras-func.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras import backend as K 4 | from keras.engine.topology import Layer 5 | from keras.layers.core import Dropout, Reshape 6 | from keras.layers.convolutional import ZeroPadding2D 7 | from keras.models import Sequential 8 | import numpy as np 9 | 10 | # test harness 11 | # creates a Sequential model out of a single layer and passes the 12 | # input through it to produce output 13 | 14 | 15 | def test_layer(layer, x): 16 | layer_config = layer.get_config() 17 | layer_config["input_shape"] = x.shape 18 | layer = layer.__class__.from_config(layer_config) 19 | model = Sequential() 20 | model.add(layer) 21 | model.compile("rmsprop", "mse") 22 | x_ = np.expand_dims(x, axis=0) 23 | return model.predict(x_)[0] 24 | 25 | 26 | # custom layer 27 | class LocalResponseNormalization(Layer): 28 | 29 | def __init__(self, n=5, alpha=0.0005, beta=0.75, k=2, **kwargs): 30 | self.n = n 31 | self.alpha = alpha 32 | self.beta = beta 33 | self.k = k 34 | super(LocalResponseNormalization, self).__init__(**kwargs) 35 | 36 | def build(self, input_shape): 37 | self.shape = input_shape 38 | super(LocalResponseNormalization, self).build(input_shape) 39 | 40 | def call(self, x, mask=None): 41 | if K.image_dim_ordering == "th": 42 | _, f, r, c = self.shape 43 | else: 44 | _, r, c, f = self.shape 45 | half_n = self.n // 2 46 | squared = K.square(x) 47 | pooled = K.pool2d(squared, (half_n, half_n), strides=(1, 1), 48 | padding="same", pool_mode="avg") 49 | if K.image_dim_ordering == "th": 50 | summed = K.sum(pooled, axis=1, keepdims=True) 51 | averaged = (self.alpha / self.n) * K.repeat_elements(summed, 52 | f, axis=1) 53 | else: 54 | summed = K.sum(pooled, axis=3, keepdims=True) 55 | averaged = (self.alpha / self.n) * K.repeat_elements(summed, 56 | f, axis=3) 57 | denom = K.pow(self.k + averaged, self.beta) 58 | return x / denom 59 | 60 | def compute_output_shape(self, input_shape): 61 | return input_shape 62 | 63 | 64 | # test the test harness 65 | x = np.random.randn(10, 10) 66 | layer = Dropout(0.5) 67 | y = test_layer(layer, x) 68 | assert(x.shape == y.shape) 69 | 70 | x = np.random.randn(10, 10, 3) 71 | layer = ZeroPadding2D(padding=(1, 1)) 72 | y = test_layer(layer, x) 73 | assert(x.shape[0] + 2 == y.shape[0]) 74 | assert(x.shape[1] + 2 == y.shape[1]) 75 | 76 | x = np.random.randn(10, 10) 77 | layer = Reshape((5, 20)) 78 | y = test_layer(layer, x) 79 | assert(y.shape == (5, 20)) 80 | 81 | # test custom layer 82 | x = np.random.randn(225, 225, 3) 83 | layer = LocalResponseNormalization() 84 | y = test_layer(layer, x) 85 | assert(x.shape == y.shape) 86 | -------------------------------------------------------------------------------- /Chapter08/game.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | from __future__ import division, print_function 4 | import collections 5 | import numpy as np 6 | import pygame 7 | import random 8 | 9 | class MyGame(object): 10 | 11 | def __init__(self): 12 | pygame.init() 13 | pygame.key.set_repeat(10, 100) 14 | 15 | # set constants 16 | self.COLOR_WHITE = (255, 255, 255) 17 | self.COLOR_BLACK = (0, 0, 0) 18 | self.GAME_WIDTH = 400 19 | self.GAME_HEIGHT = 400 20 | self.BALL_WIDTH = 20 21 | self.BALL_HEIGHT = 20 22 | self.PADDLE_WIDTH = 50 23 | self.PADDLE_HEIGHT = 10 24 | self.GAME_FLOOR = 350 25 | self.GAME_CEILING = 10 26 | self.BALL_VELOCITY = 10 27 | self.PADDLE_VELOCITY = 20 28 | self.FONT_SIZE = 30 29 | self.MAX_TRIES_PER_GAME = 100 30 | self.CUSTOM_EVENT = pygame.USEREVENT + 1 31 | self.font = pygame.font.SysFont("Comic Sans MS", self.FONT_SIZE) 32 | 33 | # we want to start the game remotely, so moving this block 34 | # to its own function 35 | self.frames = collections.deque(maxlen=4) 36 | self.game_over = False 37 | # initialize positions 38 | self.paddle_x = self.GAME_WIDTH // 2 39 | self.game_score = 0 40 | self.ball_x = random.randint(0, self.GAME_WIDTH) 41 | self.ball_y = self.GAME_CEILING 42 | self.num_tries = 0 43 | # set up display, clock, etc 44 | self.screen = pygame.display.set_mode( 45 | (self.GAME_WIDTH, self.GAME_HEIGHT)) 46 | self.clock = pygame.time.Clock() 47 | 48 | 49 | def play(self): 50 | 51 | # game loop 52 | while not self.game_over: 53 | if self.num_tries > self.MAX_TRIES_PER_GAME: 54 | self.game_over = True 55 | break 56 | 57 | for event in pygame.event.get(): 58 | if event.type == pygame.QUIT: 59 | break 60 | if event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE: 61 | self.game_over = True 62 | break 63 | if event.type == pygame.KEYDOWN and event.key == pygame.K_LEFT: 64 | self.paddle_x -= self.PADDLE_VELOCITY 65 | if self.paddle_x < 0: 66 | self.paddle_x = 0 67 | if event.type == pygame.KEYDOWN and event.key == pygame.K_RIGHT: 68 | self.paddle_x += self.PADDLE_VELOCITY 69 | if self.paddle_x > self.GAME_WIDTH - self.PADDLE_WIDTH: 70 | self.paddle_x = self.GAME_WIDTH - self.PADDLE_WIDTH 71 | 72 | self.screen.fill(self.COLOR_BLACK) 73 | score_text = self.font.render("Score: {:d}/{:d}, Ball: {:d}" 74 | .format(self.game_score, self.MAX_TRIES_PER_GAME, 75 | self.num_tries), True, self.COLOR_WHITE) 76 | # self.screen.blit(score_text, 77 | # ((self.GAME_WIDTH - score_text.get_width()) // 2, 78 | # (self.GAME_FLOOR + self.FONT_SIZE // 2))) 79 | 80 | # update ball position 81 | self.ball_y += self.BALL_VELOCITY 82 | ball = pygame.draw.rect(self.screen, self.COLOR_WHITE, 83 | pygame.Rect(self.ball_x, self.ball_y, 84 | self.BALL_WIDTH, 85 | self.BALL_HEIGHT)) 86 | # update paddle position 87 | paddle = pygame.draw.rect(self.screen, self.COLOR_WHITE, 88 | pygame.Rect(self.paddle_x, 89 | self.GAME_FLOOR, 90 | self.PADDLE_WIDTH, 91 | self.PADDLE_HEIGHT)) 92 | # check for collision 93 | if self.ball_y >= self.GAME_FLOOR - self.BALL_WIDTH // 2: 94 | if ball.colliderect(paddle): 95 | self.game_score += 1 96 | else: 97 | self.game_score -= 1 98 | self.ball_x = random.randint(0, self.GAME_WIDTH) 99 | self.ball_y = self.GAME_CEILING 100 | self.num_tries += 1 101 | 102 | pygame.display.flip() 103 | 104 | # save last 4 frames 105 | self.frames.append(pygame.surfarray.array2d(self.screen)) 106 | 107 | self.clock.tick(30) 108 | # save last 4 frames 109 | S = np.array(self.frames) 110 | with open("../data/game-screenshots.npy", "wb") as fscreenshot: 111 | np.save(fscreenshot, S) 112 | 113 | 114 | def get_state(self): 115 | return np.array(list(self.frames)) 116 | 117 | def set_action(self, action): 118 | pass 119 | 120 | if __name__ == "__main__": 121 | game = MyGame() 122 | game.play() 123 | -------------------------------------------------------------------------------- /Chapter08/game_screenshots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | from __future__ import division, print_function 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | 7 | X = np.load("../data/game-screenshots.npy") 8 | print(X.shape) 9 | 10 | sidx = 140 11 | for soff in range(4): 12 | plt.subplot(sidx + soff + 1) 13 | plt.imshow(X[soff].T, cmap="gray") 14 | plt.xticks([]) 15 | plt.yticks([]) 16 | plt.tight_layout() 17 | plt.show() -------------------------------------------------------------------------------- /Chapter08/rl-network-test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.models import load_model 4 | from keras.optimizers import Adam 5 | from scipy.misc import imresize 6 | import numpy as np 7 | import os 8 | 9 | import wrapped_game 10 | 11 | def preprocess_images(images): 12 | if images.shape[0] < 4: 13 | # single image 14 | x_t = images[0] 15 | x_t = imresize(x_t, (80, 80)) 16 | x_t = x_t.astype("float") 17 | x_t /= 255.0 18 | s_t = np.stack((x_t, x_t, x_t, x_t), axis=2) 19 | else: 20 | # 4 images 21 | xt_list = [] 22 | for i in range(images.shape[0]): 23 | x_t = imresize(images[i], (80, 80)) 24 | x_t = x_t.astype("float") 25 | x_t /= 255.0 26 | xt_list.append(x_t) 27 | s_t = np.stack((xt_list[0], xt_list[1], xt_list[2], xt_list[3]), 28 | axis=2) 29 | s_t = np.expand_dims(s_t, axis=0) 30 | return s_t 31 | 32 | ############################# main ############################### 33 | 34 | DATA_DIR = "../data" 35 | 36 | BATCH_SIZE = 32 37 | NUM_EPOCHS = 100 38 | 39 | model = load_model(os.path.join(DATA_DIR, "rl-network-4100.h5")) 40 | model.compile(optimizer=Adam(lr=1e-6), loss="mse") 41 | 42 | # train network 43 | game = wrapped_game.MyWrappedGame() 44 | 45 | num_games, num_wins = 0, 0 46 | for e in range(NUM_EPOCHS): 47 | loss = 0.0 48 | game.reset() 49 | 50 | # get first state 51 | a_0 = 1 # (0 = left, 1 = stay, 2 = right) 52 | x_t, r_0, game_over = game.step(a_0) 53 | s_t = preprocess_images(x_t) 54 | 55 | while not game_over: 56 | s_tm1 = s_t 57 | # next action 58 | q = model.predict(s_t)[0] 59 | a_t = np.argmax(q) 60 | # apply action, get reward 61 | x_t, r_t, game_over = game.step(a_t) 62 | s_t = preprocess_images(x_t) 63 | # if reward, increment num_wins 64 | if r_t == 1: 65 | num_wins += 1 66 | 67 | num_games += 1 68 | print("Game: {:03d}, Wins: {:03d}".format(num_games, num_wins), end="\r") 69 | 70 | print("") 71 | -------------------------------------------------------------------------------- /Chapter08/rl-network-train.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | from keras.models import Sequential 4 | from keras.layers.core import Activation, Dense, Flatten 5 | from keras.layers.convolutional import Conv2D 6 | from keras.optimizers import Adam 7 | from scipy.misc import imresize 8 | import collections 9 | import numpy as np 10 | import os 11 | 12 | import wrapped_game 13 | 14 | def preprocess_images(images): 15 | if images.shape[0] < 4: 16 | # single image 17 | x_t = images[0] 18 | x_t = imresize(x_t, (80, 80)) 19 | x_t = x_t.astype("float") 20 | x_t /= 255.0 21 | s_t = np.stack((x_t, x_t, x_t, x_t), axis=2) 22 | else: 23 | # 4 images 24 | xt_list = [] 25 | for i in range(images.shape[0]): 26 | x_t = imresize(images[i], (80, 80)) 27 | x_t = x_t.astype("float") 28 | x_t /= 255.0 29 | xt_list.append(x_t) 30 | s_t = np.stack((xt_list[0], xt_list[1], xt_list[2], xt_list[3]), 31 | axis=2) 32 | s_t = np.expand_dims(s_t, axis=0) 33 | return s_t 34 | 35 | 36 | def get_next_batch(experience, model, num_actions, gamma, batch_size): 37 | batch_indices = np.random.randint(low=0, high=len(experience), 38 | size=batch_size) 39 | batch = [experience[i] for i in batch_indices] 40 | X = np.zeros((batch_size, 80, 80, 4)) 41 | Y = np.zeros((batch_size, num_actions)) 42 | for i in range(len(batch)): 43 | s_t, a_t, r_t, s_tp1, game_over = batch[i] 44 | X[i] = s_t 45 | Y[i] = model.predict(s_t)[0] 46 | Q_sa = np.max(model.predict(s_tp1)[0]) 47 | if game_over: 48 | Y[i, a_t] = r_t 49 | else: 50 | Y[i, a_t] = r_t + gamma * Q_sa 51 | return X, Y 52 | 53 | 54 | ############################# main ############################### 55 | 56 | # initialize parameters 57 | DATA_DIR = "../data" 58 | NUM_ACTIONS = 3 # number of valid actions (left, stay, right) 59 | GAMMA = 0.99 # decay rate of past observations 60 | INITIAL_EPSILON = 0.1 # starting value of epsilon 61 | FINAL_EPSILON = 0.0001 # final value of epsilon 62 | MEMORY_SIZE = 50000 # number of previous transitions to remember 63 | NUM_EPOCHS_OBSERVE = 100 64 | NUM_EPOCHS_TRAIN = 2000 65 | 66 | BATCH_SIZE = 32 67 | NUM_EPOCHS = NUM_EPOCHS_OBSERVE + NUM_EPOCHS_TRAIN 68 | 69 | # build the model 70 | model = Sequential() 71 | model.add(Conv2D(32, kernel_size=8, strides=4, 72 | kernel_initializer="normal", 73 | padding="same", 74 | input_shape=(80, 80, 4))) 75 | model.add(Activation("relu")) 76 | model.add(Conv2D(64, kernel_size=4, strides=2, 77 | kernel_initializer="normal", 78 | padding="same")) 79 | model.add(Activation("relu")) 80 | model.add(Conv2D(64, kernel_size=3, strides=1, 81 | kernel_initializer="normal", 82 | padding="same")) 83 | model.add(Activation("relu")) 84 | model.add(Flatten()) 85 | model.add(Dense(512, kernel_initializer="normal")) 86 | model.add(Activation("relu")) 87 | model.add(Dense(3, kernel_initializer="normal")) 88 | 89 | model.compile(optimizer=Adam(lr=1e-6), loss="mse") 90 | 91 | # train network 92 | game = wrapped_game.MyWrappedGame() 93 | experience = collections.deque(maxlen=MEMORY_SIZE) 94 | 95 | fout = open(os.path.join(DATA_DIR, "rl-network-results.tsv"), "wb") 96 | num_games, num_wins = 0, 0 97 | epsilon = INITIAL_EPSILON 98 | for e in range(NUM_EPOCHS): 99 | loss = 0.0 100 | game.reset() 101 | 102 | # get first state 103 | a_0 = 1 # (0 = left, 1 = stay, 2 = right) 104 | x_t, r_0, game_over = game.step(a_0) 105 | s_t = preprocess_images(x_t) 106 | 107 | while not game_over: 108 | s_tm1 = s_t 109 | # next action 110 | if e <= NUM_EPOCHS_OBSERVE: 111 | a_t = np.random.randint(low=0, high=NUM_ACTIONS, size=1)[0] 112 | else: 113 | if np.random.rand() <= epsilon: 114 | a_t = np.random.randint(low=0, high=NUM_ACTIONS, size=1)[0] 115 | else: 116 | q = model.predict(s_t)[0] 117 | a_t = np.argmax(q) 118 | 119 | # apply action, get reward 120 | x_t, r_t, game_over = game.step(a_t) 121 | s_t = preprocess_images(x_t) 122 | # if reward, increment num_wins 123 | if r_t == 1: 124 | num_wins += 1 125 | # store experience 126 | experience.append((s_tm1, a_t, r_t, s_t, game_over)) 127 | 128 | if e > NUM_EPOCHS_OBSERVE: 129 | # finished observing, now start training 130 | # get next batch 131 | X, Y = get_next_batch(experience, model, NUM_ACTIONS, 132 | GAMMA, BATCH_SIZE) 133 | loss += model.train_on_batch(X, Y) 134 | 135 | # reduce epsilon gradually 136 | if epsilon > FINAL_EPSILON: 137 | epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / NUM_EPOCHS 138 | 139 | print("Epoch {:04d}/{:d} | Loss {:.5f} | Win Count: {:d}" 140 | .format(e + 1, NUM_EPOCHS, loss, num_wins)) 141 | fout.write("{:04d}\t{:.5f}\t{:d}\n" 142 | .format(e + 1, loss, num_wins)) 143 | 144 | if e % 100 == 0: 145 | model.save(os.path.join(DATA_DIR, "rl-network.h5"), overwrite=True) 146 | 147 | fout.close() 148 | model.save(os.path.join(DATA_DIR, "rl-network.h5"), overwrite=True) 149 | -------------------------------------------------------------------------------- /Chapter08/rl-network-viz.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import os 6 | 7 | DATA_DIR = "../data" 8 | 9 | xs, loss_ys, wins_ys = [], [], [] 10 | fin = open(os.path.join(DATA_DIR, "rl-network-results-5100.tsv"), "rb") 11 | lno = 0 12 | for line in fin: 13 | # if lno < 1000: 14 | # lno += 1 15 | # continue 16 | cols = line.strip().split("\t") 17 | epoch = int(cols[0]) 18 | loss = float(cols[1]) 19 | num_wins = int(cols[2]) 20 | xs.append(epoch) 21 | loss_ys.append(loss) 22 | wins_ys.append(num_wins) 23 | lno += 1 24 | fin.close() 25 | 26 | plt.subplot(211) 27 | plt.plot(xs, loss_ys) 28 | plt.title("Loss") 29 | plt.xlabel("epochs") 30 | plt.ylabel("loss (MSE)") 31 | 32 | plt.subplot(212) 33 | plt.plot(xs, wins_ys) 34 | plt.title("Wins") 35 | plt.xlabel("epochs") 36 | plt.ylabel("# wins (cumulative)") 37 | 38 | plt.tight_layout() 39 | plt.show() 40 | -------------------------------------------------------------------------------- /Chapter08/wrapped_game.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function 3 | import collections 4 | import numpy as np 5 | import pygame 6 | import random 7 | import os 8 | 9 | class MyWrappedGame(object): 10 | 11 | def __init__(self): 12 | # run pygame in headless mode 13 | # os.environ["SDL_VIDEODRIVER"] = "dummy" 14 | 15 | pygame.init() 16 | pygame.key.set_repeat(10, 100) 17 | 18 | # set constants 19 | self.COLOR_WHITE = (255, 255, 255) 20 | self.COLOR_BLACK = (0, 0, 0) 21 | self.GAME_WIDTH = 400 22 | self.GAME_HEIGHT = 400 23 | self.BALL_WIDTH = 20 24 | self.BALL_HEIGHT = 20 25 | self.PADDLE_WIDTH = 50 26 | self.PADDLE_HEIGHT = 10 27 | self.GAME_FLOOR = 350 28 | self.GAME_CEILING = 10 29 | # based on experimentation, the ball tends to move 4 times 30 | # between each paddle movement. Since here we alternate ball 31 | # and paddle movement, we make ball move 4x faster. 32 | self.BALL_VELOCITY = 10 33 | self.PADDLE_VELOCITY = 20 34 | self.FONT_SIZE = 30 35 | # self.MAX_TRIES_PER_GAME = 100 36 | self.MAX_TRIES_PER_GAME = 1 37 | self.CUSTOM_EVENT = pygame.USEREVENT + 1 38 | self.font = pygame.font.SysFont("Comic Sans MS", self.FONT_SIZE) 39 | 40 | 41 | def reset(self): 42 | self.frames = collections.deque(maxlen=4) 43 | self.game_over = False 44 | # initialize positions 45 | self.paddle_x = self.GAME_WIDTH // 2 46 | self.game_score = 0 47 | self.reward = 0 48 | self.ball_x = random.randint(0, self.GAME_WIDTH) 49 | self.ball_y = self.GAME_CEILING 50 | self.num_tries = 0 51 | # set up display, clock, etc 52 | self.screen = pygame.display.set_mode( 53 | (self.GAME_WIDTH, self.GAME_HEIGHT)) 54 | self.clock = pygame.time.Clock() 55 | 56 | def step(self, action): 57 | pygame.event.pump() 58 | 59 | if action == 0: # move paddle left 60 | self.paddle_x -= self.PADDLE_VELOCITY 61 | if self.paddle_x < 0: 62 | # bounce off the wall, go right 63 | self.paddle_x = self.PADDLE_VELOCITY 64 | elif action == 2: # move paddle right 65 | self.paddle_x += self.PADDLE_VELOCITY 66 | if self.paddle_x > self.GAME_WIDTH - self.PADDLE_WIDTH: 67 | # bounce off the wall, go left 68 | self.paddle_x = self.GAME_WIDTH - self.PADDLE_WIDTH - self.PADDLE_VELOCITY 69 | else: # dont move paddle 70 | pass 71 | 72 | self.screen.fill(self.COLOR_BLACK) 73 | score_text = self.font.render("Score: {:d}/{:d}, Ball: {:d}" 74 | .format(self.game_score, self.MAX_TRIES_PER_GAME, 75 | self.num_tries), True, self.COLOR_WHITE) 76 | # self.screen.blit(score_text, 77 | # ((self.GAME_WIDTH - score_text.get_width()) // 2, 78 | # (self.GAME_FLOOR + self.FONT_SIZE // 2))) 79 | 80 | # update ball position 81 | self.ball_y += self.BALL_VELOCITY 82 | ball = pygame.draw.rect(self.screen, self.COLOR_WHITE, 83 | pygame.Rect(self.ball_x, self.ball_y, 84 | self.BALL_WIDTH, 85 | self.BALL_HEIGHT)) 86 | # update paddle position 87 | paddle = pygame.draw.rect(self.screen, self.COLOR_WHITE, 88 | pygame.Rect(self.paddle_x, 89 | self.GAME_FLOOR, 90 | self.PADDLE_WIDTH, 91 | self.PADDLE_HEIGHT)) 92 | 93 | # check for collision and update reward 94 | self.reward = 0 95 | if self.ball_y >= self.GAME_FLOOR - self.BALL_WIDTH // 2: 96 | if ball.colliderect(paddle): 97 | self.reward = 1 98 | else: 99 | self.reward = -1 100 | 101 | self.game_score += self.reward 102 | self.ball_x = random.randint(0, self.GAME_WIDTH) 103 | self.ball_y = self.GAME_CEILING 104 | self.num_tries += 1 105 | 106 | pygame.display.flip() 107 | 108 | # save last 4 frames 109 | self.frames.append(pygame.surfarray.array2d(self.screen)) 110 | 111 | if self.num_tries >= self.MAX_TRIES_PER_GAME: 112 | self.game_over = True 113 | 114 | self.clock.tick(30) 115 | return self.get_frames(), self.reward, self.game_over 116 | 117 | 118 | def get_frames(self): 119 | return np.array(list(self.frames)) 120 | 121 | 122 | if __name__ == "__main__": 123 | game = MyWrappedGame() 124 | 125 | NUM_EPOCHS = 10 126 | for e in range(NUM_EPOCHS): 127 | print("Epoch: {:d}".format(e)) 128 | game.reset() 129 | input_t = game.get_frames() 130 | game_over = False 131 | while not game_over: 132 | action = np.random.randint(0, 3, size=1)[0] 133 | input_tp1, reward, game_over = game.step(action) 134 | print(action, reward, game_over) 135 | 136 | -------------------------------------------------------------------------------- /Docker/Dockerfile: -------------------------------------------------------------------------------- 1 | ################################################################################################################# 2 | # Base Images 3 | ################################################################################################################# 4 | FROM ubuntu:14.04 5 | 6 | ################################################################################################################# 7 | # ENV Setting 8 | ################################################################################################################# 9 | ENV CONDA_DIR /opt/conda 10 | ENV PATH $CONDA_DIR/bin:$PATH 11 | 12 | ################################################################################################################# 13 | # Initial Setting 14 | ################################################################################################################# 15 | RUN mkdir -p $CONDA_DIR && \ 16 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \ 17 | apt-get update && \ 18 | apt-get install -y wget git libhdf5-dev g++ graphviz && \ 19 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 20 | /bin/bash /Miniconda3-latest-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 21 | rm Miniconda3-latest-Linux-x86_64.sh 22 | RUN apt-get update -qq && \ 23 | apt-get install --no-install-recommends -y \ 24 | build-essential cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev libgl1-mesa-glx && \ 25 | apt-get clean && \ 26 | rm -rf /var/lib/apt/lists/* 27 | 28 | ################################################################################################################# 29 | # User Setting 30 | ################################################################################################################# 31 | ENV NB_USER keras 32 | ENV NB_UID 1000 33 | 34 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 35 | mkdir -p $CONDA_DIR && \ 36 | chown keras $CONDA_DIR -R && \ 37 | mkdir -p /src && \ 38 | chown keras /src 39 | 40 | 41 | ################################################################################################################# 42 | # Python Setting 43 | ################################################################################################################# 44 | # Python 45 | RUN conda update -y python 46 | RUN pip install numpy six tensorflow==1.3.0 keras==2.0.8 prettytensor && \ 47 | pip install Cython ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \ 48 | conda install Pillow scikit-learn notebook pandas matplotlib seaborn pyyaml h5py nltk gensim && \ 49 | conda install -y pyqt && \ 50 | conda clean -yt 51 | 52 | RUN conda install -y --channel https://conda.anaconda.org/menpo opencv3 && \ 53 | conda clean -yt 54 | 55 | ENV PYTHONPATH='/src/:$PYTHONPATH' 56 | 57 | ################################################################################################################# 58 | # NLTK DOWNLOAD 59 | ################################################################################################################# 60 | RUN python -m nltk.downloader punkt && \ 61 | python -m nltk.downloader -d /usr/share/nltk_data brown && \ 62 | python -m nltk.downloader -d /usr/share/nltk_data punkt && \ 63 | python -m nltk.downloader -d /usr/share/nltk_data treebank && \ 64 | python -m nltk.downloader -d /usr/share/nltk_data sinica_treebank && \ 65 | python -m nltk.downloader -d /usr/share/nltk_data hmm_treebank_pos_tagger && \ 66 | python -m nltk.downloader -d /usr/share/nltk_data maxent_treebank_pos_tagger && \ 67 | python -m nltk.downloader -d /usr/share/nltk_data words && \ 68 | python -m nltk.downloader -d /usr/share/nltk_data stopwords && \ 69 | python -m nltk.downloader -d /usr/share/nltk_data names && \ 70 | python -m nltk.downloader -d /usr/share/nltk_data wordnet 71 | ################################################################################################################# 72 | # WORK Jupyter 73 | ################################################################################################################# 74 | WORKDIR /src 75 | # USER keras 76 | 77 | EXPOSE 8888 78 | 79 | CMD jupyter notebook --port=8888 --ip=0.0.0.0 -------------------------------------------------------------------------------- /Docker/Dockerfile-gpu: -------------------------------------------------------------------------------- 1 | ################################################################################################################# 2 | # Base Images 3 | ################################################################################################################# 4 | FROM nvidia/cuda:8.0-cudnn6-devel 5 | 6 | ################################################################################################################# 7 | # ENV Setting 8 | ################################################################################################################# 9 | ENV CONDA_DIR /opt/conda 10 | ENV PATH $CONDA_DIR/bin:$PATH 11 | 12 | ################################################################################################################# 13 | # Initial Setting 14 | ################################################################################################################# 15 | RUN mkdir -p $CONDA_DIR && \ 16 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \ 17 | apt-get update && \ 18 | apt-get install -y wget git libhdf5-dev g++ graphviz && \ 19 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 20 | /bin/bash /Miniconda3-latest-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 21 | rm Miniconda3-latest-Linux-x86_64.sh 22 | RUN apt-get update -qq && \ 23 | apt-get install --no-install-recommends -y \ 24 | build-essential cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev libgl1-mesa-glx && \ 25 | apt-get clean && \ 26 | rm -rf /var/lib/apt/lists/* 27 | 28 | ################################################################################################################# 29 | # User Setting 30 | ################################################################################################################# 31 | ENV NB_USER keras 32 | ENV NB_UID 1000 33 | 34 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 35 | mkdir -p $CONDA_DIR && \ 36 | chown keras $CONDA_DIR -R && \ 37 | mkdir -p /src && \ 38 | chown keras /src 39 | ################################################################################################################# 40 | # Python Setting 41 | ################################################################################################################# 42 | # Python 43 | RUN conda update -y python 44 | RUN pip install numpy six tensorflow==1.3.0 keras==2.0.8 prettytensor && \ 45 | pip install Cython ipdb pytest pytest-cov python-coveralls coverage pytest-xdist pep8 pytest-pep8 pydot_ng && \ 46 | conda install Pillow scikit-learn notebook pandas matplotlib seaborn pyyaml h5py nltk gensim && \ 47 | conda install -y pyqt && \ 48 | conda clean -yt 49 | 50 | RUN conda install -y --channel https://conda.anaconda.org/menpo opencv3 && \ 51 | conda clean -yt 52 | 53 | ENV PYTHONPATH='/src/:$PYTHONPATH' 54 | ################################################################################################################# 55 | # Quiver 56 | ################################################################################################################# 57 | RUN pip install quiver_engine 58 | ################################################################################################################# 59 | # Keras Adversarial Models 60 | # https://github.com/bstriner/keras-adversarial 61 | ################################################################################################################# 62 | RUN git clone https://github.com/bstriner/keras_adversarial.git && \ 63 | cd keras_adversarial && \ 64 | python setup.py install 65 | ################################################################################################################# 66 | # NLTK DOWNLOAD 67 | ################################################################################################################# 68 | RUN python -m nltk.downloader punkt && \ 69 | python -m nltk.downloader -d /usr/share/nltk_data brown && \ 70 | python -m nltk.downloader -d /usr/share/nltk_data punkt && \ 71 | python -m nltk.downloader -d /usr/share/nltk_data treebank && \ 72 | python -m nltk.downloader -d /usr/share/nltk_data sinica_treebank && \ 73 | python -m nltk.downloader -d /usr/share/nltk_data hmm_treebank_pos_tagger && \ 74 | python -m nltk.downloader -d /usr/share/nltk_data maxent_treebank_pos_tagger && \ 75 | python -m nltk.downloader -d /usr/share/nltk_data words && \ 76 | python -m nltk.downloader -d /usr/share/nltk_data stopwords && \ 77 | python -m nltk.downloader -d /usr/share/nltk_data names && \ 78 | python -m nltk.downloader -d /usr/share/nltk_data wordnet 79 | ################################################################################################################# 80 | # WORK Jupyter 81 | ################################################################################################################# 82 | ENV PTYHONPATH=$PYTHONPATH:/src/cython_train/ 83 | WORKDIR /src 84 | USER keras 85 | 86 | EXPOSE 8888 87 | 88 | CMD jupyter notebook --port=8888 --ip=0.0.0.0 89 | -------------------------------------------------------------------------------- /Docker/Dockerfile-wavnet: -------------------------------------------------------------------------------- 1 | ################################################################################################################# 2 | # Base Images 3 | ################################################################################################################# 4 | FROM ubuntu:14.04 5 | 6 | ################################################################################################################# 7 | # ENV Setting 8 | ################################################################################################################# 9 | ENV CONDA_DIR /opt/conda 10 | ENV PATH $CONDA_DIR/bin:$PATH 11 | 12 | ################################################################################################################# 13 | # Initial Setting 14 | ################################################################################################################# 15 | RUN mkdir -p $CONDA_DIR && \ 16 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \ 17 | apt-get update && \ 18 | apt-get install -y wget git libhdf5-dev g++ graphviz && \ 19 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 20 | /bin/bash /Miniconda3-latest-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 21 | rm Miniconda3-latest-Linux-x86_64.sh 22 | RUN apt-get update -qq && \ 23 | apt-get install --no-install-recommends -y \ 24 | build-essential cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev && \ 25 | apt-get clean && \ 26 | rm -rf /var/lib/apt/lists/* 27 | 28 | ################################################################################################################# 29 | # User Setting 30 | ################################################################################################################# 31 | ENV NB_USER keras 32 | ENV NB_UID 1000 33 | 34 | RUN useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ 35 | mkdir -p $CONDA_DIR && \ 36 | chown keras $CONDA_DIR -R && \ 37 | mkdir -p /src && \ 38 | chown keras /src 39 | 40 | 41 | ################################################################################################################# 42 | # Python Setting 43 | ################################################################################################################# 44 | # Python 45 | ARG python_version=3.5.3-0 46 | ARG python_qt_version=4 47 | RUN conda install -y python=${python_version} && \ 48 | pip install numpy six tensorflow==1.2.0 keras==2.0.5 prettytensor && \ 49 | pip install Cython ipdb pytest pytest-cov python-coveralls coverage==3.7.1 pytest-xdist pep8 pytest-pep8 pydot_ng && \ 50 | conda install -y Pillow scikit-learn notebook pandas matplotlib seaborn pyyaml h5py && \ 51 | conda install -y pyqt=${python_qt_version} && \ 52 | conda clean -yt 53 | 54 | RUN conda install -y --channel https://conda.anaconda.org/menpo opencv3 && \ 55 | conda clean -yt 56 | 57 | ENV PYTHONPATH='/src/:$PYTHONPATH' 58 | ################################################################################################################# 59 | # Quiver 60 | ################################################################################################################# 61 | RUN pip install quiver_engine 62 | ################################################################################################################# 63 | # Keras Adversarial Models 64 | # https://github.com/bstriner/keras-adversarial 65 | ################################################################################################################# 66 | RUN git clone https://github.com/bstriner/keras_adversarial.git && \ 67 | cd keras_adversarial && \ 68 | python setup.py install 69 | 70 | ################################################################################################################# 71 | # Wave Net 72 | # https://github.com/basveeling/wavenet 73 | ################################################################################################################# 74 | ARG python_version=2.7.0 75 | ARG python_qt_version=4 76 | RUN conda install -y python=${python_version} && \ 77 | git clone https://github.com/basveeling/wavenet.git && \ 78 | cd wavenet && \ 79 | pip install -r requirements.txt 80 | ################################################################################################################# 81 | # WORK Jupyter 82 | ################################################################################################################# 83 | WORKDIR /src 84 | # USER keras 85 | 86 | EXPOSE 8888 87 | 88 | CMD jupyter notebook --port=8888 --ip=0.0.0.0 --allow-root 89 | -------------------------------------------------------------------------------- /Docker/Makefile: -------------------------------------------------------------------------------- 1 | help: 2 | @cat Makefile 3 | 4 | DATA?="${HOME}/Data" 5 | GPU?=0 6 | DOCKER_FILE=Dockerfile 7 | DOCKER_FILE_GPU=Dockerfile-gpu 8 | DOCKER_FILE_WAVNET=Dockerfile-wavnet 9 | DOCKER=docker 10 | DOCKER_GPU=GPU=$(GPU) nvidia-docker 11 | BACKEND=tensorflow 12 | TEST=tests/ 13 | SRC=$(shell dirname `pwd`) 14 | CUDA=/usr/local/cuda 15 | LD_LIBRARY=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64 16 | 17 | # CPU environment 18 | 19 | build: 20 | $(DOCKER) build -t deep-learning-with-keras --build-arg python_version=3.5 -f $(DOCKER_FILE) . 21 | 22 | bash: build 23 | $(DOCKER) run -p 8889:8889 -p 6007:6007 -it -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix -v $(SRC):/src -v $(DATA):/data deep-learning-with-keras bash 24 | 25 | ipython: build 26 | $(DOCKER) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src -v $(DATA):/data deep-learning-with-keras ipython 27 | 28 | notebook: build 29 | $(DOCKER) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src -v $(DATA):/data deep-learning-with-keras 30 | 31 | test: build 32 | $(DOCKER) run -it -v $(SRC):/src -v $(DATA):/data deep-learning-with-keras pytest $(TEST) 33 | 34 | # GPU environment 35 | 36 | build-gpu: 37 | $(DOCKER_GPU) build -t deep-learning-with-keras-gpu --build-arg python_version=3.5 -f $(DOCKER_FILE_GPU) . 38 | 39 | bash-gpu: build-gpu 40 | $(DOCKER_GPU) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src -v $(DATA):/data --env CUDA_HOME=${CUDA} \ 41 | -e DISPLAY=$DISPLAY \ 42 | -v /tmp/.X11-unix:/tmp/.X11-unix \ 43 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 44 | deep-learning-with-keras-gpu bash 45 | ipython-gpu: build-gpu 46 | $(DOCKER_GPU) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src -v $(DATA):/data --env CUDA_HOME=${CUDA} \ 47 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 48 | deep-learning-with-keras-gpu ipython 49 | 50 | notebook-gpu: build-gpu 51 | $(DOCKER_GPU) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src -v $(DATA):/data --env CUDA_HOME=${CUDA} \ 52 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 53 | deep-learning-with-keras-gpu 54 | 55 | test-gpu: build-gpu 56 | $(DOCKER_GPU) run -it -v $(SRC):/src -v $(DATA):/data --env CUDA_HOME=${CUDA} \ 57 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 58 | deep-learning-with-keras-gpu pytest $(TEST) 59 | 60 | # CPU environment Wave net 61 | 62 | build-wavenet: 63 | $(DOCKER_GPU) build -t deep-learning-with-keras-wavenet --build-arg python_version=3.5 -f $(DOCKER_FILE_WAVNET) . 64 | 65 | bash-wavenet: build-wavenet 66 | $(DOCKER_GPU) run -it -v $(SRC):/src --env CUDA_HOME=${CUDA} \ 67 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 68 | deep-learning-with-keras-wavenet bash 69 | 70 | ipython-wavenet: build-wavenet 71 | $(DOCKER_GPU) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src --env CUDA_HOME=${CUDA} \ 72 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 73 | deep-learning-with-keras-wavenet ipython 74 | 75 | notebook-wavenet: build-wavenet 76 | $(DOCKER_GPU) run -p 8888:8888 -p 6006:6006 -it -v $(SRC):/src --env CUDA_HOME=${CUDA} \ 77 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 78 | deep-learning-with-keras-wavenet 79 | 80 | test-wavenet: build-wavenet 81 | $(DOCKER_GPU) run -it -v $(SRC):/src --env CUDA_HOME=${CUDA} \ 82 | --env LD_LIBRARY_PATH=${LD_LIBRARY} \ 83 | deep-learning-with-keras-wavenet pytest $(TEST) -------------------------------------------------------------------------------- /Docker/README.md: -------------------------------------------------------------------------------- 1 | # Using Keras and OpenCV via Docker 2 | 3 | This directory contains `Dockerfile` to make it easy to get up and running with 4 | Keras via [Docker](http://www.docker.com/). 5 | 6 | ## Installing Docker 7 | 8 | General installation instructions are 9 | [on the Docker site](https://docs.docker.com/installation/), but we give some 10 | quick links here: 11 | 12 | * [OSX](https://docs.docker.com/installation/mac/): [docker toolbox](https://www.docker.com/toolbox) 13 | * [ubuntu](https://docs.docker.com/installation/ubuntulinux/) 14 | 15 | ## Installing NVIDIA Docker (GPU Environment) 16 | 17 | General installation instructions are 18 | [on the NVIDIA Docker site](https://github.com/NVIDIA/nvidia-docker) 19 | 20 | ## Running the container 21 | 22 | We are using `Makefile` to simplify docker commands within make commands. 23 | 24 | ### CPU environment 25 | 26 | Build the container and start a jupyter notebook 27 | 28 | $ make notebook 29 | 30 | Build the container and start an iPython shell 31 | 32 | $ make ipython 33 | 34 | Build the container and start a bash 35 | 36 | $ make bash 37 | 38 | Build the container and start a test 39 | 40 | $ make test 41 | 42 | ### GPU environment 43 | 44 | Build the container and start a jupyter notebook 45 | 46 | $ make notebook-gpu 47 | 48 | Build the container and start an iPython shell 49 | 50 | $ make ipython-gpu 51 | 52 | Build the container and start a bash 53 | 54 | $ make bash-gpu 55 | 56 | Build the container and start a test 57 | 58 | $ make test-gpu 59 | 60 | For GPU support install NVidia drivers (ideally latest) and 61 | [nvidia-docker](https://github.com/NVIDIA/nvidia-docker). Run using 62 | 63 | $ make notebook-gpu GPU=0 # or [ipython, bash] 64 | 65 | Mount a volume for external data sets 66 | 67 | $ make DATA=~/mydata 68 | 69 | Prints all make tasks 70 | 71 | $ make help 72 | 73 | 74 | Note: If you would have a problem running nvidia-docker you may try the old way 75 | we have used. But it is not recommended. If you find a bug in the nvidia-docker report 76 | it there please and try using the nvidia-docker as described above. 77 | 78 | $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}') 79 | $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') 80 | $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Deep Learning with Keras 5 | This is the code repository for [Deep Learning with Keras](https://www.packtpub.com/big-data-and-business-intelligence/deep-learning-keras?utm_source=github&utm_medium=repository&utm_campaign=9781787128422), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the book from start to finish. 6 | ## About the Book 7 | This book starts by introducing you to supervised learning algorithms such as simple linear regression, classical multilayer perceptron, and more sophisticated Deep Convolutional Networks. In addition, you will also understand unsupervised learning algorithms such as Autoencoders, Restricted Boltzmann Machines, and Deep Belief Networks. Recurrent Networks and Long Short Term Memory (LSTM) networks are also explained in detail. You will also explore image processing involving the recognition of handwritten digital images, the classification of images into different categories, and advanced object recognition with related image annotations. An example of the identification of salient points for face detection is also provided. 8 | ## Instructions and Navigation 9 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter02. 10 | 11 | 12 | 13 | The code will look like the following: 14 | ``` 15 | from keras.models import Sequential 16 | model = Sequential() 17 | model.add(Dense(12, input_dim=8, kernel_initializer='random_uniform')) 18 | ``` 19 | 20 | To be able to smoothly follow through the chapters, you will need the following pieces of software: 21 | 22 | * TensorFlow 1.0.0 or higher 23 | * Keras 2.0.2 or higher 24 | * Matplotlib 1.5.3 or higher 25 | * Scikit-learn 0.18.1 or higher 26 | * NumPy 1.12.1 or higher 27 | 28 | The hardware specifications are as follows: 29 | 30 | * Either 32-bit or 64-bit architecture 31 | * 2+ GHz CPU 32 | * 4 GB RAM 33 | * At least 10 GB of hard disk space available 34 | 35 | ## Related Products 36 | * [Deep Learning with TensorFlow](https://www.packtpub.com/big-data-and-business-intelligence/deep-learning-tensorflow?utm_source=github&utm_medium=repository&utm_campaign=9781786469786) 37 | 38 | * [Python Deep Learning](https://www.packtpub.com/big-data-and-business-intelligence/python-deep-learning?utm_source=github&utm_medium=repository&utm_campaign=9781786464453) 39 | 40 | * [Deep Learning with Hadoop](https://www.packtpub.com/big-data-and-business-intelligence/deep-learning-hadoop?utm_source=github&utm_medium=repository&utm_campaign=9781787124769) 41 | 42 | ### Suggestions and Feedback 43 | [Click here](https://docs.google.com/forms/d/e/1FAIpQLSe5qwunkGf6PUvzPirPDtuy1Du5Rlzew23UBp2S-P3wB-GcwQ/viewform) if you have any feedback or suggestions. 44 | ### Download a free PDF 45 | 46 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF. 47 |

https://packt.link/free-ebook/9781787128422

--------------------------------------------------------------------------------