├── 2 ├── 00_hello_world.py ├── 01_if.py ├── 02_if_else.py ├── 03_if_elif.py ├── 04_if_elif_else.py ├── 05_while.py ├── 06_while_break.py ├── 07_while_else.py ├── 08_for.py ├── 09_for_break.py ├── 10_for_dict.py ├── 11_for_dict_key_value.py ├── 12_function_print_x2.py ├── 13_function_return_x2.py ├── 14_function_return_ax2_1.py └── 15_function_return_ax2_2.py ├── 3 ├── 00_simple_perceptron.py ├── keras │ ├── 01_logistic_regressioni_or_keras.py │ ├── 02_multi_class_logistic_regression_keras.py │ ├── 03_mlp_xor_keras.py │ └── 04_toy_problem_keras.py └── tensorflow │ ├── 01_logistic_regression_or_tensorflow.py │ ├── 02_multi_class_logistic_regression_tensorflow.py │ ├── 03_mlp_xor_tensorflow.py │ └── 04_toy_problem_tensorflow.py ├── 4 ├── keras │ ├── 00_mnist_sigmoid_keras.py │ ├── 01_mnist_tanh_keras.py │ ├── 02_mnist_relu_keras.py │ ├── 03_mnist_lrelu_keras.py │ ├── 04_mnist_prelu_keras.py │ ├── 05_mnist_dropout_keras.py │ ├── 06_mnist_plot_keras.py │ ├── 07_mnist_momentum_keras.py │ ├── 08_mnist_nesterov_keras.py │ ├── 09_mnist_adagrad_keras.py │ ├── 10_mnist_adadelta_keras.py │ ├── 11_mnist_rmsprop_keras.py │ ├── 12_mnist_adam_keras.py │ ├── 13_mnist_early_stopping_keras.py │ └── 14_mnist_batch_normalization_keras.py └── tensorflow │ ├── 01_mnist_tanh_tensorflow.py │ ├── 02_mnist_relu_tensorflow.py │ ├── 03_mnist_lrelu_tensorflow.py │ ├── 04_mnist_prelu_tensorflow.py │ ├── 05_mnist_dropout_tensorflow.py │ ├── 06_mnist_plot_tensorflow.py │ ├── 07_mnist_momentum_tensorflow.py │ ├── 08_mnist_nesterov_tensorflow.py │ ├── 09_mnist_adagrad_tensorflow.py │ ├── 10_mnist_adadelta_tensorflow.py │ ├── 11_mnist_rmsprop_tensorflow.py │ ├── 12_mnist_adam_tensorflow.py │ ├── 13_mnist_early_stopping_tensorflow.py │ ├── 14_mnist_batch_normalization_tensorflow.py │ └── 99_mnist_mock_contrib_tensorflow.py ├── 5 ├── keras │ ├── 00_sin_simple_rnn_keras.py │ ├── 01_sin_lstm_keras.py │ ├── 02_sin_gru_keras.py │ ├── 03_adding_problem_simple_rnn_keras.py │ ├── 04_adding_problem_lstm_keras.py │ └── 05_adding_problem_gru_keras.py ├── sin.mp3 ├── sin_noise.mp3 └── tensorflow │ ├── 00_sin_simple_rnn_tensorflow.py │ ├── 01_00_sin_simple_lstm_tensorflow.py │ ├── 01_01_sin_lstm_tensorflow.py │ ├── 02_sin_gru_tensorflow.py │ ├── 03_adding_problem_simple_rnn_tensorflow.py │ ├── 04_adding_problem_lstm_tensorflow.py │ └── 05_adding_problem_gru_tensorflow.py ├── 6 ├── keras │ ├── 00_mnist_bidirectional_rnn_keras.py │ └── 01_adding_task_rnn_encoder_decoder_keras.py └── tensorflow │ ├── 00_mnist_bidirectional_rnn_tensorflow.py │ ├── 01_adding_task_rnn_encoder_decoder_tensorflow.py │ ├── 02_adding_task_attention_tensorflow.py │ ├── 03_babi_memory_networks_tensorflow.py │ └── utils │ ├── __init__.py │ └── data.py ├── .gitignore ├── README.md ├── appendix ├── 1 │ ├── keras │ │ ├── 00_save_model_simple_keras.py │ │ ├── 01_restore_model_simple_keras.py │ │ └── 02_save_model_keras.py │ └── tensorflow │ │ ├── 00_save_model_simple_tensorlow.py │ │ ├── 01_restore_model_simple_tensorflow.py │ │ ├── 02_save_model_tensorflow.py │ │ └── 03_restore_model_tensorflow.py ├── 2 │ ├── 00_tensorboard_logistic_regression.py │ ├── 01_tensorboard_adam.py │ └── log │ │ └── .keep └── 3 │ └── 00_mnist_dnn_classifier.py └── mldata └── mnist-original.mat /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store -------------------------------------------------------------------------------- /2/00_hello_world.py: -------------------------------------------------------------------------------- 1 | print("hello, world!") 2 | -------------------------------------------------------------------------------- /2/01_if.py: -------------------------------------------------------------------------------- 1 | a = 10 2 | 3 | if a > 1: 4 | print("a > 1") 5 | -------------------------------------------------------------------------------- /2/02_if_else.py: -------------------------------------------------------------------------------- 1 | a = -10 2 | 3 | if a > 1: 4 | print("a > 1") 5 | else: 6 | print("a <= 1") 7 | -------------------------------------------------------------------------------- /2/03_if_elif.py: -------------------------------------------------------------------------------- 1 | a = 0 2 | 3 | if a > 1: 4 | print("a > 1") 5 | elif a > -1: 6 | print("1 >= a > -1") 7 | -------------------------------------------------------------------------------- /2/04_if_elif_else.py: -------------------------------------------------------------------------------- 1 | a = -2 2 | 3 | if a > 1: 4 | print("a > 1") 5 | elif a > -1: 6 | print("1 >= a > -1") 7 | elif a > -3: 8 | print("-1 >= a > -3") 9 | else: 10 | print("a <= -3") 11 | -------------------------------------------------------------------------------- /2/05_while.py: -------------------------------------------------------------------------------- 1 | a = 5 2 | 3 | while a > 0: 4 | print("a =", a) 5 | a -= 1 6 | -------------------------------------------------------------------------------- /2/06_while_break.py: -------------------------------------------------------------------------------- 1 | a = 5 2 | 3 | while a > 0: 4 | print("a =", a) 5 | a -= 1 6 | 7 | if a == 4: 8 | break 9 | -------------------------------------------------------------------------------- /2/07_while_else.py: -------------------------------------------------------------------------------- 1 | a = 5 2 | 3 | while a > 0: 4 | print("a =", a) 5 | a -= 1 6 | else: 7 | print("end of while") 8 | -------------------------------------------------------------------------------- /2/08_for.py: -------------------------------------------------------------------------------- 1 | data = [0, 1, 2, 3, 4, 5] 2 | 3 | for x in data: 4 | print(x, end=' ') 5 | -------------------------------------------------------------------------------- /2/09_for_break.py: -------------------------------------------------------------------------------- 1 | data = [0, 1, 2, 3, 4, 5] 2 | 3 | for x in data: 4 | print(x, end=' ') 5 | 6 | if x == 1: 7 | break 8 | -------------------------------------------------------------------------------- /2/10_for_dict.py: -------------------------------------------------------------------------------- 1 | data = {'tokyo': 1, 'new york': 2} 2 | 3 | for x in data: 4 | print(x) 5 | -------------------------------------------------------------------------------- /2/11_for_dict_key_value.py: -------------------------------------------------------------------------------- 1 | data = {'tokyo': 1, 'new york': 2} 2 | 3 | for key, value in data.items(): 4 | print(key, end=': ') 5 | print(value) 6 | -------------------------------------------------------------------------------- /2/12_function_print_x2.py: -------------------------------------------------------------------------------- 1 | def f(x): 2 | print(x ** 2) 3 | 4 | 5 | f(1) 6 | f(2) 7 | f(3) 8 | -------------------------------------------------------------------------------- /2/13_function_return_x2.py: -------------------------------------------------------------------------------- 1 | def f(x): 2 | return x ** 2 3 | 4 | 5 | print(f(1) + f(2)) 6 | -------------------------------------------------------------------------------- /2/14_function_return_ax2_1.py: -------------------------------------------------------------------------------- 1 | def f(x, a): 2 | return a * x ** 2, 2 * a * x 3 | 4 | 5 | y, y_prime = f(1, 2) 6 | 7 | print(y) 8 | print(y_prime) 9 | -------------------------------------------------------------------------------- /2/15_function_return_ax2_2.py: -------------------------------------------------------------------------------- 1 | def f(x, a=2): 2 | return a * x ** 2, 2 * a * x 3 | 4 | 5 | y, y_prime = f(1) 6 | 7 | print(y) 8 | print(y_prime) 9 | -------------------------------------------------------------------------------- /3/00_simple_perceptron.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ''' 4 | データの生成 5 | ''' 6 | rng = np.random.RandomState(123) 7 | 8 | d = 2 9 | N = 10 10 | 11 | mean = 5 12 | 13 | x1 = rng.randn(N, d) + np.array([0, 0]) 14 | x2 = rng.randn(N, d) + np.array([mean, mean]) 15 | 16 | x = np.concatenate((x1, x2), axis=0) 17 | 18 | ''' 19 | 単純パーセプトロン 20 | ''' 21 | w = np.zeros(d) 22 | b = 0 23 | 24 | 25 | def y(x): 26 | return step(np.dot(w, x) + b) 27 | 28 | 29 | def step(x): 30 | return 1 * (x > 0) 31 | 32 | 33 | def t(i): 34 | if i < N: 35 | return 0 36 | else: 37 | return 1 38 | 39 | 40 | while True: 41 | classified = True 42 | for i in range(N * 2): 43 | delta_w = (t(i) - y(x[i])) * x[i] 44 | delta_b = (t(i) - y(x[i])) 45 | w += delta_w 46 | b += delta_b 47 | classified *= all(delta_w == 0) * (delta_b == 0) 48 | if classified: 49 | break 50 | 51 | print('w:', w) 52 | print('b:', b) 53 | 54 | print('\nTest:') 55 | print(y([0, 0])) # => 0 56 | print(y([5, 5])) # => 1 57 | -------------------------------------------------------------------------------- /3/keras/01_logistic_regressioni_or_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers import Dense, Activation 4 | from keras.optimizers import SGD 5 | 6 | np.random.seed(0) # 乱数シード 7 | 8 | ''' 9 | モデル設定 10 | ''' 11 | model = Sequential([ 12 | # Dense(input_dim=2, output_dim=1), # Keras 1 13 | Dense(input_dim=2, units=1), # Keras 2 14 | Activation('sigmoid') 15 | ]) 16 | 17 | model.compile(loss='binary_crossentropy', optimizer=SGD(lr=0.1)) 18 | 19 | ''' 20 | モデル学習 21 | ''' 22 | # ORゲート 23 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 24 | Y = np.array([[0], [1], [1], [1]]) 25 | 26 | # model.fit(X, Y, nb_epoch=200, batch_size=1) # Keras 1 27 | model.fit(X, Y, epochs=200, batch_size=1) # Keras 2 28 | 29 | ''' 30 | 学習結果の確認 31 | ''' 32 | classes = model.predict_classes(X, batch_size=1) 33 | prob = model.predict_proba(X, batch_size=1) 34 | 35 | print('classified:') 36 | print(Y == classes) 37 | print() 38 | print('output probability:') 39 | print(prob) 40 | -------------------------------------------------------------------------------- /3/keras/02_multi_class_logistic_regression_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers import Dense, Activation 4 | from keras.optimizers import SGD 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) # 乱数シード 8 | 9 | M = 2 # 入力データの次元 10 | K = 3 # クラス数 11 | n = 100 # クラスごとのデータ数 12 | N = n * K # 全データ数 13 | 14 | ''' 15 | データの生成 16 | ''' 17 | X1 = np.random.randn(n, M) + np.array([0, 10]) 18 | X2 = np.random.randn(n, M) + np.array([5, 5]) 19 | X3 = np.random.randn(n, M) + np.array([10, 0]) 20 | Y1 = np.array([[1, 0, 0] for i in range(n)]) 21 | Y2 = np.array([[0, 1, 0] for i in range(n)]) 22 | Y3 = np.array([[0, 0, 1] for i in range(n)]) 23 | 24 | X = np.concatenate((X1, X2, X3), axis=0) 25 | Y = np.concatenate((Y1, Y2, Y3), axis=0) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | model = Sequential() 31 | model.add(Dense(input_dim=M, units=K)) 32 | model.add(Activation('softmax')) 33 | 34 | model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.1)) 35 | 36 | ''' 37 | モデル学習 38 | ''' 39 | minibatch_size = 50 40 | model.fit(X, Y, epochs=20, batch_size=minibatch_size) 41 | 42 | ''' 43 | 学習結果の確認 44 | ''' 45 | X_, Y_ = shuffle(X, Y) 46 | classes = model.predict_classes(X_[0:10], batch_size=minibatch_size) 47 | prob = model.predict_proba(X_[0:10], batch_size=minibatch_size) 48 | print('classified:') 49 | print(np.argmax(model.predict(X_[0:10]), axis=1) == classes) 50 | print() 51 | print('output probability:') 52 | print(prob) 53 | -------------------------------------------------------------------------------- /3/keras/03_mlp_xor_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers import Dense, Activation 4 | from keras.optimizers import SGD 5 | 6 | np.random.seed(123) 7 | 8 | ''' 9 | データの生成 10 | ''' 11 | # XORゲート 12 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 13 | Y = np.array([[0], [1], [1], [0]]) 14 | 15 | ''' 16 | モデル設定 17 | ''' 18 | model = Sequential() 19 | 20 | # 入力層 - 隠れ層 21 | model.add(Dense(input_dim=2, units=2)) 22 | model.add(Activation('sigmoid')) 23 | 24 | # 隠れ層 - 出力層 25 | model.add(Dense(units=1)) 26 | model.add(Activation('sigmoid')) 27 | 28 | model.compile(loss='binary_crossentropy', optimizer=SGD(lr=0.1)) 29 | 30 | ''' 31 | モデル学習 32 | ''' 33 | model.fit(X, Y, epochs=4000, batch_size=4) 34 | 35 | ''' 36 | 学習結果の確認 37 | ''' 38 | classes = model.predict_classes(X, batch_size=4) 39 | prob = model.predict_proba(X, batch_size=4) 40 | 41 | print('classified:') 42 | print(Y == classes) 43 | print() 44 | print('output probability:') 45 | print(prob) 46 | -------------------------------------------------------------------------------- /3/keras/04_toy_problem_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | from sklearn.utils import shuffle 8 | 9 | np.random.seed(0) 10 | 11 | ''' 12 | データ生成 13 | ''' 14 | N = 300 15 | X, y = datasets.make_moons(N, noise=0.3) 16 | 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8) 18 | 19 | ''' 20 | モデル生成 21 | ''' 22 | model = Sequential() 23 | model.add(Dense(3, input_dim=2)) 24 | model.add(Activation('sigmoid')) 25 | model.add(Dense(1)) 26 | model.add(Activation('sigmoid')) 27 | model.compile(loss='binary_crossentropy', 28 | optimizer=SGD(lr=0.05), 29 | metrics=['accuracy']) 30 | 31 | ''' 32 | モデル学習 33 | ''' 34 | model.fit(X_train, y_train, epochs=500, batch_size=20) 35 | 36 | ''' 37 | 予測精度の評価 38 | ''' 39 | loss_and_metrics = model.evaluate(X_test, y_test) 40 | print(loss_and_metrics) 41 | -------------------------------------------------------------------------------- /3/tensorflow/01_logistic_regression_or_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | ''' 5 | モデル設定 6 | ''' 7 | tf.set_random_seed(0) # 乱数シード 8 | 9 | w = tf.Variable(tf.zeros([2, 1])) 10 | b = tf.Variable(tf.zeros([1])) 11 | 12 | x = tf.placeholder(tf.float32, shape=[None, 2]) 13 | t = tf.placeholder(tf.float32, shape=[None, 1]) 14 | y = tf.nn.sigmoid(tf.matmul(x, w) + b) 15 | 16 | cross_entropy = - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 17 | train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 18 | 19 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 20 | 21 | ''' 22 | モデル学習 23 | ''' 24 | # ORゲート 25 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 26 | Y = np.array([[0], [1], [1], [1]]) 27 | 28 | # 初期化 29 | init = tf.global_variables_initializer() 30 | sess = tf.Session() 31 | sess.run(init) 32 | 33 | # 学習 34 | for epoch in range(200): 35 | sess.run(train_step, feed_dict={ 36 | x: X, 37 | t: Y 38 | }) 39 | 40 | ''' 41 | 学習結果の確認 42 | ''' 43 | classified = correct_prediction.eval(session=sess, feed_dict={ 44 | x: X, 45 | t: Y 46 | }) 47 | prob = y.eval(session=sess, feed_dict={ 48 | x: X 49 | }) 50 | 51 | print('classified:') 52 | print(classified) 53 | print() 54 | print('output probability:') 55 | print(prob) 56 | -------------------------------------------------------------------------------- /3/tensorflow/02_multi_class_logistic_regression_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn.utils import shuffle 4 | 5 | np.random.seed(0) 6 | tf.set_random_seed(0) 7 | 8 | M = 2 # 入力データの次元 9 | K = 3 # クラス数 10 | n = 100 # クラスごとのデータ数 11 | N = n * K # 全データ数 12 | 13 | ''' 14 | データの生成 15 | ''' 16 | X1 = np.random.randn(n, M) + np.array([0, 10]) 17 | X2 = np.random.randn(n, M) + np.array([5, 5]) 18 | X3 = np.random.randn(n, M) + np.array([10, 0]) 19 | Y1 = np.array([[1, 0, 0] for i in range(n)]) 20 | Y2 = np.array([[0, 1, 0] for i in range(n)]) 21 | Y3 = np.array([[0, 0, 1] for i in range(n)]) 22 | 23 | X = np.concatenate((X1, X2, X3), axis=0) 24 | Y = np.concatenate((Y1, Y2, Y3), axis=0) 25 | 26 | ''' 27 | モデル設定 28 | ''' 29 | W = tf.Variable(tf.zeros([M, K])) 30 | b = tf.Variable(tf.zeros([K])) 31 | 32 | x = tf.placeholder(tf.float32, shape=[None, M]) 33 | t = tf.placeholder(tf.float32, shape=[None, K]) 34 | y = tf.nn.softmax(tf.matmul(x, W) + b) 35 | 36 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 37 | train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 38 | 39 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 40 | 41 | ''' 42 | モデル学習 43 | ''' 44 | # 初期化 45 | init = tf.global_variables_initializer() 46 | sess = tf.Session() 47 | sess.run(init) 48 | 49 | batch_size = 50 # ミニバッチサイズ 50 | n_batches = N // batch_size 51 | 52 | # ミニバッチ学習 53 | for epoch in range(20): 54 | X_, Y_ = shuffle(X, Y) 55 | 56 | for i in range(n_batches): 57 | start = i * batch_size 58 | end = start + batch_size 59 | 60 | sess.run(train_step, feed_dict={ 61 | x: X_[start:end], 62 | t: Y_[start:end] 63 | }) 64 | 65 | ''' 66 | 学習結果の確認 67 | ''' 68 | X_, Y_ = shuffle(X, Y) 69 | 70 | classified = correct_prediction.eval(session=sess, feed_dict={ 71 | x: X_[0:10], 72 | t: Y_[0:10] 73 | }) 74 | prob = y.eval(session=sess, feed_dict={ 75 | x: X_[0:10] 76 | }) 77 | 78 | print('classified:') 79 | print(classified) 80 | print() 81 | print('output probability:') 82 | print(prob) 83 | -------------------------------------------------------------------------------- /3/tensorflow/03_mlp_xor_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | tf.set_random_seed(0) 5 | 6 | ''' 7 | データの生成 8 | ''' 9 | # XORゲート 10 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 11 | Y = np.array([[0], [1], [1], [0]]) 12 | 13 | ''' 14 | モデル設定 15 | ''' 16 | x = tf.placeholder(tf.float32, shape=[None, 2]) 17 | t = tf.placeholder(tf.float32, shape=[None, 1]) 18 | 19 | # 入力層 - 隠れ層 20 | W = tf.Variable(tf.truncated_normal([2, 2])) 21 | b = tf.Variable(tf.zeros([2])) 22 | h = tf.nn.sigmoid(tf.matmul(x, W) + b) 23 | 24 | # 隠れ層 - 出力層 25 | V = tf.Variable(tf.truncated_normal([2, 1])) 26 | c = tf.Variable(tf.zeros([1])) 27 | y = tf.nn.sigmoid(tf.matmul(h, V) + c) 28 | 29 | cross_entropy = - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 30 | train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 31 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 32 | 33 | ''' 34 | モデル学習 35 | ''' 36 | init = tf.global_variables_initializer() 37 | sess = tf.Session() 38 | sess.run(init) 39 | 40 | for epoch in range(4000): 41 | sess.run(train_step, feed_dict={ 42 | x: X, 43 | t: Y 44 | }) 45 | if epoch % 1000 == 0: 46 | print('epoch:', epoch) 47 | 48 | ''' 49 | 学習結果の確認 50 | ''' 51 | classified = correct_prediction.eval(session=sess, feed_dict={ 52 | x: X, 53 | t: Y 54 | }) 55 | prob = y.eval(session=sess, feed_dict={ 56 | x: X 57 | }) 58 | 59 | print('classified:') 60 | print(classified) 61 | print() 62 | print('output probability:') 63 | print(prob) 64 | -------------------------------------------------------------------------------- /3/tensorflow/04_toy_problem_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | ''' 11 | データ生成 12 | ''' 13 | N = 300 # 全データ数 14 | X, y = datasets.make_moons(N, noise=0.3) 15 | Y = y.reshape(N, 1) 16 | 17 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 18 | 19 | 20 | ''' 21 | モデル生成 22 | ''' 23 | num_hidden = 3 # 隠れ層の次元数 24 | # num_hidden = 2 25 | 26 | x = tf.placeholder(tf.float32, shape=[None, 2]) 27 | t = tf.placeholder(tf.float32, shape=[None, 1]) 28 | 29 | # 入力層 - 隠れ層 30 | W = tf.Variable(tf.truncated_normal([2, num_hidden])) 31 | b = tf.Variable(tf.zeros([num_hidden])) 32 | h = tf.nn.sigmoid(tf.matmul(x, W) + b) 33 | 34 | # 隠れ層 - 出力層 35 | V = tf.Variable(tf.truncated_normal([num_hidden, 1])) 36 | c = tf.Variable(tf.zeros([1])) 37 | y = tf.nn.sigmoid(tf.matmul(h, V) + c) 38 | 39 | cross_entropy = - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 40 | train_step = tf.train.GradientDescentOptimizer(0.05).minimize(cross_entropy) 41 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 42 | 43 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 44 | 45 | ''' 46 | モデル学習 47 | ''' 48 | batch_size = 20 49 | n_batches = N // batch_size 50 | 51 | init = tf.global_variables_initializer() 52 | sess = tf.Session() 53 | sess.run(init) 54 | 55 | for epoch in range(500): 56 | X_, Y_ = shuffle(X_train, Y_train) 57 | 58 | for i in range(n_batches): 59 | start = i * batch_size 60 | end = start + batch_size 61 | 62 | sess.run(train_step, feed_dict={ 63 | x: X_[start:end], 64 | t: Y_[start:end] 65 | }) 66 | 67 | ''' 68 | 予測精度の評価 69 | ''' 70 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 71 | x: X_test, 72 | t: Y_test 73 | }) 74 | print('accuracy: ', accuracy_rate) 75 | -------------------------------------------------------------------------------- /4/keras/00_mnist_sigmoid_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | 8 | np.random.seed(0) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 18 | 19 | X = mnist.data[indices] 20 | y = mnist.target[indices] 21 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 22 | 23 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 24 | 25 | ''' 26 | モデル設定 27 | ''' 28 | n_in = len(X[0]) # 784 29 | n_hidden = 200 30 | # n_hidden = 4000 31 | n_out = len(Y[0]) # 10 32 | 33 | model = Sequential() 34 | model.add(Dense(n_hidden, input_dim=n_in)) 35 | model.add(Activation('sigmoid')) 36 | 37 | # model.add(Dense(n_hidden)) 38 | # model.add(Activation('sigmoid')) 39 | 40 | # model.add(Dense(n_hidden)) 41 | # model.add(Activation('sigmoid')) 42 | 43 | # model.add(Dense(n_hidden)) 44 | # model.add(Activation('sigmoid')) 45 | 46 | model.add(Dense(n_out)) 47 | model.add(Activation('softmax')) 48 | 49 | model.compile(loss='categorical_crossentropy', 50 | optimizer=SGD(lr=0.01), 51 | metrics=['accuracy']) 52 | 53 | ''' 54 | モデル学習 55 | ''' 56 | epochs = 100 57 | batch_size = 200 58 | 59 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 60 | 61 | ''' 62 | 予測精度の評価 63 | ''' 64 | loss_and_metrics = model.evaluate(X_test, Y_test) 65 | print(loss_and_metrics) 66 | -------------------------------------------------------------------------------- /4/keras/01_mnist_tanh_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | 8 | np.random.seed(0) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 18 | 19 | X = mnist.data[indices] 20 | y = mnist.target[indices] 21 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 22 | 23 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 24 | 25 | ''' 26 | モデル設定 27 | ''' 28 | n_in = len(X[0]) # 784 29 | n_hidden = 200 30 | n_out = len(Y[0]) # 10 31 | 32 | model = Sequential() 33 | model.add(Dense(n_hidden, input_dim=n_in)) 34 | model.add(Activation('tanh')) 35 | 36 | model.add(Dense(n_hidden)) 37 | model.add(Activation('tanh')) 38 | 39 | model.add(Dense(n_hidden)) 40 | model.add(Activation('tanh')) 41 | 42 | model.add(Dense(n_hidden)) 43 | model.add(Activation('tanh')) 44 | 45 | model.add(Dense(n_out)) 46 | model.add(Activation('softmax')) 47 | 48 | model.compile(loss='categorical_crossentropy', 49 | optimizer=SGD(lr=0.01), 50 | metrics=['accuracy']) 51 | 52 | ''' 53 | モデル学習 54 | ''' 55 | epochs = 100 56 | batch_size = 200 57 | 58 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 59 | 60 | ''' 61 | 予測精度の評価 62 | ''' 63 | loss_and_metrics = model.evaluate(X_test, Y_test) 64 | print(loss_and_metrics) 65 | -------------------------------------------------------------------------------- /4/keras/02_mnist_relu_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | 8 | np.random.seed(1234) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 18 | 19 | X = mnist.data[indices] 20 | y = mnist.target[indices] 21 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 22 | 23 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 24 | 25 | ''' 26 | モデル設定 27 | ''' 28 | n_in = len(X[0]) # 784 29 | n_hidden = 200 30 | n_out = len(Y[0]) # 10 31 | 32 | model = Sequential() 33 | model.add(Dense(n_hidden, input_dim=n_in)) 34 | model.add(Activation('relu')) 35 | 36 | model.add(Dense(n_hidden)) 37 | model.add(Activation('relu')) 38 | 39 | model.add(Dense(n_hidden)) 40 | model.add(Activation('relu')) 41 | 42 | model.add(Dense(n_hidden)) 43 | model.add(Activation('relu')) 44 | 45 | model.add(Dense(n_out)) 46 | model.add(Activation('softmax')) 47 | 48 | model.compile(loss='categorical_crossentropy', 49 | optimizer=SGD(lr=0.01), 50 | metrics=['accuracy']) 51 | 52 | ''' 53 | モデル学習 54 | ''' 55 | # epochs = 100 56 | epochs = 50 # ReLU では学習が早く進む 57 | batch_size = 200 58 | 59 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 60 | 61 | ''' 62 | 予測精度の評価 63 | ''' 64 | loss_and_metrics = model.evaluate(X_test, Y_test) 65 | print(loss_and_metrics) 66 | -------------------------------------------------------------------------------- /4/keras/03_mnist_lrelu_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.layers.advanced_activations import LeakyReLU 5 | from keras.optimizers import SGD 6 | from sklearn import datasets 7 | from sklearn.model_selection import train_test_split 8 | 9 | np.random.seed(1234) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 10000 # MNISTの一部を使う 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 25 | 26 | ''' 27 | モデル設定 28 | ''' 29 | n_in = len(X[0]) # 784 30 | n_hidden = 200 31 | n_out = len(Y[0]) # 10 32 | 33 | alpha = 0.01 34 | 35 | model = Sequential() 36 | model.add(Dense(n_hidden, input_dim=n_in)) 37 | model.add(LeakyReLU(alpha=alpha)) 38 | 39 | model.add(Dense(n_hidden)) 40 | model.add(LeakyReLU(alpha=alpha)) 41 | 42 | model.add(Dense(n_hidden)) 43 | model.add(LeakyReLU(alpha=alpha)) 44 | 45 | model.add(Dense(n_hidden)) 46 | model.add(LeakyReLU(alpha=alpha)) 47 | 48 | model.add(Dense(n_out)) 49 | model.add(Activation('softmax')) 50 | 51 | model.compile(loss='categorical_crossentropy', 52 | optimizer=SGD(lr=0.01), 53 | metrics=['accuracy']) 54 | 55 | ''' 56 | モデル学習 57 | ''' 58 | epochs = 20 59 | batch_size = 200 60 | 61 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 62 | 63 | ''' 64 | 予測精度の評価 65 | ''' 66 | loss_and_metrics = model.evaluate(X_test, Y_test) 67 | print(loss_and_metrics) 68 | -------------------------------------------------------------------------------- /4/keras/04_mnist_prelu_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.layers.advanced_activations import PReLU 5 | from keras.optimizers import SGD 6 | from sklearn import datasets 7 | from sklearn.model_selection import train_test_split 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 10000 # MNISTの一部を使う 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 25 | 26 | ''' 27 | モデル設定 28 | ''' 29 | n_in = len(X[0]) # 784 30 | n_hidden = 200 31 | n_out = len(Y[0]) # 10 32 | 33 | model = Sequential() 34 | model.add(Dense(n_hidden, input_dim=n_in)) 35 | model.add(PReLU()) 36 | 37 | model.add(Dense(n_hidden)) 38 | model.add(PReLU()) 39 | 40 | model.add(Dense(n_hidden)) 41 | model.add(PReLU()) 42 | 43 | model.add(Dense(n_hidden)) 44 | model.add(PReLU()) 45 | 46 | model.add(Dense(n_out)) 47 | model.add(Activation('softmax')) 48 | 49 | model.compile(loss='categorical_crossentropy', 50 | optimizer=SGD(lr=0.01), 51 | metrics=['accuracy']) 52 | 53 | ''' 54 | モデル学習 55 | ''' 56 | epochs = 20 57 | batch_size = 200 58 | 59 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 60 | 61 | ''' 62 | 予測精度の評価 63 | ''' 64 | loss_and_metrics = model.evaluate(X_test, Y_test) 65 | print(loss_and_metrics) 66 | -------------------------------------------------------------------------------- /4/keras/05_mnist_dropout_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | 8 | np.random.seed(0) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 18 | 19 | X = mnist.data[indices] 20 | y = mnist.target[indices] 21 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 22 | 23 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 24 | 25 | ''' 26 | モデル設定 27 | ''' 28 | n_in = len(X[0]) # 784 29 | n_hidden = 200 30 | n_out = len(Y[0]) # 10 31 | 32 | model = Sequential() 33 | model.add(Dense(n_hidden, input_dim=n_in)) 34 | model.add(Activation('tanh')) 35 | model.add(Dropout(0.5)) 36 | 37 | model.add(Dense(n_hidden)) 38 | model.add(Activation('tanh')) 39 | model.add(Dropout(0.5)) 40 | 41 | model.add(Dense(n_hidden)) 42 | model.add(Activation('tanh')) 43 | model.add(Dropout(0.5)) 44 | 45 | model.add(Dense(n_out)) 46 | model.add(Activation('softmax')) 47 | 48 | model.compile(loss='categorical_crossentropy', 49 | optimizer=SGD(lr=0.01), 50 | metrics=['accuracy']) 51 | 52 | ''' 53 | モデル学習 54 | ''' 55 | epochs = 150 56 | batch_size = 200 57 | 58 | model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size) 59 | 60 | ''' 61 | 予測精度の評価 62 | ''' 63 | loss_and_metrics = model.evaluate(X_test, Y_test) 64 | print(loss_and_metrics) 65 | -------------------------------------------------------------------------------- /4/keras/06_mnist_plot_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import SGD 5 | from keras import backend as K 6 | from sklearn import datasets 7 | from sklearn.model_selection import train_test_split 8 | import matplotlib.pyplot as plt 9 | 10 | np.random.seed(123) 11 | 12 | ''' 13 | データの生成 14 | ''' 15 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 16 | 17 | n = len(mnist.data) 18 | N = 30000 # MNISTの一部を使う 19 | N_train = 20000 20 | N_validation = 4000 21 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 22 | 23 | X = mnist.data[indices] 24 | y = mnist.target[indices] 25 | Y = np.eye(10)[y.astype(int)] 26 | 27 | X_train, X_test, Y_train, Y_test = \ 28 | train_test_split(X, Y, train_size=N_train) 29 | X_train, X_validation, Y_train, Y_validation = \ 30 | train_test_split(X_train, Y_train, test_size=N_validation) 31 | 32 | ''' 33 | モデル設定 34 | ''' 35 | n_in = len(X[0]) # 784 36 | n_hiddens = [200, 200, 200] 37 | n_out = len(Y[0]) # 10 38 | p_keep = 0.5 39 | activation = 'relu' 40 | 41 | 42 | def weight_variable(shape): 43 | return K.truncated_normal(shape, stddev=0.01) 44 | # return np.random.normal(scale=0.01, size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=SGD(lr=0.01), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_acc, label='acc', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | # plt.savefig('mnist_keras.eps') 83 | 84 | ''' 85 | 予測精度の評価 86 | ''' 87 | loss_and_metrics = model.evaluate(X_test, Y_test) 88 | print(loss_and_metrics) 89 | -------------------------------------------------------------------------------- /4/keras/07_mnist_momentum_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=SGD(lr=0.01, momentum=0.9), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/08_mnist_nesterov_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import SGD 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=SGD(lr=0.01, momentum=0.9, nesterov=True), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/09_mnist_adagrad_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import Adagrad 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=Adagrad(lr=0.01), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/10_mnist_adadelta_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import Adadelta 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=Adadelta(rho=0.95), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/11_mnist_rmsprop_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import RMSprop 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=RMSprop(lr=0.001), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/12_mnist_adam_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import Adam 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | import matplotlib.pyplot as plt 8 | 9 | np.random.seed(123) 10 | 11 | ''' 12 | データの生成 13 | ''' 14 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 15 | 16 | n = len(mnist.data) 17 | N = 30000 # MNISTの一部を使う 18 | N_train = 20000 19 | N_validation = 4000 20 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 21 | 22 | X = mnist.data[indices] 23 | X = X / 255.0 24 | X = X - X.mean(axis=1).reshape(len(X), 1) 25 | y = mnist.target[indices] 26 | Y = np.eye(10)[y.astype(int)] 27 | 28 | X_train, X_test, Y_train, Y_test = \ 29 | train_test_split(X, Y, train_size=N_train) 30 | X_train, X_validation, Y_train, Y_validation = \ 31 | train_test_split(X_train, Y_train, test_size=N_validation) 32 | 33 | ''' 34 | モデル設定 35 | ''' 36 | n_in = len(X[0]) # 784 37 | n_hiddens = [200, 200, 200] 38 | n_out = len(Y[0]) # 10 39 | p_keep = 0.5 40 | activation = 'relu' 41 | 42 | 43 | def weight_variable(shape, name=None): 44 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 45 | 46 | 47 | model = Sequential() 48 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 49 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 50 | kernel_initializer=weight_variable)) 51 | model.add(Activation(activation)) 52 | model.add(Dropout(p_keep)) 53 | 54 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 55 | model.add(Activation('softmax')) 56 | 57 | model.compile(loss='categorical_crossentropy', 58 | optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999), 59 | metrics=['accuracy']) 60 | 61 | ''' 62 | モデル学習 63 | ''' 64 | epochs = 50 65 | batch_size = 200 66 | 67 | hist = model.fit(X_train, Y_train, epochs=epochs, 68 | batch_size=batch_size, 69 | validation_data=(X_validation, Y_validation)) 70 | 71 | ''' 72 | 学習の進み具合を可視化 73 | ''' 74 | val_acc = hist.history['val_acc'] 75 | val_loss = hist.history['val_loss'] 76 | 77 | plt.rc('font', family='serif') 78 | fig = plt.figure() 79 | plt.plot(range(epochs), val_loss, label='loss', color='black') 80 | plt.xlabel('epochs') 81 | plt.show() 82 | 83 | ''' 84 | 予測精度の評価 85 | ''' 86 | loss_and_metrics = model.evaluate(X_test, Y_test) 87 | print(loss_and_metrics) 88 | -------------------------------------------------------------------------------- /4/keras/13_mnist_early_stopping_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, Dropout 4 | from keras.optimizers import Adam 5 | from keras.callbacks import EarlyStopping 6 | from sklearn import datasets 7 | from sklearn.model_selection import train_test_split 8 | import matplotlib.pyplot as plt 9 | 10 | np.random.seed(123) 11 | 12 | ''' 13 | データの生成 14 | ''' 15 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 16 | 17 | n = len(mnist.data) 18 | N = 30000 # MNISTの一部を使う 19 | N_train = 20000 20 | N_validation = 4000 21 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 22 | 23 | X = mnist.data[indices] 24 | X = X / 255.0 25 | X = X - X.mean(axis=1).reshape(len(X), 1) 26 | y = mnist.target[indices] 27 | Y = np.eye(10)[y.astype(int)] 28 | 29 | X_train, X_test, Y_train, Y_test = \ 30 | train_test_split(X, Y, train_size=N_train) 31 | X_train, X_validation, Y_train, Y_validation = \ 32 | train_test_split(X_train, Y_train, test_size=N_validation) 33 | 34 | ''' 35 | モデル設定 36 | ''' 37 | n_in = len(X[0]) # 784 38 | n_hiddens = [200, 200, 200] 39 | n_out = len(Y[0]) # 10 40 | p_keep = 0.5 41 | activation = 'relu' 42 | 43 | 44 | def weight_variable(shape, name=None): 45 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 46 | 47 | 48 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 49 | 50 | model = Sequential() 51 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 52 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 53 | kernel_initializer=weight_variable)) 54 | model.add(Activation(activation)) 55 | model.add(Dropout(p_keep)) 56 | 57 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 58 | model.add(Activation('softmax')) 59 | 60 | model.compile(loss='categorical_crossentropy', 61 | optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999), 62 | metrics=['accuracy']) 63 | 64 | ''' 65 | モデル学習 66 | ''' 67 | epochs = 200 68 | batch_size = 200 69 | 70 | hist = model.fit(X_train, Y_train, epochs=epochs, 71 | batch_size=batch_size, 72 | validation_data=(X_validation, Y_validation), 73 | callbacks=[early_stopping]) 74 | 75 | ''' 76 | 学習の進み具合を可視化 77 | ''' 78 | val_acc = hist.history['val_acc'] 79 | val_loss = hist.history['val_loss'] 80 | 81 | plt.rc('font', family='serif') 82 | fig = plt.figure() 83 | plt.plot(range(len(val_loss)), val_loss, label='loss', color='black') 84 | plt.xlabel('epochs') 85 | plt.show() 86 | 87 | ''' 88 | 予測精度の評価 89 | ''' 90 | loss_and_metrics = model.evaluate(X_test, Y_test) 91 | print(loss_and_metrics) 92 | -------------------------------------------------------------------------------- /4/keras/14_mnist_batch_normalization_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation 4 | from keras.layers.normalization import BatchNormalization 5 | from keras.optimizers import Adam 6 | from keras.callbacks import EarlyStopping 7 | from sklearn import datasets 8 | from sklearn.model_selection import train_test_split 9 | import matplotlib.pyplot as plt 10 | 11 | np.random.seed(123) 12 | 13 | ''' 14 | データの生成 15 | ''' 16 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 17 | 18 | n = len(mnist.data) 19 | N = 30000 # MNISTの一部を使う 20 | N_train = 20000 21 | N_validation = 4000 22 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 23 | 24 | X = mnist.data[indices] 25 | X = X / 255.0 26 | X = X - X.mean(axis=1).reshape(len(X), 1) 27 | y = mnist.target[indices] 28 | Y = np.eye(10)[y.astype(int)] 29 | 30 | X_train, X_test, Y_train, Y_test = \ 31 | train_test_split(X, Y, train_size=N_train) 32 | X_train, X_validation, Y_train, Y_validation = \ 33 | train_test_split(X_train, Y_train, test_size=N_validation) 34 | 35 | ''' 36 | モデル設定 37 | ''' 38 | n_in = len(X[0]) # 784 39 | n_hiddens = [200, 200, 200] 40 | n_out = len(Y[0]) # 10 41 | activation = 'relu' 42 | 43 | 44 | def weight_variable(shape, name=None): 45 | return np.sqrt(2.0 / shape[0]) * np.random.normal(size=shape) 46 | 47 | 48 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 49 | 50 | model = Sequential() 51 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 52 | model.add(Dense(n_hiddens[i], input_dim=input_dim, 53 | kernel_initializer=weight_variable)) 54 | model.add(BatchNormalization()) 55 | model.add(Activation(activation)) 56 | 57 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 58 | model.add(Activation('softmax')) 59 | 60 | model.compile(loss='categorical_crossentropy', 61 | optimizer=Adam(lr=0.01, beta_1=0.9, beta_2=0.999), 62 | metrics=['accuracy']) 63 | 64 | ''' 65 | モデル学習 66 | ''' 67 | epochs = 200 68 | batch_size = 200 69 | 70 | hist = model.fit(X_train, Y_train, epochs=epochs, 71 | batch_size=batch_size, 72 | validation_data=(X_validation, Y_validation), 73 | callbacks=[early_stopping]) 74 | 75 | ''' 76 | 学習の進み具合を可視化 77 | ''' 78 | val_acc = hist.history['val_acc'] 79 | val_loss = hist.history['val_loss'] 80 | 81 | plt.rc('font', family='serif') 82 | fig = plt.figure() 83 | plt.plot(range(len(val_loss)), val_loss, label='loss', color='black') 84 | plt.xlabel('epochs') 85 | plt.show() 86 | 87 | ''' 88 | 予測精度の評価 89 | ''' 90 | loss_and_metrics = model.evaluate(X_test, Y_test) 91 | print(loss_and_metrics) 92 | -------------------------------------------------------------------------------- /4/tensorflow/01_mnist_tanh_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(123) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | train_size = 0.8 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test =\ 25 | train_test_split(X, Y, train_size=train_size) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = len(X[0]) # 784 31 | n_hidden = 200 32 | n_out = len(Y[0]) # 10 33 | 34 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 35 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 36 | 37 | # 入力層 - 隠れ層 38 | W0 = tf.Variable(tf.truncated_normal([n_in, n_hidden], stddev=0.1)) 39 | b0 = tf.Variable(tf.zeros([n_hidden])) 40 | h0 = tf.nn.tanh(tf.matmul(x, W0) + b0) 41 | 42 | # 隠れ層 - 隠れ層 43 | W1 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.1)) 44 | b1 = tf.Variable(tf.zeros([n_hidden])) 45 | h1 = tf.nn.tanh(tf.matmul(h0, W1) + b1) 46 | 47 | W2 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.1)) 48 | b2 = tf.Variable(tf.zeros([n_hidden])) 49 | h2 = tf.nn.tanh(tf.matmul(h1, W2) + b2) 50 | 51 | W3 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.1)) 52 | b3 = tf.Variable(tf.zeros([n_hidden])) 53 | h3 = tf.nn.tanh(tf.matmul(h2, W3) + b3) 54 | 55 | # 隠れ層 - 出力層 56 | W4 = tf.Variable(tf.truncated_normal([n_hidden, n_out], stddev=0.1)) 57 | b4 = tf.Variable(tf.zeros([n_out])) 58 | y = tf.nn.softmax(tf.matmul(h3, W4) + b4) 59 | 60 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 61 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) 62 | 63 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 64 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 65 | 66 | ''' 67 | モデル学習 68 | ''' 69 | epochs = 100 70 | batch_size = 200 71 | 72 | init = tf.global_variables_initializer() 73 | sess = tf.Session() 74 | sess.run(init) 75 | 76 | n_batches = (int)(N * train_size) // batch_size 77 | 78 | for epoch in range(epochs): 79 | X_, Y_ = shuffle(X_train, Y_train) 80 | 81 | for i in range(n_batches): 82 | start = i * batch_size 83 | end = start + batch_size 84 | 85 | sess.run(train_step, feed_dict={ 86 | x: X_[start:end], 87 | t: Y_[start:end] 88 | }) 89 | 90 | # 訓練データに対する学習の進み具合を出力 91 | loss = cross_entropy.eval(session=sess, feed_dict={ 92 | x: X_, 93 | t: Y_ 94 | }) 95 | acc = accuracy.eval(session=sess, feed_dict={ 96 | x: X_, 97 | t: Y_ 98 | }) 99 | print('epoch:', epoch, ' loss:', loss, ' accuracy:', acc) 100 | 101 | ''' 102 | 予測精度の評価 103 | ''' 104 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 105 | x: X_test, 106 | t: Y_test 107 | }) 108 | print('accuracy: ', accuracy_rate) 109 | -------------------------------------------------------------------------------- /4/tensorflow/02_mnist_relu_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | train_size = 0.8 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test =\ 25 | train_test_split(X, Y, train_size=train_size) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = len(X[0]) # 784 31 | n_hidden = 200 32 | n_out = len(Y[0]) # 10 33 | 34 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 35 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 36 | 37 | # 入力層 - 隠れ層 38 | W0 = tf.Variable(tf.truncated_normal([n_in, n_hidden], stddev=0.01)) 39 | b0 = tf.Variable(tf.zeros([n_hidden])) 40 | h0 = tf.nn.relu(tf.matmul(x, W0) + b0) 41 | 42 | # 隠れ層 - 隠れ層 43 | W1 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 44 | b1 = tf.Variable(tf.zeros([n_hidden])) 45 | h1 = tf.nn.relu(tf.matmul(h0, W1) + b1) 46 | 47 | W2 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 48 | b2 = tf.Variable(tf.zeros([n_hidden])) 49 | h2 = tf.nn.relu(tf.matmul(h1, W2) + b2) 50 | 51 | W3 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 52 | b3 = tf.Variable(tf.zeros([n_hidden])) 53 | h3 = tf.nn.relu(tf.matmul(h2, W3) + b3) 54 | 55 | # 隠れ層 - 出力層 56 | W4 = tf.Variable(tf.truncated_normal([n_hidden, n_out], stddev=0.01)) 57 | b4 = tf.Variable(tf.zeros([n_out])) 58 | y = tf.nn.softmax(tf.matmul(h3, W4) + b4) 59 | 60 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 61 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) 62 | 63 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 64 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 65 | 66 | ''' 67 | モデル学習 68 | ''' 69 | epochs = 50 70 | batch_size = 200 71 | 72 | init = tf.global_variables_initializer() 73 | sess = tf.Session() 74 | sess.run(init) 75 | 76 | n_batches = (int)(N * train_size) // batch_size 77 | 78 | for epoch in range(epochs): 79 | X_, Y_ = shuffle(X_train, Y_train) 80 | 81 | for i in range(n_batches): 82 | start = i * batch_size 83 | end = start + batch_size 84 | 85 | sess.run(train_step, feed_dict={ 86 | x: X_[start:end], 87 | t: Y_[start:end] 88 | }) 89 | 90 | # 訓練データに対する学習の進み具合を出力 91 | loss = cross_entropy.eval(session=sess, feed_dict={ 92 | x: X_, 93 | t: Y_ 94 | }) 95 | acc = accuracy.eval(session=sess, feed_dict={ 96 | x: X_, 97 | t: Y_ 98 | }) 99 | print('epoch:', epoch, ' loss:', loss, ' accuracy:', acc) 100 | 101 | ''' 102 | 予測精度の評価 103 | ''' 104 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 105 | x: X_test, 106 | t: Y_test 107 | }) 108 | print('accuracy: ', accuracy_rate) 109 | -------------------------------------------------------------------------------- /4/tensorflow/03_mnist_lrelu_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(123) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | train_size = 0.8 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test =\ 25 | train_test_split(X, Y, train_size=train_size) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = len(X[0]) # 784 31 | n_hidden = 200 32 | n_out = len(Y[0]) # 10 33 | 34 | 35 | def lrelu(x, alpha=0.01): 36 | return tf.maximum(alpha * x, x) 37 | 38 | 39 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 40 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 41 | 42 | # 入力層 - 隠れ層 43 | W0 = tf.Variable(tf.truncated_normal([n_in, n_hidden], stddev=0.01)) 44 | b0 = tf.Variable(tf.zeros([n_hidden])) 45 | h0 = lrelu(tf.matmul(x, W0) + b0) 46 | 47 | # 隠れ層 - 隠れ層 48 | W1 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 49 | b1 = tf.Variable(tf.zeros([n_hidden])) 50 | h1 = lrelu(tf.matmul(h0, W1) + b1) 51 | 52 | W2 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 53 | b2 = tf.Variable(tf.zeros([n_hidden])) 54 | h2 = lrelu(tf.matmul(h1, W2) + b2) 55 | 56 | W3 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 57 | b3 = tf.Variable(tf.zeros([n_hidden])) 58 | h3 = lrelu(tf.matmul(h2, W3) + b3) 59 | 60 | # 隠れ層 - 出力層 61 | W4 = tf.Variable(tf.truncated_normal([n_hidden, n_out], stddev=0.01)) 62 | b4 = tf.Variable(tf.zeros([n_out])) 63 | y = tf.nn.softmax(tf.matmul(h3, W4) + b4) 64 | 65 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 66 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) 67 | 68 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 69 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 70 | 71 | ''' 72 | モデル学習 73 | ''' 74 | epochs = 50 75 | batch_size = 200 76 | 77 | init = tf.global_variables_initializer() 78 | sess = tf.Session() 79 | sess.run(init) 80 | 81 | n_batches = (int)(N * train_size) // batch_size 82 | 83 | for epoch in range(epochs): 84 | X_, Y_ = shuffle(X_train, Y_train) 85 | 86 | for i in range(n_batches): 87 | start = i * batch_size 88 | end = start + batch_size 89 | 90 | sess.run(train_step, feed_dict={ 91 | x: X_[start:end], 92 | t: Y_[start:end] 93 | }) 94 | 95 | # 訓練データに対する学習の進み具合を出力 96 | loss = cross_entropy.eval(session=sess, feed_dict={ 97 | x: X_, 98 | t: Y_ 99 | }) 100 | acc = accuracy.eval(session=sess, feed_dict={ 101 | x: X_, 102 | t: Y_ 103 | }) 104 | print('epoch:', epoch, ' loss:', loss, ' accuracy:', acc) 105 | 106 | ''' 107 | 予測精度の評価 108 | ''' 109 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 110 | x: X_test, 111 | t: Y_test 112 | }) 113 | print('accuracy: ', accuracy_rate) 114 | -------------------------------------------------------------------------------- /4/tensorflow/04_mnist_prelu_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(123) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | train_size = 0.8 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test =\ 25 | train_test_split(X, Y, train_size=train_size) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = len(X[0]) # 784 31 | n_hidden = 200 32 | n_out = len(Y[0]) # 10 33 | 34 | 35 | def prelu(x, alpha): 36 | return tf.maximum(tf.zeros(tf.shape(x)), x) \ 37 | + alpha * tf.minimum(tf.zeros(tf.shape(x)), x) 38 | 39 | 40 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 41 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 42 | 43 | # 入力層 - 隠れ層 44 | W0 = tf.Variable(tf.truncated_normal([n_in, n_hidden], stddev=0.01)) 45 | b0 = tf.Variable(tf.zeros([n_hidden])) 46 | alpha0 = tf.Variable(tf.zeros([n_hidden])) 47 | h0 = prelu(tf.matmul(x, W0) + b0, alpha0) 48 | 49 | # 隠れ層 - 隠れ層 50 | W1 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 51 | b1 = tf.Variable(tf.zeros([n_hidden])) 52 | alpha1 = tf.Variable(tf.zeros([n_hidden])) 53 | h1 = prelu(tf.matmul(h0, W1) + b1, alpha1) 54 | 55 | W2 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 56 | b2 = tf.Variable(tf.zeros([n_hidden])) 57 | alpha2 = tf.Variable(tf.zeros([n_hidden])) 58 | h2 = prelu(tf.matmul(h1, W2) + b2, alpha2) 59 | 60 | W3 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 61 | b3 = tf.Variable(tf.zeros([n_hidden])) 62 | alpha3 = tf.Variable(tf.zeros([n_hidden])) 63 | h3 = prelu(tf.matmul(h2, W3) + b3, alpha3) 64 | 65 | # 隠れ層 - 出力層 66 | W4 = tf.Variable(tf.truncated_normal([n_hidden, n_out], stddev=0.01)) 67 | b4 = tf.Variable(tf.zeros([n_out])) 68 | y = tf.nn.softmax(tf.matmul(h3, W4) + b4) 69 | 70 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 71 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) 72 | 73 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 74 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 75 | 76 | ''' 77 | モデル学習 78 | ''' 79 | epochs = 50 80 | batch_size = 200 81 | 82 | init = tf.global_variables_initializer() 83 | sess = tf.Session() 84 | sess.run(init) 85 | 86 | n_batches = (int)(N * train_size) // batch_size 87 | 88 | for epoch in range(epochs): 89 | X_, Y_ = shuffle(X_train, Y_train) 90 | 91 | for i in range(n_batches): 92 | start = i * batch_size 93 | end = start + batch_size 94 | 95 | sess.run(train_step, feed_dict={ 96 | x: X_[start:end], 97 | t: Y_[start:end] 98 | }) 99 | 100 | # 訓練データに対する学習の進み具合を出力 101 | loss = cross_entropy.eval(session=sess, feed_dict={ 102 | x: X_, 103 | t: Y_ 104 | }) 105 | acc = accuracy.eval(session=sess, feed_dict={ 106 | x: X_, 107 | t: Y_ 108 | }) 109 | print('epoch:', epoch, ' loss:', loss, ' accuracy:', acc) 110 | 111 | ''' 112 | 予測精度の評価 113 | ''' 114 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 115 | x: X_test, 116 | t: Y_test 117 | }) 118 | print('accuracy: ', accuracy_rate) 119 | 120 | # 学習後の alpha の値を確認 121 | # print(sess.run(alpha0)) 122 | # print(sess.run(alpha1)) 123 | # print(sess.run(alpha2)) 124 | # print(sess.run(alpha3)) 125 | -------------------------------------------------------------------------------- /4/tensorflow/05_mnist_dropout_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | ''' 11 | データの生成 12 | ''' 13 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 14 | 15 | n = len(mnist.data) 16 | N = 10000 # MNISTの一部を使う 17 | train_size = 0.8 18 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 19 | 20 | X = mnist.data[indices] 21 | y = mnist.target[indices] 22 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 23 | 24 | X_train, X_test, Y_train, Y_test =\ 25 | train_test_split(X, Y, train_size=train_size) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = len(X[0]) # 784 31 | n_hidden = 200 32 | n_out = len(Y[0]) # 10 33 | 34 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 35 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 36 | keep_prob = tf.placeholder(tf.float32) # ドロップアウトしない確率 37 | 38 | # 入力層 - 隠れ層 39 | W0 = tf.Variable(tf.truncated_normal([n_in, n_hidden], stddev=0.01)) 40 | b0 = tf.Variable(tf.zeros([n_hidden])) 41 | h0 = tf.nn.relu(tf.matmul(x, W0) + b0) 42 | h0_drop = tf.nn.dropout(h0, keep_prob) 43 | 44 | # 隠れ層 - 隠れ層 45 | W1 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 46 | b1 = tf.Variable(tf.zeros([n_hidden])) 47 | h1 = tf.nn.relu(tf.matmul(h0_drop, W1) + b1) 48 | h1_drop = tf.nn.dropout(h1, keep_prob) 49 | 50 | W2 = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], stddev=0.01)) 51 | b2 = tf.Variable(tf.zeros([n_hidden])) 52 | h2 = tf.nn.relu(tf.matmul(h1_drop, W2) + b2) 53 | h2_drop = tf.nn.dropout(h2, keep_prob) 54 | 55 | # 隠れ層 - 出力層 56 | W3 = tf.Variable(tf.truncated_normal([n_hidden, n_out], stddev=0.01)) 57 | b3 = tf.Variable(tf.zeros([n_out])) 58 | y = tf.nn.softmax(tf.matmul(h2_drop, W3) + b3) 59 | 60 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 61 | train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) 62 | 63 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 64 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 65 | 66 | ''' 67 | モデル学習 68 | ''' 69 | epochs = 30 70 | batch_size = 200 71 | 72 | init = tf.global_variables_initializer() 73 | sess = tf.Session() 74 | sess.run(init) 75 | 76 | n_batches = (int)(N * train_size) // batch_size 77 | 78 | for epoch in range(epochs): 79 | X_, Y_ = shuffle(X_train, Y_train) 80 | 81 | for i in range(n_batches): 82 | start = i * batch_size 83 | end = start + batch_size 84 | 85 | sess.run(train_step, feed_dict={ 86 | x: X_[start:end], 87 | t: Y_[start:end], 88 | keep_prob: 0.5 89 | }) 90 | 91 | # 訓練データに対する学習の進み具合を出力 92 | loss = cross_entropy.eval(session=sess, feed_dict={ 93 | x: X_, 94 | t: Y_, 95 | keep_prob: 1.0 96 | }) 97 | acc = accuracy.eval(session=sess, feed_dict={ 98 | x: X_, 99 | t: Y_, 100 | keep_prob: 1.0 101 | }) 102 | print('epoch:', epoch, ' loss:', loss, ' accuracy:', acc) 103 | 104 | ''' 105 | 予測精度の評価 106 | ''' 107 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 108 | x: X_test, 109 | t: Y_test, 110 | keep_prob: 1.0 111 | }) 112 | print('accuracy: ', accuracy_rate) 113 | -------------------------------------------------------------------------------- /4/tensorflow/06_mnist_plot_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = tf.truncated_normal(shape, stddev=0.01) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.GradientDescentOptimizer(0.01) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | y = mnist.target[indices] 77 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 78 | 79 | X_train, X_test, Y_train, Y_test = \ 80 | train_test_split(X, Y, train_size=N_train) 81 | 82 | X_train, X_validation, Y_train, Y_validation = \ 83 | train_test_split(X_train, Y_train, test_size=N_validation) 84 | 85 | ''' 86 | モデル設定 87 | ''' 88 | n_in = len(X[0]) 89 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 90 | n_out = len(Y[0]) 91 | p_keep = 0.5 92 | 93 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 94 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 95 | keep_prob = tf.placeholder(tf.float32) 96 | 97 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 98 | loss = loss(y, t) 99 | train_step = training(loss) 100 | 101 | accuracy = accuracy(y, t) 102 | 103 | history = { 104 | 'val_loss': [], 105 | 'val_acc': [] 106 | } 107 | 108 | ''' 109 | モデル学習 110 | ''' 111 | epochs = 50 112 | batch_size = 200 113 | 114 | init = tf.global_variables_initializer() 115 | sess = tf.Session() 116 | sess.run(init) 117 | 118 | n_batches = N_train // batch_size 119 | 120 | for epoch in range(epochs): 121 | X_, Y_ = shuffle(X_train, Y_train) 122 | 123 | for i in range(n_batches): 124 | start = i * batch_size 125 | end = start + batch_size 126 | 127 | sess.run(train_step, feed_dict={ 128 | x: X_[start:end], 129 | t: Y_[start:end], 130 | keep_prob: p_keep 131 | }) 132 | 133 | # 検証データを用いた評価 134 | val_loss = loss.eval(session=sess, feed_dict={ 135 | x: X_validation, 136 | t: Y_validation, 137 | keep_prob: 1.0 138 | }) 139 | val_acc = accuracy.eval(session=sess, feed_dict={ 140 | x: X_validation, 141 | t: Y_validation, 142 | keep_prob: 1.0 143 | }) 144 | 145 | # 検証データに対する学習の進み具合を記録 146 | history['val_loss'].append(val_loss) 147 | history['val_acc'].append(val_acc) 148 | 149 | print('epoch:', epoch, 150 | ' validation loss:', val_loss, 151 | ' validation accuracy:', val_acc) 152 | 153 | ''' 154 | 学習の進み具合を可視化 155 | ''' 156 | plt.rc('font', family='serif') 157 | fig = plt.figure() 158 | ax_acc = fig.add_subplot(111) 159 | ax_acc.plot(range(epochs), history['val_acc'], 160 | label='acc', color='black') 161 | ax_loss = ax_acc.twinx() 162 | ax_loss.plot(range(epochs), history['val_loss'], 163 | label='loss', color='gray') 164 | plt.xlabel('epochs') 165 | # plt.show() 166 | plt.savefig('mnist_tensorflow.eps') 167 | 168 | ''' 169 | 予測精度の評価 170 | ''' 171 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 172 | x: X_test, 173 | t: Y_test, 174 | keep_prob: 1.0 175 | }) 176 | print('accuracy: ', accuracy_rate) 177 | -------------------------------------------------------------------------------- /4/tensorflow/07_mnist_momentum_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.MomentumOptimizer(0.01, 0.9) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | X = X / 255.0 77 | X = X - X.mean(axis=1).reshape(len(X), 1) 78 | y = mnist.target[indices] 79 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 80 | 81 | X_train, X_test, Y_train, Y_test = \ 82 | train_test_split(X, Y, train_size=N_train) 83 | 84 | X_train, X_validation, Y_train, Y_validation = \ 85 | train_test_split(X_train, Y_train, test_size=N_validation) 86 | 87 | ''' 88 | モデル設定 89 | ''' 90 | n_in = len(X[0]) 91 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 92 | n_out = len(Y[0]) 93 | p_keep = 0.5 94 | 95 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 96 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 97 | keep_prob = tf.placeholder(tf.float32) 98 | 99 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 100 | loss = loss(y, t) 101 | train_step = training(loss) 102 | 103 | accuracy = accuracy(y, t) 104 | 105 | history = { 106 | 'val_loss': [], 107 | 'val_acc': [] 108 | } 109 | 110 | ''' 111 | モデル学習 112 | ''' 113 | epochs = 50 114 | batch_size = 200 115 | 116 | init = tf.global_variables_initializer() 117 | sess = tf.Session() 118 | sess.run(init) 119 | 120 | n_batches = N_train // batch_size 121 | 122 | for epoch in range(epochs): 123 | X_, Y_ = shuffle(X_train, Y_train) 124 | 125 | for i in range(n_batches): 126 | start = i * batch_size 127 | end = start + batch_size 128 | 129 | sess.run(train_step, feed_dict={ 130 | x: X_[start:end], 131 | t: Y_[start:end], 132 | keep_prob: p_keep 133 | }) 134 | 135 | # 検証データを用いた評価 136 | val_loss = loss.eval(session=sess, feed_dict={ 137 | x: X_validation, 138 | t: Y_validation, 139 | keep_prob: 1.0 140 | }) 141 | val_acc = accuracy.eval(session=sess, feed_dict={ 142 | x: X_validation, 143 | t: Y_validation, 144 | keep_prob: 1.0 145 | }) 146 | 147 | # 検証データに対する学習の進み具合を記録 148 | history['val_loss'].append(val_loss) 149 | history['val_acc'].append(val_acc) 150 | 151 | print('epoch:', epoch, 152 | ' validation loss:', val_loss, 153 | ' validation accuracy:', val_acc) 154 | 155 | ''' 156 | 学習の進み具合を可視化 157 | ''' 158 | plt.rc('font', family='serif') 159 | fig = plt.figure() 160 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 161 | plt.xlabel('epochs') 162 | plt.show() 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /4/tensorflow/08_mnist_nesterov_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.MomentumOptimizer(0.01, 0.9, use_nesterov=True) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | X = X / 255.0 77 | X = X - X.mean(axis=1).reshape(len(X), 1) 78 | y = mnist.target[indices] 79 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 80 | 81 | X_train, X_test, Y_train, Y_test = \ 82 | train_test_split(X, Y, train_size=N_train) 83 | 84 | X_train, X_validation, Y_train, Y_validation = \ 85 | train_test_split(X_train, Y_train, test_size=N_validation) 86 | 87 | ''' 88 | モデル設定 89 | ''' 90 | n_in = len(X[0]) 91 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 92 | n_out = len(Y[0]) 93 | p_keep = 0.5 94 | 95 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 96 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 97 | keep_prob = tf.placeholder(tf.float32) 98 | 99 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 100 | loss = loss(y, t) 101 | train_step = training(loss) 102 | 103 | accuracy = accuracy(y, t) 104 | 105 | history = { 106 | 'val_loss': [], 107 | 'val_acc': [] 108 | } 109 | 110 | ''' 111 | モデル学習 112 | ''' 113 | epochs = 50 114 | batch_size = 200 115 | 116 | init = tf.global_variables_initializer() 117 | sess = tf.Session() 118 | sess.run(init) 119 | 120 | n_batches = N_train // batch_size 121 | 122 | for epoch in range(epochs): 123 | X_, Y_ = shuffle(X_train, Y_train) 124 | 125 | for i in range(n_batches): 126 | start = i * batch_size 127 | end = start + batch_size 128 | 129 | sess.run(train_step, feed_dict={ 130 | x: X_[start:end], 131 | t: Y_[start:end], 132 | keep_prob: p_keep 133 | }) 134 | 135 | # 検証データを用いた評価 136 | val_loss = loss.eval(session=sess, feed_dict={ 137 | x: X_validation, 138 | t: Y_validation, 139 | keep_prob: 1.0 140 | }) 141 | val_acc = accuracy.eval(session=sess, feed_dict={ 142 | x: X_validation, 143 | t: Y_validation, 144 | keep_prob: 1.0 145 | }) 146 | 147 | # 検証データに対する学習の進み具合を記録 148 | history['val_loss'].append(val_loss) 149 | history['val_acc'].append(val_acc) 150 | 151 | print('epoch:', epoch, 152 | ' validation loss:', val_loss, 153 | ' validation accuracy:', val_acc) 154 | 155 | ''' 156 | 学習の進み具合を可視化 157 | ''' 158 | plt.rc('font', family='serif') 159 | fig = plt.figure() 160 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 161 | plt.xlabel('epochs') 162 | plt.show() 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /4/tensorflow/09_mnist_adagrad_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.AdagradOptimizer(0.01) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | X = X / 255.0 77 | X = X - X.mean(axis=1).reshape(len(X), 1) 78 | y = mnist.target[indices] 79 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 80 | 81 | X_train, X_test, Y_train, Y_test = \ 82 | train_test_split(X, Y, train_size=N_train) 83 | 84 | X_train, X_validation, Y_train, Y_validation = \ 85 | train_test_split(X_train, Y_train, test_size=N_validation) 86 | 87 | ''' 88 | モデル設定 89 | ''' 90 | n_in = len(X[0]) 91 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 92 | n_out = len(Y[0]) 93 | p_keep = 0.5 94 | 95 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 96 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 97 | keep_prob = tf.placeholder(tf.float32) 98 | 99 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 100 | loss = loss(y, t) 101 | train_step = training(loss) 102 | 103 | accuracy = accuracy(y, t) 104 | 105 | history = { 106 | 'val_loss': [], 107 | 'val_acc': [] 108 | } 109 | 110 | ''' 111 | モデル学習 112 | ''' 113 | epochs = 50 114 | batch_size = 200 115 | 116 | init = tf.global_variables_initializer() 117 | sess = tf.Session() 118 | sess.run(init) 119 | 120 | n_batches = N_train // batch_size 121 | 122 | for epoch in range(epochs): 123 | X_, Y_ = shuffle(X_train, Y_train) 124 | 125 | for i in range(n_batches): 126 | start = i * batch_size 127 | end = start + batch_size 128 | 129 | sess.run(train_step, feed_dict={ 130 | x: X_[start:end], 131 | t: Y_[start:end], 132 | keep_prob: p_keep 133 | }) 134 | 135 | # 検証データを用いた評価 136 | val_loss = loss.eval(session=sess, feed_dict={ 137 | x: X_validation, 138 | t: Y_validation, 139 | keep_prob: 1.0 140 | }) 141 | val_acc = accuracy.eval(session=sess, feed_dict={ 142 | x: X_validation, 143 | t: Y_validation, 144 | keep_prob: 1.0 145 | }) 146 | 147 | # 検証データに対する学習の進み具合を記録 148 | history['val_loss'].append(val_loss) 149 | history['val_acc'].append(val_acc) 150 | 151 | print('epoch:', epoch, 152 | ' validation loss:', val_loss, 153 | ' validation accuracy:', val_acc) 154 | 155 | ''' 156 | 学習の進み具合を可視化 157 | ''' 158 | plt.rc('font', family='serif') 159 | fig = plt.figure() 160 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 161 | plt.xlabel('epochs') 162 | plt.show() 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /4/tensorflow/10_mnist_adadelta_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.AdadeltaOptimizer(learning_rate=1.0, rho=0.95) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | X = X / 255.0 77 | X = X - X.mean(axis=1).reshape(len(X), 1) 78 | y = mnist.target[indices] 79 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 80 | 81 | X_train, X_test, Y_train, Y_test = \ 82 | train_test_split(X, Y, train_size=N_train) 83 | 84 | X_train, X_validation, Y_train, Y_validation = \ 85 | train_test_split(X_train, Y_train, test_size=N_validation) 86 | 87 | ''' 88 | モデル設定 89 | ''' 90 | n_in = len(X[0]) 91 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 92 | n_out = len(Y[0]) 93 | p_keep = 0.5 94 | 95 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 96 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 97 | keep_prob = tf.placeholder(tf.float32) 98 | 99 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 100 | loss = loss(y, t) 101 | train_step = training(loss) 102 | 103 | accuracy = accuracy(y, t) 104 | 105 | history = { 106 | 'val_loss': [], 107 | 'val_acc': [] 108 | } 109 | 110 | ''' 111 | モデル学習 112 | ''' 113 | epochs = 50 114 | batch_size = 200 115 | 116 | init = tf.global_variables_initializer() 117 | sess = tf.Session() 118 | sess.run(init) 119 | 120 | n_batches = N_train // batch_size 121 | 122 | for epoch in range(epochs): 123 | X_, Y_ = shuffle(X_train, Y_train) 124 | 125 | for i in range(n_batches): 126 | start = i * batch_size 127 | end = start + batch_size 128 | 129 | sess.run(train_step, feed_dict={ 130 | x: X_[start:end], 131 | t: Y_[start:end], 132 | keep_prob: p_keep 133 | }) 134 | 135 | # 検証データを用いた評価 136 | val_loss = loss.eval(session=sess, feed_dict={ 137 | x: X_validation, 138 | t: Y_validation, 139 | keep_prob: 1.0 140 | }) 141 | val_acc = accuracy.eval(session=sess, feed_dict={ 142 | x: X_validation, 143 | t: Y_validation, 144 | keep_prob: 1.0 145 | }) 146 | 147 | # 検証データに対する学習の進み具合を記録 148 | history['val_loss'].append(val_loss) 149 | history['val_acc'].append(val_acc) 150 | 151 | print('epoch:', epoch, 152 | ' validation loss:', val_loss, 153 | ' validation accuracy:', val_acc) 154 | 155 | ''' 156 | 学習の進み具合を可視化 157 | ''' 158 | plt.rc('font', family='serif') 159 | fig = plt.figure() 160 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 161 | plt.xlabel('epochs') 162 | plt.show() 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /4/tensorflow/11_mnist_rmsprop_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.RMSPropOptimizer(0.001) 53 | train_step = optimizer.minimize(loss) 54 | return train_step 55 | 56 | 57 | def accuracy(y, t): 58 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 59 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 60 | return accuracy 61 | 62 | 63 | if __name__ == '__main__': 64 | ''' 65 | データの生成 66 | ''' 67 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 68 | 69 | n = len(mnist.data) 70 | N = 30000 # MNISTの一部を使う 71 | N_train = 20000 72 | N_validation = 4000 73 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 74 | 75 | X = mnist.data[indices] 76 | X = X / 255.0 77 | X = X - X.mean(axis=1).reshape(len(X), 1) 78 | y = mnist.target[indices] 79 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 80 | 81 | X_train, X_test, Y_train, Y_test = \ 82 | train_test_split(X, Y, train_size=N_train) 83 | 84 | X_train, X_validation, Y_train, Y_validation = \ 85 | train_test_split(X_train, Y_train, test_size=N_validation) 86 | 87 | ''' 88 | モデル設定 89 | ''' 90 | n_in = len(X[0]) 91 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 92 | n_out = len(Y[0]) 93 | p_keep = 0.5 94 | 95 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 96 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 97 | keep_prob = tf.placeholder(tf.float32) 98 | 99 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 100 | loss = loss(y, t) 101 | train_step = training(loss) 102 | 103 | accuracy = accuracy(y, t) 104 | 105 | history = { 106 | 'val_loss': [], 107 | 'val_acc': [] 108 | } 109 | 110 | ''' 111 | モデル学習 112 | ''' 113 | epochs = 50 114 | batch_size = 200 115 | 116 | init = tf.global_variables_initializer() 117 | sess = tf.Session() 118 | sess.run(init) 119 | 120 | n_batches = N_train // batch_size 121 | 122 | for epoch in range(epochs): 123 | X_, Y_ = shuffle(X_train, Y_train) 124 | 125 | for i in range(n_batches): 126 | start = i * batch_size 127 | end = start + batch_size 128 | 129 | sess.run(train_step, feed_dict={ 130 | x: X_[start:end], 131 | t: Y_[start:end], 132 | keep_prob: p_keep 133 | }) 134 | 135 | # 検証データを用いた評価 136 | val_loss = loss.eval(session=sess, feed_dict={ 137 | x: X_validation, 138 | t: Y_validation, 139 | keep_prob: 1.0 140 | }) 141 | val_acc = accuracy.eval(session=sess, feed_dict={ 142 | x: X_validation, 143 | t: Y_validation, 144 | keep_prob: 1.0 145 | }) 146 | 147 | # 検証データに対する学習の進み具合を記録 148 | history['val_loss'].append(val_loss) 149 | history['val_acc'].append(val_acc) 150 | 151 | print('epoch:', epoch, 152 | ' validation loss:', val_loss, 153 | ' validation accuracy:', val_acc) 154 | 155 | ''' 156 | 学習の進み具合を可視化 157 | ''' 158 | plt.rc('font', family='serif') 159 | fig = plt.figure() 160 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 161 | plt.xlabel('epochs') 162 | plt.show() 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /4/tensorflow/12_mnist_adam_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001, 53 | beta1=0.9, 54 | beta2=0.999) 55 | train_step = optimizer.minimize(loss) 56 | return train_step 57 | 58 | 59 | def accuracy(y, t): 60 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 61 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 62 | return accuracy 63 | 64 | 65 | if __name__ == '__main__': 66 | ''' 67 | データの生成 68 | ''' 69 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 70 | 71 | n = len(mnist.data) 72 | N = 30000 # MNISTの一部を使う 73 | N_train = 20000 74 | N_validation = 4000 75 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 76 | 77 | X = mnist.data[indices] 78 | X = X / 255.0 79 | X = X - X.mean(axis=1).reshape(len(X), 1) 80 | y = mnist.target[indices] 81 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 82 | 83 | X_train, X_test, Y_train, Y_test = \ 84 | train_test_split(X, Y, train_size=N_train) 85 | 86 | X_train, X_validation, Y_train, Y_validation = \ 87 | train_test_split(X_train, Y_train, test_size=N_validation) 88 | 89 | ''' 90 | モデル設定 91 | ''' 92 | n_in = len(X[0]) 93 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 94 | n_out = len(Y[0]) 95 | p_keep = 0.5 96 | 97 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 98 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 99 | keep_prob = tf.placeholder(tf.float32) 100 | 101 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 102 | loss = loss(y, t) 103 | train_step = training(loss) 104 | 105 | accuracy = accuracy(y, t) 106 | 107 | history = { 108 | 'val_loss': [], 109 | 'val_acc': [] 110 | } 111 | 112 | ''' 113 | モデル学習 114 | ''' 115 | epochs = 50 116 | batch_size = 200 117 | 118 | init = tf.global_variables_initializer() 119 | sess = tf.Session() 120 | sess.run(init) 121 | 122 | n_batches = N_train // batch_size 123 | 124 | for epoch in range(epochs): 125 | X_, Y_ = shuffle(X_train, Y_train) 126 | 127 | for i in range(n_batches): 128 | start = i * batch_size 129 | end = start + batch_size 130 | 131 | sess.run(train_step, feed_dict={ 132 | x: X_[start:end], 133 | t: Y_[start:end], 134 | keep_prob: p_keep 135 | }) 136 | 137 | # 検証データを用いた評価 138 | val_loss = loss.eval(session=sess, feed_dict={ 139 | x: X_validation, 140 | t: Y_validation, 141 | keep_prob: 1.0 142 | }) 143 | val_acc = accuracy.eval(session=sess, feed_dict={ 144 | x: X_validation, 145 | t: Y_validation, 146 | keep_prob: 1.0 147 | }) 148 | 149 | # 検証データに対する学習の進み具合を記録 150 | history['val_loss'].append(val_loss) 151 | history['val_acc'].append(val_acc) 152 | 153 | print('epoch:', epoch, 154 | ' validation loss:', val_loss, 155 | ' validation accuracy:', val_acc) 156 | 157 | ''' 158 | 学習の進み具合を可視化 159 | ''' 160 | plt.rc('font', family='serif') 161 | fig = plt.figure() 162 | plt.plot(range(epochs), history['val_loss'], label='loss', color='black') 163 | plt.xlabel('epochs') 164 | plt.show() 165 | 166 | ''' 167 | 予測精度の評価 168 | ''' 169 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 170 | x: X_test, 171 | t: Y_test, 172 | keep_prob: 1.0 173 | }) 174 | print('accuracy: ', accuracy_rate) 175 | -------------------------------------------------------------------------------- /4/tensorflow/13_mnist_early_stopping_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 22 | for i, n_hidden in enumerate(n_hiddens): 23 | if i == 0: 24 | input = x 25 | input_dim = n_in 26 | else: 27 | input = output 28 | input_dim = n_hiddens[i-1] 29 | 30 | W = weight_variable([input_dim, n_hidden]) 31 | b = bias_variable([n_hidden]) 32 | 33 | h = tf.nn.relu(tf.matmul(input, W) + b) 34 | output = tf.nn.dropout(h, keep_prob) 35 | 36 | # 隠れ層 - 出力層 37 | W_out = weight_variable([n_hiddens[-1], n_out]) 38 | b_out = bias_variable([n_out]) 39 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | axis=1)) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001, 53 | beta1=0.9, 54 | beta2=0.999) 55 | train_step = optimizer.minimize(loss) 56 | return train_step 57 | 58 | 59 | def accuracy(y, t): 60 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 61 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 62 | return accuracy 63 | 64 | 65 | class EarlyStopping(): 66 | def __init__(self, patience=0, verbose=0): 67 | self._step = 0 68 | self._loss = float('inf') 69 | self.patience = patience 70 | self.verbose = verbose 71 | 72 | def validate(self, loss): 73 | if self._loss < loss: 74 | self._step += 1 75 | if self._step > self.patience: 76 | if self.verbose: 77 | print('early stopping') 78 | return True 79 | else: 80 | self._step = 0 81 | self._loss = loss 82 | 83 | return False 84 | 85 | 86 | if __name__ == '__main__': 87 | ''' 88 | データの生成 89 | ''' 90 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 91 | 92 | n = len(mnist.data) 93 | N = 30000 # MNISTの一部を使う 94 | N_train = 20000 95 | N_validation = 4000 96 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 97 | 98 | X = mnist.data[indices] 99 | X = X / 255.0 100 | X = X - X.mean(axis=1).reshape(len(X), 1) 101 | y = mnist.target[indices] 102 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 103 | 104 | X_train, X_test, Y_train, Y_test = \ 105 | train_test_split(X, Y, train_size=N_train) 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X_train, Y_train, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = len(X[0]) 114 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 115 | n_out = len(Y[0]) 116 | p_keep = 0.5 117 | 118 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 119 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 120 | keep_prob = tf.placeholder(tf.float32) 121 | 122 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 123 | loss = loss(y, t) 124 | train_step = training(loss) 125 | 126 | accuracy = accuracy(y, t) 127 | early_stopping = EarlyStopping(patience=10, verbose=1) 128 | 129 | history = { 130 | 'val_loss': [], 131 | 'val_acc': [] 132 | } 133 | 134 | ''' 135 | モデル学習 136 | ''' 137 | epochs = 200 138 | batch_size = 200 139 | 140 | init = tf.global_variables_initializer() 141 | sess = tf.Session() 142 | sess.run(init) 143 | 144 | n_batches = N_train // batch_size 145 | 146 | for epoch in range(epochs): 147 | X_, Y_ = shuffle(X_train, Y_train) 148 | 149 | for i in range(n_batches): 150 | start = i * batch_size 151 | end = start + batch_size 152 | 153 | sess.run(train_step, feed_dict={ 154 | x: X_[start:end], 155 | t: Y_[start:end], 156 | keep_prob: p_keep 157 | }) 158 | 159 | # 検証データを用いた評価 160 | val_loss = loss.eval(session=sess, feed_dict={ 161 | x: X_validation, 162 | t: Y_validation, 163 | keep_prob: 1.0 164 | }) 165 | val_acc = accuracy.eval(session=sess, feed_dict={ 166 | x: X_validation, 167 | t: Y_validation, 168 | keep_prob: 1.0 169 | }) 170 | 171 | # 検証データに対する学習の進み具合を記録 172 | history['val_loss'].append(val_loss) 173 | history['val_acc'].append(val_acc) 174 | 175 | print('epoch:', epoch, 176 | ' validation loss:', val_loss, 177 | ' validation accuracy:', val_acc) 178 | 179 | # Early Stopping チェック 180 | if early_stopping.validate(val_loss): 181 | break 182 | 183 | ''' 184 | 学習の進み具合を可視化 185 | ''' 186 | plt.rc('font', family='serif') 187 | fig = plt.figure() 188 | plt.plot(range(len(history['val_loss'])), history['val_loss'], 189 | label='loss', color='black') 190 | plt.xlabel('epochs') 191 | plt.show() 192 | 193 | ''' 194 | 予測精度の評価 195 | ''' 196 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 197 | x: X_test, 198 | t: Y_test, 199 | keep_prob: 1.0 200 | }) 201 | print('accuracy: ', accuracy_rate) 202 | -------------------------------------------------------------------------------- /4/tensorflow/14_mnist_batch_normalization_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | def inference(x, n_in, n_hiddens, n_out): 13 | def weight_variable(shape): 14 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 15 | return tf.Variable(initial) 16 | 17 | def bias_variable(shape): 18 | initial = tf.zeros(shape) 19 | return tf.Variable(initial) 20 | 21 | def batch_normalization(shape, x): 22 | eps = 1e-8 23 | beta = tf.Variable(tf.zeros(shape)) 24 | gamma = tf.Variable(tf.ones(shape)) 25 | mean, var = tf.nn.moments(x, [0]) 26 | return gamma * (x - mean) / tf.sqrt(var + eps) + beta 27 | 28 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 29 | for i, n_hidden in enumerate(n_hiddens): 30 | if i == 0: 31 | input = x 32 | input_dim = n_in 33 | else: 34 | input = output 35 | input_dim = n_hiddens[i-1] 36 | 37 | W = weight_variable([input_dim, n_hidden]) 38 | u = tf.matmul(input, W) 39 | h = batch_normalization([n_hidden], u) 40 | output = tf.nn.relu(h) 41 | 42 | # 隠れ層 - 出力層 43 | W_out = weight_variable([n_hiddens[-1], n_out]) 44 | b_out = bias_variable([n_out]) 45 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 46 | return y 47 | 48 | 49 | def loss(y, t): 50 | cross_entropy = \ 51 | tf.reduce_mean(-tf.reduce_sum( 52 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 53 | axis=1)) 54 | return cross_entropy 55 | 56 | 57 | def training(loss): 58 | optimizer = tf.train.AdamOptimizer(learning_rate=0.1, 59 | beta1=0.9, 60 | beta2=0.999) 61 | train_step = optimizer.minimize(loss) 62 | return train_step 63 | 64 | 65 | def accuracy(y, t): 66 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 67 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 68 | return accuracy 69 | 70 | 71 | class EarlyStopping(): 72 | def __init__(self, patience=0, verbose=0): 73 | self._step = 0 74 | self._loss = float('inf') 75 | self.patience = patience 76 | self.verbose = verbose 77 | 78 | def validate(self, loss): 79 | if self._loss < loss: 80 | self._step += 1 81 | if self._step > self.patience: 82 | if self.verbose: 83 | print('early stopping') 84 | return True 85 | else: 86 | self._step = 0 87 | self._loss = loss 88 | 89 | return False 90 | 91 | 92 | if __name__ == '__main__': 93 | ''' 94 | データの生成 95 | ''' 96 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 97 | 98 | n = len(mnist.data) 99 | N = 30000 # MNISTの一部を使う 100 | N_train = 20000 101 | N_validation = 4000 102 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 103 | 104 | X = mnist.data[indices] 105 | # X = X / 255.0 106 | # X = X - X.mean(axis=1).reshape(len(X), 1) 107 | y = mnist.target[indices] 108 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 109 | 110 | X_train, X_test, Y_train, Y_test = \ 111 | train_test_split(X, Y, train_size=N_train) 112 | 113 | X_train, X_validation, Y_train, Y_validation = \ 114 | train_test_split(X_train, Y_train, test_size=N_validation) 115 | 116 | ''' 117 | モデル設定 118 | ''' 119 | n_in = len(X[0]) 120 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 121 | n_out = len(Y[0]) 122 | 123 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 124 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 125 | 126 | y = inference(x, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 127 | loss = loss(y, t) 128 | train_step = training(loss) 129 | 130 | accuracy = accuracy(y, t) 131 | early_stopping = EarlyStopping(patience=10, verbose=1) 132 | 133 | history = { 134 | 'val_loss': [], 135 | 'val_acc': [] 136 | } 137 | 138 | ''' 139 | モデル学習 140 | ''' 141 | epochs = 50 142 | batch_size = 200 143 | 144 | init = tf.global_variables_initializer() 145 | sess = tf.Session() 146 | sess.run(init) 147 | 148 | n_batches = N_train // batch_size 149 | 150 | for epoch in range(epochs): 151 | X_, Y_ = shuffle(X_train, Y_train) 152 | 153 | for i in range(n_batches): 154 | start = i * batch_size 155 | end = start + batch_size 156 | 157 | sess.run(train_step, feed_dict={ 158 | x: X_[start:end], 159 | t: Y_[start:end] 160 | }) 161 | 162 | # 検証データを用いた評価 163 | val_loss = loss.eval(session=sess, feed_dict={ 164 | x: X_validation, 165 | t: Y_validation 166 | }) 167 | val_acc = accuracy.eval(session=sess, feed_dict={ 168 | x: X_validation, 169 | t: Y_validation 170 | }) 171 | 172 | # 検証データに対する学習の進み具合を記録 173 | history['val_loss'].append(val_loss) 174 | history['val_acc'].append(val_acc) 175 | 176 | print('epoch:', epoch, 177 | ' validation loss:', val_loss, 178 | ' validation accuracy:', val_acc) 179 | 180 | # Early Stopping チェック 181 | if early_stopping.validate(val_loss): 182 | break 183 | 184 | ''' 185 | 学習の進み具合を可視化 186 | ''' 187 | plt.rc('font', family='serif') 188 | fig = plt.figure() 189 | plt.plot(range(len(history['val_loss'])), history['val_loss'], 190 | label='loss', color='black') 191 | plt.xlabel('epochs') 192 | plt.show() 193 | 194 | ''' 195 | 予測精度の評価 196 | ''' 197 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 198 | x: X_test, 199 | t: Y_test 200 | }) 201 | print('accuracy: ', accuracy_rate) 202 | -------------------------------------------------------------------------------- /4/tensorflow/99_mnist_mock_contrib_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn import datasets 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | import matplotlib.pyplot as plt 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | 12 | class DNN(object): 13 | def __init__(self, n_in, n_hiddens, n_out): 14 | self.n_in = n_in 15 | self.n_hiddens = n_hiddens 16 | self.n_out = n_out 17 | self.weights = [] 18 | self.biases = [] 19 | 20 | self._x = None 21 | self._y = None 22 | self._t = None, 23 | self._keep_prob = None 24 | self._sess = None 25 | self._history = { 26 | 'accuracy': [], 27 | 'loss': [] 28 | } 29 | 30 | def weight_variable(self, shape): 31 | initial = tf.truncated_normal(shape, stddev=0.01) 32 | return tf.Variable(initial) 33 | 34 | def bias_variable(self, shape): 35 | initial = tf.zeros(shape) 36 | return tf.Variable(initial) 37 | 38 | def inference(self, x, keep_prob): 39 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 40 | for i, n_hidden in enumerate(self.n_hiddens): 41 | if i == 0: 42 | input = x 43 | input_dim = self.n_in 44 | else: 45 | input = output 46 | input_dim = self.n_hiddens[i-1] 47 | 48 | self.weights.append(self.weight_variable([input_dim, n_hidden])) 49 | self.biases.append(self.bias_variable([n_hidden])) 50 | 51 | h = tf.nn.relu(tf.matmul( 52 | input, self.weights[-1]) + self.biases[-1]) 53 | output = tf.nn.dropout(h, keep_prob) 54 | 55 | # 隠れ層 - 出力層 56 | self.weights.append( 57 | self.weight_variable([self.n_hiddens[-1], self.n_out])) 58 | self.biases.append(self.bias_variable([self.n_out])) 59 | 60 | y = tf.nn.softmax(tf.matmul( 61 | output, self.weights[-1]) + self.biases[-1]) 62 | return y 63 | 64 | def loss(self, y, t): 65 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(t * tf.log(y), axis=1)) 66 | return cross_entropy 67 | 68 | def training(self, loss): 69 | optimizer = tf.train.GradientDescentOptimizer(0.01) 70 | train_step = optimizer.minimize(loss) 71 | return train_step 72 | 73 | def accuracy(self, y, t): 74 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 75 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 76 | return accuracy 77 | 78 | def fit(self, X_train, Y_train, 79 | nb_epoch=100, batch_size=100, p_keep=0.5, 80 | verbose=1): 81 | x = tf.placeholder(tf.float32, shape=[None, self.n_in]) 82 | t = tf.placeholder(tf.float32, shape=[None, self.n_out]) 83 | keep_prob = tf.placeholder(tf.float32) 84 | 85 | self._x = x 86 | self._t = t 87 | self._keep_prob = keep_prob 88 | 89 | y = self.inference(x, keep_prob) 90 | loss = self.loss(y, t) 91 | train_step = self.training(loss) 92 | accuracy = self.accuracy(y, t) 93 | 94 | init = tf.global_variables_initializer() 95 | sess = tf.Session() 96 | sess.run(init) 97 | 98 | self._y = y 99 | self._sess = sess 100 | 101 | N_train = len(X_train) 102 | n_batches = N_train // batch_size 103 | 104 | for epoch in range(nb_epoch): 105 | X_, Y_ = shuffle(X_train, Y_train) 106 | 107 | for i in range(n_batches): 108 | start = i * batch_size 109 | end = start + batch_size 110 | 111 | sess.run(train_step, feed_dict={ 112 | x: X_[start:end], 113 | t: Y_[start:end], 114 | keep_prob: p_keep 115 | }) 116 | loss_ = loss.eval(session=sess, feed_dict={ 117 | x: X_train, 118 | t: Y_train, 119 | keep_prob: 1.0 120 | }) 121 | accuracy_ = accuracy.eval(session=sess, feed_dict={ 122 | x: X_train, 123 | t: Y_train, 124 | keep_prob: 1.0 125 | }) 126 | self._history['loss'].append(loss_) 127 | self._history['accuracy'].append(accuracy_) 128 | 129 | if verbose: 130 | print('epoch:', epoch, 131 | ' loss:', loss_, 132 | ' accuracy:', accuracy_) 133 | 134 | return self._history 135 | 136 | def evaluate(self, X_test, Y_test): 137 | accuracy = self.accuracy(self._y, self._t) 138 | return accuracy.eval(session=self._sess, feed_dict={ 139 | self._x: X_test, 140 | self._t: Y_test, 141 | self._keep_prob: 1.0 142 | }) 143 | 144 | 145 | if __name__ == '__main__': 146 | ''' 147 | データの生成 148 | ''' 149 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 150 | 151 | n = len(mnist.data) 152 | N = 10000 # MNISTの一部を使う 153 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 154 | 155 | X = mnist.data[indices] 156 | y = mnist.target[indices] 157 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 158 | 159 | X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8) 160 | 161 | ''' 162 | モデル設定 163 | ''' 164 | model = DNN(n_in=len(X[0]), 165 | n_hiddens=[200, 200, 200], 166 | n_out=len(Y[0])) 167 | 168 | ''' 169 | モデル学習 170 | ''' 171 | model.fit(X_train, Y_train, 172 | nb_epoch=30, 173 | batch_size=200, 174 | p_keep=0.5) 175 | 176 | ''' 177 | 予測精度の評価 178 | ''' 179 | accuracy = model.evaluate(X_test, Y_test) 180 | print('accuracy: ', accuracy) 181 | -------------------------------------------------------------------------------- /5/keras/00_sin_simple_rnn_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import SimpleRNN 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def sin(x, T=100): 15 | return np.sin(2.0 * np.pi * x / T) 16 | 17 | 18 | def toy_problem(T=100, ampl=0.05): 19 | x = np.arange(0, 2 * T + 1) 20 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 21 | return sin(x) + noise 22 | 23 | 24 | ''' 25 | データの生成 26 | ''' 27 | T = 100 28 | f = toy_problem(T) 29 | 30 | length_of_sequences = 2 * T 31 | maxlen = 25 # ひとつの時系列データの長さ 32 | 33 | data = [] 34 | target = [] 35 | 36 | for i in range(0, length_of_sequences - maxlen + 1): 37 | data.append(f[i: i + maxlen]) 38 | target.append(f[i + maxlen]) 39 | 40 | X = np.array(data).reshape(len(data), maxlen, 1) 41 | Y = np.array(target).reshape(len(data), 1) 42 | 43 | # データ設定 44 | N_train = int(len(data) * 0.9) 45 | N_validation = len(data) - N_train 46 | 47 | X_train, X_validation, Y_train, Y_validation = \ 48 | train_test_split(X, Y, test_size=N_validation) 49 | 50 | ''' 51 | モデル設定 52 | ''' 53 | n_in = len(X[0][0]) # 1 54 | n_hidden = 20 55 | n_out = len(Y[0]) # 1 56 | 57 | 58 | def weight_variable(shape, name=None): 59 | return np.random.normal(scale=.01, size=shape) 60 | 61 | 62 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 63 | 64 | model = Sequential() 65 | model.add(SimpleRNN(n_hidden, 66 | kernel_initializer=weight_variable, 67 | input_shape=(maxlen, n_in))) 68 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 69 | model.add(Activation('linear')) 70 | 71 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 72 | model.compile(loss='mean_squared_error', 73 | optimizer=optimizer) 74 | 75 | ''' 76 | モデル学習 77 | ''' 78 | epochs = 500 79 | batch_size = 10 80 | 81 | model.fit(X_train, Y_train, 82 | batch_size=batch_size, 83 | epochs=epochs, 84 | validation_data=(X_validation, Y_validation), 85 | callbacks=[early_stopping]) 86 | 87 | ''' 88 | 出力を用いて予測 89 | ''' 90 | truncate = maxlen 91 | Z = X[:1] # 元データの最初の一部だけ切り出し 92 | 93 | original = [f[i] for i in range(maxlen)] 94 | predicted = [None for i in range(maxlen)] 95 | 96 | for i in range(length_of_sequences - maxlen + 1): 97 | z_ = Z[-1:] 98 | y_ = model.predict(z_) 99 | sequence_ = np.concatenate( 100 | (z_.reshape(maxlen, n_in)[1:], y_), 101 | axis=0).reshape(1, maxlen, n_in) 102 | Z = np.append(Z, sequence_, axis=0) 103 | predicted.append(y_.reshape(-1)) 104 | 105 | ''' 106 | グラフで可視化 107 | ''' 108 | plt.rc('font', family='serif') 109 | plt.figure() 110 | plt.ylim([-1.5, 1.5]) 111 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 112 | plt.plot(original, linestyle='dashed', color='black') 113 | plt.plot(predicted, color='black') 114 | plt.show() 115 | -------------------------------------------------------------------------------- /5/keras/01_sin_lstm_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import LSTM 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def sin(x, T=100): 15 | return np.sin(2.0 * np.pi * x / T) 16 | 17 | 18 | def toy_problem(T=100, ampl=0.05): 19 | x = np.arange(0, 2 * T + 1) 20 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 21 | return sin(x) + noise 22 | 23 | 24 | ''' 25 | データの生成 26 | ''' 27 | T = 100 28 | f = toy_problem(T) 29 | 30 | length_of_sequences = 2 * T 31 | maxlen = 25 # ひとつの時系列データの長さ 32 | 33 | data = [] 34 | target = [] 35 | 36 | for i in range(0, length_of_sequences - maxlen + 1): 37 | data.append(f[i: i + maxlen]) 38 | target.append(f[i + maxlen]) 39 | 40 | X = np.array(data).reshape(len(data), maxlen, 1) 41 | Y = np.array(target).reshape(len(data), 1) 42 | 43 | # データ設定 44 | N_train = int(len(data) * 0.9) 45 | N_validation = len(data) - N_train 46 | 47 | X_train, X_validation, Y_train, Y_validation = \ 48 | train_test_split(X, Y, test_size=N_validation) 49 | 50 | ''' 51 | モデル設定 52 | ''' 53 | n_in = len(X[0][0]) # 1 54 | n_hidden = 30 55 | n_out = len(Y[0]) # 1 56 | 57 | 58 | def weight_variable(shape, name=None): 59 | return np.random.normal(scale=.01, size=shape) 60 | 61 | 62 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 63 | 64 | model = Sequential() 65 | model.add(LSTM(n_hidden, 66 | kernel_initializer=weight_variable, 67 | input_shape=(maxlen, n_in))) 68 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 69 | model.add(Activation('linear')) 70 | 71 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 72 | model.compile(loss='mean_squared_error', 73 | optimizer=optimizer) 74 | 75 | ''' 76 | モデル学習 77 | ''' 78 | epochs = 500 79 | batch_size = 10 80 | 81 | model.fit(X_train, Y_train, 82 | batch_size=batch_size, 83 | epochs=epochs, 84 | validation_data=(X_validation, Y_validation), 85 | callbacks=[early_stopping]) 86 | 87 | ''' 88 | 出力を用いて予測 89 | ''' 90 | truncate = maxlen 91 | Z = X[:1] # 元データの最初の一部だけ切り出し 92 | 93 | original = [f[i] for i in range(maxlen)] 94 | predicted = [None for i in range(maxlen)] 95 | 96 | for i in range(length_of_sequences - maxlen + 1): 97 | z_ = Z[-1:] 98 | y_ = model.predict(z_) 99 | sequence_ = np.concatenate( 100 | (z_.reshape(maxlen, n_in)[1:], y_), 101 | axis=0).reshape(1, maxlen, n_in) 102 | Z = np.append(Z, sequence_, axis=0) 103 | predicted.append(y_.reshape(-1)) 104 | 105 | ''' 106 | グラフで可視化 107 | ''' 108 | plt.rc('font', family='serif') 109 | plt.figure() 110 | plt.ylim([-1.5, 1.5]) 111 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 112 | plt.plot(original, linestyle='dashed', color='black') 113 | plt.plot(predicted, color='black') 114 | plt.show() 115 | -------------------------------------------------------------------------------- /5/keras/02_sin_gru_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import GRU 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(123) 12 | 13 | 14 | def sin(x, T=100): 15 | return np.sin(2.0 * np.pi * x / T) 16 | 17 | 18 | def toy_problem(T=100, ampl=0.05): 19 | x = np.arange(0, 2 * T + 1) 20 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 21 | return sin(x) + noise 22 | 23 | 24 | ''' 25 | データの生成 26 | ''' 27 | T = 100 28 | f = toy_problem(T) 29 | 30 | length_of_sequences = 2 * T 31 | maxlen = 25 # ひとつの時系列データの長さ 32 | 33 | data = [] 34 | target = [] 35 | 36 | for i in range(0, length_of_sequences - maxlen + 1): 37 | data.append(f[i: i + maxlen]) 38 | target.append(f[i + maxlen]) 39 | 40 | X = np.array(data).reshape(len(data), maxlen, 1) 41 | Y = np.array(target).reshape(len(data), 1) 42 | 43 | # データ設定 44 | N_train = int(len(data) * 0.9) 45 | N_validation = len(data) - N_train 46 | 47 | X_train, X_validation, Y_train, Y_validation = \ 48 | train_test_split(X, Y, test_size=N_validation) 49 | 50 | ''' 51 | モデル設定 52 | ''' 53 | n_in = len(X[0][0]) # 1 54 | n_hidden = 60 55 | n_out = len(Y[0]) # 1 56 | 57 | 58 | def weight_variable(shape, name=None): 59 | return np.random.normal(scale=.01, size=shape) 60 | 61 | 62 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 63 | 64 | model = Sequential() 65 | model.add(GRU(n_hidden, 66 | kernel_initializer=weight_variable, 67 | input_shape=(maxlen, n_in))) 68 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 69 | model.add(Activation('linear')) 70 | 71 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 72 | model.compile(loss='mean_squared_error', 73 | optimizer=optimizer) 74 | 75 | ''' 76 | モデル学習 77 | ''' 78 | epochs = 500 79 | batch_size = 10 80 | 81 | model.fit(X_train, Y_train, 82 | batch_size=batch_size, 83 | epochs=epochs, 84 | validation_data=(X_validation, Y_validation), 85 | callbacks=[early_stopping]) 86 | 87 | ''' 88 | 出力を用いて予測 89 | ''' 90 | truncate = maxlen 91 | Z = X[:1] # 元データの最初の一部だけ切り出し 92 | 93 | original = [f[i] for i in range(maxlen)] 94 | predicted = [None for i in range(maxlen)] 95 | 96 | for i in range(length_of_sequences - maxlen + 1): 97 | z_ = Z[-1:] 98 | y_ = model.predict(z_) 99 | sequence_ = np.concatenate( 100 | (z_.reshape(maxlen, n_in)[1:], y_), 101 | axis=0).reshape(1, maxlen, n_in) 102 | Z = np.append(Z, sequence_, axis=0) 103 | predicted.append(y_.reshape(-1)) 104 | 105 | ''' 106 | グラフで可視化 107 | ''' 108 | plt.rc('font', family='serif') 109 | plt.figure() 110 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 111 | plt.plot(original, linestyle='dashed', color='black') 112 | plt.plot(predicted, color='black') 113 | plt.show() 114 | -------------------------------------------------------------------------------- /5/keras/03_adding_problem_simple_rnn_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import SimpleRNN 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def mask(T=200): 15 | mask = np.zeros(T) 16 | indices = np.random.permutation(np.arange(T))[:2] 17 | mask[indices] = 1 18 | return mask 19 | 20 | 21 | def toy_problem(N=10, T=200): 22 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 23 | masks = np.zeros((N, T)) 24 | for i in range(N): 25 | masks[i] = mask(T) 26 | 27 | data = np.zeros((N, T, 2)) 28 | data[:, :, 0] = signals[:] 29 | data[:, :, 1] = masks[:] 30 | target = (signals * masks).sum(axis=1).reshape(N, 1) 31 | 32 | return (data, target) 33 | 34 | 35 | ''' 36 | データの生成 37 | ''' 38 | N = 10000 39 | T = 200 40 | maxlen = T 41 | 42 | X, Y = toy_problem(N=N, T=T) 43 | 44 | N_train = int(N * 0.9) 45 | N_validation = N - N_train 46 | 47 | 48 | ''' 49 | モデル設定 50 | ''' 51 | n_in = len(X[0][0]) # 2 52 | n_hidden = 100 53 | n_out = len(Y[0]) # 1 54 | 55 | 56 | def weight_variable(shape, name=None): 57 | return np.random.normal(scale=.01, size=shape) 58 | 59 | 60 | early_stopping = EarlyStopping(monitor='loss', patience=100, verbose=1) 61 | 62 | model = Sequential() 63 | model.add(SimpleRNN(n_hidden, 64 | kernel_initializer=weight_variable, 65 | input_shape=(maxlen, n_in))) 66 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 67 | model.add(Activation('linear')) 68 | 69 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 70 | model.compile(loss='mean_squared_error', 71 | optimizer=optimizer) 72 | 73 | ''' 74 | モデル学習 75 | ''' 76 | epochs = 1000 77 | batch_size = 100 78 | 79 | hist = model.fit(X, Y, 80 | batch_size=batch_size, 81 | epochs=epochs, 82 | callbacks=[early_stopping]) 83 | 84 | ''' 85 | 学習の進み具合を可視化 86 | ''' 87 | loss = hist.history['loss'] 88 | 89 | plt.rc('font', family='serif') 90 | fig = plt.figure() 91 | plt.plot(range(len(loss)), loss, label='loss', color='black') 92 | plt.xlabel('epochs') 93 | plt.show() 94 | plt.savefig(__file__ + '.eps') 95 | -------------------------------------------------------------------------------- /5/keras/04_adding_problem_lstm_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import LSTM 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def mask(T=200): 15 | mask = np.zeros(T) 16 | indices = np.random.permutation(np.arange(T))[:2] 17 | mask[indices] = 1 18 | return mask 19 | 20 | 21 | def toy_problem(N=10, T=200): 22 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 23 | masks = np.zeros((N, T)) 24 | for i in range(N): 25 | masks[i] = mask(T) 26 | 27 | data = np.zeros((N, T, 2)) 28 | data[:, :, 0] = signals[:] 29 | data[:, :, 1] = masks[:] 30 | target = (signals * masks).sum(axis=1).reshape(N, 1) 31 | 32 | return (data, target) 33 | 34 | 35 | ''' 36 | データの生成 37 | ''' 38 | N = 10000 39 | T = 200 40 | maxlen = T 41 | 42 | X, Y = toy_problem(N=N, T=T) 43 | 44 | N_train = int(N * 0.9) 45 | N_validation = N - N_train 46 | 47 | 48 | ''' 49 | モデル設定 50 | ''' 51 | n_in = len(X[0][0]) # 2 52 | n_hidden = 100 53 | n_out = len(Y[0]) # 1 54 | 55 | 56 | def weight_variable(shape, name=None): 57 | return np.random.normal(scale=.01, size=shape) 58 | 59 | 60 | early_stopping = EarlyStopping(monitor='loss', patience=100, verbose=1) 61 | 62 | model = Sequential() 63 | model.add(LSTM(n_hidden, 64 | kernel_initializer=weight_variable, 65 | input_shape=(maxlen, n_in))) 66 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 67 | model.add(Activation('linear')) 68 | 69 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 70 | model.compile(loss='mean_squared_error', 71 | optimizer=optimizer) 72 | 73 | ''' 74 | モデル学習 75 | ''' 76 | epochs = 1000 77 | batch_size = 100 78 | 79 | hist = model.fit(X, Y, 80 | batch_size=batch_size, 81 | epochs=epochs, 82 | callbacks=[early_stopping]) 83 | 84 | ''' 85 | 学習の進み具合を可視化 86 | ''' 87 | loss = hist.history['loss'] 88 | 89 | plt.rc('font', family='serif') 90 | fig = plt.figure() 91 | plt.plot(range(len(loss)), loss, label='loss', color='black') 92 | plt.xlabel('epochs') 93 | plt.show() 94 | plt.savefig(__file__ + '.eps') 95 | -------------------------------------------------------------------------------- /5/keras/05_adding_problem_gru_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import GRU 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def mask(T=200): 15 | mask = np.zeros(T) 16 | indices = np.random.permutation(np.arange(T))[:2] 17 | mask[indices] = 1 18 | return mask 19 | 20 | 21 | def toy_problem(N=10, T=200): 22 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 23 | masks = np.zeros((N, T)) 24 | for i in range(N): 25 | masks[i] = mask(T) 26 | 27 | data = np.zeros((N, T, 2)) 28 | data[:, :, 0] = signals[:] 29 | data[:, :, 1] = masks[:] 30 | target = (signals * masks).sum(axis=1).reshape(N, 1) 31 | 32 | return (data, target) 33 | 34 | 35 | ''' 36 | データの生成 37 | ''' 38 | N = 10000 39 | T = 200 40 | maxlen = T 41 | 42 | X, Y = toy_problem(N=N, T=T) 43 | 44 | N_train = int(N * 0.9) 45 | N_validation = N - N_train 46 | 47 | 48 | ''' 49 | モデル設定 50 | ''' 51 | n_in = len(X[0][0]) # 2 52 | n_hidden = 100 53 | n_out = len(Y[0]) # 1 54 | 55 | 56 | def weight_variable(shape, name=None): 57 | return np.random.normal(scale=.01, size=shape) 58 | 59 | 60 | early_stopping = EarlyStopping(monitor='loss', patience=100, verbose=1) 61 | 62 | model = Sequential() 63 | model.add(GRU(n_hidden, 64 | kernel_initializer=weight_variable, 65 | input_shape=(maxlen, n_in))) 66 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 67 | model.add(Activation('linear')) 68 | 69 | optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999) 70 | model.compile(loss='mean_squared_error', 71 | optimizer=optimizer) 72 | 73 | ''' 74 | モデル学習 75 | ''' 76 | epochs = 1000 77 | batch_size = 100 78 | 79 | hist = model.fit(X, Y, 80 | batch_size=batch_size, 81 | epochs=epochs, 82 | callbacks=[early_stopping]) 83 | 84 | ''' 85 | 学習の進み具合を可視化 86 | ''' 87 | loss = hist.history['loss'] 88 | 89 | plt.rc('font', family='serif') 90 | fig = plt.figure() 91 | plt.plot(range(len(loss)), loss, label='loss', color='black') 92 | plt.xlabel('epochs') 93 | plt.show() 94 | plt.savefig(__file__ + '.eps') 95 | -------------------------------------------------------------------------------- /5/sin.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yusugomori/deeplearning-tensorflow-keras/34eb2eb70746b76abb2bc1178c721f0bf7b0c353/5/sin.mp3 -------------------------------------------------------------------------------- /5/sin_noise.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yusugomori/deeplearning-tensorflow-keras/34eb2eb70746b76abb2bc1178c721f0bf7b0c353/5/sin_noise.mp3 -------------------------------------------------------------------------------- /5/tensorflow/00_sin_simple_rnn_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('RNN'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | class EarlyStopping(): 55 | def __init__(self, patience=0, verbose=0): 56 | self._step = 0 57 | self._loss = float('inf') 58 | self.patience = patience 59 | self.verbose = verbose 60 | 61 | def validate(self, loss): 62 | if self._loss < loss: 63 | self._step += 1 64 | if self._step > self.patience: 65 | if self.verbose: 66 | print('early stopping') 67 | return True 68 | else: 69 | self._step = 0 70 | self._loss = loss 71 | 72 | return False 73 | 74 | 75 | if __name__ == '__main__': 76 | def sin(x, T=100): 77 | return np.sin(2.0 * np.pi * x / T) 78 | 79 | def toy_problem(T=100, ampl=0.05): 80 | x = np.arange(0, 2 * T + 1) 81 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 82 | return sin(x) + noise 83 | 84 | ''' 85 | データの生成 86 | ''' 87 | T = 100 88 | f = toy_problem(T) 89 | 90 | length_of_sequences = 2 * T # 全時系列の長さ 91 | maxlen = 25 # ひとつの時系列データの長さ 92 | 93 | data = [] 94 | target = [] 95 | 96 | for i in range(0, length_of_sequences - maxlen + 1): 97 | data.append(f[i: i + maxlen]) 98 | target.append(f[i + maxlen]) 99 | 100 | X = np.array(data).reshape(len(data), maxlen, 1) 101 | Y = np.array(target).reshape(len(data), 1) 102 | 103 | # データ設定 104 | N_train = int(len(data) * 0.9) 105 | N_validation = len(data) - N_train 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X, Y, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = len(X[0][0]) # 1 114 | n_hidden = 30 115 | n_out = len(Y[0]) # 1 116 | 117 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 118 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 119 | n_batch = tf.placeholder(tf.int32, shape=[]) 120 | 121 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 122 | loss = loss(y, t) 123 | train_step = training(loss) 124 | 125 | early_stopping = EarlyStopping(patience=10, verbose=1) 126 | history = { 127 | 'val_loss': [] 128 | } 129 | 130 | ''' 131 | モデル学習 132 | ''' 133 | epochs = 500 134 | batch_size = 10 135 | 136 | init = tf.global_variables_initializer() 137 | sess = tf.Session() 138 | sess.run(init) 139 | 140 | n_batches = N_train // batch_size 141 | 142 | for epoch in range(epochs): 143 | X_, Y_ = shuffle(X_train, Y_train) 144 | 145 | for i in range(n_batches): 146 | start = i * batch_size 147 | end = start + batch_size 148 | 149 | sess.run(train_step, feed_dict={ 150 | x: X_[start:end], 151 | t: Y_[start:end], 152 | n_batch: batch_size 153 | }) 154 | 155 | # 検証データを用いた評価 156 | val_loss = loss.eval(session=sess, feed_dict={ 157 | x: X_validation, 158 | t: Y_validation, 159 | n_batch: N_validation 160 | }) 161 | 162 | history['val_loss'].append(val_loss) 163 | print('epoch:', epoch, 164 | ' validation loss:', val_loss) 165 | 166 | # Early Stopping チェック 167 | if early_stopping.validate(val_loss): 168 | break 169 | 170 | ''' 171 | 出力を用いて予測 172 | ''' 173 | truncate = maxlen 174 | Z = X[:1] # 元データの最初の一部だけ切り出し 175 | 176 | original = [f[i] for i in range(maxlen)] 177 | predicted = [None for i in range(maxlen)] 178 | 179 | for i in range(length_of_sequences - maxlen + 1): 180 | # 最後の時系列データから未来を予測 181 | z_ = Z[-1:] 182 | y_ = y.eval(session=sess, feed_dict={ 183 | x: Z[-1:], 184 | n_batch: 1 185 | }) 186 | # 予測結果を用いて新しい時系列データを生成 187 | sequence_ = np.concatenate( 188 | (z_.reshape(maxlen, n_in)[1:], y_), axis=0) \ 189 | .reshape(1, maxlen, n_in) 190 | Z = np.append(Z, sequence_, axis=0) 191 | predicted.append(y_.reshape(-1)) 192 | 193 | ''' 194 | グラフで可視化 195 | ''' 196 | plt.rc('font', family='serif') 197 | plt.figure() 198 | plt.ylim([-1.5, 1.5]) 199 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 200 | plt.plot(original, linestyle='dashed', color='black') 201 | plt.plot(predicted, color='black') 202 | plt.show() 203 | -------------------------------------------------------------------------------- /5/tensorflow/01_00_sin_simple_lstm_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('LSTM'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | class EarlyStopping(): 55 | def __init__(self, patience=0, verbose=0): 56 | self._step = 0 57 | self._loss = float('inf') 58 | self.patience = patience 59 | self.verbose = verbose 60 | 61 | def validate(self, loss): 62 | if self._loss < loss: 63 | self._step += 1 64 | if self._step > self.patience: 65 | if self.verbose: 66 | print('early stopping') 67 | return True 68 | else: 69 | self._step = 0 70 | self._loss = loss 71 | 72 | return False 73 | 74 | 75 | if __name__ == '__main__': 76 | def sin(x, T=100): 77 | return np.sin(2.0 * np.pi * x / T) 78 | 79 | def toy_problem(T=100, ampl=0.05): 80 | x = np.arange(0, 2 * T + 1) 81 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 82 | return sin(x) + noise 83 | 84 | ''' 85 | データの生成 86 | ''' 87 | T = 100 88 | f = toy_problem(T) 89 | 90 | length_of_sequences = 2 * T 91 | maxlen = 25 92 | 93 | data = [] 94 | target = [] 95 | 96 | for i in range(0, length_of_sequences - maxlen + 1): 97 | data.append(f[i: i + maxlen]) 98 | target.append(f[i + maxlen]) 99 | 100 | X = np.array(data).reshape(len(data), maxlen, 1) 101 | Y = np.array(target).reshape(len(data), 1) 102 | 103 | # データ設定 104 | N_train = int(len(data) * 0.9) 105 | N_validation = len(data) - N_train 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X, Y, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = len(X[0][0]) # 1 114 | n_hidden = 30 115 | n_out = len(Y[0]) # 1 116 | 117 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 118 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 119 | n_batch = tf.placeholder(tf.int32, shape=[]) 120 | 121 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 122 | loss = loss(y, t) 123 | train_step = training(loss) 124 | 125 | early_stopping = EarlyStopping(patience=10, verbose=1) 126 | history = { 127 | 'val_loss': [] 128 | } 129 | 130 | ''' 131 | モデル学習 132 | ''' 133 | epochs = 500 134 | batch_size = 10 135 | 136 | init = tf.global_variables_initializer() 137 | sess = tf.Session() 138 | sess.run(init) 139 | 140 | n_batches = N_train // batch_size 141 | 142 | for epoch in range(epochs): 143 | X_, Y_ = shuffle(X_train, Y_train) 144 | 145 | for i in range(n_batches): 146 | start = i * batch_size 147 | end = start + batch_size 148 | 149 | sess.run(train_step, feed_dict={ 150 | x: X_[start:end], 151 | t: Y_[start:end], 152 | n_batch: batch_size 153 | }) 154 | 155 | # 検証データを用いた評価 156 | val_loss = loss.eval(session=sess, feed_dict={ 157 | x: X_validation, 158 | t: Y_validation, 159 | n_batch: N_validation 160 | }) 161 | 162 | history['val_loss'].append(val_loss) 163 | print('epoch:', epoch, 164 | ' validation loss:', val_loss) 165 | 166 | # Early Stopping チェック 167 | if early_stopping.validate(val_loss): 168 | break 169 | 170 | ''' 171 | 出力を用いて予測 172 | ''' 173 | truncate = maxlen 174 | Z = X[:1] # 元データの最初の一部だけ切り出し 175 | 176 | original = [f[i] for i in range(maxlen)] 177 | predicted = [None for i in range(maxlen)] 178 | 179 | for i in range(length_of_sequences - maxlen + 1): 180 | # 最後の時系列データから未来を予測 181 | z_ = Z[-1:] 182 | y_ = y.eval(session=sess, feed_dict={ 183 | x: Z[-1:], 184 | n_batch: 1 185 | }) 186 | # 予測結果を用いて新しい時系列データを生成 187 | sequence_ = np.concatenate( 188 | (z_.reshape(maxlen, n_in)[1:], y_), axis=0) \ 189 | .reshape(1, maxlen, n_in) 190 | Z = np.append(Z, sequence_, axis=0) 191 | predicted.append(y_.reshape(-1)) 192 | 193 | ''' 194 | グラフで可視化 195 | ''' 196 | plt.rc('font', family='serif') 197 | plt.figure() 198 | plt.ylim([-1.5, 1.5]) 199 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 200 | plt.plot(original, linestyle='dashed', color='black') 201 | plt.plot(predicted, color='black') 202 | plt.show() 203 | -------------------------------------------------------------------------------- /5/tensorflow/01_01_sin_lstm_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.LSTMCell(n_hidden, forget_bias=1.0) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('LSTM'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | class EarlyStopping(): 55 | def __init__(self, patience=0, verbose=0): 56 | self._step = 0 57 | self._loss = float('inf') 58 | self.patience = patience 59 | self.verbose = verbose 60 | 61 | def validate(self, loss): 62 | if self._loss < loss: 63 | self._step += 1 64 | if self._step > self.patience: 65 | if self.verbose: 66 | print('early stopping') 67 | return True 68 | else: 69 | self._step = 0 70 | self._loss = loss 71 | 72 | return False 73 | 74 | 75 | if __name__ == '__main__': 76 | def sin(x, T=100): 77 | return np.sin(2.0 * np.pi * x / T) 78 | 79 | def toy_problem(T=100, ampl=0.05): 80 | x = np.arange(0, 2 * T + 1) 81 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 82 | return sin(x) + noise 83 | 84 | ''' 85 | データの生成 86 | ''' 87 | T = 100 88 | f = toy_problem(T) 89 | 90 | length_of_sequences = 2 * T 91 | maxlen = 25 92 | 93 | data = [] 94 | target = [] 95 | 96 | for i in range(0, length_of_sequences - maxlen + 1): 97 | data.append(f[i: i + maxlen]) 98 | target.append(f[i + maxlen]) 99 | 100 | X = np.array(data).reshape(len(data), maxlen, 1) 101 | Y = np.array(target).reshape(len(data), 1) 102 | 103 | # データ設定 104 | N_train = int(len(data) * 0.9) 105 | N_validation = len(data) - N_train 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X, Y, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = len(X[0][0]) # 1 114 | n_hidden = 30 115 | n_out = len(Y[0]) # 1 116 | 117 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 118 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 119 | n_batch = tf.placeholder(tf.int32, shape=[]) 120 | 121 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 122 | loss = loss(y, t) 123 | train_step = training(loss) 124 | 125 | early_stopping = EarlyStopping(patience=10, verbose=1) 126 | history = { 127 | 'val_loss': [] 128 | } 129 | 130 | ''' 131 | モデル学習 132 | ''' 133 | epochs = 500 134 | batch_size = 10 135 | 136 | init = tf.global_variables_initializer() 137 | sess = tf.Session() 138 | sess.run(init) 139 | 140 | n_batches = N_train // batch_size 141 | 142 | for epoch in range(epochs): 143 | X_, Y_ = shuffle(X_train, Y_train) 144 | 145 | for i in range(n_batches): 146 | start = i * batch_size 147 | end = start + batch_size 148 | 149 | sess.run(train_step, feed_dict={ 150 | x: X_[start:end], 151 | t: Y_[start:end], 152 | n_batch: batch_size 153 | }) 154 | 155 | # 検証データを用いた評価 156 | val_loss = loss.eval(session=sess, feed_dict={ 157 | x: X_validation, 158 | t: Y_validation, 159 | n_batch: N_validation 160 | }) 161 | 162 | history['val_loss'].append(val_loss) 163 | print('epoch:', epoch, 164 | ' validation loss:', val_loss) 165 | 166 | # Early Stopping チェック 167 | if early_stopping.validate(val_loss): 168 | break 169 | 170 | ''' 171 | 出力を用いて予測 172 | ''' 173 | truncate = maxlen 174 | Z = X[:1] # 元データの最初の一部だけ切り出し 175 | 176 | original = [f[i] for i in range(maxlen)] 177 | predicted = [None for i in range(maxlen)] 178 | 179 | for i in range(length_of_sequences - maxlen + 1): 180 | # 最後の時系列データから未来を予測 181 | z_ = Z[-1:] 182 | y_ = y.eval(session=sess, feed_dict={ 183 | x: Z[-1:], 184 | n_batch: 1 185 | }) 186 | # 予測結果を用いて新しい時系列データを生成 187 | sequence_ = np.concatenate( 188 | (z_.reshape(maxlen, n_in)[1:], y_), axis=0) \ 189 | .reshape(1, maxlen, n_in) 190 | Z = np.append(Z, sequence_, axis=0) 191 | predicted.append(y_.reshape(-1)) 192 | 193 | ''' 194 | グラフで可視化 195 | ''' 196 | plt.rc('font', family='serif') 197 | plt.figure() 198 | plt.ylim([-1.5, 1.5]) 199 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 200 | plt.plot(original, linestyle='dashed', color='black') 201 | plt.plot(predicted, color='black') 202 | plt.show() 203 | -------------------------------------------------------------------------------- /5/tensorflow/02_sin_gru_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.GRUCell(n_hidden) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('GRU'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | class EarlyStopping(): 55 | def __init__(self, patience=0, verbose=0): 56 | self._step = 0 57 | self._loss = float('inf') 58 | self.patience = patience 59 | self.verbose = verbose 60 | 61 | def validate(self, loss): 62 | if self._loss < loss: 63 | self._step += 1 64 | if self._step > self.patience: 65 | if self.verbose: 66 | print('early stopping') 67 | return True 68 | else: 69 | self._step = 0 70 | self._loss = loss 71 | 72 | return False 73 | 74 | 75 | if __name__ == '__main__': 76 | def sin(x, T=100): 77 | return np.sin(2.0 * np.pi * x / T) 78 | 79 | def toy_problem(T=100, ampl=0.05): 80 | x = np.arange(0, 2 * T + 1) 81 | noise = ampl * np.random.uniform(low=-1.0, high=1.0, size=len(x)) 82 | return sin(x) + noise 83 | 84 | ''' 85 | データの生成 86 | ''' 87 | T = 100 88 | f = toy_problem(T) 89 | 90 | length_of_sequences = 2 * T 91 | maxlen = 25 92 | 93 | data = [] 94 | target = [] 95 | 96 | for i in range(0, length_of_sequences - maxlen + 1): 97 | data.append(f[i: i + maxlen]) 98 | target.append(f[i + maxlen]) 99 | 100 | X = np.array(data).reshape(len(data), maxlen, 1) 101 | Y = np.array(target).reshape(len(data), 1) 102 | 103 | # データ設定 104 | N_train = int(len(data) * 0.9) 105 | N_validation = len(data) - N_train 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X, Y, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = len(X[0][0]) # 1 114 | n_hidden = 30 115 | n_out = len(Y[0]) # 1 116 | 117 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 118 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 119 | n_batch = tf.placeholder(tf.int32, shape=[]) 120 | 121 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 122 | loss = loss(y, t) 123 | train_step = training(loss) 124 | 125 | early_stopping = EarlyStopping(patience=10, verbose=1) 126 | history = { 127 | 'val_loss': [] 128 | } 129 | 130 | ''' 131 | モデル学習 132 | ''' 133 | epochs = 500 134 | batch_size = 10 135 | 136 | init = tf.global_variables_initializer() 137 | sess = tf.Session() 138 | sess.run(init) 139 | 140 | n_batches = N_train // batch_size 141 | 142 | for epoch in range(epochs): 143 | X_, Y_ = shuffle(X_train, Y_train) 144 | 145 | for i in range(n_batches): 146 | start = i * batch_size 147 | end = start + batch_size 148 | 149 | sess.run(train_step, feed_dict={ 150 | x: X_[start:end], 151 | t: Y_[start:end], 152 | n_batch: batch_size 153 | }) 154 | 155 | # 検証データを用いた評価 156 | val_loss = loss.eval(session=sess, feed_dict={ 157 | x: X_validation, 158 | t: Y_validation, 159 | n_batch: N_validation 160 | }) 161 | 162 | history['val_loss'].append(val_loss) 163 | print('epoch:', epoch, 164 | ' validation loss:', val_loss) 165 | 166 | # Early Stopping チェック 167 | if early_stopping.validate(val_loss): 168 | break 169 | 170 | ''' 171 | 出力を用いて予測 172 | ''' 173 | truncate = maxlen 174 | Z = X[:1] # 元データの最初の一部だけ切り出し 175 | 176 | original = [f[i] for i in range(maxlen)] 177 | predicted = [None for i in range(maxlen)] 178 | 179 | for i in range(length_of_sequences - maxlen + 1): 180 | # 最後の時系列データから未来を予測 181 | z_ = Z[-1:] 182 | y_ = y.eval(session=sess, feed_dict={ 183 | x: Z[-1:], 184 | n_batch: 1 185 | }) 186 | # 予測結果を用いて新しい時系列データを生成 187 | sequence_ = np.concatenate( 188 | (z_.reshape(maxlen, n_in)[1:], y_), axis=0) \ 189 | .reshape(1, maxlen, n_in) 190 | Z = np.append(Z, sequence_, axis=0) 191 | predicted.append(y_.reshape(-1)) 192 | 193 | ''' 194 | グラフで可視化 195 | ''' 196 | plt.rc('font', family='serif') 197 | plt.figure() 198 | plt.ylim([-1.5, 1.5]) 199 | plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa') 200 | plt.plot(original, linestyle='dashed', color='black') 201 | plt.plot(predicted, color='black') 202 | plt.show() 203 | -------------------------------------------------------------------------------- /5/tensorflow/03_adding_problem_simple_rnn_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('LSTM'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | if __name__ == '__main__': 55 | def mask(T=200): 56 | mask = np.zeros(T) 57 | indices = np.random.permutation(np.arange(T))[:2] 58 | mask[indices] = 1 59 | return mask 60 | 61 | def toy_problem(N=10, T=200): 62 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 63 | masks = np.zeros((N, T)) 64 | for i in range(N): 65 | masks[i] = mask(T) 66 | 67 | data = np.zeros((N, T, 2)) 68 | data[:, :, 0] = signals[:] 69 | data[:, :, 1] = masks[:] 70 | target = (signals * masks).sum(axis=1).reshape(N, 1) 71 | 72 | return (data, target) 73 | 74 | ''' 75 | データの生成 76 | ''' 77 | N = 10000 78 | T = 200 79 | maxlen = T 80 | 81 | X, Y = toy_problem(N=N, T=T) 82 | 83 | N_train = int(N * 0.9) 84 | N_validation = N - N_train 85 | 86 | X_train, X_validation, Y_train, Y_validation = \ 87 | train_test_split(X, Y, test_size=N_validation) 88 | 89 | ''' 90 | モデル設定 91 | ''' 92 | n_in = len(X[0][0]) # 2 93 | n_hidden = 100 94 | n_out = len(Y[0]) # 1 95 | 96 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 97 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 98 | n_batch = tf.placeholder(tf.int32, shape=[]) 99 | 100 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 101 | loss = loss(y, t) 102 | train_step = training(loss) 103 | 104 | history = { 105 | 'val_loss': [] 106 | } 107 | 108 | ''' 109 | モデル学習 110 | ''' 111 | epochs = 300 112 | batch_size = 100 113 | 114 | init = tf.global_variables_initializer() 115 | sess = tf.Session() 116 | sess.run(init) 117 | 118 | n_batches = N_train // batch_size 119 | 120 | for epoch in range(epochs): 121 | X_, Y_ = shuffle(X_train, Y_train) 122 | 123 | for i in range(n_batches): 124 | start = i * batch_size 125 | end = start + batch_size 126 | 127 | sess.run(train_step, feed_dict={ 128 | x: X_[start:end], 129 | t: Y_[start:end], 130 | n_batch: batch_size 131 | }) 132 | 133 | # 検証データを用いた評価 134 | val_loss = loss.eval(session=sess, feed_dict={ 135 | x: X_validation, 136 | t: Y_validation, 137 | n_batch: N_validation 138 | }) 139 | 140 | history['val_loss'].append(val_loss) 141 | print('epoch:', epoch, 142 | ' validation loss:', val_loss) 143 | 144 | ''' 145 | 学習の進み具合を可視化 146 | ''' 147 | loss = history['val_loss'] 148 | 149 | plt.rc('font', family='serif') 150 | fig = plt.figure() 151 | plt.plot(range(len(loss)), loss, label='loss', color='black') 152 | plt.xlabel('epochs') 153 | plt.show() 154 | -------------------------------------------------------------------------------- /5/tensorflow/04_adding_problem_lstm_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) 21 | # cell = tf.nn.rnn_cell.LSTMCell(n_hidden, forget_bias=1.0) 22 | initial_state = cell.zero_state(n_batch, tf.float32) 23 | 24 | state = initial_state 25 | outputs = [] # 過去の隠れ層の出力を保存 26 | with tf.variable_scope('LSTM'): 27 | for t in range(maxlen): 28 | if t > 0: 29 | tf.get_variable_scope().reuse_variables() 30 | (cell_output, state) = cell(x[:, t, :], state) 31 | outputs.append(cell_output) 32 | 33 | output = outputs[-1] 34 | 35 | V = weight_variable([n_hidden, n_out]) 36 | c = bias_variable([n_out]) 37 | y = tf.matmul(output, V) + c # 線形活性 38 | 39 | return y 40 | 41 | 42 | def loss(y, t): 43 | mse = tf.reduce_mean(tf.square(y - t)) 44 | return mse 45 | 46 | 47 | def training(loss): 48 | optimizer = \ 49 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 50 | 51 | train_step = optimizer.minimize(loss) 52 | return train_step 53 | 54 | 55 | if __name__ == '__main__': 56 | def mask(T=200): 57 | mask = np.zeros(T) 58 | indices = np.random.permutation(np.arange(T))[:2] 59 | mask[indices] = 1 60 | return mask 61 | 62 | def toy_problem(N=10, T=200): 63 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 64 | masks = np.zeros((N, T)) 65 | for i in range(N): 66 | masks[i] = mask(T) 67 | 68 | data = np.zeros((N, T, 2)) 69 | data[:, :, 0] = signals[:] 70 | data[:, :, 1] = masks[:] 71 | target = (signals * masks).sum(axis=1).reshape(N, 1) 72 | 73 | return (data, target) 74 | 75 | ''' 76 | データの生成 77 | ''' 78 | N = 10000 79 | T = 200 80 | maxlen = T 81 | 82 | X, Y = toy_problem(N=N, T=T) 83 | 84 | N_train = int(N * 0.9) 85 | N_validation = N - N_train 86 | 87 | X_train, X_validation, Y_train, Y_validation = \ 88 | train_test_split(X, Y, test_size=N_validation) 89 | 90 | ''' 91 | モデル設定 92 | ''' 93 | n_in = len(X[0][0]) # 2 94 | n_hidden = 100 95 | n_out = len(Y[0]) # 1 96 | 97 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 98 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 99 | n_batch = tf.placeholder(tf.int32, shape=[]) 100 | 101 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 102 | loss = loss(y, t) 103 | train_step = training(loss) 104 | 105 | history = { 106 | 'val_loss': [] 107 | } 108 | 109 | ''' 110 | モデル学習 111 | ''' 112 | epochs = 300 113 | batch_size = 100 114 | 115 | init = tf.global_variables_initializer() 116 | sess = tf.Session() 117 | sess.run(init) 118 | 119 | n_batches = N_train // batch_size 120 | 121 | for epoch in range(epochs): 122 | X_, Y_ = shuffle(X_train, Y_train) 123 | 124 | for i in range(n_batches): 125 | start = i * batch_size 126 | end = start + batch_size 127 | 128 | sess.run(train_step, feed_dict={ 129 | x: X_[start:end], 130 | t: Y_[start:end], 131 | n_batch: batch_size 132 | }) 133 | 134 | # 検証データを用いた評価 135 | val_loss = loss.eval(session=sess, feed_dict={ 136 | x: X_validation, 137 | t: Y_validation, 138 | n_batch: N_validation 139 | }) 140 | 141 | history['val_loss'].append(val_loss) 142 | print('epoch:', epoch, 143 | ' validation loss:', val_loss) 144 | 145 | ''' 146 | 学習の進み具合を可視化 147 | ''' 148 | loss = history['val_loss'] 149 | 150 | plt.rc('font', family='serif') 151 | fig = plt.figure() 152 | plt.plot(range(len(loss)), loss, label='loss', color='black') 153 | plt.xlabel('epochs') 154 | plt.show() 155 | -------------------------------------------------------------------------------- /5/tensorflow/05_adding_problem_gru_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.utils import shuffle 6 | 7 | np.random.seed(0) 8 | tf.set_random_seed(1234) 9 | 10 | 11 | def inference(x, n_batch, maxlen=None, n_hidden=None, n_out=None): 12 | def weight_variable(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.01) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.zeros(shape, dtype=tf.float32) 18 | return tf.Variable(initial) 19 | 20 | cell = tf.nn.rnn_cell.GRUCell(n_hidden) 21 | initial_state = cell.zero_state(n_batch, tf.float32) 22 | 23 | state = initial_state 24 | outputs = [] # 過去の隠れ層の出力を保存 25 | with tf.variable_scope('GRU'): 26 | for t in range(maxlen): 27 | if t > 0: 28 | tf.get_variable_scope().reuse_variables() 29 | (cell_output, state) = cell(x[:, t, :], state) 30 | outputs.append(cell_output) 31 | 32 | output = outputs[-1] 33 | 34 | V = weight_variable([n_hidden, n_out]) 35 | c = bias_variable([n_out]) 36 | y = tf.matmul(output, V) + c # 線形活性 37 | 38 | return y 39 | 40 | 41 | def loss(y, t): 42 | mse = tf.reduce_mean(tf.square(y - t)) 43 | return mse 44 | 45 | 46 | def training(loss): 47 | optimizer = \ 48 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 49 | 50 | train_step = optimizer.minimize(loss) 51 | return train_step 52 | 53 | 54 | if __name__ == '__main__': 55 | def mask(T=200): 56 | mask = np.zeros(T) 57 | indices = np.random.permutation(np.arange(T))[:2] 58 | mask[indices] = 1 59 | return mask 60 | 61 | def toy_problem(N=10, T=200): 62 | signals = np.random.uniform(low=0.0, high=1.0, size=(N, T)) 63 | masks = np.zeros((N, T)) 64 | for i in range(N): 65 | masks[i] = mask(T) 66 | 67 | data = np.zeros((N, T, 2)) 68 | data[:, :, 0] = signals[:] 69 | data[:, :, 1] = masks[:] 70 | target = (signals * masks).sum(axis=1).reshape(N, 1) 71 | 72 | return (data, target) 73 | 74 | ''' 75 | データの生成 76 | ''' 77 | N = 10000 78 | T = 200 79 | maxlen = T 80 | 81 | X, Y = toy_problem(N=N, T=T) 82 | 83 | N_train = int(N * 0.9) 84 | N_validation = N - N_train 85 | 86 | X_train, X_validation, Y_train, Y_validation = \ 87 | train_test_split(X, Y, test_size=N_validation) 88 | 89 | ''' 90 | モデル設定 91 | ''' 92 | n_in = len(X[0][0]) # 2 93 | n_hidden = 100 94 | n_out = len(Y[0]) # 1 95 | 96 | x = tf.placeholder(tf.float32, shape=[None, maxlen, n_in]) 97 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 98 | n_batch = tf.placeholder(tf.int32, shape=[]) 99 | 100 | y = inference(x, n_batch, maxlen=maxlen, n_hidden=n_hidden, n_out=n_out) 101 | loss = loss(y, t) 102 | train_step = training(loss) 103 | 104 | history = { 105 | 'val_loss': [] 106 | } 107 | 108 | ''' 109 | モデル学習 110 | ''' 111 | epochs = 300 112 | batch_size = 100 113 | 114 | init = tf.global_variables_initializer() 115 | sess = tf.Session() 116 | sess.run(init) 117 | 118 | n_batches = N_train // batch_size 119 | 120 | for epoch in range(epochs): 121 | X_, Y_ = shuffle(X_train, Y_train) 122 | 123 | for i in range(n_batches): 124 | start = i * batch_size 125 | end = start + batch_size 126 | 127 | sess.run(train_step, feed_dict={ 128 | x: X_[start:end], 129 | t: Y_[start:end], 130 | n_batch: batch_size 131 | }) 132 | 133 | # 検証データを用いた評価 134 | val_loss = loss.eval(session=sess, feed_dict={ 135 | x: X_validation, 136 | t: Y_validation, 137 | n_batch: N_validation 138 | }) 139 | 140 | history['val_loss'].append(val_loss) 141 | print('epoch:', epoch, 142 | ' validation loss:', val_loss) 143 | 144 | ''' 145 | 学習の進み具合を可視化 146 | ''' 147 | loss = history['val_loss'] 148 | 149 | plt.rc('font', family='serif') 150 | fig = plt.figure() 151 | plt.plot(range(len(loss)), loss, label='loss', color='black') 152 | plt.xlabel('epochs') 153 | plt.show() 154 | -------------------------------------------------------------------------------- /6/keras/00_mnist_bidirectional_rnn_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation 5 | from keras.layers.recurrent import LSTM 6 | from keras.layers.wrappers import Bidirectional 7 | from keras.optimizers import Adam 8 | from keras.callbacks import EarlyStopping 9 | from sklearn import datasets 10 | from sklearn.model_selection import train_test_split 11 | from sklearn.utils import shuffle 12 | 13 | np.random.seed(0) 14 | 15 | ''' 16 | データの生成 17 | ''' 18 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 19 | 20 | n = len(mnist.data) 21 | N = 30000 # MNISTの一部を使う 22 | N_train = 20000 23 | N_validation = 4000 24 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 25 | 26 | X = mnist.data[indices] 27 | X = X / 255.0 28 | X = X - X.mean(axis=1).reshape(len(X), 1) 29 | X = X.reshape(len(X), 28, 28) # 時系列データに変換 30 | y = mnist.target[indices] 31 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 32 | 33 | X_train, X_test, Y_train, Y_test = \ 34 | train_test_split(X, Y, train_size=N_train) 35 | 36 | X_train, X_validation, Y_train, Y_validation = \ 37 | train_test_split(X_train, Y_train, test_size=N_validation) 38 | 39 | ''' 40 | モデル設定 41 | ''' 42 | n_in = 28 43 | n_time = 28 44 | n_hidden = 128 45 | n_out = 10 46 | 47 | 48 | def weight_variable(shape, name=None): 49 | return np.random.normal(scale=.01, size=shape) 50 | 51 | 52 | early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1) 53 | 54 | model = Sequential() 55 | model.add(Bidirectional(LSTM(n_hidden), 56 | input_shape=(n_time, n_in))) 57 | model.add(Dense(n_out, kernel_initializer=weight_variable)) 58 | model.add(Activation('softmax')) 59 | 60 | model.compile(loss='categorical_crossentropy', 61 | optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999), 62 | metrics=['accuracy']) 63 | 64 | ''' 65 | モデル学習 66 | ''' 67 | epochs = 300 68 | batch_size = 250 69 | 70 | hist = model.fit(X_train, Y_train, 71 | batch_size=batch_size, 72 | epochs=epochs, 73 | validation_data=(X_validation, Y_validation), 74 | callbacks=[early_stopping]) 75 | 76 | ''' 77 | 学習の進み具合を可視化 78 | ''' 79 | acc = hist.history['val_acc'] 80 | loss = hist.history['val_loss'] 81 | 82 | plt.rc('font', family='serif') 83 | fig = plt.figure() 84 | plt.plot(range(len(loss)), loss, 85 | label='loss', color='black') 86 | plt.xlabel('epochs') 87 | plt.show() 88 | 89 | ''' 90 | 予測精度の評価 91 | ''' 92 | loss_and_metrics = model.evaluate(X_test, Y_test) 93 | print(loss_and_metrics) 94 | -------------------------------------------------------------------------------- /6/keras/01_adding_task_rnn_encoder_decoder_keras.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from keras.models import Sequential 3 | from keras.layers.core import Dense, Activation, RepeatVector 4 | from keras.layers.recurrent import LSTM 5 | from keras.layers.wrappers import TimeDistributed 6 | from keras.optimizers import Adam 7 | from keras.callbacks import EarlyStopping 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.utils import shuffle 10 | 11 | np.random.seed(0) 12 | 13 | 14 | def n(digits=3): 15 | number = '' 16 | for i in range(np.random.randint(1, digits + 1)): 17 | number += np.random.choice(list('0123456789')) 18 | return int(number) 19 | 20 | 21 | def padding(chars, maxlen): 22 | return chars + ' ' * (maxlen - len(chars)) 23 | 24 | 25 | ''' 26 | データの生成 27 | ''' 28 | N = 20000 29 | N_train = int(N * 0.9) 30 | N_validation = N - N_train 31 | 32 | digits = 3 # 最大の桁数 33 | input_digits = digits * 2 + 1 # 例： 123+456 34 | output_digits = digits + 1 # 500+500 = 1000 以上で４桁になる 35 | 36 | added = set() 37 | questions = [] 38 | answers = [] 39 | 40 | while len(questions) < N: 41 | a, b = n(), n() # 適当な数を２つ生成 42 | 43 | pair = tuple(sorted((a, b))) 44 | if pair in added: 45 | continue 46 | 47 | question = '{}+{}'.format(a, b) 48 | question = padding(question, input_digits) 49 | answer = str(a + b) 50 | answer = padding(answer, output_digits) 51 | 52 | added.add(pair) 53 | questions.append(question) 54 | answers.append(answer) 55 | 56 | chars = '0123456789+ ' 57 | char_indices = dict((c, i) for i, c in enumerate(chars)) 58 | indices_char = dict((i, c) for i, c in enumerate(chars)) 59 | 60 | X = np.zeros((len(questions), input_digits, len(chars)), dtype=np.integer) 61 | Y = np.zeros((len(questions), digits + 1, len(chars)), dtype=np.integer) 62 | 63 | for i in range(N): 64 | for t, char in enumerate(questions[i]): 65 | X[i, t, char_indices[char]] = 1 66 | for t, char in enumerate(answers[i]): 67 | Y[i, t, char_indices[char]] = 1 68 | 69 | X_train, X_validation, Y_train, Y_validation = \ 70 | train_test_split(X, Y, train_size=N_train) 71 | 72 | ''' 73 | モデル設定 74 | ''' 75 | n_in = len(chars) 76 | n_hidden = 128 77 | n_out = len(chars) 78 | 79 | model = Sequential() 80 | 81 | # Encoder 82 | model.add(LSTM(n_hidden, input_shape=(input_digits, n_in))) 83 | 84 | # Decoder 85 | model.add(RepeatVector(output_digits)) 86 | model.add(LSTM(n_hidden, return_sequences=True)) 87 | 88 | model.add(TimeDistributed(Dense(n_out))) 89 | model.add(Activation('softmax')) 90 | model.compile(loss='categorical_crossentropy', 91 | optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999), 92 | metrics=['accuracy']) 93 | 94 | ''' 95 | モデル学習 96 | ''' 97 | epochs = 200 98 | batch_size = 200 99 | 100 | for epoch in range(epochs): 101 | model.fit(X_train, Y_train, batch_size=batch_size, epochs=1, 102 | validation_data=(X_validation, Y_validation)) 103 | 104 | # 検証データからランダムに問題を選んで答え合わせ 105 | for i in range(10): 106 | index = np.random.randint(0, N_validation) 107 | question = X_validation[np.array([index])] 108 | answer = Y_validation[np.array([index])] 109 | prediction = model.predict_classes(question, verbose=0) 110 | 111 | question = question.argmax(axis=-1) 112 | answer = answer.argmax(axis=-1) 113 | 114 | q = ''.join(indices_char[i] for i in question[0]) 115 | a = ''.join(indices_char[i] for i in answer[0]) 116 | p = ''.join(indices_char[i] for i in prediction[0]) 117 | 118 | print('-' * 10) 119 | print('Q: ', q) 120 | print('A: ', p) 121 | print('T/F:', end=' ') 122 | if a == p: 123 | print('T') 124 | else: 125 | print('F') 126 | print('-' * 10) 127 | -------------------------------------------------------------------------------- /6/tensorflow/00_mnist_bidirectional_rnn_tensorflow.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from tensorflow.nn import rnn_cell 5 | from sklearn import datasets 6 | from sklearn.model_selection import train_test_split 7 | from sklearn.utils import shuffle 8 | 9 | np.random.seed(0) 10 | tf.set_random_seed(1234) 11 | 12 | 13 | def inference(x, n_in=None, n_time=None, n_hidden=None, n_out=None): 14 | def weight_variable(shape): 15 | initial = tf.truncated_normal(shape, stddev=0.01) 16 | return tf.Variable(initial) 17 | 18 | def bias_variable(shape): 19 | initial = tf.zeros(shape, dtype=tf.float32) 20 | return tf.Variable(initial) 21 | 22 | # 時系列データの形式をAPIの仕様に合わせるため、最終的に 23 | # (ミニバッチサイズ, 入力次元数) が時間長分ある形に変形 24 | x = tf.transpose(x, [1, 0, 2]) 25 | x = tf.reshape(x, [-1, n_in]) 26 | x = tf.split(x, n_time, 0) 27 | 28 | cell_forward = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) 29 | cell_backward = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) 30 | 31 | outputs, _, _ = \ 32 | tf.nn.static_bidirectional_rnn(cell_forward, cell_backward, x, 33 | dtype=tf.float32) 34 | 35 | W = weight_variable([n_hidden * 2, n_out]) 36 | b = bias_variable([n_out]) 37 | 38 | y = tf.nn.softmax(tf.matmul(outputs[-1], W) + b) 39 | 40 | return y 41 | 42 | 43 | def loss(y, t): 44 | cross_entropy = \ 45 | tf.reduce_mean(-tf.reduce_sum( 46 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 47 | reduction_indices=[1])) 48 | return cross_entropy 49 | 50 | 51 | def training(loss): 52 | optimizer = \ 53 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 54 | train_step = optimizer.minimize(loss) 55 | return train_step 56 | 57 | 58 | def accuracy(y, t): 59 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 60 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 61 | return accuracy 62 | 63 | 64 | class EarlyStopping(): 65 | def __init__(self, patience=0, verbose=0): 66 | self._step = 0 67 | self._loss = float('inf') 68 | self.patience = patience 69 | self.verbose = verbose 70 | 71 | def validate(self, loss): 72 | if self._loss < loss: 73 | self._step += 1 74 | if self._step > self.patience: 75 | if self.verbose: 76 | print('early stopping') 77 | return True 78 | else: 79 | self._step = 0 80 | self._loss = loss 81 | 82 | return False 83 | 84 | 85 | if __name__ == '__main__': 86 | ''' 87 | データの生成 88 | ''' 89 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 90 | 91 | n = len(mnist.data) 92 | N = 30000 # MNISTの一部を使う 93 | N_train = 20000 94 | N_validation = 4000 95 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 96 | 97 | X = mnist.data[indices] 98 | X = X / 255.0 99 | X = X - X.mean(axis=1).reshape(len(X), 1) 100 | X = X.reshape(len(X), 28, 28) # 時系列データに変換 101 | y = mnist.target[indices] 102 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 103 | 104 | X_train, X_test, Y_train, Y_test = \ 105 | train_test_split(X, Y, train_size=N_train) 106 | 107 | X_train, X_validation, Y_train, Y_validation = \ 108 | train_test_split(X_train, Y_train, test_size=N_validation) 109 | 110 | ''' 111 | モデル設定 112 | ''' 113 | n_in = 28 114 | n_time = 28 115 | n_hidden = 128 116 | n_out = 10 117 | 118 | x = tf.placeholder(tf.float32, shape=[None, n_time, n_in]) 119 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 120 | 121 | y = inference(x, n_in=n_in, n_time=n_time, n_hidden=n_hidden, n_out=n_out) 122 | loss = loss(y, t) 123 | train_step = training(loss) 124 | 125 | accuracy = accuracy(y, t) 126 | 127 | early_stopping = EarlyStopping(patience=10, verbose=1) 128 | history = { 129 | 'val_loss': [], 130 | 'val_acc': [] 131 | } 132 | 133 | ''' 134 | モデル学習 135 | ''' 136 | epochs = 300 137 | batch_size = 250 138 | 139 | init = tf.global_variables_initializer() 140 | sess = tf.Session() 141 | sess.run(init) 142 | 143 | n_batches = N_train // batch_size 144 | 145 | for epoch in range(epochs): 146 | X_, Y_ = shuffle(X_train, Y_train) 147 | 148 | for i in range(n_batches): 149 | start = i * batch_size 150 | end = start + batch_size 151 | 152 | sess.run(train_step, feed_dict={ 153 | x: X_[start:end], 154 | t: Y_[start:end] 155 | }) 156 | 157 | val_loss = loss.eval(session=sess, feed_dict={ 158 | x: X_validation, 159 | t: Y_validation 160 | }) 161 | val_acc = accuracy.eval(session=sess, feed_dict={ 162 | x: X_validation, 163 | t: Y_validation 164 | }) 165 | 166 | history['val_loss'].append(val_loss) 167 | history['val_acc'].append(val_acc) 168 | 169 | print('epoch:', epoch, 170 | ' validation loss:', val_loss, 171 | ' validation accuracy:', val_acc) 172 | 173 | if early_stopping.validate(val_loss): 174 | break 175 | 176 | ''' 177 | 学習の進み具合を可視化 178 | ''' 179 | loss = history['val_loss'] 180 | 181 | plt.rc('font', family='serif') 182 | fig = plt.figure() 183 | plt.plot(range(len(loss)), loss, 184 | label='loss', color='black') 185 | plt.xlabel('epochs') 186 | plt.show() 187 | 188 | ''' 189 | 予測精度の評価 190 | ''' 191 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 192 | x: X_test, 193 | t: Y_test 194 | }) 195 | print('accuracy: ', accuracy_rate) 196 | -------------------------------------------------------------------------------- /6/tensorflow/03_babi_memory_networks_tensorflow.py: -------------------------------------------------------------------------------- 1 | import re 2 | import tarfile 3 | import numpy as np 4 | import tensorflow as tf 5 | from sklearn.utils import shuffle 6 | from functools import reduce 7 | from utils.data import get_file 8 | 9 | np.random.seed(0) 10 | tf.set_random_seed(1234) 11 | 12 | 13 | def inference(x, q, n_batch, 14 | vocab_size=None, 15 | embedding_dim=None, 16 | story_maxlen=None, 17 | question_maxlen=None): 18 | def weight_variable(shape, stddev=0.08): 19 | initial = tf.truncated_normal(shape, stddev=stddev) 20 | return tf.Variable(initial) 21 | 22 | def bias_variable(shape): 23 | initial = tf.zeros(shape, dtype=tf.float32) 24 | return tf.Variable(initial) 25 | 26 | A = weight_variable([vocab_size, embedding_dim]) 27 | B = weight_variable([vocab_size, embedding_dim]) 28 | C = weight_variable([vocab_size, question_maxlen]) 29 | m = tf.nn.embedding_lookup(A, x) 30 | u = tf.nn.embedding_lookup(B, q) 31 | c = tf.nn.embedding_lookup(C, x) 32 | p = tf.nn.softmax(tf.einsum('ijk,ilk->ijl', m, u)) 33 | o = tf.add(p, c) 34 | o = tf.transpose(o, perm=[0, 2, 1]) 35 | ou = tf.concat([o, u], axis=-1) 36 | 37 | cell = tf.nn.rnn_cell.BasicLSTMCell(embedding_dim//2, forget_bias=1.0) 38 | initial_state = cell.zero_state(n_batch, tf.float32) 39 | state = initial_state 40 | outputs = [] 41 | with tf.variable_scope('LSTM'): 42 | for t in range(question_maxlen): 43 | if t > 0: 44 | tf.get_variable_scope().reuse_variables() 45 | (cell_output, state) = cell(ou[:, t, :], state) 46 | outputs.append(cell_output) 47 | output = outputs[-1] 48 | W = weight_variable([embedding_dim//2, vocab_size], stddev=0.01) 49 | a = tf.nn.softmax(tf.matmul(output, W)) 50 | 51 | return a 52 | 53 | 54 | def loss(y, t): 55 | cross_entropy = \ 56 | tf.reduce_mean(-tf.reduce_sum( 57 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 58 | reduction_indices=[1])) 59 | return cross_entropy 60 | 61 | 62 | def training(loss): 63 | optimizer = \ 64 | tf.train.AdamOptimizer(learning_rate=0.001, beta1=0.9, beta2=0.999) 65 | train_step = optimizer.minimize(loss) 66 | return train_step 67 | 68 | 69 | def accuracy(y, t): 70 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 71 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 72 | return accuracy 73 | 74 | 75 | def tokenize(sent): 76 | return [x.strip() for x in re.split('(\W+)', sent) if x.strip()] 77 | 78 | 79 | def parse_stories(lines): 80 | data = [] 81 | story = [] 82 | for line in lines: 83 | line = line.decode('utf-8').strip() 84 | nid, line = line.split(' ', 1) 85 | nid = int(nid) 86 | if nid == 1: 87 | story = [] 88 | if '\t' in line: 89 | q, a, supporting = line.split('\t') 90 | q = tokenize(q) 91 | substory = [x for x in story if x] 92 | data.append((substory, q, a)) 93 | story.append('') 94 | else: 95 | sent = tokenize(line) 96 | story.append(sent) 97 | return data 98 | 99 | 100 | def get_stories(f, max_length=None): 101 | def flatten(data): 102 | return reduce(lambda x, y: x + y, data) 103 | 104 | data = parse_stories(f.readlines()) 105 | data = [(flatten(story), q, answer) 106 | for story, q, answer in data 107 | if not max_length or len(flatten(story)) < max_length] 108 | return data 109 | 110 | 111 | def vectorize_stories(data, word_indices, story_maxlen, question_maxlen): 112 | X = [] 113 | Q = [] 114 | A = [] 115 | for story, question, answer in data: 116 | x = [word_indices[w] for w in story] 117 | q = [word_indices[w] for w in question] 118 | a = np.zeros(len(word_indices) + 1) # パディング用に +1 119 | a[word_indices[answer]] = 1 120 | X.append(x) 121 | Q.append(q) 122 | A.append(a) 123 | 124 | return (padding(X, maxlen=story_maxlen), 125 | padding(Q, maxlen=question_maxlen), np.array(A)) 126 | 127 | 128 | def padding(words, maxlen): 129 | for i, word in enumerate(words): 130 | words[i] = [0] * (maxlen - len(word)) + word 131 | return np.array(words) 132 | 133 | 134 | if __name__ == '__main__': 135 | ''' 136 | データ読み込み 137 | ''' 138 | print('Fetching data...') 139 | try: 140 | path = \ 141 | get_file('babi-tasks-v1-2.tar.gz', 142 | url='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz') 143 | except Exception as e: 144 | raise 145 | tar = tarfile.open(path) 146 | 147 | challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt' 148 | train_stories = get_stories(tar.extractfile(challenge.format('train'))) 149 | test_stories = get_stories(tar.extractfile(challenge.format('test'))) 150 | 151 | vocab = set() 152 | for story, q, answer in train_stories + test_stories: 153 | vocab |= set(story + q + [answer]) 154 | vocab = sorted(vocab) 155 | vocab_size = len(vocab) + 1 # パディング用に +1 156 | 157 | story_maxlen = \ 158 | max(map(len, (x for x, _, _ in train_stories + test_stories))) 159 | question_maxlen = \ 160 | max(map(len, (x for _, x, _ in train_stories + test_stories))) 161 | 162 | print('Vectorizing data...') 163 | word_indices = dict((c, i + 1) for i, c in enumerate(vocab)) 164 | inputs_train, questions_train, answers_train = \ 165 | vectorize_stories(train_stories, word_indices, 166 | story_maxlen, question_maxlen) 167 | 168 | inputs_test, questions_test, answers_test = \ 169 | vectorize_stories(test_stories, word_indices, 170 | story_maxlen, question_maxlen) 171 | 172 | ''' 173 | モデル設定 174 | ''' 175 | print('Building model...') 176 | x = tf.placeholder(tf.int32, shape=[None, story_maxlen]) 177 | q = tf.placeholder(tf.int32, shape=[None, question_maxlen]) 178 | a = tf.placeholder(tf.float32, shape=[None, vocab_size]) 179 | n_batch = tf.placeholder(tf.int32, shape=[]) 180 | 181 | y = inference(x, q, n_batch, 182 | vocab_size=vocab_size, 183 | embedding_dim=64, 184 | story_maxlen=story_maxlen, 185 | question_maxlen=question_maxlen) 186 | loss = loss(y, a) 187 | train_step = training(loss) 188 | acc = accuracy(y, a) 189 | history = { 190 | 'val_loss': [], 191 | 'val_acc': [] 192 | } 193 | 194 | ''' 195 | モデル学習 196 | ''' 197 | print('Training model...') 198 | epochs = 120 199 | batch_size = 100 200 | 201 | init = tf.global_variables_initializer() 202 | sess = tf.Session() 203 | sess.run(init) 204 | 205 | n_batches = len(inputs_train) // batch_size 206 | 207 | for epoch in range(epochs): 208 | inputs_train_, questions_train_, answers_train_ = \ 209 | shuffle(inputs_train, questions_train, answers_train) 210 | 211 | for i in range(n_batches): 212 | start = i * batch_size 213 | end = start + batch_size 214 | 215 | sess.run(train_step, feed_dict={ 216 | x: inputs_train_[start:end], 217 | q: questions_train_[start:end], 218 | a: answers_train_[start:end], 219 | n_batch: batch_size 220 | }) 221 | 222 | # テストデータを用いた評価 223 | val_loss = loss.eval(session=sess, feed_dict={ 224 | x: inputs_test, 225 | q: questions_test, 226 | a: answers_test, 227 | n_batch: len(inputs_test) 228 | }) 229 | val_acc = acc.eval(session=sess, feed_dict={ 230 | x: inputs_test, 231 | q: questions_test, 232 | a: answers_test, 233 | n_batch: len(inputs_test) 234 | }) 235 | 236 | history['val_loss'].append(val_loss) 237 | history['val_acc'].append(val_acc) 238 | print('epoch:', epoch, 239 | ' validation loss:', val_loss, 240 | ' validation accuracy:', val_acc) 241 | -------------------------------------------------------------------------------- /6/tensorflow/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from . import data 2 | -------------------------------------------------------------------------------- /6/tensorflow/utils/data.py: -------------------------------------------------------------------------------- 1 | import os 2 | from urllib.request import urlretrieve 3 | 4 | base_dir = os.path.join(os.path.dirname(__file__), '..', 'data') 5 | 6 | 7 | def get_file(filename, url=None, datadir=None): 8 | if url is None: 9 | raise 10 | if datadir is None: 11 | datadir = base_dir 12 | if not os.path.exists(datadir): 13 | os.makedirs(datadir) 14 | 15 | fpath = os.path.join(datadir, filename) 16 | 17 | download = False 18 | if os.path.exists(fpath): 19 | pass 20 | else: 21 | download = True 22 | 23 | if download: 24 | print('Downloading data from', url) 25 | try: 26 | try: 27 | urlretrieve(url, fpath) 28 | except URLError as e: 29 | raise 30 | except HTTPError as e: 31 | raise 32 | except (Exception, KeyboardInterrupt) as e: 33 | if os.path.exists(fpath): 34 | os.remove(fpath) 35 | raise 36 | 37 | return fpath 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 詳解ディープラーニング 2 | 3 | ディープラーニング書籍 [詳解ディープラーニング \~TensorFlow・Kerasによる時系列データ処理\~](https://book.mynavi.jp/ec/products/detail/id=72995) 4 | の中で紹介しているコード集です。 5 | 書籍執筆時は TensorFlow のAPIバージョンが r1.0 でしたので、書籍内のコードは r1.0 での実行を前提としているものですが、 6 | バージョンアップに伴う仕様変更に対応したコードをそれぞれブランチに分けて公開しています。 7 | 8 | [r1.0](https://github.com/yusugomori/deeplearning-tensorflow-keras/tree/r1.0) (r1.0, r1.1) 9 | 10 | [r1.2](https://github.com/yusugomori/deeplearning-tensorflow-keras/tree/r1.2) (r1.2, r1.3) 11 | 12 | [r1.4](https://github.com/yusugomori/deeplearning-tensorflow-keras/tree/r1.4) (r1.4, r1.5, r1.6, r1.7) 13 | 14 | [r1.8](https://github.com/yusugomori/deeplearning-tensorflow-keras/tree/r1.8) (r1.8, r1.9, r1.10) 15 | -------------------------------------------------------------------------------- /appendix/1/keras/00_save_model_simple_keras.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from keras.models import Sequential 4 | from keras.layers import Dense, Activation 5 | from keras.optimizers import SGD 6 | 7 | np.random.seed(0) 8 | 9 | ''' 10 | モデルファイル用設定 11 | ''' 12 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 13 | 14 | if os.path.exists(MODEL_DIR) is False: 15 | os.mkdir(MODEL_DIR) 16 | 17 | ''' 18 | データの生成 19 | ''' 20 | # ORゲート 21 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 22 | Y = np.array([[0], [1], [1], [1]]) 23 | 24 | ''' 25 | モデル設定 26 | ''' 27 | model = Sequential([ 28 | Dense(1, input_dim=2), 29 | Activation('sigmoid') 30 | ]) 31 | 32 | model.compile(loss='binary_crossentropy', optimizer=SGD(lr=0.1)) 33 | 34 | ''' 35 | モデル学習 36 | ''' 37 | model.fit(X, Y, epochs=200, batch_size=1) 38 | model.save(MODEL_DIR + '/model.hdf5') 39 | print('Model saved') 40 | -------------------------------------------------------------------------------- /appendix/1/keras/01_restore_model_simple_keras.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from keras.models import load_model 4 | from keras.models import Sequential 5 | from keras.layers import Dense, Activation 6 | from keras.optimizers import SGD 7 | 8 | np.random.seed(0) 9 | 10 | ''' 11 | モデルファイル用設定 12 | ''' 13 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 14 | 15 | if os.path.exists(MODEL_DIR) is False: 16 | os.mkdir(MODEL_DIR) 17 | 18 | ''' 19 | データの生成 20 | ''' 21 | # ORゲート 22 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 23 | Y = np.array([[0], [1], [1], [1]]) 24 | 25 | ''' 26 | 保存したモデルを読み込み再実験 27 | ''' 28 | model = load_model(MODEL_DIR + '/model.hdf5') 29 | 30 | ''' 31 | 学習済モデルで実験 32 | ''' 33 | classes = model.predict_classes(X, batch_size=1, verbose=0) 34 | prob = model.predict_proba(X, batch_size=1, verbose=0) 35 | 36 | print('classified:') 37 | print(Y == classes) 38 | print() 39 | print('output probability:') 40 | print(prob) 41 | -------------------------------------------------------------------------------- /appendix/1/keras/02_save_model_keras.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from keras.models import Sequential 4 | from keras.layers.core import Dense, Activation, Dropout 5 | from keras.optimizers import Adam 6 | from keras.callbacks import ModelCheckpoint 7 | from keras import backend as K 8 | from sklearn import datasets 9 | from sklearn.model_selection import train_test_split 10 | import matplotlib.pyplot as plt 11 | 12 | np.random.seed(123) 13 | 14 | ''' 15 | モデルファイル用設定 16 | ''' 17 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 18 | 19 | if os.path.exists(MODEL_DIR) is False: 20 | os.mkdir(MODEL_DIR) 21 | 22 | ''' 23 | データの生成 24 | ''' 25 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 26 | 27 | n = len(mnist.data) 28 | N = 30000 # MNISTの一部を使う 29 | N_train = 20000 30 | N_validation = 4000 31 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 32 | 33 | X = mnist.data[indices] 34 | X = X / 255.0 35 | X = X - X.mean(axis=1).reshape(len(X), 1) 36 | y = mnist.target[indices] 37 | Y = np.eye(10)[y.astype(int)] 38 | 39 | X_train, X_test, Y_train, Y_test = \ 40 | train_test_split(X, Y, train_size=N_train) 41 | X_train, X_validation, Y_train, Y_validation = \ 42 | train_test_split(X_train, Y_train, test_size=N_validation) 43 | 44 | ''' 45 | モデル設定 46 | ''' 47 | n_in = len(X[0]) # 784 48 | n_hiddens = [200, 200, 200] 49 | n_out = len(Y[0]) # 10 50 | p_keep = 0.5 51 | activation = 'relu' 52 | 53 | checkpoint = ModelCheckpoint( 54 | filepath=os.path.join( 55 | MODEL_DIR, 56 | 'model_{epoch:02d}_vloss{val_loss:.2f}.hdf5'), 57 | save_best_only=True) 58 | 59 | model = Sequential() 60 | for i, input_dim in enumerate(([n_in] + n_hiddens)[:-1]): 61 | model.add(Dense(n_hiddens[i], input_dim=input_dim)) 62 | model.add(Activation(activation)) 63 | model.add(Dropout(p_keep)) 64 | 65 | model.add(Dense(n_out)) 66 | model.add(Activation('softmax')) 67 | 68 | model.compile(loss='categorical_crossentropy', 69 | optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999), 70 | metrics=['accuracy']) 71 | 72 | ''' 73 | モデル学習 74 | ''' 75 | epochs = 50 76 | batch_size = 200 77 | 78 | model.fit(X_train, Y_train, epochs=epochs, 79 | batch_size=batch_size, 80 | validation_data=(X_validation, Y_validation), 81 | callbacks=[checkpoint]) 82 | -------------------------------------------------------------------------------- /appendix/1/tensorflow/00_save_model_simple_tensorlow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | tf.set_random_seed(0) 6 | 7 | ''' 8 | モデルファイル用設定 9 | ''' 10 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 11 | 12 | if os.path.exists(MODEL_DIR) is False: 13 | os.mkdir(MODEL_DIR) 14 | 15 | ''' 16 | データの生成 17 | ''' 18 | # ORゲート 19 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 20 | Y = np.array([[0], [1], [1], [1]]) 21 | 22 | ''' 23 | モデル設定 24 | ''' 25 | w = tf.Variable(tf.zeros([2, 1]), name='w') 26 | b = tf.Variable(tf.zeros([1]), name='b') 27 | 28 | x = tf.placeholder(tf.float32, shape=[None, 2]) 29 | t = tf.placeholder(tf.float32, shape=[None, 1]) 30 | y = tf.nn.sigmoid(tf.matmul(x, w) + b) 31 | 32 | cross_entropy = - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 33 | 34 | train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 35 | 36 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 37 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 38 | 39 | ''' 40 | モデル学習 41 | ''' 42 | init = tf.global_variables_initializer() 43 | saver = tf.train.Saver() # モデル保存用 44 | sess = tf.Session() 45 | sess.run(init) 46 | 47 | # 学習 48 | for epoch in range(200): 49 | sess.run(train_step, feed_dict={ 50 | x: X, 51 | t: Y 52 | }) 53 | 54 | # モデル保存 55 | model_path = saver.save(sess, MODEL_DIR + '/model.ckpt') 56 | print('Model saved to:', model_path) 57 | -------------------------------------------------------------------------------- /appendix/1/tensorflow/01_restore_model_simple_tensorflow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | tf.set_random_seed(0) 6 | 7 | ''' 8 | モデルファイル用設定 9 | ''' 10 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 11 | 12 | if os.path.exists(MODEL_DIR) is False: 13 | os.mkdir(MODEL_DIR) 14 | 15 | ''' 16 | データの生成 17 | ''' 18 | # ORゲート 19 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 20 | Y = np.array([[0], [1], [1], [1]]) 21 | 22 | 23 | ''' 24 | 保存したモデルを読み込み再実験 25 | ''' 26 | 27 | ''' 28 | モデル再設定 29 | ''' 30 | w = tf.Variable(tf.zeros([2, 1]), name='w') 31 | b = tf.Variable(tf.zeros([1]), name='b') 32 | 33 | x = tf.placeholder(tf.float32, shape=[None, 2]) 34 | t = tf.placeholder(tf.float32, shape=[None, 1]) 35 | y = tf.nn.sigmoid(tf.matmul(x, w) + b) 36 | 37 | cross_entropy = - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 38 | 39 | train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 40 | 41 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 42 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 43 | 44 | ''' 45 | 学習済モデルで実験 46 | ''' 47 | # init = tf.global_variables_initializer() # 初期化は不要 48 | saver = tf.train.Saver() # モデル読み込み用 49 | sess = tf.Session() 50 | # sess.run(init) 51 | 52 | # モデル読み込み 53 | saver.restore(sess, MODEL_DIR + '/model.ckpt') 54 | print('Model restored.') 55 | 56 | acc = accuracy.eval(session=sess, feed_dict={ 57 | x: X, 58 | t: Y 59 | }) 60 | print('accuracy:', acc) 61 | -------------------------------------------------------------------------------- /appendix/1/tensorflow/02_save_model_tensorflow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | from sklearn import datasets 5 | from sklearn.model_selection import train_test_split 6 | from sklearn.utils import shuffle 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | ''' 12 | モデルファイル用設定 13 | ''' 14 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 15 | 16 | if os.path.exists(MODEL_DIR) is False: 17 | os.mkdir(MODEL_DIR) 18 | 19 | 20 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 21 | def weight_variable(shape, name=None): 22 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 23 | return tf.Variable(initial, name=name) 24 | 25 | def bias_variable(shape, name=None): 26 | initial = tf.zeros(shape) 27 | return tf.Variable(initial, name=name) 28 | 29 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 30 | for i, n_hidden in enumerate(n_hiddens): 31 | if i == 0: 32 | input = x 33 | input_dim = n_in 34 | else: 35 | input = output 36 | input_dim = n_hiddens[i-1] 37 | 38 | W = weight_variable([input_dim, n_hidden], 39 | name='W_{}'.format(i)) 40 | b = bias_variable([n_hidden], 41 | name='b_{}'.format(i)) 42 | 43 | h = tf.nn.relu(tf.matmul(input, W) + b) 44 | output = tf.nn.dropout(h, keep_prob) 45 | 46 | # 隠れ層 - 出力層 47 | W_out = weight_variable([n_hiddens[-1], n_out], name='W_out') 48 | b_out = bias_variable([n_out], name='b_out') 49 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 50 | return y 51 | 52 | 53 | def loss(y, t): 54 | cross_entropy = \ 55 | tf.reduce_mean(-tf.reduce_sum( 56 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 57 | axis=1)) 58 | return cross_entropy 59 | 60 | 61 | def training(loss): 62 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001, 63 | beta1=0.9, 64 | beta2=0.999) 65 | train_step = optimizer.minimize(loss) 66 | return train_step 67 | 68 | 69 | def accuracy(y, t): 70 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 71 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 72 | return accuracy 73 | 74 | 75 | if __name__ == '__main__': 76 | ''' 77 | データの生成 78 | ''' 79 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 80 | 81 | n = len(mnist.data) 82 | N = 30000 # MNISTの一部を使う 83 | N_train = 20000 84 | N_validation = 4000 85 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 86 | 87 | X = mnist.data[indices] 88 | X = X / 255.0 89 | X = X - X.mean(axis=1).reshape(len(X), 1) 90 | y = mnist.target[indices] 91 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 92 | 93 | X_train, X_test, Y_train, Y_test = \ 94 | train_test_split(X, Y, train_size=N_train) 95 | 96 | X_train, X_validation, Y_train, Y_validation = \ 97 | train_test_split(X_train, Y_train, test_size=N_validation) 98 | 99 | ''' 100 | モデル設定 101 | ''' 102 | n_in = len(X[0]) 103 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 104 | n_out = len(Y[0]) 105 | p_keep = 0.5 106 | 107 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 108 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 109 | keep_prob = tf.placeholder(tf.float32) 110 | 111 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 112 | loss = loss(y, t) 113 | train_step = training(loss) 114 | 115 | accuracy = accuracy(y, t) 116 | 117 | ''' 118 | モデル学習 119 | ''' 120 | epochs = 50 121 | batch_size = 200 122 | 123 | init = tf.global_variables_initializer() 124 | saver = tf.train.Saver() # モデル保存用 125 | sess = tf.Session() 126 | sess.run(init) 127 | 128 | n_batches = N_train // batch_size 129 | 130 | for epoch in range(epochs): 131 | X_, Y_ = shuffle(X_train, Y_train) 132 | 133 | for i in range(n_batches): 134 | start = i * batch_size 135 | end = start + batch_size 136 | 137 | sess.run(train_step, feed_dict={ 138 | x: X_[start:end], 139 | t: Y_[start:end], 140 | keep_prob: p_keep 141 | }) 142 | 143 | model_path = \ 144 | saver.save(sess, MODEL_DIR + '/model_{}.ckpt'.format(epoch)) 145 | print('Model saved to:', model_path) 146 | 147 | if epoch == 10: # 学習を中断 148 | break 149 | -------------------------------------------------------------------------------- /appendix/1/tensorflow/03_restore_model_tensorflow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | from sklearn import datasets 5 | from sklearn.model_selection import train_test_split 6 | from sklearn.utils import shuffle 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | ''' 12 | モデルファイル用設定 13 | ''' 14 | MODEL_DIR = os.path.join(os.path.dirname(__file__), 'model') 15 | 16 | if os.path.exists(MODEL_DIR) is False: 17 | os.mkdir(MODEL_DIR) 18 | 19 | 20 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 21 | def weight_variable(shape, name=None): 22 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 23 | return tf.Variable(initial, name=name) 24 | 25 | def bias_variable(shape, name=None): 26 | initial = tf.zeros(shape) 27 | return tf.Variable(initial, name=name) 28 | 29 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 30 | for i, n_hidden in enumerate(n_hiddens): 31 | if i == 0: 32 | input = x 33 | input_dim = n_in 34 | else: 35 | input = output 36 | input_dim = n_hiddens[i-1] 37 | 38 | W = weight_variable([input_dim, n_hidden], 39 | name='W_{}'.format(i)) 40 | b = bias_variable([n_hidden], 41 | name='b_{}'.format(i)) 42 | 43 | h = tf.nn.relu(tf.matmul(input, W) + b) 44 | output = tf.nn.dropout(h, keep_prob) 45 | 46 | # 隠れ層 - 出力層 47 | W_out = weight_variable([n_hiddens[-1], n_out], name='W_out') 48 | b_out = bias_variable([n_out], name='b_out') 49 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out) 50 | return y 51 | 52 | 53 | def loss(y, t): 54 | cross_entropy = \ 55 | tf.reduce_mean(-tf.reduce_sum( 56 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 57 | axis=1)) 58 | return cross_entropy 59 | 60 | 61 | def training(loss): 62 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001, 63 | beta1=0.9, 64 | beta2=0.999) 65 | train_step = optimizer.minimize(loss) 66 | return train_step 67 | 68 | 69 | def accuracy(y, t): 70 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 71 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 72 | return accuracy 73 | 74 | 75 | if __name__ == '__main__': 76 | ''' 77 | データの生成 78 | ''' 79 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 80 | 81 | n = len(mnist.data) 82 | N = 30000 # MNISTの一部を使う 83 | N_train = 20000 84 | N_validation = 4000 85 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 86 | 87 | X = mnist.data[indices] 88 | X = X / 255.0 89 | X = X - X.mean(axis=1).reshape(len(X), 1) 90 | y = mnist.target[indices] 91 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 92 | 93 | X_train, X_test, Y_train, Y_test = \ 94 | train_test_split(X, Y, train_size=N_train) 95 | 96 | X_train, X_validation, Y_train, Y_validation = \ 97 | train_test_split(X_train, Y_train, test_size=N_validation) 98 | 99 | ''' 100 | モデル設定 101 | ''' 102 | n_in = len(X[0]) 103 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 104 | n_out = len(Y[0]) 105 | p_keep = 0.5 106 | 107 | x = tf.placeholder(tf.float32, shape=[None, n_in]) 108 | t = tf.placeholder(tf.float32, shape=[None, n_out]) 109 | keep_prob = tf.placeholder(tf.float32) 110 | 111 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 112 | loss = loss(y, t) 113 | train_step = training(loss) 114 | 115 | accuracy = accuracy(y, t) 116 | 117 | ''' 118 | モデル学習 119 | ''' 120 | epochs = 50 121 | batch_size = 200 122 | 123 | # init = tf.global_variables_initializer() 124 | saver = tf.train.Saver() # モデル読み込み用 125 | sess = tf.Session() 126 | # sess.run(init) 127 | 128 | # 中断して保存したモデルを読み込み 129 | saver.restore(sess, MODEL_DIR + '/model_10.ckpt') 130 | print('Model restored.') 131 | 132 | n_batches = N_train // batch_size 133 | 134 | # 中断したところから学習を再開 135 | for epoch in range(11, epochs): 136 | X_, Y_ = shuffle(X_train, Y_train) 137 | 138 | for i in range(n_batches): 139 | start = i * batch_size 140 | end = start + batch_size 141 | 142 | sess.run(train_step, feed_dict={ 143 | x: X_[start:end], 144 | t: Y_[start:end], 145 | keep_prob: p_keep 146 | }) 147 | 148 | model_path = \ 149 | saver.save(sess, MODEL_DIR + '/model_{}.ckpt'.format(epoch)) 150 | print('Model saved to:', model_path) 151 | 152 | ''' 153 | 予測精度の評価 154 | ''' 155 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 156 | x: X_test, 157 | t: Y_test, 158 | keep_prob: 1.0 159 | }) 160 | print('accuracy: ', accuracy_rate) 161 | -------------------------------------------------------------------------------- /appendix/2/00_tensorboard_logistic_regression.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | tf.set_random_seed(0) 6 | 7 | ''' 8 | ログファイル用設定 9 | ''' 10 | LOG_DIR = os.path.join(os.path.dirname(__file__), 'log') 11 | 12 | if os.path.exists(LOG_DIR) is False: 13 | os.mkdir(LOG_DIR) 14 | 15 | ''' 16 | データの生成 17 | ''' 18 | # ORゲート 19 | X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) 20 | Y = np.array([[0], [1], [1], [1]]) 21 | 22 | ''' 23 | モデル設定 24 | ''' 25 | w = tf.Variable(tf.zeros([2, 1]), name='w') 26 | b = tf.Variable(tf.zeros([1]), name='b') 27 | 28 | x = tf.placeholder(tf.float32, shape=[None, 2], name='x') 29 | t = tf.placeholder(tf.float32, shape=[None, 1], name='t') 30 | y = tf.nn.sigmoid(tf.matmul(x, w) + b, name='y') 31 | 32 | with tf.name_scope('loss'): 33 | cross_entropy = \ 34 | - tf.reduce_sum(t * tf.log(y) + (1 - t) * tf.log(1 - y)) 35 | tf.summary.scalar('cross_entropy', cross_entropy) # TensorBoard 用に登録 36 | 37 | with tf.name_scope('train'): 38 | train_step = \ 39 | tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) 40 | 41 | with tf.name_scope('accuracy'): 42 | correct_prediction = tf.equal(tf.to_float(tf.greater(y, 0.5)), t) 43 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 44 | 45 | ''' 46 | モデル学習 47 | ''' 48 | init = tf.global_variables_initializer() 49 | sess = tf.Session() 50 | 51 | file_writer = tf.summary.FileWriter(LOG_DIR, sess.graph) # TensorBoardに対応 52 | summaries = tf.summary.merge_all() # 登録した変数をひとまとめにする 53 | 54 | sess.run(init) 55 | 56 | # 学習 57 | for epoch in range(200): 58 | sess.run(train_step, feed_dict={ 59 | x: X, 60 | t: Y 61 | }) 62 | 63 | summary, loss = sess.run([summaries, cross_entropy], feed_dict={ 64 | x: X, 65 | t: Y 66 | }) 67 | file_writer.add_summary(summary, epoch) # TensorBoard に記録 68 | -------------------------------------------------------------------------------- /appendix/2/01_tensorboard_adam.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | from sklearn import datasets 5 | from sklearn.model_selection import train_test_split 6 | from sklearn.utils import shuffle 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(1234) 10 | 11 | ''' 12 | ログファイル用設定 13 | ''' 14 | LOG_DIR = os.path.join(os.path.dirname(__file__), 'log') 15 | 16 | if os.path.exists(LOG_DIR) is False: 17 | os.mkdir(LOG_DIR) 18 | 19 | 20 | def inference(x, keep_prob, n_in, n_hiddens, n_out): 21 | def weight_variable(shape, name=None): 22 | initial = np.sqrt(2.0 / shape[0]) * tf.truncated_normal(shape) 23 | return tf.Variable(initial, name=name) 24 | 25 | def bias_variable(shape, name=None): 26 | initial = tf.zeros(shape) 27 | return tf.Variable(initial, name=name) 28 | 29 | with tf.name_scope('inference'): 30 | # 入力層 - 隠れ層、隠れ層 - 隠れ層 31 | for i, n_hidden in enumerate(n_hiddens): 32 | if i == 0: 33 | input = x 34 | input_dim = n_in 35 | else: 36 | input = output 37 | input_dim = n_hiddens[i-1] 38 | 39 | W = weight_variable([input_dim, n_hidden], 40 | name='W_{}'.format(i)) 41 | b = bias_variable([n_hidden], 42 | name='b_{}'.format(i)) 43 | 44 | h = tf.nn.relu(tf.matmul(input, W) + b, 45 | name='relu_{}'.format(i)) 46 | output = tf.nn.dropout(h, keep_prob, 47 | name='dropout_{}'.format(i)) 48 | 49 | # 隠れ層 - 出力層 50 | W_out = weight_variable([n_hiddens[-1], n_out], name='W_out') 51 | b_out = bias_variable([n_out], name='b_out') 52 | y = tf.nn.softmax(tf.matmul(output, W_out) + b_out, name='y') 53 | return y 54 | 55 | 56 | def loss(y, t): 57 | with tf.name_scope('loss'): 58 | cross_entropy = \ 59 | tf.reduce_mean(-tf.reduce_sum( 60 | t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), 61 | axis=1)) 62 | tf.summary.scalar('cross_entropy', cross_entropy) 63 | return cross_entropy 64 | 65 | 66 | def training(loss): 67 | with tf.name_scope('training'): 68 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001, 69 | beta1=0.9, 70 | beta2=0.999) 71 | train_step = optimizer.minimize(loss) 72 | return train_step 73 | 74 | 75 | def accuracy(y, t): 76 | with tf.name_scope('accuracy'): 77 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(t, 1)) 78 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 79 | return accuracy 80 | 81 | 82 | if __name__ == '__main__': 83 | ''' 84 | データの生成 85 | ''' 86 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 87 | 88 | n = len(mnist.data) 89 | N = 30000 # MNISTの一部を使う 90 | N_train = 20000 91 | N_validation = 4000 92 | indices = np.random.permutation(range(n))[:N] # ランダムにN枚を選択 93 | 94 | X = mnist.data[indices] 95 | X = X / 255.0 96 | X = X - X.mean(axis=1).reshape(len(X), 1) 97 | y = mnist.target[indices] 98 | Y = np.eye(10)[y.astype(int)] # 1-of-K 表現に変換 99 | 100 | X_train, X_test, Y_train, Y_test = \ 101 | train_test_split(X, Y, train_size=N_train) 102 | 103 | X_train, X_validation, Y_train, Y_validation = \ 104 | train_test_split(X_train, Y_train, test_size=N_validation) 105 | 106 | ''' 107 | モデル設定 108 | ''' 109 | n_in = len(X[0]) 110 | n_hiddens = [200, 200, 200] # 各隠れ層の次元数 111 | n_out = len(Y[0]) 112 | p_keep = 0.5 113 | 114 | x = tf.placeholder(tf.float32, shape=[None, n_in], name='x') 115 | t = tf.placeholder(tf.float32, shape=[None, n_out], name='t') 116 | keep_prob = tf.placeholder(tf.float32, name='keep_prob') 117 | 118 | y = inference(x, keep_prob, n_in=n_in, n_hiddens=n_hiddens, n_out=n_out) 119 | loss = loss(y, t) 120 | train_step = training(loss) 121 | 122 | accuracy = accuracy(y, t) 123 | 124 | ''' 125 | モデル学習 126 | ''' 127 | epochs = 50 128 | batch_size = 200 129 | 130 | init = tf.global_variables_initializer() 131 | sess = tf.Session() 132 | 133 | summaries = tf.summary.merge_all() 134 | file_writer = tf.summary.FileWriter(LOG_DIR, sess.graph) 135 | 136 | sess.run(init) 137 | 138 | n_batches = N_train // batch_size 139 | 140 | for epoch in range(epochs): 141 | X_, Y_ = shuffle(X_train, Y_train) 142 | 143 | for i in range(n_batches): 144 | start = i * batch_size 145 | end = start + batch_size 146 | 147 | sess.run(train_step, feed_dict={ 148 | x: X_[start:end], 149 | t: Y_[start:end], 150 | keep_prob: p_keep 151 | }) 152 | 153 | # 検証データを用いた評価 154 | summary, val_loss = sess.run([summaries, loss], feed_dict={ 155 | x: X_validation, 156 | t: Y_validation, 157 | keep_prob: 1.0 158 | }) 159 | file_writer.add_summary(summary, epoch) 160 | 161 | print('epoch:', epoch, 162 | ' validation loss:', val_loss) 163 | 164 | ''' 165 | 予測精度の評価 166 | ''' 167 | accuracy_rate = accuracy.eval(session=sess, feed_dict={ 168 | x: X_test, 169 | t: Y_test, 170 | keep_prob: 1.0 171 | }) 172 | print('accuracy: ', accuracy_rate) 173 | -------------------------------------------------------------------------------- /appendix/2/log/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yusugomori/deeplearning-tensorflow-keras/34eb2eb70746b76abb2bc1178c721f0bf7b0c353/appendix/2/log/.keep -------------------------------------------------------------------------------- /appendix/3/00_mnist_dnn_classifier.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | from sklearn import datasets 5 | from sklearn.model_selection import train_test_split 6 | from sklearn.utils import shuffle 7 | 8 | np.random.seed(0) 9 | tf.set_random_seed(123) 10 | tf.logging.set_verbosity(tf.logging.ERROR) 11 | 12 | ''' 13 | データの生成 14 | ''' 15 | mnist = datasets.fetch_mldata('MNIST original', data_home='.') 16 | 17 | N_train = 60000 18 | 19 | X = mnist.data.astype(np.float32) 20 | X = X / 255.0 21 | X = X - X.mean(axis=1).reshape(len(X), 1) 22 | y = mnist.target.astype(int) 23 | 24 | X_train, X_test, y_train, y_test = \ 25 | train_test_split(X, y, train_size=N_train) 26 | 27 | ''' 28 | モデル設定 29 | ''' 30 | n_in = 784 31 | n_hiddens = [200, 200, 200] 32 | n_out = 10 33 | 34 | feature_columns = \ 35 | [tf.contrib.layers.real_valued_column('', dimension=n_in)] 36 | 37 | model = \ 38 | tf.contrib.learn.DNNClassifier( 39 | feature_columns=feature_columns, 40 | hidden_units=n_hiddens, 41 | n_classes=n_out) 42 | 43 | ''' 44 | モデル学習 45 | ''' 46 | model.fit(x=X_train, 47 | y=y_train, 48 | steps=300, 49 | batch_size=250) 50 | 51 | 52 | accuracy = model.evaluate(x=X_test, 53 | y=y_test)['accuracy'] 54 | print('accuracy:', accuracy) 55 | -------------------------------------------------------------------------------- /mldata/mnist-original.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yusugomori/deeplearning-tensorflow-keras/34eb2eb70746b76abb2bc1178c721f0bf7b0c353/mldata/mnist-original.mat --------------------------------------------------------------------------------