├── README.md ├── argmax.py ├── checkpoint └── test.py ├── cpp ├── BUILD └── example.cc ├── gpu ├── 02.py ├── log_device.py └── use_gpu.md ├── mnist ├── cnn.py ├── cnn_gpu.py ├── cnn_gpu2.py └── mnist_softmax.py ├── mnist_distributed ├── README.md ├── dist_fifo.py └── dist_join.py └── sess.py /README.md: -------------------------------------------------------------------------------- 1 | # tensorflow_examples 2 | -------------------------------------------------------------------------------- /argmax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | sess = tf.InteractiveSession() 5 | 6 | arr = np.array([[31, 23, 4, 24, 27, 34], 7 | [18, 3, 25, 0, 6, 35], 8 | [28, 14, 33, 22, 20, 8], 9 | [13, 30, 21, 19, 7, 9], 10 | [16, 1, 26, 32, 2, 29], 11 | [17, 12, 5, 11, 10, 15]]) 12 | 13 | # arr has rank 2 and shape (6, 6) 14 | print(tf.rank(arr).eval()) 15 | 16 | print(tf.shape(arr).eval()) 17 | 18 | print(tf.argmax(arr, 0).eval()) 19 | -------------------------------------------------------------------------------- /checkpoint/test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | 6 | isTrain = False 7 | train_steps = 100 8 | checkpoint_steps = 50 9 | checkpoint_dir = './chk_dir' 10 | 11 | x = tf.placeholder(tf.float32, shape=[None, 1]) 12 | y = 4 * x + 4 13 | 14 | w = tf.Variable(tf.random_normal([1], -1, 1)) 15 | b = tf.Variable(tf.zeros([1])) 16 | y_predict = w * x + b 17 | 18 | 19 | loss = tf.reduce_mean(tf.square(y - y_predict)) 20 | optimizer = tf.train.GradientDescentOptimizer(0.5) 21 | train = optimizer.minimize(loss) 22 | 23 | saver = tf.train.Saver() # defaults to saving all variables - in this case w and b 24 | x_data = np.reshape(np.random.rand(10).astype(np.float32), (10, 1)) 25 | 26 | with tf.Session() as sess: 27 | sess.run(tf.global_variables_initializer()) 28 | if isTrain: 29 | for i in xrange(train_steps): 30 | sess.run(train, feed_dict={x: x_data}) 31 | if (i + 1) % checkpoint_steps == 0: 32 | checkpoint_file = os.path.join(checkpoint_dir, 'model.ckpt') 33 | saver.save(sess, checkpoint_file, global_step=i+1) 34 | else: 35 | ckpt = tf.train.get_checkpoint_state(checkpoint_dir) 36 | if ckpt and ckpt.model_checkpoint_path: 37 | saver.restore(sess, ckpt.model_checkpoint_path) 38 | else: 39 | pass 40 | print(sess.run(w)) 41 | print(sess.run(b)) 42 | -------------------------------------------------------------------------------- /cpp/BUILD: -------------------------------------------------------------------------------- 1 | load("//tensorflow:tensorflow.bzl", "tf_cc_binary") 2 | 3 | tf_cc_binary( 4 | name = "example", 5 | srcs = ["example.cc"], 6 | deps = [ 7 | "//tensorflow/cc:cc_ops", 8 | "//tensorflow/cc:client_session", 9 | "//tensorflow/core:tensorflow", 10 | ], 11 | ) 12 | -------------------------------------------------------------------------------- /cpp/example.cc: -------------------------------------------------------------------------------- 1 | #include "tensorflow/cc/client/client_session.h" 2 | #include "tensorflow/cc/ops/standard_ops.h" 3 | #include "tensorflow/core/framework/tensor.h" 4 | 5 | int main() { 6 | using namespace tensorflow; 7 | using namespace tensorflow::ops; 8 | Scope root = Scope::NewRootScope(); 9 | // Matrix A = [3 2; -1 0] 10 | auto A = Const(root, { {3.f, 2.f}, {-1.f, 0.f} }); 11 | // Vector b = [3 5] 12 | auto b = Const(root, { {3.f, 5.f} }); 13 | // v = Ab^T 14 | auto v = MatMul(root.WithOpName("v"), A, b, MatMul::TransposeB(true)); 15 | std::vector outputs; 16 | ClientSession session(root); 17 | // Run and fetch v 18 | TF_CHECK_OK(session.Run({v}, &outputs)); 19 | // Expect outputs[0] == [19; -3] 20 | LOG(INFO) << outputs[0].matrix(); 21 | return 0; 22 | } 23 | -------------------------------------------------------------------------------- /gpu/02.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | # Creates a graph. 4 | with tf.device('/cpu:0'): 5 | a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') 6 | b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') 7 | c = tf.matmul(a, b) 8 | # Creates a session with log_device_placement set to True. 9 | sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) 10 | # Runs the op. 11 | print(sess.run(c)) 12 | -------------------------------------------------------------------------------- /gpu/log_device.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | # Creates a graph. 4 | a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') 5 | b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') 6 | c = tf.matmul(a, b) 7 | # Creates a session with log_device_placement set to True. 8 | sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) 9 | # Runs the op. 10 | print(sess.run(c)) 11 | -------------------------------------------------------------------------------- /gpu/use_gpu.md: -------------------------------------------------------------------------------- 1 | ## Logging Device placement 2 | 3 | ``` 4 | # python log_device.py 5 | ... 6 | MatMul: (MatMul): /job:localhost/replica:0/task:0/device:GPU:0 7 | 2018-04-03 14:57:00.495959: I tensorflow/core/common_runtime/placer.cc:875] MatMul: (MatMul)/job:localhost/replica:0/task:0/device:GPU:0 8 | b: (Const): /job:localhost/replica:0/task:0/device:GPU:0 9 | 2018-04-03 14:57:00.496005: I tensorflow/core/common_runtime/placer.cc:875] b: (Const)/job:localhost/replica:0/task:0/device:GPU:0 10 | a: (Const): /job:localhost/replica:0/task:0/device:GPU:0 11 | 2018-04-03 14:57:00.496037: I tensorflow/core/common_runtime/placer.cc:875] a: (Const)/job:localhost/replica:0/task:0/device:GPU:0 12 | [[ 22. 28.] 13 | [ 49. 64.]] 14 | ``` 15 | 16 | ## Refs 17 | 18 | * [Using GPUs](https://www.tensorflow.org/programmers_guide/using_gpu) 19 | -------------------------------------------------------------------------------- /mnist/cnn.py: -------------------------------------------------------------------------------- 1 | __author__ = 'chapter' 2 | 3 | from datetime import datetime 4 | import time 5 | 6 | import argparse 7 | import sys 8 | 9 | import tensorflow as tf 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | def weight_varible(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.1) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.constant(0.1, shape=shape) 18 | return tf.Variable(initial) 19 | 20 | def conv2d(x, W): 21 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 22 | 23 | def max_pool_2x2(x): 24 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 25 | 26 | FLAGS = None 27 | 28 | def main(_): 29 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 30 | print("Download Done!") 31 | 32 | sess = tf.InteractiveSession() 33 | # paras 34 | W_conv1 = weight_varible([5, 5, 1, 32]) 35 | b_conv1 = bias_variable([32]) 36 | 37 | # conv layer-1 38 | x = tf.placeholder(tf.float32, [None, 784]) 39 | x_image = tf.reshape(x, [-1, 28, 28, 1]) 40 | 41 | h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) 42 | h_pool1 = max_pool_2x2(h_conv1) 43 | 44 | # conv layer-2 45 | W_conv2 = weight_varible([5, 5, 32, 64]) 46 | b_conv2 = bias_variable([64]) 47 | 48 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) 49 | h_pool2 = max_pool_2x2(h_conv2) 50 | 51 | # full connection 52 | W_fc1 = weight_varible([7 * 7 * 64, 1024]) 53 | b_fc1 = bias_variable([1024]) 54 | 55 | h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) 56 | h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) 57 | 58 | # dropout 59 | keep_prob = tf.placeholder(tf.float32) 60 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 61 | 62 | # output layer: softmax 63 | W_fc2 = weight_varible([1024, 10]) 64 | b_fc2 = bias_variable([10]) 65 | 66 | y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) 67 | y_ = tf.placeholder(tf.float32, [None, 10]) 68 | 69 | # model training 70 | cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv)) 71 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) 72 | 73 | correct_prediction = tf.equal(tf.arg_max(y_conv, 1), tf.arg_max(y_, 1)) 74 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 75 | 76 | sess.run(tf.global_variables_initializer()) 77 | 78 | start_time = time.time() 79 | for i in range(20000): 80 | batch = mnist.train.next_batch(50) 81 | 82 | if i % 100 == 0: 83 | train_accuacy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0}) 84 | print("step %d, training accuracy %g"%(i, train_accuacy)) 85 | train_step.run(feed_dict = {x: batch[0], y_: batch[1], keep_prob: 0.5}) 86 | 87 | duration = time.time() - start_time 88 | # accuacy on test 89 | print("test accuracy %g, used %d seconds"%(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}), duration)) 90 | 91 | if __name__ == '__main__': 92 | parser = argparse.ArgumentParser() 93 | parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', 94 | help='Directory for storing input data') 95 | FLAGS, unparsed = parser.parse_known_args() 96 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 97 | -------------------------------------------------------------------------------- /mnist/cnn_gpu.py: -------------------------------------------------------------------------------- 1 | __author__ = 'chapter' 2 | 3 | from datetime import datetime 4 | import time 5 | 6 | import argparse 7 | import sys 8 | 9 | import tensorflow as tf 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | def weight_varible(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.1) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.constant(0.1, shape=shape) 18 | return tf.Variable(initial) 19 | 20 | def conv2d(x, W): 21 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 22 | 23 | def max_pool_2x2(x): 24 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 25 | 26 | FLAGS = None 27 | 28 | 29 | def model(x_image, W1, b1, W2, b2, Wc1, bc1, keep_prob, Wc2, bc2): 30 | with tf.device('/gpu:0'): 31 | 32 | # layer-1 33 | h_conv1 = tf.nn.relu(conv2d(x_image, W1) + b1) 34 | h_pool1 = max_pool_2x2(h_conv1) 35 | 36 | # layer-2 37 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W2) + b2) 38 | h_pool2 = max_pool_2x2(h_conv2) 39 | 40 | # full connection 41 | h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) 42 | h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, Wc1) + bc1) 43 | 44 | # dropout 45 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 46 | 47 | # output layer: softmax 48 | Wc2 = weight_varible([1024, 10]) 49 | bc2 = bias_variable([10]) 50 | y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, Wc2) + bc2) 51 | return y_conv 52 | 53 | def main(_): 54 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 55 | print("Download Done!") 56 | 57 | # input 58 | x = tf.placeholder(tf.float32, [None, 784]) 59 | x_image = tf.reshape(x, [-1, 28, 28, 1]) 60 | 61 | # conv layer-1 62 | W1 = weight_varible([5, 5, 1, 32]) 63 | b1 = bias_variable([32]) 64 | 65 | # conv layer-2 66 | W2 = weight_varible([5, 5, 32, 64]) 67 | b2 = bias_variable([64]) 68 | 69 | # full connection 70 | W_fc1 = weight_varible([7 * 7 * 64, 1024]) 71 | b_fc1 = bias_variable([1024]) 72 | 73 | # dropout 74 | keep_prob = tf.placeholder(tf.float32) 75 | 76 | # output layer: softmax 77 | W_fc2 = weight_varible([1024, 10]) 78 | b_fc2 = bias_variable([10]) 79 | 80 | y_conv = model(x_image, W1, b1, W2, b2, W_fc1, b_fc1, keep_prob, W_fc2, b_fc2) 81 | y_ = tf.placeholder(tf.float32, [None, 10]) 82 | 83 | sess = tf.InteractiveSession(config=tf.ConfigProto( 84 | log_device_placement=True)) 85 | 86 | # model training 87 | cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv)) 88 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) 89 | 90 | correct_prediction = tf.equal(tf.arg_max(y_conv, 1), tf.arg_max(y_, 1)) 91 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 92 | 93 | sess.run(tf.global_variables_initializer()) 94 | 95 | start_time = time.time() 96 | for i in range(20000): 97 | batch = mnist.train.next_batch(50) 98 | 99 | if i % 100 == 0: 100 | train_accuacy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0}) 101 | print("step %d, training accuracy %g"%(i, train_accuacy)) 102 | train_step.run(feed_dict = {x: batch[0], y_: batch[1], keep_prob: 0.5}) 103 | 104 | duration = time.time() - start_time 105 | # accuacy on test 106 | print("test accuracy %g, used %d seconds"%(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}), duration)) 107 | 108 | if __name__ == '__main__': 109 | parser = argparse.ArgumentParser() 110 | parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', 111 | help='Directory for storing input data') 112 | FLAGS, unparsed = parser.parse_known_args() 113 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 114 | -------------------------------------------------------------------------------- /mnist/cnn_gpu2.py: -------------------------------------------------------------------------------- 1 | __author__ = 'chapter' 2 | 3 | from datetime import datetime 4 | import time 5 | 6 | import argparse 7 | import sys 8 | 9 | import tensorflow as tf 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | def weight_varible(shape): 13 | initial = tf.truncated_normal(shape, stddev=0.1) 14 | return tf.Variable(initial) 15 | 16 | def bias_variable(shape): 17 | initial = tf.constant(0.1, shape=shape) 18 | return tf.Variable(initial) 19 | 20 | def conv2d(x, W): 21 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 22 | 23 | def max_pool_2x2(x): 24 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 25 | 26 | FLAGS = None 27 | 28 | 29 | def model(x_image, W1, b1, W2, b2, Wc1, bc1, keep_prob, Wc2, bc2): 30 | with tf.device('/gpu:0'): 31 | 32 | # layer-1 33 | h_conv1 = tf.nn.relu(conv2d(x_image, W1) + b1) 34 | h_pool1 = max_pool_2x2(h_conv1) 35 | 36 | # layer-2 37 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W2) + b2) 38 | h_pool2 = max_pool_2x2(h_conv2) 39 | 40 | # full connection 41 | h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) 42 | h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, Wc1) + bc1) 43 | 44 | # dropout 45 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 46 | 47 | # output layer: softmax 48 | Wc2 = weight_varible([1024, 10]) 49 | bc2 = bias_variable([10]) 50 | y_conv = tf.matmul(h_fc1_drop, Wc2) + bc2 51 | return y_conv 52 | 53 | def main(_): 54 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 55 | print("Download Done!") 56 | 57 | # input 58 | x = tf.placeholder(tf.float32, [None, 784]) 59 | x_image = tf.reshape(x, [-1, 28, 28, 1]) 60 | 61 | # conv layer-1 62 | W1 = weight_varible([5, 5, 1, 32]) 63 | b1 = bias_variable([32]) 64 | 65 | # conv layer-2 66 | W2 = weight_varible([5, 5, 32, 64]) 67 | b2 = bias_variable([64]) 68 | 69 | # full connection 70 | W_fc1 = weight_varible([7 * 7 * 64, 1024]) 71 | b_fc1 = bias_variable([1024]) 72 | 73 | # dropout 74 | keep_prob = tf.placeholder(tf.float32) 75 | 76 | # output layer: softmax 77 | W_fc2 = weight_varible([1024, 10]) 78 | b_fc2 = bias_variable([10]) 79 | 80 | y_conv = model(x_image, W1, b1, W2, b2, W_fc1, b_fc1, keep_prob, W_fc2, b_fc2) 81 | y_ = tf.placeholder(tf.float32, [None, 10]) 82 | 83 | sess = tf.InteractiveSession(config=tf.ConfigProto( 84 | log_device_placement=True)) 85 | 86 | # model training 87 | #cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv)) 88 | cross_entropy = tf.reduce_mean( 89 | tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) 90 | 91 | train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) 92 | 93 | correct_prediction = tf.equal(tf.arg_max(y_conv, 1), tf.arg_max(y_, 1)) 94 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 95 | 96 | sess.run(tf.global_variables_initializer()) 97 | 98 | start_time = time.time() 99 | for i in range(20000): 100 | batch = mnist.train.next_batch(50) 101 | 102 | if i % 100 == 0: 103 | train_accuacy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0}) 104 | print("step %d, training accuracy %g"%(i, train_accuacy)) 105 | train_step.run(feed_dict = {x: batch[0], y_: batch[1], keep_prob: 0.5}) 106 | 107 | duration = time.time() - start_time 108 | # accuacy on test 109 | print("test accuracy %g, used %d seconds"%(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}), duration)) 110 | 111 | if __name__ == '__main__': 112 | parser = argparse.ArgumentParser() 113 | parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', 114 | help='Directory for storing input data') 115 | FLAGS, unparsed = parser.parse_known_args() 116 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 117 | -------------------------------------------------------------------------------- /mnist/mnist_softmax.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """A very simple MNIST classifier. 17 | See extensive documentation at 18 | http://tensorflow.org/tutorials/mnist/beginners/index.md 19 | """ 20 | from __future__ import absolute_import 21 | from __future__ import division 22 | from __future__ import print_function 23 | 24 | import argparse 25 | import sys 26 | 27 | from tensorflow.examples.tutorials.mnist import input_data 28 | 29 | import tensorflow as tf 30 | 31 | FLAGS = None 32 | 33 | 34 | def main(_): 35 | # Import data 36 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 37 | 38 | # Create the model 39 | x = tf.placeholder(tf.float32, [None, 784]) 40 | W = tf.Variable(tf.zeros([784, 10])) 41 | b = tf.Variable(tf.zeros([10])) 42 | y = tf.matmul(x, W) + b 43 | 44 | # Define loss and optimizer 45 | y_ = tf.placeholder(tf.float32, [None, 10]) 46 | 47 | # The raw formulation of cross-entropy, 48 | # 49 | # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), 50 | # reduction_indices=[1])) 51 | # 52 | # can be numerically unstable. 53 | # 54 | # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw 55 | # outputs of 'y', and then average across the batch. 56 | cross_entropy = tf.reduce_mean( 57 | tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) 58 | train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) 59 | 60 | sess = tf.InteractiveSession() 61 | tf.global_variables_initializer().run() 62 | # Train 63 | for _ in range(1000): 64 | batch_xs, batch_ys = mnist.train.next_batch(100) 65 | sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) 66 | 67 | # Test trained model 68 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 69 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 70 | print(sess.run(accuracy, feed_dict={x: mnist.test.images, 71 | y_: mnist.test.labels})) 72 | 73 | if __name__ == '__main__': 74 | parser = argparse.ArgumentParser() 75 | parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', 76 | help='Directory for storing input data') 77 | FLAGS, unparsed = parser.parse_known_args() 78 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 79 | -------------------------------------------------------------------------------- /mnist_distributed/README.md: -------------------------------------------------------------------------------- 1 | ## Usage 2 | 3 | ```sh 4 | # python mnist_dist.py --ps_hosts=127.0.0.1:2222 --worker_hosts=127.0.0.1:2224,127.0.0.1:2225 --job_name=ps --task_index=0 5 | # python mnist_dist.py --ps_hosts=127.0.0.1:2222 --worker_hosts=127.0.0.1:2224,127.0.0.1:2225 --job_name=worker --task_index=0 6 | # python mnist_dist.py --ps_hosts=127.0.0.1:2222 --worker_hosts=127.0.0.1:2224,127.0.0.1:2225 --job_name=worker --task_index=1 7 | ``` 8 | -------------------------------------------------------------------------------- /mnist_distributed/dist_fifo.py: -------------------------------------------------------------------------------- 1 | import math 2 | import sys 3 | import time 4 | 5 | import tensorflow as tf 6 | from tensorflow.examples.tutorials.mnist import input_data 7 | 8 | # Flags for defining the tf.train.ClusterSpec 9 | tf.app.flags.DEFINE_string("ps_hosts", "", 10 | "Comma-separated list of hostname:port pairs") 11 | tf.app.flags.DEFINE_string("worker_hosts", "", 12 | "Comma-separated list of hostname:port pairs") 13 | 14 | # Flags for defining the tf.train.Server 15 | tf.app.flags.DEFINE_string("job_name", "", "One of 'ps', 'worker'") 16 | tf.app.flags.DEFINE_integer("task_index", 0, "Index of task within the job") 17 | tf.app.flags.DEFINE_integer("hidden_units", 100, 18 | "Number of units in the hidden layer of the NN") 19 | tf.app.flags.DEFINE_string("data_dir", "/tmp/tensorflow/mnist/input_data/", 20 | "Directory for storing mnist data") 21 | tf.app.flags.DEFINE_integer("batch_size", 100, "Training batch size") 22 | tf.app.flags.DEFINE_integer("workers", 2, "Number of workers") 23 | tf.app.flags.DEFINE_integer("ps", 1, "Number of ps") 24 | tf.app.flags.DEFINE_integer("max_step", 2000, "Number of max steps") 25 | 26 | FLAGS = tf.app.flags.FLAGS 27 | 28 | IMAGE_PIXELS = 28 29 | 30 | def create_done_queue(i): 31 | """Queue used to signal death for i'th ps shard. Intended to have 32 | all workers enqueue an item onto it to signal doneness.""" 33 | 34 | with tf.device("/job:ps/task:%d" % (i)): 35 | return tf.FIFOQueue(FLAGS.workers, tf.int32, shared_name="done_queue"+ 36 | str(i)) 37 | 38 | def create_done_queues(): 39 | return [create_done_queue(i) for i in range(FLAGS.ps)] 40 | 41 | def main(_): 42 | ps_hosts = FLAGS.ps_hosts.split(",") 43 | worker_hosts = FLAGS.worker_hosts.split(",") 44 | 45 | # Create a cluster from the parameter server and worker hosts. 46 | cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) 47 | 48 | # Create and start a server for the local task. 49 | server = tf.train.Server(cluster, 50 | job_name=FLAGS.job_name, 51 | task_index=FLAGS.task_index) 52 | 53 | if FLAGS.job_name == "ps": 54 | sess = tf.Session(server.target) 55 | queue = create_done_queue(FLAGS.task_index) 56 | 57 | # wait until all workers are done 58 | for i in range(FLAGS.workers): 59 | sess.run(queue.dequeue()) 60 | print("ps %d received done %d" % (FLAGS.task_index, i)) 61 | 62 | print("ps %d: quitting"%(FLAGS.task_index)) 63 | elif FLAGS.job_name == "worker": 64 | 65 | # Assigns ops to the local worker by default. 66 | with tf.device(tf.train.replica_device_setter( 67 | worker_device="/job:worker/task:%d" % FLAGS.task_index, 68 | cluster=cluster)): 69 | 70 | # Variables of the hidden layer 71 | hid_w = tf.Variable( 72 | tf.truncated_normal([IMAGE_PIXELS * IMAGE_PIXELS, FLAGS.hidden_units], 73 | stddev=1.0 / IMAGE_PIXELS), name="hid_w") 74 | hid_b = tf.Variable(tf.zeros([FLAGS.hidden_units]), name="hid_b") 75 | 76 | # Variables of the softmax layer 77 | sm_w = tf.Variable( 78 | tf.truncated_normal([FLAGS.hidden_units, 10], 79 | stddev=1.0 / math.sqrt(FLAGS.hidden_units)), 80 | name="sm_w") 81 | sm_b = tf.Variable(tf.zeros([10]), name="sm_b") 82 | 83 | x = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS]) 84 | y_ = tf.placeholder(tf.float32, [None, 10]) 85 | 86 | hid_lin = tf.nn.xw_plus_b(x, hid_w, hid_b) 87 | hid = tf.nn.relu(hid_lin) 88 | 89 | y = tf.nn.softmax(tf.nn.xw_plus_b(hid, sm_w, sm_b)) 90 | loss = -tf.reduce_sum(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0))) 91 | 92 | global_step = tf.Variable(0) 93 | 94 | train_op = tf.train.AdagradOptimizer(0.01).minimize( 95 | loss, global_step=global_step) 96 | 97 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 98 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 99 | 100 | saver = tf.train.Saver() 101 | summary_op = tf.merge_all_summaries() 102 | init_op = tf.initialize_all_variables() 103 | 104 | 105 | enq_ops = [] 106 | for q in create_done_queues(): 107 | qop = q.enqueue(1) 108 | enq_ops.append(qop) 109 | 110 | # Create a "supervisor", which oversees the training process. 111 | sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0), 112 | logdir="./logs_%d" % FLAGS.task_index, 113 | init_op=init_op, 114 | summary_op=summary_op, 115 | saver=saver, 116 | global_step=global_step, 117 | save_model_secs=60) 118 | 119 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 120 | 121 | begin_time = time.time() 122 | frequency = 100 123 | # The supervisor takes care of session initialization, restoring from 124 | # a checkpoint, and closing when done or an error occurs. 125 | with sv.managed_session(server.target) as sess: 126 | # Loop until the supervisor shuts down or 100000 steps have completed. 127 | step = 0 128 | while not sv.should_stop() and step < FLAGS.max_step: 129 | # Run a training step asynchronously. 130 | # See `tf.train.SyncReplicasOptimizer` for additional details on how to 131 | # perform *synchronous* training. 132 | start_time = time.time() 133 | 134 | batch_xs, batch_ys = mnist.train.next_batch(FLAGS.batch_size) 135 | train_feed = {x: batch_xs, y_: batch_ys} 136 | 137 | _, step = sess.run([train_op, global_step], feed_dict=train_feed) 138 | elapsed_time = time.time() - start_time 139 | if step % frequency == 0: 140 | print ("Done step %d" % step, " AvgTime: %3.2fms" % float(elapsed_time*1000/frequency)) 141 | 142 | # Test trained model 143 | print("Test-Accuracy: %2.4f" % sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})) 144 | 145 | # signal to ps shards that we are done 146 | #for q in create_done_queues(): 147 | #sess.run(q.enqueue(1)) 148 | for op in enq_ops: 149 | sess.run(op) 150 | print("Total Time: %3.2fs" % float(time.time() - begin_time)) 151 | 152 | # Ask for all the services to stop. 153 | sv.stop() 154 | 155 | if __name__ == "__main__": 156 | tf.app.run() 157 | -------------------------------------------------------------------------------- /mnist_distributed/dist_join.py: -------------------------------------------------------------------------------- 1 | import math 2 | import sys 3 | import time 4 | 5 | import tensorflow as tf 6 | from tensorflow.examples.tutorials.mnist import input_data 7 | 8 | # Flags for defining the tf.train.ClusterSpec 9 | tf.app.flags.DEFINE_string("ps_hosts", "", 10 | "Comma-separated list of hostname:port pairs") 11 | tf.app.flags.DEFINE_string("worker_hosts", "", 12 | "Comma-separated list of hostname:port pairs") 13 | 14 | # Flags for defining the tf.train.Server 15 | tf.app.flags.DEFINE_string("job_name", "", "One of 'ps', 'worker'") 16 | tf.app.flags.DEFINE_integer("task_index", 0, "Index of task within the job") 17 | tf.app.flags.DEFINE_integer("hidden_units", 100, 18 | "Number of units in the hidden layer of the NN") 19 | tf.app.flags.DEFINE_string("data_dir", "/tmp/tensorflow/mnist/input_data/", 20 | "Directory for storing mnist data") 21 | tf.app.flags.DEFINE_integer("batch_size", 100, "Training batch size") 22 | tf.app.flags.DEFINE_integer("max_step", 100, "Training batch size") 23 | 24 | FLAGS = tf.app.flags.FLAGS 25 | 26 | IMAGE_PIXELS = 28 27 | 28 | def main(_): 29 | ps_hosts = FLAGS.ps_hosts.split(",") 30 | worker_hosts = FLAGS.worker_hosts.split(",") 31 | 32 | # Create a cluster from the parameter server and worker hosts. 33 | cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) 34 | 35 | # Create and start a server for the local task. 36 | server = tf.train.Server(cluster, 37 | job_name=FLAGS.job_name, 38 | task_index=FLAGS.task_index) 39 | 40 | if FLAGS.job_name == "ps": 41 | server.join() 42 | elif FLAGS.job_name == "worker": 43 | 44 | # Assigns ops to the local worker by default. 45 | with tf.device(tf.train.replica_device_setter( 46 | worker_device="/job:worker/task:%d" % FLAGS.task_index, 47 | cluster=cluster)): 48 | 49 | # Variables of the hidden layer 50 | hid_w = tf.Variable( 51 | tf.truncated_normal([IMAGE_PIXELS * IMAGE_PIXELS, FLAGS.hidden_units], 52 | stddev=1.0 / IMAGE_PIXELS), name="hid_w") 53 | hid_b = tf.Variable(tf.zeros([FLAGS.hidden_units]), name="hid_b") 54 | 55 | # Variables of the softmax layer 56 | sm_w = tf.Variable( 57 | tf.truncated_normal([FLAGS.hidden_units, 10], 58 | stddev=1.0 / math.sqrt(FLAGS.hidden_units)), 59 | name="sm_w") 60 | sm_b = tf.Variable(tf.zeros([10]), name="sm_b") 61 | 62 | x = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS]) 63 | y_ = tf.placeholder(tf.float32, [None, 10]) 64 | 65 | hid_lin = tf.nn.xw_plus_b(x, hid_w, hid_b) 66 | hid = tf.nn.relu(hid_lin) 67 | 68 | y = tf.nn.softmax(tf.nn.xw_plus_b(hid, sm_w, sm_b)) 69 | loss = -tf.reduce_sum(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0))) 70 | 71 | global_step = tf.Variable(0) 72 | 73 | train_op = tf.train.AdagradOptimizer(0.01).minimize( 74 | loss, global_step=global_step) 75 | 76 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 77 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 78 | 79 | saver = tf.train.Saver() 80 | summary_op = tf.merge_all_summaries() 81 | init_op = tf.initialize_all_variables() 82 | 83 | # Create a "supervisor", which oversees the training process. 84 | sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0), 85 | logdir="./logs_%d" % FLAGS.task_index, 86 | init_op=init_op, 87 | summary_op=summary_op, 88 | saver=saver, 89 | global_step=global_step, 90 | save_model_secs=60) 91 | 92 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) 93 | 94 | begin_time = time.time() 95 | frequency = 100 96 | # The supervisor takes care of session initialization, restoring from 97 | # a checkpoint, and closing when done or an error occurs. 98 | with sv.managed_session(server.target) as sess: 99 | # Loop until the supervisor shuts down or 100000 steps have completed. 100 | step = 0 101 | while not sv.should_stop() and step < 100000: 102 | # Run a training step asynchronously. 103 | # See `tf.train.SyncReplicasOptimizer` for additional details on how to 104 | # perform *synchronous* training. 105 | start_time = time.time() 106 | 107 | batch_xs, batch_ys = mnist.train.next_batch(FLAGS.batch_size) 108 | train_feed = {x: batch_xs, y_: batch_ys} 109 | 110 | _, step = sess.run([train_op, global_step], feed_dict=train_feed) 111 | elapsed_time = time.time() - start_time 112 | if step % frequency == 0: 113 | print ("Done step %d" % step, " AvgTime: %3.2fms" % float(elapsed_time*1000/frequency)) 114 | 115 | 116 | # Test trained model 117 | print("Test-Accuracy: %2.4f" % sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})) 118 | 119 | print("Total Time: %3.2fs" % float(time.time() - begin_time)) 120 | # Ask for all the services to stop. 121 | sv.stop() 122 | 123 | if __name__ == "__main__": 124 | tf.app.run() 125 | -------------------------------------------------------------------------------- /sess.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | sess = tf.InteractiveSession() 5 | 6 | a = tf.constant([10, 20]) 7 | b = tf.constant([1.0, 2.0]) 8 | c = tf.add(a, b) 9 | v = sess.run(c) 10 | print(v) 11 | --------------------------------------------------------------------------------