├── .gitignore ├── README.md ├── auto_encoder ├── AdditiveGaussianNoiseAutoencoderRunner.py ├── __init__.py ├── autoencoder_models │ ├── Autoencoder.py │ ├── DenoisingAutoencoder.py │ ├── VariationalAutoencoder.py │ └── __init__.py ├── show_mnist_data.py ├── test.py ├── test_of_random_normal.py └── test_of_softplus.py ├── cnns ├── SimpleCNN.py └── __init__.py ├── dist_test ├── __init__.py └── mnist_replica.py ├── multi_gpu_train ├── __init__.py ├── cifar10.py ├── cifar10_input.py ├── cifar10_multi_gpu_train.py ├── shape_test.py └── show_gpu_count.py ├── multi_layer_perception ├── __init__.py ├── mlp_mnist.py ├── test_of_dropout.py └── test_of_relu.py ├── project_utils.py ├── requirements.txt ├── root_dir.py ├── tensor_board ├── __init__.py ├── mnist_with_summaries.py └── test_of_resharp.py ├── tests ├── __init__.py ├── android_test.py └── ocr_test.py ├── tf_tests ├── HelloWorld.py ├── __init__.py ├── mnist_softmax.py ├── test_of_am.py ├── test_of_argparse.py ├── test_of_mat.py ├── test_of_rm.py └── test_of_sc.py └── wilson_score ├── __init__.py ├── wilson_score_model.py └── ws_img.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Python template 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | MLT_ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | ### Example user template template 99 | ### Example user template 100 | 101 | # IntelliJ project files 102 | .idea/ 103 | *.iml 104 | out 105 | gen 106 | 107 | # 自定义忽略 108 | 109 | # 忽略数据 110 | MNIST_data/ 111 | auto_encoder/IMAGE_data/ 112 | multi_gpu_train/cifar10_data/ 113 | tests/data/ 114 | venv/ 115 | 116 | 117 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MachineLearningTutorial 2 | 3 | 机器学习指南 4 | 5 | Created By C. L. Wang, Email: morndragon@126.com 6 | 7 | 1. [TensorFlow 框架初探](http://www.jianshu.com/p/73ae26b3ea70) （含有项目配置，虚拟环境） 8 | 9 | 2. [自编码器 AutoEncoder](http://www.jianshu.com/p/9ccf67ccd44b) 10 | 11 | 3. [多层感知机 MLP](http://www.jianshu.com/p/ac5c1d83dc71) 12 | 13 | 4. [模型可视化 TensorBoard](http://www.jianshu.com/p/f3e51ee564ab) 14 | 15 | 5. [TensorFlow 的 GPU](http://www.jianshu.com/p/2ccfa4170750) 16 | 17 | 6. [TensorFlow 的 分布式架构](http://www.jianshu.com/p/b6e25d0a9399) 18 | 19 | 7. [TensorFlow Android 版](http://www.jianshu.com/p/dff6ad105c8e) 20 | 21 | 其余： 22 | 23 | - [威尔逊得分 Wilson Score 排序算法](http://www.jianshu.com/p/4d2b45918958) 24 | 25 | - [百度 OCR 文字识别的接口测试](http://www.jianshu.com/p/a1688fed63e3) 26 | 27 | - [TensorFlow集成Android工程的框架](http://www.jianshu.com/p/870e9a54749a) 28 | -------------------------------------------------------------------------------- /auto_encoder/AdditiveGaussianNoiseAutoencoderRunner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import numpy as np 7 | import sklearn.preprocessing as prep 8 | import tensorflow as tf 9 | from PIL import Image 10 | from scipy import misc 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | from autoencoder_models.DenoisingAutoencoder import AdditiveGaussianNoiseAutoencoder 14 | from project_utils import listdir_files 15 | 16 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True) 17 | 18 | 19 | def standard_scale(X_train, X_test): 20 | preprocessor = prep.StandardScaler().fit(X_train) 21 | X_train = preprocessor.transform(X_train) 22 | X_test = preprocessor.transform(X_test) 23 | return X_train, X_test 24 | 25 | 26 | def get_random_block_from_data(data, batch_size): 27 | start_index = np.random.randint(0, len(data) - batch_size) # 随机获取区块 28 | return data[start_index:(start_index + batch_size)] # batch_size大小的区块 29 | pi 30 | 31 | X_train, X_test = standard_scale(mnist.train.images, mnist.test.images) 32 | 33 | n_samples = int(mnist.train.num_examples) 34 | training_epochs = 20 35 | batch_size = 128 36 | display_step = 1 37 | 38 | autoencoder = AdditiveGaussianNoiseAutoencoder( 39 | n_input=784, n_hidden=200, transfer_function=tf.nn.softplus, 40 | optimizer=tf.train.AdamOptimizer(learning_rate=0.001), scale=0.01) 41 | 42 | for epoch in range(training_epochs): 43 | avg_cost = 0. 44 | total_batch = int(n_samples / batch_size) 45 | # Loop over all batches 46 | for i in range(total_batch): 47 | batch_xs = get_random_block_from_data(X_train, batch_size) 48 | 49 | # Fit training using batch data 50 | cost = autoencoder.partial_fit(batch_xs) 51 | # Compute average loss 52 | avg_cost += cost / n_samples * batch_size 53 | 54 | # Display logs per epoch step 55 | if epoch % display_step == 0: 56 | print("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(avg_cost)) 57 | 58 | print("Total cost: " + str(autoencoder.calc_total_cost(X_test))) 59 | 60 | en_images = autoencoder.reconstruct(mnist.test.images) 61 | labels = mnist.test.labels # 标签 62 | 63 | for i in range(len(en_images)): 64 | img = np.array(en_images[i]).reshape((28, 28)) 65 | lbl = np.argmax(labels[i]) 66 | misc.imsave('./IMAGE_data/en_test/' + str(i) + '_' + str(lbl) + '.png', img) # scipy的存储模式 67 | if i == 100: 68 | break 69 | 70 | paths_list, _, __ = listdir_files('./IMAGE_data/en_test/') 71 | large_size = 28 * 10 72 | large_img = Image.new('RGBA', (large_size, large_size)) 73 | for i in range(100): 74 | img = Image.open(paths_list[i]) 75 | loc = ((int(i / 10) * 28), (i % 10) * 28) 76 | large_img.paste(img, loc) 77 | large_img.save('./IMAGE_data/en_merged.png') 78 | -------------------------------------------------------------------------------- /auto_encoder/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /auto_encoder/autoencoder_models/Autoencoder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class Autoencoder(object): 4 | 5 | def __init__(self, n_input, n_hidden, transfer_function=tf.nn.softplus, optimizer = tf.train.AdamOptimizer()): 6 | self.n_input = n_input 7 | self.n_hidden = n_hidden 8 | self.transfer = transfer_function 9 | 10 | network_weights = self._initialize_weights() 11 | self.weights = network_weights 12 | 13 | # model 14 | self.x = tf.placeholder(tf.float32, [None, self.n_input]) 15 | self.hidden = self.transfer(tf.add(tf.matmul(self.x, self.weights['w1']), self.weights['b1'])) 16 | self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2']) 17 | 18 | # cost 19 | self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) 20 | self.optimizer = optimizer.minimize(self.cost) 21 | 22 | init = tf.global_variables_initializer() 23 | self.sess = tf.Session() 24 | self.sess.run(init) 25 | 26 | 27 | def _initialize_weights(self): 28 | all_weights = dict() 29 | all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden], 30 | initializer=tf.contrib.layers.xavier_initializer()) 31 | all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32)) 32 | all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype=tf.float32)) 33 | all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32)) 34 | return all_weights 35 | 36 | def partial_fit(self, X): 37 | cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict={self.x: X}) 38 | return cost 39 | 40 | def calc_total_cost(self, X): 41 | return self.sess.run(self.cost, feed_dict = {self.x: X}) 42 | 43 | def transform(self, X): 44 | return self.sess.run(self.hidden, feed_dict={self.x: X}) 45 | 46 | def generate(self, hidden = None): 47 | if hidden is None: 48 | hidden = self.sess.run(tf.random_normal([1, self.n_hidden])) 49 | return self.sess.run(self.reconstruction, feed_dict={self.hidden: hidden}) 50 | 51 | def reconstruct(self, X): 52 | return self.sess.run(self.reconstruction, feed_dict={self.x: X}) 53 | 54 | def getWeights(self): 55 | return self.sess.run(self.weights['w1']) 56 | 57 | def getBiases(self): 58 | return self.sess.run(self.weights['b1']) 59 | 60 | -------------------------------------------------------------------------------- /auto_encoder/autoencoder_models/DenoisingAutoencoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import tensorflow as tf 5 | 6 | 7 | class AdditiveGaussianNoiseAutoencoder(object): 8 | def __init__(self, n_input, n_hidden, transfer_function=tf.nn.softplus, optimizer=tf.train.AdamOptimizer(), 9 | scale=0.1): 10 | self.n_input = n_input # 输入的节点数 11 | self.n_hidden = n_hidden # 隐含层节点数，小于输入节点数 12 | self.transfer = transfer_function # 激活函数 13 | self.scale = tf.placeholder(tf.float32) # 系数，待训练的参数，初始的feed数据是training_scale 14 | self.training_scale = scale # 高斯噪声系数 15 | network_weights = self._initialize_weights() # 初始化权重系数，输入层w1/b1，输出层w2/b2 16 | self.weights = network_weights # 权重 17 | 18 | # model 19 | self.x = tf.placeholder(tf.float32, [None, self.n_input]) # 需要feed的数据 20 | self.hidden = self.transfer(tf.add(tf.matmul(self.x + scale * tf.random_normal((n_input,)), 21 | self.weights['w1']), 22 | self.weights['b1'])) 23 | self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2']) 24 | 25 | # cost，0.5*(x - x_)^2，求和 26 | self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) 27 | self.optimizer = optimizer.minimize(self.cost) 28 | 29 | init = tf.global_variables_initializer() 30 | self.sess = tf.Session() 31 | self.sess.run(init) # 执行图 32 | 33 | def _initialize_weights(self): 34 | all_weights = dict() 35 | # 使用xavier_initializer初始化 36 | all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden], 37 | initializer=tf.contrib.layers.xavier_initializer()) 38 | all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32)) 39 | all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype=tf.float32)) 40 | all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32)) 41 | return all_weights 42 | 43 | def partial_fit(self, X): 44 | cost, opt = self.sess.run((self.cost, self.optimizer), 45 | feed_dict={self.x: X, self.scale: self.training_scale}) 46 | return cost 47 | 48 | def calc_total_cost(self, X): 49 | return self.sess.run(self.cost, feed_dict={self.x: X, 50 | self.scale: self.training_scale 51 | }) 52 | 53 | def transform(self, X): 54 | return self.sess.run(self.hidden, feed_dict={self.x: X, 55 | self.scale: self.training_scale 56 | }) 57 | 58 | def generate(self, hidden=None): 59 | if hidden is None: 60 | hidden = self.sess.run(tf.random_normal([1, self.n_hidden])) 61 | return self.sess.run(self.reconstruction, feed_dict={self.hidden: hidden}) 62 | 63 | def reconstruct(self, X): 64 | return self.sess.run(self.reconstruction, feed_dict={self.x: X, 65 | self.scale: self.training_scale 66 | }) 67 | 68 | def getWeights(self): 69 | return self.sess.run(self.weights['w1']) 70 | 71 | def getBiases(self): 72 | return self.sess.run(self.weights['b1']) 73 | 74 | 75 | class MaskingNoiseAutoencoder(object): 76 | def __init__(self, n_input, n_hidden, transfer_function=tf.nn.softplus, optimizer=tf.train.AdamOptimizer(), 77 | dropout_probability=0.95): 78 | self.n_input = n_input 79 | self.n_hidden = n_hidden 80 | self.transfer = transfer_function 81 | self.dropout_probability = dropout_probability 82 | self.keep_prob = tf.placeholder(tf.float32) 83 | 84 | network_weights = self._initialize_weights() 85 | self.weights = network_weights 86 | 87 | # model 88 | self.x = tf.placeholder(tf.float32, [None, self.n_input]) 89 | self.hidden = self.transfer(tf.add(tf.matmul(tf.nn.dropout(self.x, self.keep_prob), self.weights['w1']), 90 | self.weights['b1'])) 91 | self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2']) 92 | 93 | # cost 94 | self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) 95 | self.optimizer = optimizer.minimize(self.cost) 96 | 97 | init = tf.global_variables_initializer() 98 | self.sess = tf.Session() 99 | self.sess.run(init) 100 | 101 | def _initialize_weights(self): 102 | all_weights = dict() 103 | all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden], 104 | initializer=tf.contrib.layers.xavier_initializer()) 105 | all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32)) 106 | all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype=tf.float32)) 107 | all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32)) 108 | return all_weights 109 | 110 | def partial_fit(self, X): 111 | cost, opt = self.sess.run((self.cost, self.optimizer), 112 | feed_dict={self.x: X, self.keep_prob: self.dropout_probability}) 113 | return cost 114 | 115 | def calc_total_cost(self, X): 116 | return self.sess.run(self.cost, feed_dict={self.x: X, self.keep_prob: 1.0}) 117 | 118 | def transform(self, X): 119 | return self.sess.run(self.hidden, feed_dict={self.x: X, self.keep_prob: 1.0}) 120 | 121 | def generate(self, hidden=None): 122 | if hidden is None: 123 | hidden = self.sess.run(tf.random_normal([1, self.n_hidden])) 124 | return self.sess.run(self.reconstruction, feed_dict={self.hidden: hidden}) 125 | 126 | def reconstruct(self, X): 127 | return self.sess.run(self.reconstruction, feed_dict={self.x: X, self.keep_prob: 1.0}) 128 | 129 | def getWeights(self): 130 | return self.sess.run(self.weights['w1']) 131 | 132 | def getBiases(self): 133 | return self.sess.run(self.weights['b1']) 134 | -------------------------------------------------------------------------------- /auto_encoder/autoencoder_models/VariationalAutoencoder.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class VariationalAutoencoder(object): 4 | 5 | def __init__(self, n_input, n_hidden, optimizer = tf.train.AdamOptimizer()): 6 | self.n_input = n_input 7 | self.n_hidden = n_hidden 8 | 9 | network_weights = self._initialize_weights() 10 | self.weights = network_weights 11 | 12 | # model 13 | self.x = tf.placeholder(tf.float32, [None, self.n_input]) 14 | self.z_mean = tf.add(tf.matmul(self.x, self.weights['w1']), self.weights['b1']) 15 | self.z_log_sigma_sq = tf.add(tf.matmul(self.x, self.weights['log_sigma_w1']), self.weights['log_sigma_b1']) 16 | 17 | # sample from gaussian distribution 18 | eps = tf.random_normal(tf.stack([tf.shape(self.x)[0], self.n_hidden]), 0, 1, dtype = tf.float32) 19 | self.z = tf.add(self.z_mean, tf.multiply(tf.sqrt(tf.exp(self.z_log_sigma_sq)), eps)) 20 | 21 | self.reconstruction = tf.add(tf.matmul(self.z, self.weights['w2']), self.weights['b2']) 22 | 23 | # cost 24 | reconstr_loss = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) 25 | latent_loss = -0.5 * tf.reduce_sum(1 + self.z_log_sigma_sq 26 | - tf.square(self.z_mean) 27 | - tf.exp(self.z_log_sigma_sq), 1) 28 | self.cost = tf.reduce_mean(reconstr_loss + latent_loss) 29 | self.optimizer = optimizer.minimize(self.cost) 30 | 31 | init = tf.global_variables_initializer() 32 | self.sess = tf.Session() 33 | self.sess.run(init) 34 | 35 | def _initialize_weights(self): 36 | all_weights = dict() 37 | all_weights['w1'] = tf.get_variable("w1", shape=[self.n_input, self.n_hidden], 38 | initializer=tf.contrib.layers.xavier_initializer()) 39 | all_weights['log_sigma_w1'] = tf.get_variable("log_sigma_w1", shape=[self.n_input, self.n_hidden], 40 | initializer=tf.contrib.layers.xavier_initializer()) 41 | all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32)) 42 | all_weights['log_sigma_b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype=tf.float32)) 43 | all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype=tf.float32)) 44 | all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype=tf.float32)) 45 | return all_weights 46 | 47 | def partial_fit(self, X): 48 | cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict={self.x: X}) 49 | return cost 50 | 51 | def calc_total_cost(self, X): 52 | return self.sess.run(self.cost, feed_dict = {self.x: X}) 53 | 54 | def transform(self, X): 55 | return self.sess.run(self.z_mean, feed_dict={self.x: X}) 56 | 57 | def generate(self, hidden = None): 58 | if hidden is None: 59 | hidden = self.sess.run(tf.random_normal([1, self.n_hidden])) 60 | return self.sess.run(self.reconstruction, feed_dict={self.z: hidden}) 61 | 62 | def reconstruct(self, X): 63 | return self.sess.run(self.reconstruction, feed_dict={self.x: X}) 64 | 65 | def getWeights(self): 66 | return self.sess.run(self.weights['w1']) 67 | 68 | def getBiases(self): 69 | return self.sess.run(self.weights['b1']) 70 | 71 | -------------------------------------------------------------------------------- /auto_encoder/autoencoder_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SpikeKing/MachineLearningTutorial/2c23b2c7d92db54829508558089c1427d4548830/auto_encoder/autoencoder_models/__init__.py -------------------------------------------------------------------------------- /auto_encoder/show_mnist_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import cv2.cv2 as cv2 5 | # Created by C.L.Wang 6 | # 7 | import matplotlib 8 | import scipy.misc as misc 9 | from PIL import Image 10 | 11 | matplotlib.use('TkAgg') 12 | 13 | from project_utils import * 14 | import matplotlib.pyplot as plt 15 | from tensorflow.examples.tutorials.mnist import input_data 16 | 17 | # 加载数据 18 | mnist = input_data.read_data_sets('../MNIST_data', one_hot=True) 19 | images = mnist.test.images # 图片 20 | labels = mnist.test.labels # 标签 21 | 22 | # 存储图片 23 | size = len(labels) 24 | for i in range(size): 25 | pxl = np.array(images[i]) # 像素 26 | img = pxl.reshape((28, 28)) # 图片 27 | lbl = np.argmax(labels[i]) # 标签 28 | misc.imsave('./IMAGE_data/test/' + str(i) + '_' + str(lbl) + '.png', img) # scipy的存储模式 29 | if i == 100: 30 | break 31 | 32 | # 合并图片 33 | large_size = 28 * 10 34 | large_img = Image.new('RGBA', (large_size, large_size)) 35 | paths_list, _, __ = listdir_files('./IMAGE_data/test/') 36 | for i in range(100): 37 | img = Image.open(paths_list[i]) 38 | loc = ((int(i / 10) * 28), (i % 10) * 28) 39 | large_img.paste(img, loc) 40 | large_img.save('./IMAGE_data/merged.png') 41 | 42 | # 其他的图片存储方式 43 | pixel = np.array(images[0]) # 784维的数据 44 | label = np.argmax(labels[0]) # 找到标签 45 | image = pixel.reshape((28, 28)) # 转换成28*28维的矩阵 46 | 47 | # -------------------- scipy模式 -------------------- # 48 | misc.imsave('./IMAGE_data/scipy.png', image) # scipy的存储模式 49 | # -------------------- scipy模式 -------------------- # 50 | 51 | # -------------------- matplotlib模式 -------------------- # 52 | plt.gray() # 转变为灰度图片 53 | plt.imshow(image) 54 | plt.savefig("./IMAGE_data/plt.png") 55 | # plt.show() 56 | # -------------------- matplotlib模式 -------------------- # 57 | 58 | # -------------------- opencv模式 -------------------- # 59 | image = image * 255 # 数据是0~1的浮点数 60 | cv2.imwrite("./IMAGE_data/opencv.png", image) 61 | # cv2.imshow('hah', pixels) 62 | # cv2.waitKey(0) 63 | # -------------------- opencv模式 -------------------- # 64 | -------------------------------------------------------------------------------- /auto_encoder/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | for i in range(4): 7 | # loc = ((i % 2) * 200, (int(i/2) * 200)) 8 | loc = ((int(i/2) * 200), (i % 2) * 200) 9 | print(loc) -------------------------------------------------------------------------------- /auto_encoder/test_of_random_normal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | rn = tf.random_normal((100000,)) # 一行，指定seed，防止均值的时候随机 9 | mean, variance = tf.nn.moments(rn, 0) # 计算均值和方差，预期均值约等于是0，方差是1 10 | print tf.Session().run(tf.nn.moments(rn, 0)) 11 | -------------------------------------------------------------------------------- /auto_encoder/test_of_softplus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | mat = [1., 2., 3.] # 需要使用小数 9 | # softplus: [ln(e^1 + 1), ln(e^2 + 1), ln(e^3 + 1)] 10 | print tf.Session().run(tf.nn.softplus(mat)) 11 | -------------------------------------------------------------------------------- /cnns/SimpleCNN.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | """ 6 | 简易的卷积神经网络（CNN） 7 | """ 8 | import os 9 | import sys 10 | 11 | import tensorflow as tf 12 | 13 | p = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 14 | if p not in sys.path: 15 | sys.path.append(p) 16 | 17 | from tensorflow.examples.tutorials.mnist import input_data 18 | 19 | from root_dir import ROOT_DIR, MNIST_DIR 20 | 21 | 22 | def test_simple_cnn(): 23 | """ 24 | 测试简易的CNN 25 | :return: None 26 | """ 27 | mnist_path = os.path.join(ROOT_DIR, MNIST_DIR) 28 | mnist = input_data.read_data_sets(mnist_path, one_hot=True) # 读取MNIST数据 29 | sess = tf.InteractiveSession() 30 | 31 | x = tf.placeholder(tf.float32, [None, 784]) # 28*28维的输入图像 32 | y_ = tf.placeholder(tf.float32, [None, 10]) # 10维的输入标签 33 | x_image = tf.reshape(x, [-1, 28, 28, 1]) # 将784维数据转换为28*28维的图片, -1表示数量不固定, 1表示图片的维度为1 34 | 35 | # print mnist.test.images[0] # 测试读取数据 36 | 37 | # ************ 第一层卷积操作 ************ 38 | W_conv1 = weight_var([5, 5, 1, 32]) # 5*5的卷积核, 1个颜色通道, 32个不同的核 39 | b_conv1 = bias_var([32]) # 32维的偏移 40 | h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) # 在x_image中执行卷积操作, 再使用ReLU的激活函数 41 | h_pool1 = max_pool_2x2(h_conv1) # 执行最大的池化操作 42 | 43 | # ************ 第二层卷积操作 ************ 44 | W_conv2 = weight_var([5, 5, 32, 64]) # 第二层卷积, 核5*5, 32个通道(与第一层对应), 64个不同核 45 | b_conv2 = bias_var([64]) 46 | h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) # 第二层卷积 47 | h_pool2 = max_pool_2x2(h_conv2) # 执行最大的池化操作 48 | 49 | # ************ 全连接层 ************ 50 | W_fc1 = weight_var([7 * 7 * 64, 1024]) # 隐含节点, 1024个 51 | b_fc1 = bias_var([1024]) 52 | h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) # 展开 53 | h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # 全连接层 54 | 55 | # ************ Dropout ************ 56 | keep_prob = tf.placeholder(tf.float32) 57 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob=keep_prob) # 随机遗忘 58 | 59 | # ************ SoftMax ************ 60 | W_fc2 = weight_var([1024, 10]) 61 | b_fc2 = weight_var([10]) 62 | y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) 63 | 64 | # ************ 训练步骤, 最小化交差熵 ************ 65 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1])) 66 | train_step = tf.train.AdadeltaOptimizer(1e-4).minimize(cross_entropy) # 最小化交差熵 67 | 68 | correct_prediction = tf.equal(tf.arg_max(y_conv, 1), tf.arg_max(y_, 1)) 69 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # 平均值 70 | 71 | tf.global_variables_initializer().run() # 初始化变量 72 | 73 | # ************ Feed数据, 训练模型, 验证 ************ 74 | for i in range(20000): 75 | batch = mnist.train.next_batch(50) 76 | if i % 100 == 0: 77 | train_accuracy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0}) 78 | print "step %d, training accuracy %g" % (i, train_accuracy) 79 | train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) 80 | 81 | print "test accuracy %g" % accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}) 82 | 83 | 84 | def weight_var(shape): 85 | """ 86 | wx + b 的w, 权重 87 | :param shape: w的维度 88 | :return: tf的var 89 | """ 90 | initial = tf.truncated_normal(shape=shape, stddev=0.1) # 截断的正态分布 91 | return tf.Variable(initial_value=initial) 92 | 93 | 94 | def bias_var(shape): 95 | """ 96 | wx + b 的b, 偏移 97 | :param shape: b的维度 98 | :return: tf的var 99 | """ 100 | initial = tf.constant(0.1, shape=shape) 101 | return tf.Variable(initial_value=initial) 102 | 103 | 104 | def conv2d(x, W): 105 | """ 106 | 二维卷积操作, 步长为1, 维度不变 107 | :param x: 输入矩阵 108 | :param W: 卷积参数, 如[5, 5, 1, 32], 卷积维度5*5, 1维, 32个卷积核 109 | :return: 卷积之后的矩阵 110 | """ 111 | return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') 112 | 113 | 114 | def max_pool_2x2(x): 115 | """ 116 | 2x2的核维度, 步长是横向和竖向2步, 全部维度 117 | :param x: 输入矩阵 118 | :return: 池化之后的矩阵 119 | """ 120 | return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 121 | 122 | 123 | if __name__ == '__main__': 124 | test_simple_cnn() # 测试简易的CNN 125 | -------------------------------------------------------------------------------- /cnns/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /dist_test/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /dist_test/mnist_replica.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ============================================================================== 17 | 18 | """Distributed MNIST training and validation, with model replicas. 19 | 20 | A simple softmax model with one hidden layer is defined. The parameters 21 | (weights and biases) are located on one parameter server (ps), while the ops 22 | are executed on two worker nodes by default. The TF sessions also run on the 23 | worker node. 24 | Multiple invocations of this script can be done in parallel, with different 25 | values for --task_index. There should be exactly one invocation with 26 | --task_index, which will create a master session that carries out variable 27 | initialization. The other, non-master, sessions will wait for the master 28 | session to finish the initialization before proceeding to the training stage. 29 | 30 | The coordination between the multiple worker invocations occurs due to 31 | the definition of the parameters on the same ps devices. The parameter updates 32 | from one worker is visible to all other workers. As such, the workers can 33 | perform forward computation and gradient calculation in parallel, which 34 | should lead to increased training speed for the simple model. 35 | """ 36 | 37 | from __future__ import absolute_import 38 | from __future__ import division 39 | from __future__ import print_function 40 | 41 | import math 42 | import sys 43 | import tempfile 44 | import time 45 | 46 | import tensorflow as tf 47 | from tensorflow.examples.tutorials.mnist import input_data 48 | 49 | flags = tf.app.flags 50 | flags.DEFINE_string("data_dir", "/tmp/mnist-data", 51 | "Directory for storing mnist data") 52 | flags.DEFINE_boolean("download_only", False, 53 | "Only perform downloading of data; Do not proceed to " 54 | "session preparation, model definition or training") 55 | flags.DEFINE_integer("task_index", None, 56 | "Worker task index, should be >= 0. task_index=0 is " 57 | "the master worker task the performs the variable " 58 | "initialization ") 59 | flags.DEFINE_integer("num_gpus", 1, 60 | "Total number of gpus for each machine." 61 | "If you don't use GPU, please set it to '0'") 62 | flags.DEFINE_integer("replicas_to_aggregate", None, 63 | "Number of replicas to aggregate before parameter update" 64 | "is applied (For sync_replicas mode only; default: " 65 | "num_workers)") 66 | flags.DEFINE_integer("hidden_units", 100, 67 | "Number of units in the hidden layer of the NN") 68 | flags.DEFINE_integer("train_steps", 20000000, 69 | "Number of (global) training steps to perform") 70 | flags.DEFINE_integer("batch_size", 100, "Training batch size") 71 | flags.DEFINE_float("learning_rate", 0.01, "Learning rate") 72 | flags.DEFINE_boolean("sync_replicas", False, 73 | "Use the sync_replicas (synchronized replicas) mode, " 74 | "wherein the parameter updates from workers are aggregated " 75 | "before applied to avoid stale gradients") 76 | flags.DEFINE_boolean( 77 | "existing_servers", False, "Whether servers already exists. If True, " 78 | "will use the worker hosts via their GRPC URLs (one client process " 79 | "per worker host). Otherwise, will create an in-process TensorFlow " 80 | "server.") 81 | # flags.DEFINE_string("ps_hosts", "localhost:2222", 82 | # "Comma-separated list of hostname:port pairs") 83 | # flags.DEFINE_string("worker_hosts", "localhost:2223,localhost:2224", 84 | # "Comma-separated list of hostname:port pairs") 85 | flags.DEFINE_string("ps_hosts", "hd1:2222", 86 | "Comma-separated list of hostname:port pairs") 87 | flags.DEFINE_string("worker_hosts", "learn:2222,docker01:2222", 88 | "Comma-separated list of hostname:port pairs") 89 | 90 | flags.DEFINE_string("job_name", None, "job name: worker or ps") 91 | 92 | FLAGS = flags.FLAGS 93 | 94 | IMAGE_PIXELS = 28 95 | 96 | 97 | def main(unused_argv): 98 | """ 99 | 执行命令，worker服务器与ps服务器需要提前设定 100 | python mnist_replica.py --job_name=ps --task_index=0 101 | python mnist_replica.py --job_name=worker --task_index=0 102 | python mnist_replica.py --job_name=worker --task_index=1 103 | :param unused_argv: 104 | :return: 105 | """ 106 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) # 下载数据 107 | if FLAGS.download_only: 108 | sys.exit(0) 109 | 110 | # 执行时，需要制定名称和任务ID，任务id为0，是主任务，名称ps是参数服务器，worker是计算服务器 111 | if FLAGS.job_name is None or FLAGS.job_name == "": 112 | raise ValueError("Must specify an explicit `job_name`") 113 | if FLAGS.task_index is None or FLAGS.task_index == "": 114 | raise ValueError("Must specify an explicit `task_index`") 115 | 116 | print("job name = %s" % FLAGS.job_name) 117 | print("task index = %d" % FLAGS.task_index) 118 | 119 | # Construct the cluster and start the server 120 | ps_spec = FLAGS.ps_hosts.split(",") 121 | worker_spec = FLAGS.worker_hosts.split(",") 122 | 123 | # Get the number of workers. 124 | num_workers = len(worker_spec) # worker的数量 125 | 126 | # 创建集群 127 | cluster = tf.train.ClusterSpec({"ps": ps_spec, "worker": worker_spec}) 128 | 129 | if not FLAGS.existing_servers: 130 | # Not using existing servers. Create an in-process server. 131 | server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) 132 | if FLAGS.job_name == "ps": 133 | server.join() 134 | 135 | is_chief = (FLAGS.task_index == 0) # id为0是主worker 136 | 137 | # 含有GPU使用GPU，没有使用CPU 138 | if FLAGS.num_gpus > 0: 139 | # Avoid gpu allocation conflict: now allocate task_num -> #gpu 140 | # for each worker in the corresponding machine 141 | gpu = (FLAGS.task_index % FLAGS.num_gpus) 142 | worker_device = "/job:worker/task:%d/gpu:%d" % (FLAGS.task_index, gpu) 143 | elif FLAGS.num_gpus == 0: 144 | # Just allocate the CPU to worker server 145 | cpu = 0 146 | worker_device = "/job:worker/task:%d/cpu:%d" % (FLAGS.task_index, cpu) 147 | 148 | # The device setter will automatically place Variables ops on separate 149 | # parameter servers (ps). The non-Variable ops will be placed on the workers. 150 | # The ps use CPU and workers use corresponding GPU 151 | with tf.device(tf.train.replica_device_setter( 152 | worker_device=worker_device, 153 | ps_device="/job:ps/cpu:0", 154 | cluster=cluster)): 155 | 156 | global_step = tf.Variable(0, name="global_step", trainable=False) 157 | 158 | # Variables of the hidden layer 159 | hid_w = tf.Variable( 160 | tf.truncated_normal( 161 | [IMAGE_PIXELS * IMAGE_PIXELS, FLAGS.hidden_units], 162 | stddev=1.0 / IMAGE_PIXELS), 163 | name="hid_w") 164 | hid_b = tf.Variable(tf.zeros([FLAGS.hidden_units]), name="hid_b") 165 | 166 | # Variables of the softmax layer 167 | sm_w = tf.Variable( 168 | tf.truncated_normal( 169 | [FLAGS.hidden_units, 10], 170 | stddev=1.0 / math.sqrt(FLAGS.hidden_units)), 171 | name="sm_w") 172 | sm_b = tf.Variable(tf.zeros([10]), name="sm_b") 173 | 174 | # Ops: located on the worker specified with FLAGS.task_index 175 | x = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS]) 176 | y_ = tf.placeholder(tf.float32, [None, 10]) 177 | 178 | hid_lin = tf.nn.xw_plus_b(x, hid_w, hid_b) 179 | hid = tf.nn.relu(hid_lin) 180 | 181 | y = tf.nn.softmax(tf.nn.xw_plus_b(hid, sm_w, sm_b)) 182 | cross_entropy = -tf.reduce_sum(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0))) 183 | 184 | opt = tf.train.AdamOptimizer(FLAGS.learning_rate) 185 | 186 | # 同步训练模式 187 | if FLAGS.sync_replicas: 188 | if FLAGS.replicas_to_aggregate is None: 189 | replicas_to_aggregate = num_workers 190 | else: 191 | replicas_to_aggregate = FLAGS.replicas_to_aggregate 192 | 193 | opt = tf.train.SyncReplicasOptimizer( 194 | opt, 195 | replicas_to_aggregate=replicas_to_aggregate, 196 | total_num_replicas=num_workers, 197 | name="mnist_sync_replicas") 198 | 199 | train_step = opt.minimize(cross_entropy, global_step=global_step) 200 | 201 | if FLAGS.sync_replicas: 202 | local_init_op = opt.local_step_init_op 203 | if is_chief: 204 | local_init_op = opt.chief_init_op 205 | 206 | ready_for_local_init_op = opt.ready_for_local_init_op 207 | 208 | # Initial token and chief queue runners required by the sync_replicas mode 209 | chief_queue_runner = opt.get_chief_queue_runner() 210 | sync_init_op = opt.get_init_tokens_op() 211 | 212 | init_op = tf.global_variables_initializer() 213 | train_dir = tempfile.mkdtemp() 214 | 215 | if FLAGS.sync_replicas: 216 | sv = tf.train.Supervisor( 217 | is_chief=is_chief, 218 | logdir=train_dir, 219 | init_op=init_op, 220 | local_init_op=local_init_op, 221 | ready_for_local_init_op=ready_for_local_init_op, 222 | recovery_wait_secs=1, 223 | global_step=global_step) 224 | else: 225 | sv = tf.train.Supervisor( 226 | is_chief=is_chief, 227 | logdir=train_dir, 228 | init_op=init_op, 229 | recovery_wait_secs=1, 230 | global_step=global_step) 231 | 232 | sess_config = tf.ConfigProto( 233 | allow_soft_placement=True, 234 | log_device_placement=False, 235 | device_filters=["/job:ps", "/job:worker/task:%d" % FLAGS.task_index]) 236 | 237 | # The chief worker (task_index==0) session will prepare the session, 238 | # while the remaining workers will wait for the preparation to complete. 239 | if is_chief: 240 | print("Worker %d: Initializing session..." % FLAGS.task_index) 241 | else: 242 | print("Worker %d: Waiting for session to be initialized..." % 243 | FLAGS.task_index) 244 | 245 | if FLAGS.existing_servers: 246 | server_grpc_url = "grpc://" + worker_spec[FLAGS.task_index] 247 | print("Using existing server at: %s" % server_grpc_url) 248 | 249 | sess = sv.prepare_or_wait_for_session(server_grpc_url, 250 | config=sess_config) 251 | else: 252 | sess = sv.prepare_or_wait_for_session(server.target, config=sess_config) 253 | 254 | print("Worker %d: Session initialization complete." % FLAGS.task_index) 255 | 256 | if FLAGS.sync_replicas and is_chief: 257 | # Chief worker will start the chief queue runner and call the init op. 258 | sess.run(sync_init_op) 259 | sv.start_queue_runners(sess, [chief_queue_runner]) 260 | 261 | # Perform training 262 | time_begin = time.time() 263 | print("Training begins @ %f" % time_begin) 264 | 265 | # 执行过程 266 | local_step = 0 267 | while True: 268 | # Training feed 269 | batch_xs, batch_ys = mnist.train.next_batch(FLAGS.batch_size) 270 | train_feed = {x: batch_xs, y_: batch_ys} 271 | 272 | _, step = sess.run([train_step, global_step], feed_dict=train_feed) 273 | local_step += 1 274 | 275 | now = time.time() 276 | time.sleep(2) # 延迟两秒处理 277 | print("%f: Worker %d: training step %d done (global step: %d)" % 278 | (now, FLAGS.task_index, local_step, step)) 279 | 280 | if step >= FLAGS.train_steps: 281 | break 282 | 283 | time_end = time.time() 284 | print("Training ends @ %f" % time_end) 285 | training_time = time_end - time_begin 286 | print("Training elapsed time: %f s" % training_time) 287 | 288 | # Validation feed 289 | val_feed = {x: mnist.validation.images, y_: mnist.validation.labels} 290 | val_xent = sess.run(cross_entropy, feed_dict=val_feed) 291 | print("After %d training step(s), validation cross entropy = %g" % 292 | (FLAGS.train_steps, val_xent)) 293 | 294 | 295 | if __name__ == "__main__": 296 | tf.app.run() 297 | -------------------------------------------------------------------------------- /multi_gpu_train/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Makes helper libraries available in the cifar10 package.""" 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from multi_gpu_train import cifar10 22 | from multi_gpu_train import cifar10_input 23 | -------------------------------------------------------------------------------- /multi_gpu_train/cifar10.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Builds the CIFAR-10 network. 17 | 18 | Summary of available functions: 19 | 20 | # Compute input images and labels for training. If you would like to run 21 | # evaluations, use inputs() instead. 22 | inputs, labels = distorted_inputs() 23 | 24 | # Compute inference on the model inputs to make a prediction. 25 | predictions = inference(inputs) 26 | 27 | # Compute the total loss of the prediction with respect to the labels. 28 | loss = loss(predictions, labels) 29 | 30 | # Create a graph to run one step of training with respect to the loss. 31 | train_op = train(loss, global_step) 32 | """ 33 | # pylint: disable=missing-docstring 34 | from __future__ import absolute_import 35 | from __future__ import division 36 | from __future__ import print_function 37 | 38 | import os 39 | import re 40 | import sys 41 | import tarfile 42 | import sys 43 | 44 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 45 | 46 | import tensorflow as tf 47 | from six.moves import urllib 48 | 49 | from multi_gpu_train import cifar10_input 50 | 51 | FLAGS = tf.app.flags.FLAGS 52 | 53 | # Basic model parameters. 54 | tf.app.flags.DEFINE_integer('batch_size', 128, 55 | """Number of images to process in a batch.""") 56 | tf.app.flags.DEFINE_string('data_dir', './cifar10_data', 57 | """Path to the CIFAR-10 data directory.""") 58 | tf.app.flags.DEFINE_boolean('use_fp16', False, 59 | """Train the model using fp16.""") 60 | 61 | # Global constants describing the CIFAR-10 data set. 62 | IMAGE_SIZE = cifar10_input.IMAGE_SIZE 63 | NUM_CLASSES = cifar10_input.NUM_CLASSES 64 | NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN 65 | NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL 66 | 67 | # Constants describing the training process. 68 | MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. 69 | NUM_EPOCHS_PER_DECAY = 350.0 # Epochs after which learning rate decays. 70 | LEARNING_RATE_DECAY_FACTOR = 0.1 # Learning rate decay factor. 71 | INITIAL_LEARNING_RATE = 0.1 # Initial learning rate. 72 | 73 | # If a model is trained with multiple GPUs, prefix all Op names with tower_name 74 | # to differentiate the operations. Note that this prefix is removed from the 75 | # names of the summaries when visualizing a model. 76 | TOWER_NAME = 'tower' 77 | 78 | DATA_URL = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' 79 | 80 | 81 | def _activation_summary(x): 82 | """Helper to create summaries for activations. 83 | 84 | Creates a summary that provides a histogram of activations. 85 | Creates a summary that measures the sparsity of activations. 86 | 87 | Args: 88 | x: Tensor 89 | Returns: 90 | nothing 91 | """ 92 | # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training 93 | # session. This helps the clarity of presentation on tensorboard. 94 | tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name) 95 | tf.summary.histogram(tensor_name + '/activations', x) 96 | tf.summary.scalar(tensor_name + '/sparsity', 97 | tf.nn.zero_fraction(x)) 98 | 99 | 100 | def _variable_on_cpu(name, shape, initializer): 101 | """Helper to create a Variable stored on CPU memory. 102 | 103 | Args: 104 | name: name of the variable 105 | shape: list of ints 106 | initializer: initializer for Variable 107 | 108 | Returns: 109 | Variable Tensor 110 | """ 111 | with tf.device('/cpu:0'): 112 | dtype = tf.float16 if FLAGS.use_fp16 else tf.float32 113 | var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype) 114 | return var 115 | 116 | 117 | def _variable_with_weight_decay(name, shape, stddev, wd): 118 | """Helper to create an initialized Variable with weight decay. 119 | 120 | Note that the Variable is initialized with a truncated normal distribution. 121 | A weight decay is added only if one is specified. 122 | 123 | Args: 124 | name: name of the variable 125 | shape: list of ints 126 | stddev: standard deviation of a truncated Gaussian 127 | wd: add L2Loss weight decay multiplied by this float. If None, weight 128 | decay is not added for this Variable. 129 | 130 | Returns: 131 | Variable Tensor 132 | """ 133 | dtype = tf.float16 if FLAGS.use_fp16 else tf.float32 134 | var = _variable_on_cpu( 135 | name, 136 | shape, 137 | tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) 138 | if wd is not None: 139 | weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') 140 | tf.add_to_collection('losses', weight_decay) 141 | return var 142 | 143 | 144 | def distorted_inputs(): 145 | """Construct distorted input for CIFAR training using the Reader ops. 146 | 147 | Returns: 148 | images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. 149 | labels: Labels. 1D tensor of [batch_size] size. 150 | 151 | Raises: 152 | ValueError: If no data_dir 153 | """ 154 | if not FLAGS.data_dir: 155 | raise ValueError('Please supply a data_dir') 156 | data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin') 157 | images, labels = cifar10_input.distorted_inputs(data_dir=data_dir, 158 | batch_size=FLAGS.batch_size) 159 | if FLAGS.use_fp16: 160 | images = tf.cast(images, tf.float16) 161 | labels = tf.cast(labels, tf.float16) 162 | return images, labels 163 | 164 | 165 | def inputs(eval_data): 166 | """Construct input for CIFAR evaluation using the Reader ops. 167 | 168 | Args: 169 | eval_data: bool, indicating if one should use the train or eval data set. 170 | 171 | Returns: 172 | images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. 173 | labels: Labels. 1D tensor of [batch_size] size. 174 | 175 | Raises: 176 | ValueError: If no data_dir 177 | """ 178 | if not FLAGS.data_dir: 179 | raise ValueError('Please supply a data_dir') 180 | data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin') 181 | images, labels = cifar10_input.inputs(eval_data=eval_data, 182 | data_dir=data_dir, 183 | batch_size=FLAGS.batch_size) 184 | if FLAGS.use_fp16: 185 | images = tf.cast(images, tf.float16) 186 | labels = tf.cast(labels, tf.float16) 187 | return images, labels 188 | 189 | 190 | def inference(images): 191 | """Build the CIFAR-10 model. 192 | 193 | Args: 194 | images: Images returned from distorted_inputs() or inputs(). 195 | 196 | Returns: 197 | Logits. 198 | """ 199 | # We instantiate all variables using tf.get_variable() instead of 200 | # tf.Variable() in order to share variables across multiple GPU training runs. 201 | # If we only ran this model on a single GPU, we could simplify this function 202 | # by replacing all instances of tf.get_variable() with tf.Variable(). 203 | # 204 | # conv1 205 | with tf.variable_scope('conv1') as scope: 206 | kernel = _variable_with_weight_decay('weights', 207 | shape=[5, 5, 3, 64], 208 | stddev=5e-2, 209 | wd=0.0) 210 | conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME') 211 | biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.0)) 212 | pre_activation = tf.nn.bias_add(conv, biases) 213 | conv1 = tf.nn.relu(pre_activation, name=scope.name) 214 | _activation_summary(conv1) 215 | 216 | # pool1 217 | pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], 218 | padding='SAME', name='pool1') 219 | # norm1 220 | norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, 221 | name='norm1') 222 | 223 | # conv2 224 | with tf.variable_scope('conv2') as scope: 225 | kernel = _variable_with_weight_decay('weights', 226 | shape=[5, 5, 64, 64], 227 | stddev=5e-2, 228 | wd=0.0) 229 | conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME') 230 | biases = _variable_on_cpu('biases', [64], tf.constant_initializer(0.1)) 231 | pre_activation = tf.nn.bias_add(conv, biases) 232 | conv2 = tf.nn.relu(pre_activation, name=scope.name) 233 | _activation_summary(conv2) 234 | 235 | # norm2 236 | norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, 237 | name='norm2') 238 | # pool2 239 | pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], 240 | strides=[1, 2, 2, 1], padding='SAME', name='pool2') 241 | 242 | # local3 243 | with tf.variable_scope('local3') as scope: 244 | # Move everything into depth so we can perform a single matrix multiply. 245 | reshape = tf.reshape(pool2, [FLAGS.batch_size, -1]) 246 | dim = reshape.get_shape()[1].value 247 | weights = _variable_with_weight_decay('weights', shape=[dim, 384], 248 | stddev=0.04, wd=0.004) 249 | biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1)) 250 | local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name) 251 | _activation_summary(local3) 252 | 253 | # local4 254 | with tf.variable_scope('local4') as scope: 255 | weights = _variable_with_weight_decay('weights', shape=[384, 192], 256 | stddev=0.04, wd=0.004) 257 | biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1)) 258 | local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name) 259 | _activation_summary(local4) 260 | 261 | # linear layer(WX + b), 262 | # We don't apply softmax here because 263 | # tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits 264 | # and performs the softmax internally for efficiency. 265 | with tf.variable_scope('softmax_linear') as scope: 266 | weights = _variable_with_weight_decay('weights', [192, NUM_CLASSES], 267 | stddev=1 / 192.0, wd=0.0) 268 | biases = _variable_on_cpu('biases', [NUM_CLASSES], 269 | tf.constant_initializer(0.0)) 270 | softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name) 271 | _activation_summary(softmax_linear) 272 | 273 | return softmax_linear 274 | 275 | 276 | def loss(logits, labels): 277 | """Add L2Loss to all the trainable variables. 278 | 279 | Add summary for "Loss" and "Loss/avg". 280 | Args: 281 | logits: Logits from inference(). 282 | labels: Labels from distorted_inputs or inputs(). 1-D tensor 283 | of shape [batch_size] 284 | 285 | Returns: 286 | Loss tensor of type float. 287 | """ 288 | # Calculate the average cross entropy loss across the batch. 289 | labels = tf.cast(labels, tf.int64) 290 | cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( 291 | labels=labels, logits=logits, name='cross_entropy_per_example') 292 | cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') 293 | tf.add_to_collection('losses', cross_entropy_mean) 294 | 295 | # The total loss is defined as the cross entropy loss plus all of the weight 296 | # decay terms (L2 loss). 297 | return tf.add_n(tf.get_collection('losses'), name='total_loss') 298 | 299 | 300 | def _add_loss_summaries(total_loss): 301 | """Add summaries for losses in CIFAR-10 model. 302 | 303 | Generates moving average for all losses and associated summaries for 304 | visualizing the performance of the network. 305 | 306 | Args: 307 | total_loss: Total loss from loss(). 308 | Returns: 309 | loss_averages_op: op for generating moving averages of losses. 310 | """ 311 | # Compute the moving average of all individual losses and the total loss. 312 | loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg') 313 | losses = tf.get_collection('losses') 314 | loss_averages_op = loss_averages.apply(losses + [total_loss]) 315 | 316 | # Attach a scalar summary to all individual losses and the total loss; do the 317 | # same for the averaged version of the losses. 318 | for l in losses + [total_loss]: 319 | # Name each loss as '(raw)' and name the moving average version of the loss 320 | # as the original loss name. 321 | tf.summary.scalar(l.op.name + ' (raw)', l) 322 | tf.summary.scalar(l.op.name, loss_averages.average(l)) 323 | 324 | return loss_averages_op 325 | 326 | 327 | def train(total_loss, global_step): 328 | """Train CIFAR-10 model. 329 | 330 | Create an optimizer and apply to all trainable variables. Add moving 331 | average for all trainable variables. 332 | 333 | Args: 334 | total_loss: Total loss from loss(). 335 | global_step: Integer Variable counting the number of training steps 336 | processed. 337 | Returns: 338 | train_op: op for training. 339 | """ 340 | # Variables that affect learning rate. 341 | num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / FLAGS.batch_size 342 | decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) 343 | 344 | # Decay the learning rate exponentially based on the number of steps. 345 | lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, 346 | global_step, 347 | decay_steps, 348 | LEARNING_RATE_DECAY_FACTOR, 349 | staircase=True) 350 | tf.summary.scalar('learning_rate', lr) 351 | 352 | # Generate moving averages of all losses and associated summaries. 353 | loss_averages_op = _add_loss_summaries(total_loss) 354 | 355 | # Compute gradients. 356 | with tf.control_dependencies([loss_averages_op]): 357 | opt = tf.train.GradientDescentOptimizer(lr) 358 | grads = opt.compute_gradients(total_loss) 359 | 360 | # Apply gradients. 361 | apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) 362 | 363 | # Add histograms for trainable variables. 364 | for var in tf.trainable_variables(): 365 | tf.summary.histogram(var.op.name, var) 366 | 367 | # Add histograms for gradients. 368 | for grad, var in grads: 369 | if grad is not None: 370 | tf.summary.histogram(var.op.name + '/gradients', grad) 371 | 372 | # Track the moving averages of all trainable variables. 373 | variable_averages = tf.train.ExponentialMovingAverage( 374 | MOVING_AVERAGE_DECAY, global_step) 375 | variables_averages_op = variable_averages.apply(tf.trainable_variables()) 376 | 377 | with tf.control_dependencies([apply_gradient_op, variables_averages_op]): 378 | train_op = tf.no_op(name='train') 379 | 380 | return train_op 381 | 382 | 383 | def maybe_download_and_extract(): 384 | """Download and extract the tarball from Alex's website.""" 385 | dest_directory = FLAGS.data_dir 386 | if not os.path.exists(dest_directory): 387 | os.makedirs(dest_directory) 388 | filename = DATA_URL.split('/')[-1] 389 | filepath = os.path.join(dest_directory, filename) 390 | if not os.path.exists(filepath): 391 | def _progress(count, block_size, total_size): 392 | sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename, 393 | float(count * block_size) / float(total_size) * 100.0)) 394 | sys.stdout.flush() 395 | 396 | filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) 397 | print() 398 | statinfo = os.stat(filepath) 399 | print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') 400 | extracted_dir_path = os.path.join(dest_directory, 'cifar-10-batches-bin') 401 | if not os.path.exists(extracted_dir_path): 402 | tarfile.open(filepath, 'r:gz').extractall(dest_directory) 403 | -------------------------------------------------------------------------------- /multi_gpu_train/cifar10_input.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Routine for decoding the CIFAR-10 binary file format.""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import os 23 | 24 | from six.moves import xrange # pylint: disable=redefined-builtin 25 | import tensorflow as tf 26 | 27 | # Process images of this size. Note that this differs from the original CIFAR 28 | # image size of 32 x 32. If one alters this number, then the entire model 29 | # architecture will change and any model would need to be retrained. 30 | IMAGE_SIZE = 24 31 | 32 | # Global constants describing the CIFAR-10 data set. 33 | NUM_CLASSES = 10 34 | NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 35 | NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000 36 | 37 | 38 | def read_cifar10(filename_queue): 39 | """Reads and parses examples from CIFAR10 data files. 40 | 41 | Recommendation: if you want N-way read parallelism, call this function 42 | N times. This will give you N independent Readers reading different 43 | files & positions within those files, which will give better mixing of 44 | examples. 45 | 46 | Args: 47 | filename_queue: A queue of strings with the filenames to read from. 48 | 49 | Returns: 50 | An object representing a single example, with the following fields: 51 | height: number of rows in the result (32) 52 | width: number of columns in the result (32) 53 | depth: number of color channels in the result (3) 54 | key: a scalar string Tensor describing the filename & record number 55 | for this example. 56 | label: an int32 Tensor with the label in the range 0..9. 57 | uint8image: a [height, width, depth] uint8 Tensor with the image data 58 | """ 59 | 60 | class CIFAR10Record(object): 61 | pass 62 | result = CIFAR10Record() 63 | 64 | # Dimensions of the images in the CIFAR-10 dataset. 65 | # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the 66 | # input format. 67 | label_bytes = 1 # 2 for CIFAR-100 68 | result.height = 32 69 | result.width = 32 70 | result.depth = 3 71 | image_bytes = result.height * result.width * result.depth 72 | # Every record consists of a label followed by the image, with a 73 | # fixed number of bytes for each. 74 | record_bytes = label_bytes + image_bytes 75 | 76 | # Read a record, getting filenames from the filename_queue. No 77 | # header or footer in the CIFAR-10 format, so we leave header_bytes 78 | # and footer_bytes at their default of 0. 79 | reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) 80 | result.key, value = reader.read(filename_queue) 81 | 82 | # Convert from a string to a vector of uint8 that is record_bytes long. 83 | record_bytes = tf.decode_raw(value, tf.uint8) 84 | 85 | # The first bytes represent the label, which we convert from uint8->int32. 86 | result.label = tf.cast( 87 | tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) 88 | 89 | # The remaining bytes after the label represent the image, which we reshape 90 | # from [depth * height * width] to [depth, height, width]. 91 | depth_major = tf.reshape( 92 | tf.strided_slice(record_bytes, [label_bytes], 93 | [label_bytes + image_bytes]), 94 | [result.depth, result.height, result.width]) 95 | # Convert from [depth, height, width] to [height, width, depth]. 96 | result.uint8image = tf.transpose(depth_major, [1, 2, 0]) 97 | 98 | return result 99 | 100 | 101 | def _generate_image_and_label_batch(image, label, min_queue_examples, 102 | batch_size, shuffle): 103 | """Construct a queued batch of images and labels. 104 | 105 | Args: 106 | image: 3-D Tensor of [height, width, 3] of type.float32. 107 | label: 1-D Tensor of type.int32 108 | min_queue_examples: int32, minimum number of samples to retain 109 | in the queue that provides of batches of examples. 110 | batch_size: Number of images per batch. 111 | shuffle: boolean indicating whether to use a shuffling queue. 112 | 113 | Returns: 114 | images: Images. 4D tensor of [batch_size, height, width, 3] size. 115 | labels: Labels. 1D tensor of [batch_size] size. 116 | """ 117 | # Create a queue that shuffles the examples, and then 118 | # read 'batch_size' images + labels from the example queue. 119 | num_preprocess_threads = 16 120 | if shuffle: 121 | images, label_batch = tf.train.shuffle_batch( 122 | [image, label], 123 | batch_size=batch_size, 124 | num_threads=num_preprocess_threads, 125 | capacity=min_queue_examples + 3 * batch_size, 126 | min_after_dequeue=min_queue_examples) 127 | else: 128 | images, label_batch = tf.train.batch( 129 | [image, label], 130 | batch_size=batch_size, 131 | num_threads=num_preprocess_threads, 132 | capacity=min_queue_examples + 3 * batch_size) 133 | 134 | # Display the training images in the visualizer. 135 | tf.summary.image('images', images) 136 | 137 | return images, tf.reshape(label_batch, [batch_size]) 138 | 139 | 140 | def distorted_inputs(data_dir, batch_size): 141 | """Construct distorted input for CIFAR training using the Reader ops. 142 | 143 | Args: 144 | data_dir: Path to the CIFAR-10 data directory. 145 | batch_size: Number of images per batch. 146 | 147 | Returns: 148 | images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. 149 | labels: Labels. 1D tensor of [batch_size] size. 150 | """ 151 | filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i) 152 | for i in xrange(1, 6)] 153 | for f in filenames: 154 | if not tf.gfile.Exists(f): 155 | raise ValueError('Failed to find file: ' + f) 156 | 157 | # Create a queue that produces the filenames to read. 158 | filename_queue = tf.train.string_input_producer(filenames) 159 | 160 | # Read examples from files in the filename queue. 161 | read_input = read_cifar10(filename_queue) 162 | reshaped_image = tf.cast(read_input.uint8image, tf.float32) 163 | 164 | height = IMAGE_SIZE 165 | width = IMAGE_SIZE 166 | 167 | # Image processing for training the network. Note the many random 168 | # distortions applied to the image. 169 | 170 | # Randomly crop a [height, width] section of the image. 171 | distorted_image = tf.random_crop(reshaped_image, [height, width, 3]) 172 | 173 | # Randomly flip the image horizontally. 174 | distorted_image = tf.image.random_flip_left_right(distorted_image) 175 | 176 | # Because these operations are not commutative, consider randomizing 177 | # the order their operation. 178 | # NOTE: since per_image_standardization zeros the mean and makes 179 | # the stddev unit, this likely has no effect see tensorflow#1458. 180 | distorted_image = tf.image.random_brightness(distorted_image, 181 | max_delta=63) 182 | distorted_image = tf.image.random_contrast(distorted_image, 183 | lower=0.2, upper=1.8) 184 | 185 | # Subtract off the mean and divide by the variance of the pixels. 186 | float_image = tf.image.per_image_standardization(distorted_image) 187 | 188 | # Set the shapes of tensors. 189 | float_image.set_shape([height, width, 3]) 190 | read_input.label.set_shape([1]) 191 | 192 | # Ensure that the random shuffling has good mixing properties. 193 | min_fraction_of_examples_in_queue = 0.4 194 | min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * 195 | min_fraction_of_examples_in_queue) 196 | print ('Filling queue with %d CIFAR images before starting to train. ' 197 | 'This will take a few minutes.' % min_queue_examples) 198 | 199 | # Generate a batch of images and labels by building up a queue of examples. 200 | return _generate_image_and_label_batch(float_image, read_input.label, 201 | min_queue_examples, batch_size, 202 | shuffle=True) 203 | 204 | 205 | def inputs(eval_data, data_dir, batch_size): 206 | """Construct input for CIFAR evaluation using the Reader ops. 207 | 208 | Args: 209 | eval_data: bool, indicating if one should use the train or eval data set. 210 | data_dir: Path to the CIFAR-10 data directory. 211 | batch_size: Number of images per batch. 212 | 213 | Returns: 214 | images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. 215 | labels: Labels. 1D tensor of [batch_size] size. 216 | """ 217 | if not eval_data: 218 | filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i) 219 | for i in xrange(1, 6)] 220 | num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN 221 | else: 222 | filenames = [os.path.join(data_dir, 'test_batch.bin')] 223 | num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_EVAL 224 | 225 | for f in filenames: 226 | if not tf.gfile.Exists(f): 227 | raise ValueError('Failed to find file: ' + f) 228 | 229 | # Create a queue that produces the filenames to read. 230 | filename_queue = tf.train.string_input_producer(filenames) 231 | 232 | # Read examples from files in the filename queue. 233 | read_input = read_cifar10(filename_queue) 234 | reshaped_image = tf.cast(read_input.uint8image, tf.float32) 235 | 236 | height = IMAGE_SIZE 237 | width = IMAGE_SIZE 238 | 239 | # Image processing for evaluation. 240 | # Crop the central [height, width] of the image. 241 | resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, 242 | height, width) 243 | 244 | # Subtract off the mean and divide by the variance of the pixels. 245 | float_image = tf.image.per_image_standardization(resized_image) 246 | 247 | # Set the shapes of tensors. 248 | float_image.set_shape([height, width, 3]) 249 | read_input.label.set_shape([1]) 250 | 251 | # Ensure that the random shuffling has good mixing properties. 252 | min_fraction_of_examples_in_queue = 0.4 253 | min_queue_examples = int(num_examples_per_epoch * 254 | min_fraction_of_examples_in_queue) 255 | 256 | # Generate a batch of images and labels by building up a queue of examples. 257 | return _generate_image_and_label_batch(float_image, read_input.label, 258 | min_queue_examples, batch_size, 259 | shuffle=False) 260 | -------------------------------------------------------------------------------- /multi_gpu_train/cifar10_multi_gpu_train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # ============================================================================== 20 | 21 | """A binary to train CIFAR-10 using multiple GPUs with synchronous updates. 22 | 23 | Accuracy: 24 | cifar10_multi_gpu_train.py achieves ~86% accuracy after 100K steps (256 25 | epochs of data) as judged by cifar10_eval.py. 26 | 27 | Speed: With batch_size 128. 28 | 29 | System | Step Time (sec/batch) | Accuracy 30 | -------------------------------------------------------------------- 31 | 1 Tesla K20m | 0.35-0.60 | ~86% at 60K steps (5 hours) 32 | 1 Tesla K40m | 0.25-0.35 | ~86% at 100K steps (4 hours) 33 | 2 Tesla K20m | 0.13-0.20 | ~84% at 30K steps (2.5 hours) 34 | 3 Tesla K20m | 0.13-0.18 | ~84% at 30K steps 35 | 4 Tesla K20m | ~0.10 | ~84% at 30K steps 36 | 37 | Usage: 38 | Please see the tutorial and website for how to download the CIFAR-10 39 | data set, compile the program and train the model. 40 | 41 | http://tensorflow.org/tutorials/deep_cnn/ 42 | """ 43 | from __future__ import absolute_import 44 | from __future__ import division 45 | from __future__ import print_function 46 | 47 | import os.path 48 | import re 49 | import sys 50 | import time 51 | 52 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 53 | 54 | from datetime import datetime 55 | 56 | import numpy as np 57 | import tensorflow as tf 58 | from six.moves import xrange # pylint: disable=redefined-builtin 59 | 60 | from multi_gpu_train import cifar10 61 | 62 | FLAGS = tf.app.flags.FLAGS 63 | 64 | tf.app.flags.DEFINE_string('train_dir', '/tmp/cifar10_train', 65 | """Directory where to write event logs """ 66 | """and checkpoint.""") 67 | tf.app.flags.DEFINE_integer('max_steps', 1000000, 68 | """Number of batches to run.""") 69 | tf.app.flags.DEFINE_integer('num_gpus', 1, 70 | """How many GPUs to use.""") 71 | tf.app.flags.DEFINE_boolean('log_device_placement', False, 72 | """Whether to log device placement.""") 73 | 74 | 75 | def tower_loss(scope, images, labels): 76 | """Calculate the total loss on a single tower running the CIFAR model. 77 | 78 | Args: 79 | scope: unique prefix string identifying the CIFAR tower, e.g. 'tower_0' 80 | images: Images. 4D tensor of shape [batch_size, height, width, 3]. 81 | labels: Labels. 1D tensor of shape [batch_size]. 82 | 83 | Returns: 84 | Tensor of shape [] containing the total loss for a batch of data 85 | """ 86 | 87 | # Build inference Graph. 88 | logits = cifar10.inference(images) 89 | 90 | # Build the portion of the Graph calculating the losses. Note that we will 91 | # assemble the total_loss using a custom function below. 92 | _ = cifar10.loss(logits, labels) 93 | 94 | # Assemble all of the losses for the current tower only. 95 | losses = tf.get_collection('losses', scope) 96 | 97 | # Calculate the total loss for the current tower. 98 | total_loss = tf.add_n(losses, name='total_loss') 99 | 100 | # Attach a scalar summary to all individual losses and the total loss; do the 101 | # same for the averaged version of the losses. 102 | for l in losses + [total_loss]: 103 | # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training 104 | # session. This helps the clarity of presentation on tensorboard. 105 | loss_name = re.sub('%s_[0-9]*/' % cifar10.TOWER_NAME, '', l.op.name) 106 | tf.summary.scalar(loss_name, l) 107 | 108 | return total_loss 109 | 110 | 111 | def average_gradients(tower_grads): 112 | """Calculate the average gradient for each shared variable across all towers. 113 | 114 | Note that this function provides a synchronization point across all towers. 115 | 116 | Args: 117 | tower_grads: List of lists of (gradient, variable) tuples. The outer list 118 | is over individual gradients. The inner list is over the gradient 119 | calculation for each tower. 120 | Returns: 121 | List of pairs of (gradient, variable) where the gradient has been averaged 122 | across all towers. 123 | """ 124 | average_grads = [] 125 | for grad_and_vars in zip(*tower_grads): 126 | # Note that each grad_and_vars looks like the following: 127 | # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) 128 | grads = [] 129 | for g, _ in grad_and_vars: 130 | # Add 0 dimension to the gradients to represent the tower. 131 | expanded_g = tf.expand_dims(g, 0) 132 | 133 | # Append on a 'tower' dimension which we will average over below. 134 | grads.append(expanded_g) 135 | 136 | # Average over the 'tower' dimension. 137 | grad = tf.concat(axis=0, values=grads) 138 | grad = tf.reduce_mean(grad, 0) 139 | 140 | # Keep in mind that the Variables are redundant because they are shared 141 | # across towers. So .. we will just return the first tower's pointer to 142 | # the Variable. 143 | v = grad_and_vars[0][1] 144 | grad_and_var = (grad, v) 145 | average_grads.append(grad_and_var) 146 | return average_grads 147 | 148 | 149 | def train(): 150 | """Train CIFAR-10 for a number of steps.""" 151 | with tf.Graph().as_default(), tf.device('/cpu:0'): # 默认使用默认CPU0 152 | # Create a variable to count the number of train() calls. This equals the 153 | # number of batches processed * FLAGS.num_gpus. 154 | # 参数: trainable是False，不用训练，全局步数就是global_step，默认设置。 155 | global_step = tf.get_variable( 156 | 'global_step', [], 157 | initializer=tf.constant_initializer(0), trainable=False) 158 | 159 | # Calculate the learning rate schedule. 160 | # 每个批次的训练数 161 | num_batches_per_epoch = (cifar10.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / 162 | FLAGS.batch_size) # batch_size是128，50000 / 128=390.625 163 | decay_steps = int(num_batches_per_epoch * cifar10.NUM_EPOCHS_PER_DECAY) # 每个批次需要衰减的次数 164 | 165 | # Decay the learning rate exponentially based on the number of steps. 166 | lr = tf.train.exponential_decay(cifar10.INITIAL_LEARNING_RATE, 167 | global_step, 168 | decay_steps, 169 | cifar10.LEARNING_RATE_DECAY_FACTOR, 170 | staircase=True) # 计算学习率，lr=Learning Rate 171 | 172 | # Create an optimizer that performs gradient descent. 173 | opt = tf.train.GradientDescentOptimizer(lr) # 参数是学习率 174 | 175 | # Get images and labels for CIFAR-10. 176 | images, labels = cifar10.distorted_inputs() # 获取图片资源和标签 177 | batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue( 178 | [images, labels], capacity=2 * FLAGS.num_gpus) # 使用预加载的队列 179 | 180 | # Calculate the gradients for each model tower. 181 | tower_grads = [] 182 | with tf.variable_scope(tf.get_variable_scope()): # 变量的名称 183 | for i in xrange(FLAGS.num_gpus): # 创建GUP的循环 184 | with tf.device('/gpu:%d' % i): # 指定GPU 185 | with tf.name_scope('%s_%d' % (cifar10.TOWER_NAME, i)) as scope: 186 | # 含有几个GPU执行几个，没有不执行 187 | print('running: %s_%d' % (cifar10.TOWER_NAME, i)) 188 | # Dequeues one batch for the GPU 189 | image_batch, label_batch = batch_queue.dequeue() 190 | # Calculate the loss for one tower of the CIFAR model. This function 191 | # constructs the entire CIFAR model but shares the variables across 192 | # all towers. 193 | loss = tower_loss(scope, image_batch, label_batch) # 获得损失函数 194 | 195 | # Reuse variables for the next tower. 196 | tf.get_variable_scope().reuse_variables() # 重用变量 197 | 198 | # Retain the summaries from the final tower. 199 | summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope) # 创建存储信息 200 | 201 | # Calculate the gradients for the batch of data on this CIFAR tower. 202 | grads = opt.compute_gradients(loss) # 计算梯度 203 | 204 | # Keep track of the gradients across all towers. 205 | tower_grads.append(grads) # 添加梯度，tower_grads是外部变量，会存储全部梯度信息 206 | 207 | # We must calculate the mean of each gradient. Note that this is the 208 | # synchronization point across all towers. 209 | grads = average_gradients(tower_grads) # 求梯度的平均值 210 | 211 | # Add a summary to track the learning rate. 212 | summaries.append(tf.summary.scalar('learning_rate', lr)) 213 | 214 | # Add histograms for gradients. 215 | for grad, var in grads: # 存储数据 216 | if grad is not None: 217 | summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad)) 218 | 219 | # Apply the gradients to adjust the shared variables. 220 | apply_gradient_op = opt.apply_gradients(grads, global_step=global_step) # 将梯度应用于变量 221 | 222 | # Add histograms for trainable variables. 223 | for var in tf.trainable_variables(): # 存储数据 224 | summaries.append(tf.summary.histogram(var.op.name, var)) 225 | 226 | # Track the moving averages of all trainable variables. 227 | variable_averages = tf.train.ExponentialMovingAverage( 228 | cifar10.MOVING_AVERAGE_DECAY, global_step) # 求变量的均值 229 | variables_averages_op = variable_averages.apply(tf.trainable_variables()) # 将变量的均值应用于操作 230 | 231 | # Group all updates to into a single train op. 232 | train_op = tf.group(apply_gradient_op, variables_averages_op) # 训练操作 233 | 234 | # Create a saver. 235 | saver = tf.train.Saver(tf.global_variables()) # 创建变量存储器 236 | 237 | # Build the summary operation from the last tower summaries. 238 | summary_op = tf.summary.merge(summaries) # 合并统计数据 239 | 240 | # Build an initialization operation to run below. 241 | init = tf.global_variables_initializer() 242 | 243 | # Start running operations on the Graph. allow_soft_placement must be set to 244 | # True to build towers on GPU, as some of the ops do not have GPU 245 | # implementations. 246 | sess = tf.Session(config=tf.ConfigProto( 247 | allow_soft_placement=True, 248 | log_device_placement=FLAGS.log_device_placement)) 249 | sess.run(init) 250 | 251 | # Start the queue runners. 252 | tf.train.start_queue_runners(sess=sess) 253 | 254 | summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) # 写入summary的地址 255 | 256 | for step in xrange(FLAGS.max_steps): 257 | start_time = time.time() 258 | _, loss_value = sess.run([train_op, loss]) # 训练并获取单词的损失值。 259 | duration = time.time() - start_time 260 | 261 | assert not np.isnan(loss_value), 'Model diverged with loss = NaN' 262 | 263 | if step % 10 == 0: 264 | num_examples_per_step = FLAGS.batch_size * FLAGS.num_gpus 265 | examples_per_sec = num_examples_per_step / duration 266 | sec_per_batch = duration / FLAGS.num_gpus 267 | 268 | format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 269 | 'sec/batch)') 270 | print(format_str % (datetime.now(), step, loss_value, 271 | examples_per_sec, sec_per_batch)) 272 | 273 | if step % 100 == 0: 274 | summary_str = sess.run(summary_op) 275 | summary_writer.add_summary(summary_str, step) 276 | 277 | # Save the model checkpoint periodically. 278 | if step % 1000 == 0 or (step + 1) == FLAGS.max_steps: 279 | checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt') 280 | saver.save(sess, checkpoint_path, global_step=step) 281 | 282 | 283 | def main(argv=None): # pylint: disable=unused-argument 284 | cifar10.maybe_download_and_extract() # 下载数据 285 | 286 | # 目录处理的标准流程，使用tf.gfile模块 287 | if tf.gfile.Exists(FLAGS.train_dir): # 如果存在已有的训练数据 288 | tf.gfile.DeleteRecursively(FLAGS.train_dir) # 则递归删除 289 | tf.gfile.MakeDirs(FLAGS.train_dir) # 新建目录 290 | 291 | train() # 核心方法，训练 292 | 293 | 294 | if __name__ == '__main__': 295 | tf.app.run() 296 | -------------------------------------------------------------------------------- /multi_gpu_train/shape_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | var_1 = tf.get_variable(name='var_1', shape=[], initializer=tf.constant_initializer(0), trainable=False) 9 | ph_1 = tf.placeholder(name='ph_1', shape=(1, None), dtype=tf.float32) 10 | 11 | init = tf.global_variables_initializer() 12 | sess = tf.InteractiveSession() 13 | sess.run(init) 14 | rand_array = [[1, 2, 3]] 15 | 16 | print sess.run(var_1, feed_dict={var_1: 2}) # 测试变量 17 | print sess.run(ph_1, feed_dict={ph_1: [[1, 2, 3]]}) # 测试PlaceHolder 18 | -------------------------------------------------------------------------------- /multi_gpu_train/show_gpu_count.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | from tensorflow.python.client import device_lib 7 | 8 | 9 | def get_available_gpus(): 10 | """ 11 | 查看GPU的命令：nvidia-smi 12 | 查看被占用的情况：ps aux | grep PID 13 | :return: GPU个数 14 | """ 15 | local_device_protos = device_lib.list_local_devices() 16 | print "all: %s" % [x.name for x in local_device_protos] 17 | print "gpu: %s" % [x.name for x in local_device_protos if x.device_type == 'GPU'] 18 | 19 | 20 | get_available_gpus() 21 | -------------------------------------------------------------------------------- /multi_layer_perception/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /multi_layer_perception/mlp_mnist.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | """ 6 | 三层神经网络，输入层、隐含层、输出层 7 | """ 8 | 9 | import tensorflow as tf 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | mnist = input_data.read_data_sets("../MNIST_data/", one_hot=True) # Data Set 13 | sess = tf.InteractiveSession() # Session 14 | 15 | in_units = 784 # input neuron dimen 16 | h1_units = 300 # hide level neuron dimen 17 | 18 | # hide level para 19 | # truncated_normal 重新选择超过两个标准差的值 20 | # 矩阵用大写字母，向量用小写字母 21 | W1 = tf.Variable(tf.truncated_normal([in_units, h1_units], stddev=0.1)) 22 | b1 = tf.Variable(tf.zeros([h1_units])) 23 | 24 | # output para 25 | W2 = tf.Variable(tf.zeros([h1_units, 10])) 26 | b2 = tf.Variable(tf.zeros([10])) 27 | 28 | x = tf.placeholder(tf.float32, [None, in_units]) # 任意个in_units维的数 29 | keep_prob = tf.placeholder(tf.float32) # dropout的保留比率 30 | 31 | hidden1 = tf.nn.relu(tf.matmul(x, W1) + b1) # 隐含层，校正线性单元：Rectified Linear Unit 32 | hidden1_drop = tf.nn.dropout(hidden1, keep_prob) # 隐含层的dropout 33 | y = tf.nn.softmax(tf.matmul(hidden1, W2) + b2) # 输出层 34 | 35 | y_ = tf.placeholder(tf.float32, [None, 10]) # ground truth 36 | cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) 37 | train_step = tf.train.AdagradOptimizer(0.3).minimize(cross_entropy) 38 | 39 | tf.global_variables_initializer().run() # 初始化全部变量 40 | 41 | for i in range(3000): 42 | batch_xs, batch_ys = mnist.train.next_batch(100) # 随机采样 43 | train_step.run({x: batch_xs, y_: batch_ys, keep_prob: 0.75}) # Feed数据，并且训练 44 | 45 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 46 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 47 | 48 | print (accuracy.eval({x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})) # 评估数据 49 | """ 50 | 输出结果：0.9811 51 | """ -------------------------------------------------------------------------------- /multi_layer_perception/test_of_dropout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | in_mat = [2., 3., 2., 1., 4., 2., 3.] 9 | keep_prop = 0.4 10 | print tf.Session().run(tf.nn.dropout(in_mat, keep_prop)) 11 | """ 12 | 输出：[ 5. 7.5 0. 2.5 10. 0. 7.5] 13 | keep_prop 随机保留40%的数据，并且将所有值增加2.5倍，2.5*0.4=1； 14 | 如果keep_prop是0.5，则所有值增加2倍； 15 | """ 16 | -------------------------------------------------------------------------------- /multi_layer_perception/test_of_relu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | in_mat = [2, 3, 0, -1, -2] 9 | relu = tf.nn.relu(in_mat) 10 | print tf.Session().run(relu) 11 | # 输出 [2 3 0 0 0] 12 | # 将全部负数都转换为0 13 | -------------------------------------------------------------------------------- /project_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 常用方法 5 | # Created by C.L.Wang 6 | 7 | from __future__ import absolute_import 8 | 9 | import collections 10 | import os 11 | import random 12 | import shutil 13 | import sys 14 | import time 15 | from datetime import timedelta, datetime 16 | 17 | reload(sys) # 重置系统参数 18 | sys.setdefaultencoding('utf8') # 避免编码错误 19 | 20 | from dateutil.relativedelta import relativedelta 21 | 22 | p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 23 | if p not in sys.path: 24 | sys.path.append(p) 25 | 26 | import glob 27 | import json 28 | import os 29 | import re 30 | import numpy as np 31 | import operator 32 | 33 | from itertools import izip 34 | 35 | FORMAT_DATE = '%Y%m%d' 36 | FORMAT_DATE_2 = '%Y-%m-%d' 37 | FORMAT_DATE_3 = '%Y%m%d%H%M%S' 38 | 39 | 40 | def datetime_to_str(date, date_format=FORMAT_DATE): 41 | return date.strftime(date_format) 42 | 43 | 44 | def str_to_datetime(date_str, date_format=FORMAT_DATE): 45 | date = time.strptime(date_str, date_format) 46 | return datetime(*date[:6]) 47 | 48 | 49 | def get_next_half_year(): 50 | """ 51 | 当前时间的半年前 52 | :return: 半年时间 53 | """ 54 | n_days = datetime.now() - timedelta(days=178) 55 | return n_days.strftime('%Y-%m-%d') 56 | 57 | 58 | def get_person_age(born_date, is_log=False): 59 | """ 60 | 根据出生日期, 获取年龄 61 | :param born_date: 出生日期 62 | :param is_log: 日志显示 63 | :return: 异常返回-1, 其他正常 64 | """ 65 | try: 66 | if is_log: 67 | print "当前时间: %s, 出生日期: %s" % (datetime.now(), datetime.fromtimestamp(born_date / 1000.0)) 68 | time_data = relativedelta(datetime.now(), datetime.fromtimestamp(born_date / 1000.0)) 69 | years = float(time_data.years) 70 | months = float(time_data.months) 71 | age = (years * 12.0 + months) / 12.0 72 | if age < 0: 73 | print "获取年龄异常: %s" % age 74 | return -1.0 75 | else: 76 | return age 77 | except Exception as e: 78 | print "获取年龄异常: %s" % e 79 | return -1.0 80 | 81 | 82 | def timestr_2_timestamp(time_str): 83 | """ 84 | 时间字符串转换为毫秒 85 | :param time_str: 时间字符串, 2017-10-11 86 | :return: 毫秒, 如1443715200000 87 | """ 88 | return int(time.mktime(datetime.strptime(time_str, "%Y-%m-%d").timetuple()) * 1000) 89 | 90 | 91 | def create_folder(atp_out_dir): 92 | """ 93 | 创建文件夹 94 | :param atp_out_dir: 文件夹 95 | :return: 96 | """ 97 | if os.path.exists(atp_out_dir): 98 | shutil.rmtree(atp_out_dir) 99 | print '文件夹 "%s" 存在，删除文件夹。' % atp_out_dir 100 | 101 | if not os.path.exists(atp_out_dir): 102 | os.makedirs(atp_out_dir) 103 | print '文件夹 "%s" 不存在，创建文件夹。' % atp_out_dir 104 | 105 | 106 | def create_file(file_name): 107 | """ 108 | 创建文件 109 | :param file_name: 文件名 110 | :return: None 111 | """ 112 | if os.path.exists(file_name): 113 | print "文件存在，删除文件：%s" % file_name 114 | os.remove(file_name) # 删除已有文件 115 | if not os.path.exists(file_name): 116 | print "文件不存在，创建文件：%s" % file_name 117 | open(file_name, 'a').close() 118 | 119 | 120 | def remove_punctuation(line): 121 | """ 122 | 去除所有半角全角符号，只留字母、数字、中文 123 | :param line: 124 | :return: 125 | """ 126 | rule = re.compile(ur"[^a-zA-Z0-9\u4e00-\u9fa5]") 127 | line = rule.sub('', line) 128 | return line 129 | 130 | 131 | def check_punctuation(word): 132 | pattern = re.compile(ur"[^a-zA-Z0-9\u4e00-\u9fa5]") 133 | if pattern.search(word): 134 | return True 135 | else: 136 | return False 137 | 138 | 139 | def clean_text(text): 140 | """ 141 | text = "hello world nice ok done \n\n\n hhade\t\rjdla" 142 | result = hello world nice ok done hhade jdla 143 | 将多个空格换成一个 144 | :param text: 145 | :return: 146 | """ 147 | if not text: 148 | return '' 149 | return re.sub(r"\s+", " ", text) 150 | 151 | 152 | def merge_files(folder, merge_file): 153 | """ 154 | 将多个文件合并为一个文件 155 | :param folder: 文件夹 156 | :param merge_file: 合并后的文件 157 | :return: 158 | """ 159 | paths, _, _ = listdir_files(folder) 160 | with open(merge_file, 'w') as outfile: 161 | for file_path in paths: 162 | with open(file_path) as infile: 163 | for line in infile: 164 | outfile.write(line) 165 | 166 | 167 | def random_pick(some_list, probabilities): 168 | """ 169 | 根据概率随机获取元素 170 | :param some_list: 元素列表 171 | :param probabilities: 概率列表 172 | :return: 当前元素 173 | """ 174 | x = random.uniform(0, 1) 175 | cumulative_probability = 0.0 176 | item = some_list[0] 177 | for item, item_probability in zip(some_list, probabilities): 178 | cumulative_probability += item_probability 179 | if x < cumulative_probability: 180 | break 181 | return item 182 | 183 | 184 | def intersection_of_lists(l1, l2): 185 | """ 186 | 两个list的交集 187 | :param l1: 188 | :param l2: 189 | :return: 190 | """ 191 | return list(set(l1).intersection(set(l2))) 192 | 193 | 194 | def safe_div(x, y): 195 | """ 196 | 安全除法 197 | :param x: 分子 198 | :param y: 分母 199 | :return: 除法 200 | """ 201 | x = float(x) 202 | y = float(y) 203 | if y == 0.0: 204 | return 0.0 205 | return x / y 206 | 207 | 208 | def calculate_percent(x, y): 209 | """ 210 | 计算百分比 211 | :param x: 分子 212 | :param y: 分母 213 | :return: 百分比 214 | """ 215 | x = float(x) 216 | y = float(y) 217 | return safe_div(x, y) * 100 218 | 219 | 220 | def invert_dict(d): 221 | """ 222 | 当字典的元素不重复时, 反转字典 223 | :param d: 字典 224 | :return: 反转后的字典 225 | """ 226 | return dict((v, k) for k, v in d.iteritems()) 227 | 228 | 229 | def init_num_dict(): 230 | """ 231 | 初始化值是int的字典 232 | :return: 233 | """ 234 | return collections.defaultdict(int) 235 | 236 | 237 | def sort_dict_by_value(dict_, reverse=True): 238 | """ 239 | 按照values排序字典 240 | :param dict_: 待排序字典 241 | :param reverse: 默认从大到小 242 | :return: 排序后的字典 243 | """ 244 | return sorted(dict_.items(), key=operator.itemgetter(1), reverse=reverse) 245 | 246 | 247 | def get_current_time_str(): 248 | """ 249 | 输入当天的日期格式, 20170718_1137 250 | :return: 20170718_1137 251 | """ 252 | return datetime.now().strftime('%Y%m%d%H%M%S') 253 | 254 | 255 | def get_current_time_for_show(): 256 | """ 257 | 输入当天的日期格式, 20170718_1137 258 | :return: 20170718_1137 259 | """ 260 | return datetime.now().strftime('%Y-%m-%d %H:%M:%S') 261 | 262 | 263 | def get_current_day_str(): 264 | """ 265 | 输入当天的日期格式, 20170718 266 | :return: 20170718 267 | """ 268 | return datetime.now().strftime('%Y%m%d') 269 | 270 | 271 | def remove_line_of_file(ex_line, file_name): 272 | ex_line = ex_line.replace('\n', '') 273 | lines = read_file(file_name) 274 | 275 | out_file = open(file_name, "w") 276 | for line in lines: 277 | line = line.replace('\n', '') # 确认编码格式 278 | if line != ex_line: 279 | out_file.write(line + '\n') 280 | out_file.close() 281 | 282 | 283 | def map_to_ordered_list(data_dict, reverse=True): 284 | """ 285 | 将字段根据Key的值转换为有序列表 286 | :param data_dict: 字典 287 | :param reverse: 默认从大到小 288 | :return: 有序列表 289 | """ 290 | return sorted(data_dict.items(), key=operator.itemgetter(1), reverse=reverse) 291 | 292 | 293 | def map_to_index_list(data_list, all_list): 294 | """ 295 | 转换为one-hot形式 296 | :param data_list: 297 | :param all_list: 298 | :return: 299 | """ 300 | index_dict = {l.strip(): i for i, l in enumerate(all_list)} # 字典 301 | index = index_dict[data_list.strip()] 302 | index_list = np.zeros(len(all_list), np.float32) 303 | index_list[index] = 1 304 | return index_list 305 | 306 | 307 | def map_to_index(data_list, all_list): 308 | """ 309 | 转换为one-hot形式 310 | :param data_list: 311 | :param all_list: 312 | :return: 313 | """ 314 | index_dict = {l.strip(): i for i, l in enumerate(all_list)} # 字典 315 | index = index_dict[data_list.strip()] 316 | return index 317 | 318 | 319 | def n_lines_of_file(file_name): 320 | """ 321 | 获取文件行数 322 | :param file_name: 文件名 323 | :return: 数量 324 | """ 325 | return sum(1 for line in open(file_name)) 326 | 327 | 328 | def remove_file(file_name): 329 | """ 330 | 删除文件 331 | :param file_name: 文件名 332 | :return: 删除文件 333 | """ 334 | if os.path.exists(file_name): 335 | os.remove(file_name) 336 | 337 | 338 | def find_sub_in_str(string, sub_str): 339 | """ 340 | 子字符串的起始位置 341 | :param string: 字符串 342 | :param sub_str: 子字符串 343 | :return: 当前字符串 344 | """ 345 | return [m.start() for m in re.finditer(sub_str, string)] 346 | 347 | 348 | def list_has_sub_str(string_list, sub_str): 349 | """ 350 | 字符串是否在子字符串中 351 | :param string_list: 字符串列表 352 | :param sub_str: 子字符串列表 353 | :return: 是否在其中 354 | """ 355 | for string in string_list: 356 | if sub_str in string: 357 | return True 358 | return False 359 | 360 | 361 | def list_has_index(index_list, sub_index): 362 | """ 363 | 判断sub_index是否在索引列表中 364 | :param index_list: 索引列表(start_index, end_index) 365 | :param sub_index: 索引 366 | :return: 是否在其中 367 | """ 368 | for start_index, end_index in grouped(index_list, 2): 369 | if start_index <= sub_index <= end_index: 370 | return True 371 | return False 372 | 373 | 374 | def remove_last_char(str_value, num): 375 | """ 376 | 删除最后的字符串 377 | :param str_value: 字符串 378 | :param num: 删除位置 379 | :return: 新的字符串 380 | """ 381 | str_list = list(str_value) 382 | return "".join(str_list[:(-1 * num)]) 383 | 384 | 385 | def read_file(data_file, mode='more'): 386 | """ 387 | 读文件, 原文件和数据文件 388 | :return: 单行或数组 389 | """ 390 | try: 391 | with open(data_file, 'r') as f: 392 | if mode == 'one': 393 | output = f.read() 394 | return output 395 | elif mode == 'more': 396 | output = f.readlines() 397 | return map(str.strip, output) 398 | else: 399 | return list() 400 | except IOError: 401 | return list() 402 | 403 | 404 | def find_word_position(original, word): 405 | """ 406 | 查询字符串的位置 407 | :param original: 原始字符串 408 | :param word: 单词 409 | :return: [起始位置, 终止位置] 410 | """ 411 | u_original = original.decode('utf-8') 412 | u_word = word.decode('utf-8') 413 | start_indexes = find_sub_in_str(u_original, u_word) 414 | end_indexes = [x + len(u_word) - 1 for x in start_indexes] 415 | return zip(start_indexes, end_indexes) 416 | 417 | 418 | def write_list_to_file(file_name, data_list): 419 | """ 420 | 将列表写入文件 421 | :param file_name: 文件名 422 | :param data_list: 数据列表 423 | :return: None 424 | """ 425 | for data in data_list: 426 | write_line(file_name, data) 427 | 428 | 429 | def write_line(file_name, line): 430 | """ 431 | 将行数据写入文件 432 | :param file_name: 文件名 433 | :param line: 行数据 434 | :return: None 435 | """ 436 | if file_name == "": 437 | return 438 | with open(file_name, "a+") as fs: 439 | if type(line) is (tuple or list): 440 | fs.write("%s\n" % ", ".join(line)) 441 | else: 442 | fs.write("%s\n" % line) 443 | 444 | 445 | def show_set(data_set): 446 | """ 447 | 显示集合数据 448 | :param data_set: 数据集 449 | :return: None 450 | """ 451 | data_list = list(data_set) 452 | show_string(data_list) 453 | 454 | 455 | def show_string(obj): 456 | """ 457 | 用于显示UTF-8字符串, 尤其是含有中文的. 458 | :param obj: 输入对象, 可以是列表或字典 459 | :return: None 460 | """ 461 | print list_2_utf8(obj) 462 | 463 | 464 | def list_2_utf8(obj): 465 | """ 466 | 用于显示list汉字 467 | :param obj: 468 | :return: 469 | """ 470 | return json.dumps(obj, encoding="UTF-8", ensure_ascii=False) 471 | 472 | 473 | def grouped_list(iterable, n): 474 | """ 475 | "s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ..." 476 | 例子: 477 | for x, y in grouped(l, 2): 478 | print "%d + %d = %d" % (x, y, x + y) 479 | 480 | :param iterable: 迭代器 481 | :param n: 间隔 482 | :return: 组合 483 | """ 484 | return izip(*[iter(iterable)] * n) 485 | 486 | 487 | def listdir_no_hidden(root_dir): 488 | """ 489 | 显示顶层文件夹 490 | :param root_dir: 根目录 491 | :return: 文件夹列表 492 | """ 493 | return glob.glob(os.path.join(root_dir, '*')) 494 | 495 | 496 | def listdir_files(root_dir, ext=None): 497 | """ 498 | 列出文件夹中的文件 499 | :param root_dir: 根目录 500 | :param ext: 类型 501 | :return: [文件路径(相对路径), 文件夹名称, 文件名称] 502 | """ 503 | names_list = [] 504 | paths_list = [] 505 | for parent, _, fileNames in os.walk(root_dir): 506 | for name in fileNames: 507 | if name.startswith('.'): 508 | continue 509 | if ext: 510 | if name.endswith(tuple(ext)): 511 | names_list.append(name) 512 | paths_list.append(os.path.join(parent, name)) 513 | else: 514 | names_list.append(name) 515 | paths_list.append(os.path.join(parent, name)) 516 | return paths_list, names_list 517 | 518 | 519 | def time_elapsed(start, end): 520 | """ 521 | 输出时间 522 | :param start: 开始 523 | :param end: 结束 524 | :return: 525 | """ 526 | hours, rem = divmod(end - start, 3600) 527 | minutes, seconds = divmod(rem, 60) 528 | return "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds) 529 | 530 | 531 | def batch(iterable, n=1): 532 | """ 533 | 批次迭代器 534 | :param iterable: 迭代器 535 | :param n: 次数 536 | :return: 537 | """ 538 | l = len(iterable) 539 | for ndx in range(0, l, n): 540 | yield iterable[ndx:min(ndx + n, l)] 541 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.1.13 2 | astor==0.6.2 3 | backports.functools-lru-cache==1.5 4 | backports.weakref==1.0.post1 5 | bleach==1.5.0 6 | cycler==0.10.0 7 | enum34==1.1.6 8 | funcsigs==1.0.2 9 | futures==3.2.0 10 | gast==0.2.0 11 | grpcio==1.10.0 12 | html5lib==0.9999999 13 | kiwisolver==1.0.1 14 | Markdown==2.6.11 15 | matplotlib==2.2.2 16 | mock==2.0.0 17 | numpy==1.14.2 18 | pbr==4.0.1 19 | Pillow==5.1.0 20 | protobuf==3.5.2.post1 21 | pyparsing==2.2.0 22 | python-dateutil==2.7.2 23 | pytz==2018.3 24 | scikit-learn==0.19.1 25 | scipy==1.0.1 26 | six==1.11.0 27 | sklearn==0.0 28 | subprocess32==3.2.7 29 | tensorboard==1.7.0 30 | tensorflow==1.7.0 31 | termcolor==1.1.0 32 | Werkzeug==0.14.1 33 | -------------------------------------------------------------------------------- /root_dir.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | import os 4 | 5 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # 存储项目所在的绝对路径 6 | MNIST_DIR = "MNIST_data" 7 | -------------------------------------------------------------------------------- /tensor_board/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /tensor_board/mnist_with_summaries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | """A simple MNIST classifier which displays summaries in TensorBoard. 7 | 8 | This is an unimpressive MNIST model, but it is a good example of using 9 | tf.name_scope to make a graph legible in the TensorBoard graph explorer, and of 10 | naming summary tags so that they are grouped meaningfully in TensorBoard. 11 | 12 | It demonstrates the functionality of every TensorBoard dashboard. 13 | """ 14 | from __future__ import absolute_import 15 | from __future__ import division 16 | from __future__ import print_function 17 | 18 | import argparse 19 | import sys 20 | 21 | import os 22 | import tensorflow as tf 23 | from tensorflow.examples.tutorials.mnist import input_data 24 | 25 | FLAGS = None 26 | 27 | 28 | def train(): 29 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True, fake_data=FLAGS.fake_data) # 加载数据 30 | sess = tf.InteractiveSession() 31 | # Create a multilayer model. 32 | 33 | # Input placeholders 34 | with tf.name_scope('input'): # 指定命名空间 35 | x = tf.placeholder(tf.float32, [None, 784], name='x-input') 36 | y_ = tf.placeholder(tf.float32, [None, 10], name='y-input') 37 | 38 | with tf.name_scope('input_reshape'): 39 | image_shaped_input = tf.reshape(x, [-1, 28, 28, 1]) 40 | tf.summary.image('input', image_shaped_input, 10) # 10表示只存储10张 41 | 42 | # We can't initialize these variables to 0 - the network will get stuck. 43 | def weight_variable(shape): # 权重 44 | """Create a weight variable with appropriate initialization.""" 45 | initial = tf.truncated_normal(shape, stddev=0.1) 46 | return tf.Variable(initial) 47 | 48 | def bias_variable(shape): # 偏移 49 | """Create a bias variable with appropriate initialization.""" 50 | initial = tf.constant(0.1, shape=shape) 51 | return tf.Variable(initial) 52 | 53 | def variable_summaries(var): # 54 | """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" 55 | with tf.name_scope('summaries'): 56 | mean = tf.reduce_mean(var) # 均值 57 | tf.summary.scalar('mean', mean) 58 | with tf.name_scope('stddev'): 59 | stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) 60 | tf.summary.scalar('stddev', stddev) 61 | tf.summary.scalar('max', tf.reduce_max(var)) # 标量 62 | tf.summary.scalar('min', tf.reduce_min(var)) 63 | tf.summary.histogram('histogram', var) # 直方图 64 | 65 | def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu): 66 | """Reusable code for making a simple neural net layer. 67 | 68 | It does a matrix multiply, bias add, and then uses ReLU to nonlinearize. 69 | It also sets up name scoping so that the resultant graph is easy to read, 70 | and adds a number of summary ops. 71 | """ 72 | # Adding a name scope ensures logical grouping of the layers in the graph. 73 | with tf.name_scope(layer_name): 74 | # This Variable will hold the state of the weights for the layer 75 | with tf.name_scope('weights'): 76 | weights = weight_variable([input_dim, output_dim]) 77 | variable_summaries(weights) 78 | with tf.name_scope('biases'): 79 | biases = bias_variable([output_dim]) 80 | variable_summaries(biases) 81 | with tf.name_scope('Wx_plus_b'): 82 | preactivate = tf.matmul(input_tensor, weights) + biases 83 | tf.summary.histogram('pre_activations', preactivate) # 未激活的直方图 84 | activations = act(preactivate, name='activation') 85 | tf.summary.histogram('activations', activations) # 激活的直方图 86 | return activations 87 | 88 | hidden1 = nn_layer(x, 784, 500, 'layer1') # 隐藏层 89 | 90 | with tf.name_scope('dropout'): 91 | keep_prob = tf.placeholder(tf.float32) 92 | tf.summary.scalar('dropout_keep_probability', keep_prob) 93 | dropped = tf.nn.dropout(hidden1, keep_prob) # 执行dropout参数 94 | 95 | # Do not apply softmax activation yet, see below. 96 | y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity) # 未使用激活函数 97 | 98 | with tf.name_scope('cross_entropy'): 99 | # The raw formulation of cross-entropy, 100 | # 101 | # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)), 102 | # reduction_indices=[1])) 103 | # 104 | # can be numerically unstable. 105 | # 106 | # So here we use tf.nn.softmax_cross_entropy_with_logits on the 107 | # raw outputs of the nn_layer above, and then average across 108 | # the batch. 109 | diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y) 110 | with tf.name_scope('total'): 111 | cross_entropy = tf.reduce_mean(diff) 112 | tf.summary.scalar('cross_entropy', cross_entropy) 113 | 114 | with tf.name_scope('train'): 115 | train_step = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize( 116 | cross_entropy) 117 | 118 | with tf.name_scope('accuracy'): 119 | with tf.name_scope('correct_prediction'): 120 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 121 | with tf.name_scope('accuracy'): 122 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 123 | tf.summary.scalar('accuracy', accuracy) 124 | 125 | # Merge all the summaries and write them out to 126 | # /tmp/tensorflow/mnist/logs/mnist_with_summaries (by default) 127 | merged = tf.summary.merge_all() 128 | train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) 129 | test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test') 130 | 131 | # Train the model, and also write summaries. 132 | # Every 10th step, measure test-set accuracy, and write test summaries 133 | # All other steps, run train_step on training data, & add training summaries 134 | 135 | def feed_dict(train): 136 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 137 | # 训练与测试的dropout不同 138 | if train or FLAGS.fake_data: 139 | xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data) 140 | k = FLAGS.dropout 141 | else: 142 | xs, ys = mnist.test.images, mnist.test.labels 143 | k = 1.0 144 | return {x: xs, y_: ys, keep_prob: k} 145 | 146 | tf.global_variables_initializer().run() 147 | 148 | saver = tf.train.Saver() # 存储checkpoint数据 149 | for i in range(FLAGS.max_steps): 150 | if i % 10 == 0: # Record summaries and test-set accuracy 151 | summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) # feed测试数据 152 | test_writer.add_summary(summary, i) 153 | print('Accuracy at step %s: %s' % (i, acc)) 154 | else: # Record train set summaries, and train 155 | if i % 100 == 99: # Record execution stats 156 | run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) 157 | run_metadata = tf.RunMetadata() 158 | summary, _ = sess.run([merged, train_step], # feed训练数据 159 | feed_dict=feed_dict(True), 160 | options=run_options, 161 | run_metadata=run_metadata) 162 | train_writer.add_run_metadata(run_metadata, 'step%03d' % i) 163 | train_writer.add_summary(summary, i) 164 | saver.save(sess, FLAGS.log_dir + "/model.ckpt", i) # 存储checkpoint数据 165 | print('Adding run metadata for', i) 166 | else: # Record a summary 167 | summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) # feed训练数据 168 | train_writer.add_summary(summary, i) 169 | train_writer.close() 170 | test_writer.close() 171 | 172 | 173 | def main(_): 174 | if tf.gfile.Exists(FLAGS.log_dir): 175 | tf.gfile.DeleteRecursively(FLAGS.log_dir) 176 | tf.gfile.MakeDirs(FLAGS.log_dir) 177 | train() 178 | 179 | 180 | if __name__ == '__main__': 181 | parser = argparse.ArgumentParser() 182 | parser.add_argument('--fake_data', nargs='?', const=True, type=bool, default=False, 183 | help='If true, uses fake data for unit testing.') 184 | parser.add_argument('--max_steps', type=int, default=1000, # 最大步数 1000 185 | help='Number of steps to run trainer.') 186 | parser.add_argument('--learning_rate', type=float, default=0.001, # 学习率 0.001 187 | help='Initial learning rate') 188 | parser.add_argument('--dropout', type=float, default=0.9, # Dropout的保留率 0.9 189 | help='Keep probability for training dropout.') 190 | parser.add_argument('--data_dir', type=str, # 数据目录 191 | default=os.path.join(os.getenv('TEST_TMPDIR', '/tmp'), 'tensorflow/mnist/input_data'), 192 | help='Directory for storing input data') 193 | parser.add_argument('--log_dir', type=str, # Log目录 194 | default=os.path.join(os.getenv('TEST_TMPDIR', '/tmp'), 195 | 'tensorflow/mnist/logs/mnist_with_summaries'), 196 | help='Summaries log directory') 197 | FLAGS, unparsed = parser.parse_known_args() 198 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 199 | -------------------------------------------------------------------------------- /tensor_board/test_of_resharp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import tensorflow as tf 7 | 8 | in_mat = [1., 2., 3., 4., 5., 6., 7., 8., 9.] 9 | r_mat = tf.reshape(in_mat, [-1, 3, 3, 1]) 10 | print tf.Session().run(r_mat) 11 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /tests/android_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | # 生成Android集成TF的测试模型 6 | 7 | import tensorflow as tf 8 | from tensorflow.python.tools import freeze_graph, optimize_for_inference_lib 9 | 10 | MODEL_FOLDER = "./data/android/" 11 | MODEL_NAME = 'tfdroid' 12 | 13 | 14 | def gnr_graph_checkpoint(): 15 | """ 16 | 生成简单的图和checkpoint 17 | :return: 18 | """ 19 | I = tf.placeholder(tf.float32, shape=[None, 3], name='I') # input 20 | W = tf.Variable(tf.zeros(shape=[3, 2]), dtype=tf.float32, name='W') # weights 21 | b = tf.Variable(tf.zeros(shape=[2]), dtype=tf.float32, name='b') # biases 22 | O = tf.nn.relu(tf.matmul(I, W) + b, name='O') # activation / output 23 | 24 | saver = tf.train.Saver() 25 | init_op = tf.global_variables_initializer() 26 | 27 | with tf.Session() as sess: 28 | sess.run(init_op) 29 | 30 | tf.train.write_graph(sess.graph_def, MODEL_FOLDER, 'tfdroid.pbtxt') # 存储TensorFlow的图 31 | 32 | # 训练数据，本例直接赋值 33 | sess.run(tf.assign(W, [[1, 2], [4, 5], [7, 8]])) 34 | sess.run(tf.assign(b, [1, 1])) 35 | 36 | # 存储checkpoint文件，即参数信息 37 | saver.save(sess, MODEL_FOLDER + 'tfdroid.ckpt') 38 | 39 | 40 | def gnr_freeze_graph(input_graph, input_saver, input_binary, input_checkpoint, 41 | output_node_names, output_graph, clear_devices): 42 | """ 43 | 将输入图与参数结合在一起 44 | 45 | :param input_graph: 输入图 46 | :param input_saver: Saver解析器 47 | :param input_binary: 输入图的格式，false是文本，true是二进制 48 | :param input_checkpoint: checkpoint，检查点文件 49 | 50 | :param output_node_names: 输出节点名称 51 | :param output_graph: 保存输出文件 52 | :param clear_devices: 清除训练设备 53 | :return: NULL 54 | """ 55 | restore_op_name = "save/restore_all" 56 | filename_tensor_name = "save/Const:0" 57 | 58 | freeze_graph.freeze_graph( 59 | input_graph=input_graph, # 输入图 60 | input_saver=input_saver, # Saver解析器 61 | input_binary=input_binary, # 输入图的格式，false是文本，true是二进制 62 | input_checkpoint=input_checkpoint, # checkpoint，检查点文件 63 | output_node_names=output_node_names, # 输出节点名称 64 | restore_op_name=restore_op_name, # 从模型恢复节点的名字 65 | filename_tensor_name=filename_tensor_name, # tensor名称 66 | output_graph=output_graph, # 保存输出文件 67 | clear_devices=clear_devices, # 清除训练设备 68 | initializer_nodes="") # 初始化节点 69 | 70 | 71 | def gnr_optimize_graph(graph_path, optimized_graph_path): 72 | """ 73 | 优化图 74 | :param graph_path: 原始图 75 | :param optimized_graph_path: 优化的图 76 | :return: NULL 77 | """ 78 | input_graph_def = tf.GraphDef() # 读取原始图 79 | with tf.gfile.Open(graph_path, "r") as f: 80 | data = f.read() 81 | input_graph_def.ParseFromString(data) 82 | 83 | # 设置输入输出节点，剪切分支，大约节省1/4 84 | output_graph_def = optimize_for_inference_lib.optimize_for_inference( 85 | input_graph_def, 86 | ["I"], # an array of the input node(s) 87 | ["O"], # an array of output nodes 88 | tf.float32.as_datatype_enum) 89 | 90 | # 存储优化的图 91 | f = tf.gfile.FastGFile(optimized_graph_path, "w") 92 | f.write(output_graph_def.SerializeToString()) 93 | 94 | 95 | if __name__ == "__main__": 96 | gnr_graph_checkpoint() # 生成图和参数 97 | 98 | input_graph_path = MODEL_FOLDER + MODEL_NAME + '.pbtxt' # 输入图 99 | checkpoint_path = MODEL_FOLDER + MODEL_NAME + '.ckpt' # 输入参数 100 | output_path = MODEL_FOLDER + 'frozen_' + MODEL_NAME + '.pb' # Freeze模型 101 | 102 | # 生成模型 103 | gnr_freeze_graph(input_graph=input_graph_path, input_saver="", 104 | input_binary=False, input_checkpoint=checkpoint_path, 105 | output_node_names="O", output_graph=output_path, clear_devices=True) 106 | 107 | optimized_output_graph = MODEL_FOLDER + 'optimized_' + MODEL_NAME + '.pb' 108 | 109 | # 生成优化模型 110 | gnr_optimize_graph(output_path, optimized_output_graph) 111 | -------------------------------------------------------------------------------- /tests/ocr_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | import json 6 | import time 7 | import urllib 8 | import urllib2 9 | 10 | from project_utils import show_string, batch, time_elapsed 11 | 12 | app_key = '' 13 | secret_key = '' 14 | 15 | 16 | def get_access_token(app_key, secret_key): 17 | api_key_url = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=%s&client_secret=%s' 18 | 19 | # client_id 为官网获取的AK， client_secret 为官网获取的SK 20 | host = (api_key_url % (app_key, secret_key)) 21 | request = urllib2.Request(host) 22 | request.add_header('Content-Type', 'application/json; charset=UTF-8') 23 | response = urllib2.urlopen(request) 24 | content = response.read() 25 | keys = json.loads(content) 26 | access_token = keys['access_token'] 27 | 28 | return access_token 29 | 30 | 31 | def recognize_img_words(access_token, img): 32 | ocr_url = 'https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic/?access_token=%s' 33 | url = (ocr_url % access_token) 34 | 35 | # 上传的参数 36 | data = dict() 37 | data['languagetype'] = "CHN_ENG" # 识别文字 38 | 39 | # 图片数据源，网络图片或本地图片 40 | if img.startswith('http://'): 41 | data['url'] = img 42 | else: 43 | image_data = open(img, 'rb').read() 44 | data['image'] = image_data.encode('base64').replace('\n', '') 45 | 46 | # 发送请求 47 | decoded_data = urllib.urlencode(data) 48 | req = urllib2.Request(url, data=decoded_data) 49 | req.add_header("Content-Type", "application/x-www-form-urlencoded") 50 | 51 | # 获取请求的数据，并读取内容 52 | resp = urllib2.urlopen(req) 53 | content = resp.read() 54 | 55 | # 识别出的图片数据 56 | words_result = json.loads(content)['words_result'] 57 | words_list = list() 58 | for words in words_result: 59 | words_list.append(words['words']) 60 | return words_list 61 | 62 | 63 | if __name__ == '__main__': 64 | local_img = './data/text_img2.jpeg' 65 | online_img = "http://www.zhaoniupai.com/hbv/upload/20150714_LiangDuiBan.jpg" 66 | 67 | s_time = time.time() 68 | 69 | access_token = get_access_token(app_key=app_key, secret_key=secret_key) 70 | print 'access_token: %s' % access_token 71 | 72 | print '\nwords_list:' 73 | words_list = recognize_img_words(access_token, online_img) 74 | for x in batch(words_list, 5): # 每次打印5个数据 75 | show_string(x) 76 | 77 | print "time elapsed: %s" % time_elapsed(s_time, time.time()) 78 | -------------------------------------------------------------------------------- /tf_tests/HelloWorld.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import os 7 | 8 | import tensorflow as tf 9 | 10 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 11 | 12 | hello = tf.constant('Hello, TensorFlow!') 13 | sess = tf.Session() 14 | print sess.run(hello) 15 | 16 | a = tf.constant(10) 17 | b = tf.constant(32) 18 | print sess.run(a + b) 19 | -------------------------------------------------------------------------------- /tf_tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /tf_tests/mnist_softmax.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # ============================================================================== 17 | 18 | """A very simple MNIST classifier. 19 | 20 | See extensive documentation at 21 | https://www.tensorflow.org/get_started/mnist/beginners 22 | 23 | 回归分析：https://zh.wikipedia.org/wiki/%E8%BF%B4%E6%AD%B8%E5%88%86%E6%9E%90 24 | 反向传播： 25 | https://zhuanlan.zhihu.com/p/25081671 26 | https://zhuanlan.zhihu.com/p/25416673 27 | 随机梯度下降： 28 | https://www.zhihu.com/question/28728418 29 | https://www.zhihu.com/question/27012077 30 | """ 31 | from __future__ import absolute_import 32 | from __future__ import division 33 | from __future__ import print_function 34 | 35 | import argparse 36 | import sys 37 | 38 | import tensorflow as tf 39 | from tensorflow.examples.tutorials.mnist import input_data 40 | 41 | FLAGS = None # 全局变量 42 | 43 | 44 | def main(_): 45 | mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) # 加载数据源 46 | 47 | x = tf.placeholder(tf.float32, [None, 784]) # 数据输入，没有 48 | W = tf.Variable(tf.zeros([784, 10])) 49 | b = tf.Variable(tf.zeros([10])) 50 | y = tf.matmul(x, W) + b # Softmax Regression，softmax(y) 51 | 52 | y_ = tf.placeholder(tf.float32, [None, 10]) # 标签输入 53 | 54 | # cross_entropy = tf.reduce_mean(tf.reduce_sum(-1 * (y_ * tf.log(tf.nn.softmax(y))), reduction_indices=[1])) 55 | cross_entropy = tf.reduce_mean( 56 | tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) # 损失函数 57 | train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) # 优化器 58 | 59 | sess = tf.InteractiveSession() # 交互会话 60 | tf.global_variables_initializer().run() # 初始化变量 61 | 62 | # 训练模型 63 | for _ in range(1000): 64 | batch_xs, batch_ys = mnist.train.next_batch(100) 65 | sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) 66 | 67 | # 验证模型 68 | correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) 69 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 70 | print(sess.run(accuracy, feed_dict={x: mnist.test.images, 71 | y_: mnist.test.labels})) 72 | 73 | 74 | if __name__ == '__main__': 75 | parser = argparse.ArgumentParser() # 设置参数data_dir 76 | parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', 77 | help='Directory for storing input data') 78 | FLAGS, unparsed = parser.parse_known_args() 79 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 80 | -------------------------------------------------------------------------------- /tf_tests/test_of_am.py: -------------------------------------------------------------------------------- 1 | # -*-coding: utf-8-*-# 2 | 3 | # Created by C.L.Wang 4 | 5 | import os # 避免Warning 6 | 7 | import tensorflow as tf 8 | 9 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 10 | 11 | ''' 12 | 2 13 | 1 14 | False 15 | True 16 | 0.0, 1.0 17 | ''' 18 | 19 | mat = [1, 2, 3, 2, 1] 20 | mat2 = [1, 3, 2, 2, 1] 21 | mat3 = [1, 3, 2, 2, 1] 22 | 23 | am1 = tf.argmax(mat) 24 | am2 = tf.argmax(mat2) 25 | am3 = tf.argmax(mat3) 26 | 27 | equal = tf.equal(am1, am2) 28 | equal2 = tf.equal(am2, am3) 29 | 30 | val = tf.cast(equal, tf.float32) 31 | val2 = tf.cast(equal2, tf.float32) 32 | 33 | sess = tf.Session() 34 | 35 | print sess.run(am1) 36 | print sess.run(am2) 37 | print sess.run(equal) 38 | print sess.run(equal2) 39 | print '%s, %s' % (sess.run(val), sess.run(val2)) 40 | -------------------------------------------------------------------------------- /tf_tests/test_of_argparse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import argparse 7 | import sys 8 | 9 | parse = argparse.ArgumentParser() 10 | parse.add_argument("--learning_rate", type=float, default=0.01, help="initial learining rate") 11 | parse.add_argument("--max_steps", type=int, default=2000, help="max") 12 | parse.add_argument("--hidden1", type=int, default=100, help="hidden1") 13 | FLAGS, unparsed = parse.parse_known_args(sys.argv[1:]) 14 | 15 | print FLAGS.learning_rate 16 | print FLAGS.max_steps 17 | print FLAGS.hidden1 18 | print unparsed # []，表示未指定的参数 19 | print sys.argv[0] # test_of_argparse.py 20 | -------------------------------------------------------------------------------- /tf_tests/test_of_mat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import os # 避免Warning 7 | 8 | import tensorflow as tf 9 | 10 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 11 | 12 | mat = tf.zeros([2, 3]) # 第一维表示行，第二维表示列 13 | print tf.Session().run(mat) -------------------------------------------------------------------------------- /tf_tests/test_of_rm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import os 7 | 8 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 9 | 10 | import tensorflow as tf 11 | 12 | mat = [[1., 2.], [3., 4.]] 13 | 14 | rm = tf.reduce_mean(mat) 15 | rm_col = tf.reduce_mean(mat, 0) # 保留行 16 | rm_row = tf.reduce_mean(mat, 1) # 保留列 17 | 18 | sess = tf.Session() 19 | print sess.run(rm) 20 | print sess.run(rm_col) 21 | print sess.run(rm_row) 22 | 23 | # 2.5 24 | # [ 2. 3.] 25 | # [ 1.5 3.5] 26 | -------------------------------------------------------------------------------- /tf_tests/test_of_sc.py: -------------------------------------------------------------------------------- 1 | # -*-coding: utf-8-*-# 2 | 3 | # Created by C.L.Wang 4 | 5 | import os # 避免Warning 6 | 7 | import tensorflow as tf 8 | 9 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 10 | 11 | labels_ = [[0., 1.], [1., 0.]] 12 | labels = [[0.1, 0.9], [0.8, 0.2]] 13 | 14 | sc = tf.nn.softmax_cross_entropy_with_logits(labels=labels_, logits=labels) 15 | 16 | sess = tf.Session() 17 | 18 | print sess.run(sc) 19 | 20 | sc2 = tf.reduce_sum(-1 * (labels_ * tf.log(tf.nn.softmax(labels))), reduction_indices=[1]) 21 | 22 | print sess.run(sc2) 23 | 24 | # [ 0.37110069 0.43748799] 25 | # [ 0.37110075 0.43748796] 26 | -------------------------------------------------------------------------------- /wilson_score/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang -------------------------------------------------------------------------------- /wilson_score/wilson_score_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Created by C.L.Wang 5 | 6 | import numpy as np 7 | 8 | 9 | def wilson_score(pos, total, p_z=2.): 10 | """ 11 | 威尔逊得分计算函数 12 | 参考：https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval 13 | :param pos: 正例数 14 | :param total: 总数 15 | :param p_z: 正太分布的分位数 16 | :return: 威尔逊得分 17 | """ 18 | pos_rat = pos * 1. / total * 1. # 正例比率 19 | score = (pos_rat + (np.square(p_z) / (2. * total)) 20 | - ((p_z / (2. * total)) * np.sqrt(4. * total * (1. - pos_rat) * pos_rat + np.square(p_z)))) / \ 21 | (1. + np.square(p_z) / total) 22 | return score 23 | 24 | 25 | def wilson_score_norm(mean, var, total, p_z=2.): 26 | """ 27 | 威尔逊得分计算函数 正态分布版 支持如5星评价，或百分制评价 28 | :param mean: 均值 29 | :param var: 方差 30 | :param total: 总数 31 | :param p_z: 正太分布的分位数 32 | :return: 33 | """ 34 | # 归一化，符合正太分布的分位数 35 | score = (mean + (np.square(p_z) / (2. * total)) 36 | - ((p_z / (2. * total)) * np.sqrt(4. * total * var + np.square(p_z)))) / \ 37 | (1 + np.square(p_z) / total) 38 | return score 39 | 40 | 41 | def test_of_values(): 42 | """ 43 | 五星评价的归一化实例，百分制类似 44 | :return: 总数，均值，方差 45 | """ 46 | max = 5. # 五星评价的最大值 47 | min = 1. # 五星评价的最小值 48 | values = np.array([1., 2., 3., 4., 5.]) # 示例 49 | 50 | norm_values = (values - min) / (max - min) # 归一化 51 | total = norm_values.size # 总数 52 | mean = np.mean(norm_values) # 归一化后的均值 53 | var = np.var(norm_values) # 归一化后的方差 54 | return total, mean, var 55 | 56 | 57 | total, mean, var = test_of_values() 58 | print "total: %s, mean: %s, var: %s" % (total, mean, var) 59 | 60 | print 'score: %s' % wilson_score_norm(mean=mean, var=var, total=total) 61 | print 'score: %s' % wilson_score(90, 90 + 10, p_z=2.) 62 | print 'score: %s' % wilson_score(90, 90 + 10, p_z=6.) 63 | print 'score: %s' % wilson_score(900, 900 + 100, p_z=6.) 64 | -------------------------------------------------------------------------------- /wilson_score/ws_img.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Created by C.L.Wang 4 | 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | from wilson_score.wilson_score_model import wilson_score 9 | 10 | 11 | def show_wilson(): 12 | value = np.linspace(0.01, 100, 1000) 13 | u, v = np.meshgrid(value, value) 14 | 15 | fig, ax = plt.subplots(1, 2) 16 | levels = np.linspace(0, 1, 10) 17 | 18 | cs = ax[0].contourf(u, v, wilson_score(u, u + v), levels=levels) 19 | cb1 = fig.colorbar(cs, ax=ax[0], format="%.2f") 20 | 21 | cs = ax[1].contourf(u, v, wilson_score(u, u + v, 6.), levels=levels) 22 | cb2 = fig.colorbar(cs, ax=ax[1], format="%.2f") 23 | 24 | ax[0].set_xlabel(u'pos') 25 | ax[0].set_ylabel(u'neg') 26 | cb1.set_label(u'wilson(z=2)') 27 | 28 | ax[1].set_xlabel(u'pos') 29 | ax[1].set_ylabel(u'neg') 30 | cb2.set_label(u'wilson(z=6)') 31 | 32 | plt.show() 33 | 34 | 35 | if __name__ == '__main__': 36 | show_wilson() 37 | --------------------------------------------------------------------------------