├── 1.Basic └── helloworld.py ├── 10.SaveModel ├── README.md ├── checkpoint.py └── saved_model.py ├── 11.TFRecord ├── README.md ├── features.ipynb ├── read_cifar10.py ├── read_mnist.py ├── write2.py ├── write_cifar10.py └── write_mnist.py ├── 12.ImageProcess ├── 1.jpg ├── ImageProcess.ipynb ├── examples.py ├── read_dog.py └── write_dog.py ├── 14.RNN ├── BasicLSTM │ ├── bilstm_mnist.py │ ├── bilstm_mnist2.py │ ├── bilstm_mnist3.py │ ├── embedding_lookup.py │ ├── lstm.py │ ├── lstm_mnist.py │ ├── lstm_sin_prediction.py │ ├── lstm_sin_prediction_2.py │ └── lstm_sin_prediction_3.py ├── GRU │ ├── 1.basic.py │ ├── bigru_mnist.py │ └── gru.py ├── IndyLSTM │ ├── IndyLSTM.py │ ├── Indylstm_mnist.py │ └── Indylstm_mnist2.py ├── LSTM │ ├── lstm.py │ ├── lstm_cell.py │ ├── lstm_cell_mnist.py │ ├── lstm_mnist.py │ └── lstm_sin_prediction.py ├── RNN.py └── layerNorm_mnist.py ├── 15.Tricks ├── argxxx.py └── sequence_mask.py ├── 16.DataPipelines ├── Preprocess │ └── one-hot.py ├── dataset.py ├── dataset2.py ├── example1.py └── multi_gpu.py ├── 2.constant_and_Variable ├── Tensor.py └── Variable.py ├── 21.Metrics ├── 1.accuracy.py ├── 2.precision.py └── softmax.py ├── 22.Muilti_GPU ├── model.py └── train.py ├── 23.tf.layers └── MNIST_TEST │ ├── model.py │ ├── parameter.py │ ├── test.py │ └── train.py ├── 3.Graph_and_Session ├── graph.py ├── graph_test.ipynb ├── session.py └── session_test.ipynb ├── 4.Optimizer&GradientTape ├── gradient_tape.py └── optimizer.py ├── 5.math_random_shape ├── math.py ├── random.py └── shape.py ├── 6.NeuralNet_basic └── mlp.py ├── 7.keras ├── 1.basic.py ├── 2.layers.py └── 3.model.py ├── 8.Template ├── 1.module.py ├── example.py └── example2.py ├── 9.CNN ├── conv2d.py ├── example_basic.py ├── example_keras.py └── max_pool.py ├── Project ├── DeepDream │ └── README.md ├── ImageCaptioning │ ├── README.md │ └── model │ │ ├── model.py │ │ ├── parameter.py │ │ ├── preprocess.py │ │ └── train.py ├── LanguageModel │ ├── README.md │ ├── dataset │ │ └── ptb_process.py │ ├── index_files │ │ └── words_ids.csv │ ├── model │ │ ├── lstm.py │ │ ├── parameter.py │ │ ├── test.py │ │ └── train.py │ ├── ptb_corpus │ │ └── README.md │ └── utils │ │ └── generate_index.py ├── NeuralMachineTranslation │ ├── README.md │ ├── corpus │ │ ├── README.md │ │ └── spa.txt │ └── model │ │ ├── model.py │ │ ├── parameter.py │ │ ├── process.py │ │ ├── test.py │ │ └── train.py ├── README.md ├── StyleTransfer │ └── model │ │ ├── image_utils.py │ │ ├── parameter.py │ │ ├── style_transfer.py │ │ ├── test.py │ │ └── train.py └── Transformer │ ├── dataset │ └── process.py │ ├── model │ ├── test.py │ ├── train.py │ └── transformer.py │ └── utils │ ├── display.py │ └── generate_index.py └── README.md /1.Basic/helloworld.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function,division 2 | import tensorflow as tf 3 | 4 | #define the graph 5 | info=tf.constant("hello,world") 6 | a=tf.constant(10) 7 | b=tf.constant(20) 8 | c=tf.add(a,b) 9 | 10 | print("info:",info) 11 | print("type of info:",type(info)) 12 | print(info.numpy()) 13 | print("type of info.numpy()",type(info.numpy())) 14 | print("\n\n") 15 | 16 | print("a:",a) 17 | print("type of a:",type(a)) 18 | print(a.numpy()) 19 | print("type of a.numpy()",type(a.numpy())) 20 | 21 | 22 | print("b:",b) 23 | print("c:",c) -------------------------------------------------------------------------------- /10.SaveModel/README.md: -------------------------------------------------------------------------------- 1 | [**tf.train.Checkpoint**](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/train/Checkpoint) 2 | 3 | [**Training checkpoints**](https://www.tensorflow.org/alpha/guide/checkpoints) 4 | 5 | [**tf.saved_model**](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/saved_model) 6 | 7 | 8 | -------------------------------------------------------------------------------- /10.SaveModel/checkpoint.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | EPOCH=20 6 | BATCH_SIZE=100 7 | TRAIN_EXAMPLES=42000 8 | LEARNING_RATE=0.01 9 | 10 | #------------------------Generate Data---------------------------# 11 | #generate data 12 | train_frame = pd.read_csv("../Mnist/train.csv") 13 | test_frame = pd.read_csv("../Mnist/test.csv") 14 | 15 | # pop the labels and one-hot coding 16 | train_labels_frame = train_frame.pop("label") 17 | 18 | # get values one-hot on labels 19 | X_train = train_frame.astype(np.float32).values/255 20 | y_train=pd.get_dummies(data=train_labels_frame).values 21 | X_test = test_frame.astype(np.float32).values/255 22 | 23 | #trans the shape to (batch,time_steps,input_size) 24 | #X_train=np.reshape(X_train,newshape=(-1,28,28)) 25 | #X_test=np.reshape(X_test,newshape=(-1,28,28)) 26 | print(X_train.shape) 27 | print(y_train.shape) 28 | print(X_test.shape) 29 | 30 | #------------------------------------------------------------------# 31 | class MLP(tf.keras.Model): 32 | def __init__(self,hidden_dim): 33 | super(MLP,self).__init__() 34 | self.linear1=tf.keras.layers.Dense( 35 | units=hidden_dim, 36 | activation=tf.nn.relu, 37 | use_bias=True 38 | ) 39 | self.linear2=tf.keras.layers.Dense( 40 | units=10, 41 | activation=None, 42 | use_bias=True 43 | ) 44 | 45 | def __call__(self,inputs): 46 | logits_1 = self.linear1(inputs) 47 | logits_2 = self.linear2(logits_1) 48 | return logits_2 49 | 50 | def train(): 51 | mlp=MLP(hidden_dim=200) 52 | optimizer=tf.keras.optimizers.SGD(LEARNING_RATE) 53 | checkpoint=tf.train.Checkpoint(optimizer=optimizer,model=mlp) 54 | for epoch in range(1,EPOCH+1): 55 | print("epoch:",epoch) 56 | train_losses = [] 57 | accus = [] 58 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 59 | with tf.GradientTape() as tape: 60 | logits_2=mlp(X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE]) 61 | entropy=tf.nn.softmax_cross_entropy_with_logits( 62 | labels=y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 63 | logits=logits_2 64 | ) 65 | loss=tf.math.reduce_mean(entropy) 66 | #print("loss:",loss) 67 | 68 | #计算梯度 69 | gradient=tape.gradient(target=loss,sources=mlp.trainable_variables) 70 | #print("gradient:",gradient) 71 | #应用梯度 72 | optimizer.apply_gradients(zip(gradient,mlp.trainable_variables)) 73 | 74 | train_losses.append(loss.numpy()) 75 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 1)) 76 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")).numpy() 77 | accus.append(accuracy) 78 | print("average training loss:", sum(train_losses) / len(train_losses)) 79 | print("accuracy:",sum(accus)/len(accus)) 80 | 81 | #save checkpoint 82 | checkpoint.save(file_prefix="./checkpoints/test") 83 | 84 | 85 | 86 | 87 | 88 | def test(): 89 | mlp = MLP(hidden_dim=200) 90 | optimizer = tf.keras.optimizers.SGD(LEARNING_RATE) 91 | checkpoint = tf.train.Checkpoint(model=mlp) 92 | checkpoint.restore(save_path="./checkpoints/test-3") 93 | 94 | train_losses = [] 95 | accus = [] 96 | for j in range(TRAIN_EXAMPLES // BATCH_SIZE): 97 | logits_2 = mlp(X_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE]) 98 | entropy = tf.nn.softmax_cross_entropy_with_logits( 99 | labels=y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 100 | logits=logits_2 101 | ) 102 | loss = tf.math.reduce_mean(entropy) 103 | train_losses.append(loss.numpy()) 104 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), 105 | tf.argmax(y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 1)) 106 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")).numpy() 107 | accus.append(accuracy) 108 | 109 | print("average training loss:", sum(train_losses) / len(train_losses)) 110 | print("accuracy:", sum(accus) / len(accus)) 111 | 112 | 113 | 114 | 115 | 116 | if __name__=="__main__": 117 | istrain=False 118 | if istrain: 119 | train() 120 | else: 121 | test() -------------------------------------------------------------------------------- /10.SaveModel/saved_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/10.SaveModel/saved_model.py -------------------------------------------------------------------------------- /11.TFRecord/README.md: -------------------------------------------------------------------------------- 1 | https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/data/TFRecordDataset 2 | 3 | https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/data/TFRecordDataset 4 | 5 | 6 | https://www.tensorflow.org/alpha/tutorials/load_data/tf_records 7 | 8 | -------------------------------------------------------------------------------- /11.TFRecord/read_cifar10.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | import matplotlib.image as mpimg 5 | 6 | #tfrecord 文件列表 7 | file_list=["cifar_10_valid.tfrecords"] 8 | 9 | #创建dataset对象 10 | dataset=tf.data.TFRecordDataset(filenames=file_list) 11 | 12 | #定义解析和预处理函数 13 | def _parse_data(example_proto): 14 | parsed_features=tf.parse_single_example( 15 | serialized=example_proto, 16 | features={ 17 | "image_raw":tf.FixedLenFeature(shape=[],dtype=tf.string), 18 | "label":tf.FixedLenFeature(shape=[],dtype=tf.int64) 19 | } 20 | ) 21 | 22 | # get single feature 23 | raw = parsed_features["image_raw"] 24 | label = parsed_features["label"] 25 | # decode raw 26 | image = tf.decode_raw(bytes=raw, out_type=tf.float32) 27 | image=tf.reshape(tensor=image,shape=[32,32,-1]) 28 | return image,label 29 | 30 | #使用map处理得到新的dataset 31 | dataset=dataset.map(map_func=_parse_data) 32 | #使用batch_size为32生成mini-batch 33 | #dataset = dataset.batch(32) 34 | 35 | #创建迭代器 36 | iterator=dataset.make_one_shot_iterator() 37 | 38 | next_element=iterator.get_next() 39 | 40 | with tf.Session() as sess: 41 | for i in range(10): 42 | image, label = sess.run(next_element) 43 | print(label) 44 | print("image.shape:",image.shape) 45 | print("label.shape",label.shape) 46 | plt.imshow(image) 47 | plt.show() 48 | 49 | 50 | #if __name__=="__main__": 51 | 52 | -------------------------------------------------------------------------------- /11.TFRecord/read_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import matplotlib.pyplot as plt 5 | 6 | #tfrecord 文件列表 7 | file_list=["train.tfrecords"] 8 | 9 | #创建dataset对象 10 | dataset=tf.data.TFRecordDataset(filenames=file_list) 11 | 12 | #定义解析和预处理函数 13 | def _parse_data(example_proto): 14 | parsed_features=tf.parse_single_example( 15 | serialized=example_proto, 16 | features={ 17 | "image_raw":tf.FixedLenFeature(shape=(784,),dtype=tf.float32), 18 | "label":tf.FixedLenFeature(shape=(),dtype=tf.int64) 19 | } 20 | ) 21 | # get single feature 22 | raw = parsed_features["image_raw"] 23 | label = parsed_features["label"] 24 | # decode raw 25 | #image = tf.decode_raw(bytes=raw, out_type=tf.int64) 26 | image=tf.reshape(tensor=raw,shape=[28,28]) 27 | return image,label 28 | 29 | #使用map处理得到新的dataset 30 | dataset=dataset.map(map_func=_parse_data) 31 | #dataset = dataset.batch(32) 32 | 33 | #创建迭代器 34 | iterator=dataset.make_one_shot_iterator() 35 | 36 | next_element=iterator.get_next() 37 | 38 | with tf.Session() as sess: 39 | image, label = sess.run(next_element) 40 | print(label) 41 | print(label.shape) 42 | print(image.shape) 43 | print(image) 44 | plt.imshow(image) 45 | plt.show() 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /11.TFRecord/write2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | num_files=3 4 | num_instance=100 5 | 6 | for i in range(num_files): 7 | print("write ",i," file") 8 | fileName=("test.tfrecords-%.5d-of-%.5d" % (i,num_files)) 9 | writer=tf.python_io.TFRecordWriter(path=fileName) 10 | 11 | for j in range(num_instance): 12 | print("write ",j," record") 13 | example=tf.train.Example( 14 | features=tf.train.Features( 15 | feature={ 16 | "i":tf.train.Feature(int64_list=tf.train.Int64List(value=[i])), 17 | "j":tf.train.Feature(int64_list=tf.train.Int64List(value=[j])) 18 | } 19 | ) 20 | 21 | ) 22 | writer.write(record=example.SerializeToString()) 23 | writer.close() 24 | 25 | -------------------------------------------------------------------------------- /11.TFRecord/write_cifar10.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | import tensorflow as tf 5 | import matplotlib.pyplot as plt 6 | import matplotlib.image as mpimg 7 | 8 | #mapping name to number 9 | mapping_dict={ 10 | "frog":0, 11 | "truck":1, 12 | "deer":2, 13 | "automobile":3, 14 | "bird":4, 15 | "horse":5, 16 | "ship":6, 17 | "cat":7, 18 | "airplane":8, 19 | "dog":9 20 | } 21 | 22 | 23 | def pics2tfrecords(folder,is_train): 24 | ''' 25 | :param folder: folder to storage pics 26 | :param is_train:train set of validation set 27 | :return: 28 | ''' 29 | print("Trans Pictures To TFRecords") 30 | if is_train: 31 | train_labels_frame = pd.read_csv(filepath_or_buffer=folder+"trainLabels.csv") 32 | writer_train = tf.python_io.TFRecordWriter(path="cifar_10_train.tfrecords") 33 | writer_valid = tf.python_io.TFRecordWriter(path="cifar_10_valid.tfrecords") 34 | 35 | #training set 36 | for i in range(1,45000+1): 37 | pic = mpimg.imread(fname=folder+"train/"+str(i)+".png") 38 | pic_raw=pic.tostring() 39 | kind=mapping_dict[train_labels_frame["label"][i-1]] 40 | 41 | example=tf.train.Example( 42 | features=tf.train.Features( 43 | feature={ 44 | "image_raw":tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_raw])), 45 | "label":tf.train.Feature(int64_list=tf.train.Int64List(value=[kind])) 46 | } 47 | ) 48 | ) 49 | writer_train.write(record=example.SerializeToString()) 50 | writer_train.close() 51 | 52 | #validation set 53 | for i in range(45000+1,50000+1): 54 | pic = mpimg.imread(fname=folder + "train/" + str(i) + ".png") 55 | pic_raw = pic.tostring() 56 | kind = mapping_dict[train_labels_frame["label"][i - 1]] 57 | 58 | example = tf.train.Example( 59 | features=tf.train.Features( 60 | feature={ 61 | "image_raw": tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_raw])), 62 | "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[kind])) 63 | } 64 | ) 65 | ) 66 | writer_valid.write(record=example.SerializeToString()) 67 | writer_valid.close() 68 | 69 | else: 70 | pass 71 | 72 | 73 | if __name__=="__main__": 74 | #pic = mpimg.imread(fname="../CIFAR-10/train/2.png") 75 | #print(pic.shape) 76 | #train_labels_frame=pd.read_csv(filepath_or_buffer="../CIFAR-10/trainLabels.csv") 77 | #print(train_labels_frame) 78 | #print(train_labels_frame[train_labels_frame['id']==2]) 79 | #plt.imshow(pic) 80 | #plt.show() 81 | #print(maping_dict["dog"]) 82 | #print(train_labels_frame["label"][0]) 83 | 84 | pics2tfrecords(folder="../CIFAR-10/",is_train=True) 85 | 86 | 87 | -------------------------------------------------------------------------------- /11.TFRecord/write_mnist.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import pandas as pd 4 | 5 | #---------------------loading data from .csv file------------------------------# 6 | #load .csv file 7 | train_frame=pd.read_csv(filepath_or_buffer="../Mnist/train.csv") 8 | #print(train_frame.head()) 9 | train_labels_frame=train_frame.pop(item="label") 10 | #print(train_labels_frame.shape) 11 | 12 | train_values=train_frame.values.astype(np.float32) 13 | train_size=train_values.shape[0] 14 | train_labels_values=train_labels_frame.values 15 | 16 | 17 | #print(train_values[0].shape) 18 | #print(train_values[0].dtype) 19 | #print(train_labels_values[0]) 20 | 21 | #------------------------------create TFRecord file----------------------------# 22 | writer=tf.python_io.TFRecordWriter(path="train.tfrecords") 23 | for i in range(train_size): 24 | image_raw=train_values[i] 25 | example=tf.train.Example( 26 | features=tf.train.Features( 27 | feature={ 28 | "image_raw":tf.train.Feature(float_list=tf.train.FloatList(value=image_raw)), 29 | "label":tf.train.Feature(int64_list=tf.train.Int64List(value=[train_labels_values[i]])) 30 | } 31 | ) 32 | ) 33 | writer.write(record=example.SerializeToString()) 34 | writer.close() 35 | 36 | -------------------------------------------------------------------------------- /12.ImageProcess/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/12.ImageProcess/1.jpg -------------------------------------------------------------------------------- /12.ImageProcess/examples.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | import matplotlib.image as mpimg 5 | import matplotlib.pyplot as plt 6 | import tensorflow as tf 7 | 8 | #read picture 9 | pic=mpimg.imread("../../data/DogsVsCats/train/cat.0.jpg") 10 | print(pic) 11 | plt.imshow(pic) 12 | plt.show() 13 | 14 | #randon random_flip_left_right 15 | flip_pic=tf.image.random_flip_left_right(image=pic) 16 | 17 | #tf.image.random_flip_up_down 18 | flip_pic=tf.image.random_flip_up_down(image=flip_pic) 19 | 20 | new=tf.image.resize_image_with_crop_or_pad(image=pic,target_height=224,target_width=224) 21 | 22 | with tf.Session() as sess: 23 | plt.imshow(sess.run(flip_pic)) 24 | plt.show() -------------------------------------------------------------------------------- /12.ImageProcess/read_dog.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | import matplotlib.image as mpimg 5 | 6 | #tfrecord 文件列表 7 | file_list=["dog_vs_cat_valid.tfrecords"] 8 | 9 | #创建dataset对象 10 | dataset=tf.data.TFRecordDataset(filenames=file_list) 11 | 12 | #定义解析和预处理函数 13 | def _parse_data(example_proto): 14 | parsed_features=tf.parse_single_example( 15 | serialized=example_proto, 16 | features={ 17 | "image_raw":tf.FixedLenFeature(shape=[],dtype=tf.string), 18 | "label":tf.FixedLenFeature(shape=[],dtype=tf.int64) 19 | } 20 | ) 21 | 22 | # get single feature 23 | raw = parsed_features["image_raw"] 24 | label = parsed_features["label"] 25 | # decode raw 26 | image = tf.decode_raw(bytes=raw, out_type=tf.uint8) 27 | image = tf.reshape(tensor=image, shape=(250, 250, 3)) 28 | #crop 29 | image=tf.image.resize_image_with_crop_or_pad(image=image,target_height=224,target_width=224) 30 | #trans to float 31 | image=tf.image.convert_image_dtype(image=image,dtype=tf.float32) 32 | return image,label 33 | 34 | #使用map处理得到新的dataset 35 | dataset=dataset.map(map_func=_parse_data) 36 | #使用batch_size为32生成mini-batch 37 | #dataset = dataset.batch(32) 38 | 39 | #创建迭代器 40 | iterator=dataset.make_one_shot_iterator() 41 | 42 | next_element=iterator.get_next() 43 | 44 | with tf.Session() as sess: 45 | for i in range(10): 46 | element = sess.run(next_element) 47 | print("label:",element[1]) 48 | print("image.shape:",element[0].shape) 49 | print("label.shape",element[1].shape) 50 | 51 | plt.imshow(element[0]) 52 | plt.show() 53 | 54 | 55 | 56 | #if __name__=="__main__": 57 | -------------------------------------------------------------------------------- /12.ImageProcess/write_dog.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | import pandas as pd 5 | import tensorflow as tf 6 | import matplotlib.pyplot as plt 7 | import matplotlib.image as mpimg 8 | 9 | 10 | #"../../data/DogsVsCats/train/cat.0.jpg" 11 | #0~12499 12 | 13 | def pics2tfrecords(folder,is_train): 14 | ''' 15 | :param folder: folder to storage pics 16 | :param is_train:train set of validation set 17 | :return: 18 | ''' 19 | print("Trans Pictures To TFRecords") 20 | if is_train: 21 | kind_cat = 0 22 | kind_dog = 1 23 | writer_train = tf.python_io.TFRecordWriter(path="dog_vs_cat_train.tfrecords") 24 | writer_valid = tf.python_io.TFRecordWriter(path="dog_vs_cat_valid.tfrecords") 25 | 26 | #training set 27 | for i in range(10): 28 | pic_cat=cv2.imread(filename=folder+"train/cat."+str(i)+".jpg",flags=cv2.IMREAD_UNCHANGED) 29 | #resize to 250x250 30 | pic_cat=cv2.resize(src=pic_cat,dsize=(250,250),interpolation=cv2.INTER_AREA) 31 | #to string 32 | pic_cat_raw=pic_cat.tostring() 33 | print(pic_cat.shape) 34 | 35 | example=tf.train.Example( 36 | features=tf.train.Features( 37 | feature={ 38 | "image_raw":tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_cat_raw])), 39 | "label":tf.train.Feature(int64_list=tf.train.Int64List(value=[kind_cat])) 40 | } 41 | ) 42 | ) 43 | writer_train.write(record=example.SerializeToString()) 44 | 45 | pic_dog = cv2.imread(filename=folder + "train/dog." + str(i) + ".jpg", flags=cv2.IMREAD_UNCHANGED) 46 | # resize to 250x250 47 | pic_dog = cv2.resize(src=pic_dog, dsize=(250, 250), interpolation=cv2.INTER_AREA) 48 | # to string 49 | pic_dog_raw = pic_dog.tostring() 50 | print(pic_dog.shape) 51 | 52 | example = tf.train.Example( 53 | features=tf.train.Features( 54 | feature={ 55 | "image_raw": tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_dog_raw])), 56 | "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[kind_dog])) 57 | } 58 | ) 59 | ) 60 | writer_train.write(record=example.SerializeToString()) 61 | 62 | writer_train.close() 63 | 64 | #validation set 65 | for i in range(10,20): 66 | pic_cat = cv2.imread(filename=folder + "train/cat." + str(i) + ".jpg", flags=cv2.IMREAD_UNCHANGED) 67 | # resize to 300x300 68 | pic_cat = cv2.resize(src=pic_cat, dsize=(250, 250), interpolation=cv2.INTER_AREA) 69 | # to string 70 | pic_cat_raw = pic_cat.tostring() 71 | print(pic_cat.shape) 72 | 73 | example = tf.train.Example( 74 | features=tf.train.Features( 75 | feature={ 76 | "image_raw": tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_cat_raw])), 77 | "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[kind_cat])) 78 | } 79 | ) 80 | ) 81 | writer_valid.write(record=example.SerializeToString()) 82 | 83 | pic_dog = cv2.imread(filename=folder + "train/dog." + str(i) + ".jpg", flags=cv2.IMREAD_UNCHANGED) 84 | # resize to 300x300 85 | pic_dog = cv2.resize(src=pic_dog, dsize=(250, 250), interpolation=cv2.INTER_AREA) 86 | # to string 87 | pic_dog_raw = pic_dog.tostring() 88 | print(pic_dog.shape) 89 | 90 | example = tf.train.Example( 91 | features=tf.train.Features( 92 | feature={ 93 | "image_raw": tf.train.Feature(bytes_list=tf.train.BytesList(value=[pic_dog_raw])), 94 | "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[kind_dog])) 95 | } 96 | ) 97 | ) 98 | writer_valid.write(record=example.SerializeToString()) 99 | 100 | writer_valid.close() 101 | 102 | else: 103 | pass 104 | 105 | 106 | if __name__=="__main__": 107 | #pic = mpimg.imread(fname="../CIFAR-10/train/2.png") 108 | #print(pic.shape) 109 | #train_labels_frame=pd.read_csv(filepath_or_buffer="../CIFAR-10/trainLabels.csv") 110 | #print(train_labels_frame) 111 | #print(train_labels_frame[train_labels_frame['id']==2]) 112 | #plt.imshow(pic) 113 | #plt.show() 114 | #print(maping_dict["dog"]) 115 | #print(train_labels_frame["label"][0]) 116 | 117 | pics2tfrecords(folder="../../data/DogsVsCats/",is_train=True) -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/bilstm_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../../Mnist/train.csv") 21 | test_frame = pd.read_csv("../../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | #print(X_train.shape) 36 | #print(y_dummy.shape) 37 | #print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | 41 | 42 | #--------------------------------------Define Graph---------------------------------------------------# 43 | graph=tf.Graph() 44 | with graph.as_default(): 45 | 46 | #------------------------------------construct LSTM------------------------------------------# 47 | #place hoder 48 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 49 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 50 | 51 | #lstm instance 52 | lstm_forward=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 53 | lstm_backward=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 54 | 55 | outputs,(states_fw,states_bw)=tf.nn.bidirectional_dynamic_rnn( 56 | cell_fw=lstm_forward, 57 | cell_bw=lstm_backward, 58 | inputs=X_p, 59 | dtype=tf.float32 60 | ) 61 | states_concat=tf.concat(values=[states_fw,states_bw],axis=1) 62 | print(type(states_concat)) 63 | ''' 64 | print("state_concat.shape:",states_concat.shape) 65 | finale_state=(states_concat[0],states_concat[1]) 66 | print("type of final_state:",type(finale_state)) 67 | c_forward=states[0].c 68 | print(c_forward.shape) 69 | c_backward=states[1].c 70 | print(c_backward.shape) 71 | c_concat=tf.concat(values=[c_forward,c_backward],axis=-1) 72 | print(c_concat.shape) 73 | ''' 74 | 75 | 76 | #print(outputs[0].shape) 77 | #print(states[0].h) 78 | #state_h_fw=states[0].h 79 | #print(state_h_fw.shape) 80 | #outputs_fw=outputs[0] 81 | #outputs_bw = outputs[1] 82 | #output_h_fw = outputs_fw[:,-1,:] 83 | #print(output_h_fw.shape) 84 | #h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 85 | #print(h.shape) 86 | #---------------------------------------;-----------------------------------------------------# 87 | 88 | #---------------------------------define loss and optimizer----------------------------------# 89 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 90 | #print(loss.shape) 91 | 92 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 93 | #accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 94 | 95 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 96 | 97 | init=tf.global_variables_initializer() 98 | 99 | 100 | #-------------------------------------------Define Session---------------------------------------# 101 | with tf.Session(graph=graph) as sess: 102 | sess.run(init) 103 | for epoch in range(1,EPOCH+1): 104 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 105 | train_losses=[] 106 | accus=[] 107 | #test_losses=[] 108 | print("epoch:",epoch) 109 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 110 | _,train_loss,accu,s_fw,o_fw=sess.run( 111 | fetches=(optimizer,cross_loss,accuracy,state_h_fw,output_h_fw), 112 | feed_dict={ 113 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 114 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 115 | } 116 | ) 117 | train_losses.append(train_loss) 118 | accus.append(accu) 119 | #print("s_fw:",s_fw[0]) 120 | #print("o_fw:",o_fw[0]) 121 | print("average training loss:", sum(train_losses) / len(train_losses)) 122 | print("accuracy:",sum(accus)/len(accus)) 123 | 124 | -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/bilstm_mnist2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=20 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../../Mnist/train.csv") 21 | test_frame = pd.read_csv("../../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | #print(X_train.shape) 36 | #print(y_dummy.shape) 37 | #print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | 41 | 42 | #--------------------------------------Define Graph---------------------------------------------------# 43 | graph=tf.Graph() 44 | with graph.as_default(): 45 | 46 | #------------------------------------construct LSTM------------------------------------------# 47 | #place hoder 48 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 49 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 50 | 51 | #lstm instance 52 | lstm_forward_1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 53 | #加attention(这里的attention和encoder-decoder架构的attention稍有不同) 54 | lstm_forward_1=rnn.AttentionCellWrapper(cell=lstm_forward_1,attn_length=5) 55 | 56 | lstm_forward_2=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 57 | # 加attention 58 | lstm_forward_2 = rnn.AttentionCellWrapper(cell=lstm_forward_2, attn_length=5) 59 | lstm_forward=rnn.MultiRNNCell(cells=[lstm_forward_1,lstm_forward_2]) 60 | 61 | lstm_backward_1 = rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 62 | #加attention 63 | lstm_backward_1 = rnn.AttentionCellWrapper(cell=lstm_backward_1, attn_length=5) 64 | 65 | lstm_backward_2 = rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 66 | lstm_backward_2 = rnn.AttentionCellWrapper(cell=lstm_backward_2, attn_length=5) 67 | lstm_backward=rnn.MultiRNNCell(cells=[lstm_backward_1,lstm_backward_2]) 68 | 69 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 70 | cell_fw=lstm_forward, 71 | cell_bw=lstm_backward, 72 | inputs=X_p, 73 | dtype=tf.float32 74 | ) 75 | 76 | outputs_fw=outputs[0] 77 | outputs_bw = outputs[1] 78 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 79 | # print(h.shape) 80 | #--------------------------------------------------------------------------------------------# 81 | 82 | #---------------------------------define loss and optimizer----------------------------------# 83 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 84 | #print(loss.shape) 85 | 86 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 87 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 88 | 89 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 90 | 91 | init=tf.global_variables_initializer() 92 | 93 | 94 | #-------------------------------------------Define Session---------------------------------------# 95 | with tf.Session(graph=graph) as sess: 96 | sess.run(init) 97 | for epoch in range(1,EPOCH+1): 98 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 99 | train_losses=[] 100 | accus=[] 101 | #test_losses=[] 102 | print("epoch:",epoch) 103 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 104 | _,train_loss,accu=sess.run( 105 | fetches=(optimizer,cross_loss,accuracy), 106 | feed_dict={ 107 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 108 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 109 | } 110 | ) 111 | train_losses.append(train_loss) 112 | accus.append(accu) 113 | print("average training loss:", sum(train_losses) / len(train_losses)) 114 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/bilstm_mnist3.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../../Mnist/train.csv") 21 | test_frame = pd.read_csv("../../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | #print(X_train.shape) 36 | #print(y_dummy.shape) 37 | #print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | 41 | 42 | #--------------------------------------Define Graph---------------------------------------------------# 43 | graph=tf.Graph() 44 | with graph.as_default(): 45 | 46 | #------------------------------------construct LSTM------------------------------------------# 47 | #place hoder 48 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 49 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 50 | 51 | #lstm instance 52 | lstm_forward_1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 53 | lstm_forward_2=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 54 | lstm_forward=rnn.MultiRNNCell(cells=[lstm_forward_1,lstm_forward_2]) 55 | 56 | lstm_backward_1 = rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 57 | lstm_backward_2 = rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 58 | lstm_backward=rnn.MultiRNNCell(cells=[lstm_backward_1,lstm_backward_2]) 59 | 60 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 61 | cell_fw=lstm_forward, 62 | cell_bw=lstm_backward, 63 | inputs=X_p, 64 | dtype=tf.float32 65 | ) 66 | 67 | outputs_fw=outputs[0] 68 | outputs_bw = outputs[1] 69 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 70 | 71 | 72 | # print(h.shape) 73 | #---------------------------------------;-----------------------------------------------------# 74 | 75 | #---------------------------------define loss and optimizer----------------------------------# 76 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 77 | #print(loss.shape) 78 | 79 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 80 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 81 | 82 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 83 | 84 | init=tf.global_variables_initializer() 85 | 86 | 87 | #-------------------------------------------Define Session---------------------------------------# 88 | with tf.Session(graph=graph) as sess: 89 | sess.run(init) 90 | for epoch in range(1,EPOCH+1): 91 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 92 | train_losses=[] 93 | accus=[] 94 | #test_losses=[] 95 | print("epoch:",epoch) 96 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 97 | _,train_loss,accu=sess.run( 98 | fetches=(optimizer,cross_loss,accuracy), 99 | feed_dict={ 100 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 101 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 102 | } 103 | ) 104 | train_losses.append(train_loss) 105 | accus.append(accu) 106 | print("average training loss:", sum(train_losses) / len(train_losses)) 107 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/embedding_lookup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | value=np.array([[1,1,1],[2,2,2],[3,3,3],[4,4,4]]) 6 | graph=tf.Graph() 7 | with graph.as_default(): 8 | embeddings=tf.Variable(initial_value=value,name="embeddings") 9 | print("embeddings.shape",embeddings.shape) 10 | 11 | #一维ids 12 | ids1 = tf.Variable(initial_value=[1, 2], name="ids") 13 | vecs1 = tf.nn.embedding_lookup(params=embeddings, ids=ids1) 14 | 15 | #二维ids,可以做为批处理来用 16 | ids2 = tf.Variable(initial_value=[[1, 2], [2, 1], [0, 2]]) 17 | print("ids2.shape:", ids2.shape) 18 | vecs2 = tf.nn.embedding_lookup(params=embeddings, ids=ids2) 19 | 20 | init_op = tf.global_variables_initializer() 21 | init_op = tf.global_variables_initializer() 22 | 23 | with tf.Session(graph=graph) as sess: 24 | sess.run(init_op) 25 | embed= sess.run(embeddings) 26 | vec1=sess.run(vecs1) 27 | vec2=sess.run(vecs2) 28 | print("embeddings:\n",embed) 29 | print("vec1:\n",vec1) 30 | print("vec2:\n",vec2) 31 | print(vec2.shape) -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/lstm.py: -------------------------------------------------------------------------------- 1 | ''' 2 | import numpy as np 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | ''' 6 | ''' 7 | 1.tensorflow.contrib.rnn.BasicLSTMCell 8 | 2.tensorflow.contrib.rnn.MultiRNNCell 9 | 3.tf.nn.dynamic_rnn() 10 | 4.tf.nn.bidirectional_dynamic_rnn() 11 | ''' 12 | 13 | 14 | #单向LSTM 15 | 16 | 17 | 18 | #双向LSTM(biLSTM) 19 | 20 | 21 | # -*- coding:utf-8 -*- 22 | import tensorflow as tf 23 | import numpy as np 24 | from tensorflow.contrib import rnn 25 | from tensorflow.examples.tutorials.mnist import input_data 26 | 27 | # 设置 GPU 按需增长 28 | config = tf.ConfigProto() 29 | config.gpu_options.allow_growth = True 30 | sess = tf.Session(config=config) 31 | 32 | # 首先导入数据，看一下数据的形式 33 | mnist = input_data.read_data_sets('MNIST_data', one_hot=True) 34 | print(mnist.train.images.shape) 35 | 36 | lr = 1e-3 37 | # 在训练和测试的时候，我们想用不同的 batch_size.所以采用占位符的方式 38 | batch_size = tf.placeholder(tf.int32, []) 39 | 40 | # 每个时刻的输入特征是28维的，就是每个时刻输入一行，一行有 28 个像素 41 | input_size = 28 42 | 43 | # 时序持续长度为28，即每做一次预测，需要先输入28行 44 | timestep_size = 28 45 | 46 | # 每个隐含层的节点数 47 | hidden_size = 10 48 | # LSTM layer 的层数 49 | layer_num = 2 50 | # 最后输出分类类别数量，如果是回归预测的话应该是 1 51 | class_num = 10 52 | 53 | _X = tf.placeholder(tf.float32, [None, 784]) 54 | y = tf.placeholder(tf.float32, [None, class_num]) 55 | keep_prob = tf.placeholder(tf.float32, []) 56 | 57 | # 下面几个步骤是实现 RNN / LSTM 的关键 58 | #################################################################### 59 | # **步骤1：RNN 的输入shape = (batch_size, timestep_size, input_size),这里恰好把784个点的字符信息还原成 28 * 28 的图片, 60 | #本质就是每一行作为一个时间点的输入,一共有28个时间点(行数),所以对于一副图片,要经过28个时间点的输入得到一个结果. 61 | X = tf.reshape(_X, [-1, 28, 28]) 62 | print(X.shape) 63 | 64 | # **步骤2：定义一层 LSTM_cell，只需要说明 hidden_size, 它会自动匹配输入的 X 的维度 65 | lstm_cell = rnn.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0, state_is_tuple=True) 66 | 67 | # **步骤3：添加 dropout layer, 一般只设置 output_keep_prob 68 | lstm_cell = rnn.DropoutWrapper(cell=lstm_cell, input_keep_prob=1.0, output_keep_prob=keep_prob) 69 | 70 | # **步骤4：调用 MultiRNNCell 来实现多层 LSTM 71 | #mlstm_cell = rnn.MultiRNNCell([lstm_cell,lstm_cell], state_is_tuple=True) 72 | 73 | # **步骤5：用全零来初始化state 74 | init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32) 75 | 76 | # **步骤6：方法一，调用 dynamic_rnn() 来让我们构建好的网络运行起来 77 | # ** 当 time_major==False 时， outputs.shape = [batch_size, timestep_size, hidden_size] 78 | # ** 所以，可以取 h_state = outputs[:, -1, :] 作为最后输出 79 | # ** state.shape = [layer_num, 2, batch_size, hidden_size], 80 | # ** 或者，可以取 h_state = state[-1][1] 作为最后输出 81 | # ** 最后输出维度是 [batch_size, hidden_size] 82 | outputs, state = tf.nn.dynamic_rnn(lstm_cell, inputs=X, initial_state=init_state, time_major=False) 83 | 84 | print(outputs.shape) 85 | 86 | h_state = outputs[:, -1, :] # 或者 h_state = state[-1][1] 87 | 88 | print(h_state.shape) 89 | 90 | 91 | y_pre = tf.nn.softmax(h_state) 92 | cross_entropy = -tf.reduce_mean(y * tf.log(y_pre)) 93 | train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy) 94 | 95 | correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1)) 96 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 97 | 98 | 99 | sess.run(tf.global_variables_initializer()) 100 | for i in range(2000): 101 | _batch_size = 128 102 | batch = mnist.train.next_batch(_batch_size) 103 | if (i+1)%200 == 0: 104 | train_accuracy = sess.run(accuracy, feed_dict={ 105 | _X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size}) 106 | # 已经迭代完成的 epoch 数: mnist.train.epochs_completed 107 | print("Iter%d, step %d, training accuracy %g" % ( mnist.train.epochs_completed, (i+1), train_accuracy)) 108 | sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size}) 109 | 110 | # 计算测试数据的准确率 111 | print("test accuracy %g"% sess.run(accuracy, feed_dict={ 112 | _X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]})) 113 | 114 | 115 | ''' 116 | # 上面 LSTM 部分的输出会是一个 [hidden_size] 的tensor，我们要分类的话，还需要接一个 softmax 层 117 | # 首先定义 softmax 的连接权重矩阵和偏置 118 | # out_W = tf.placeholder(tf.float32, [hidden_size, class_num], name='out_Weights') 119 | # out_bias = tf.placeholder(tf.float32, [class_num], name='out_bias') 120 | # 开始训练和测试 121 | W = tf.Variable(tf.truncated_normal([hidden_size, class_num], stddev=0.1), dtype=tf.float32) 122 | bias = tf.Variable(tf.constant(0.1,shape=[class_num]), dtype=tf.float32) 123 | y_pre = tf.nn.softmax(tf.matmul(h_state, W) + bias) 124 | 125 | 126 | # 损失和评估函数 127 | cross_entropy = -tf.reduce_mean(y * tf.log(y_pre)) 128 | train_op = tf.train.AdamOptimizer(lr).minimize(cross_entropy) 129 | 130 | correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(y,1)) 131 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 132 | 133 | 134 | sess.run(tf.global_variables_initializer()) 135 | for i in range(2000): 136 | _batch_size = 128 137 | batch = mnist.train.next_batch(_batch_size) 138 | if (i+1)%200 == 0: 139 | train_accuracy = sess.run(accuracy, feed_dict={ 140 | _X:batch[0], y: batch[1], keep_prob: 1.0, batch_size: _batch_size}) 141 | # 已经迭代完成的 epoch 数: mnist.train.epochs_completed 142 | print("Iter%d, step %d, training accuracy %g" % ( mnist.train.epochs_completed, (i+1), train_accuracy)) 143 | sess.run(train_op, feed_dict={_X: batch[0], y: batch[1], keep_prob: 0.5, batch_size: _batch_size}) 144 | 145 | # 计算测试数据的准确率 146 | print("test accuracy %g"% sess.run(accuracy, feed_dict={ 147 | _X: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0, batch_size:mnist.test.images.shape[0]})) 148 | 149 | ''' 150 | 151 | -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/lstm_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../Mnist/train.csv") 21 | test_frame = pd.read_csv("../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | #print(X_train.shape) 36 | #print(y_dummy.shape) 37 | #print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | 41 | #--------------------------------------Define Graph---------------------------------------------------# 42 | graph=tf.Graph() 43 | with graph.as_default(): 44 | 45 | #------------------------------------construct LSTM------------------------------------------# 46 | #place hoder 47 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 48 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 49 | 50 | #lstm instance 51 | #lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 52 | lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 53 | 54 | #multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell]) 55 | 56 | #initialize to zero 57 | init_state=lstm_cell.zero_state(batch_size=BATCH_SIZE,dtype=tf.float32) 58 | 59 | ''' 60 | #dynamic rnn 61 | outputs,states=tf.nn.dynamic_rnn(cell=multi_lstm,inputs=X_p,initial_state=init_state,dtype=tf.float32) 62 | #print(outputs.shape) 63 | h=outputs[:,-1,:] 64 | #print(h.shape) 65 | ''' 66 | 67 | outputs=[] 68 | state=init_state 69 | with tf.variable_scope("RNN"): 70 | for time_step in range(TIME_STEPS): 71 | if time_step>0: 72 | tf.get_variable_scope().reuse_variables() 73 | #shape of output is [batch_size,units_size],input:[batch_size,input_size] 74 | output,state=lstm_cell(inputs=X_p[:,time_step,:],state=state) 75 | #print(output.shape) 76 | outputs.append(output) 77 | #print(len(outputs)) 78 | h=outputs[-1] 79 | #print(h.shape) 80 | #--------------------------------------------------------------------------------------------# 81 | 82 | #---------------------------------define loss and optimizer----------------------------------# 83 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 84 | #print(loss.shape) 85 | 86 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 87 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 88 | 89 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 90 | 91 | init=tf.global_variables_initializer() 92 | 93 | 94 | #-------------------------------------------Define Session---------------------------------------# 95 | with tf.Session(graph=graph) as sess: 96 | sess.run(init) 97 | for epoch in range(1,EPOCH+1): 98 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 99 | train_losses=[] 100 | accus=[] 101 | #test_losses=[] 102 | print("epoch:",epoch) 103 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 104 | _,train_loss,accu=sess.run( 105 | fetches=(optimizer,cross_loss,accuracy), 106 | feed_dict={ 107 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 108 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 109 | } 110 | ) 111 | train_losses.append(train_loss) 112 | accus.append(accu) 113 | print("average training loss:", sum(train_losses) / len(train_losses)) 114 | print("accuracy:",sum(accus)/len(accus)) 115 | 116 | 117 | -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/lstm_sin_prediction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow.contrib.rnn as rnn 4 | import matplotlib.pyplot as plt 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=10 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS=1 11 | LEARNING_RATE=0.001 12 | EPOCH=150 13 | 14 | TRAIN_EXAMPLES=11000 15 | TEST_EXAMPLES=1100 16 | 17 | #------------------------------------Generate Data-----------------------------------------------# 18 | #generate data 19 | def generate(seq): 20 | X=[] 21 | y=[] 22 | for i in range(len(seq)-TIME_STEPS): 23 | X.append([seq[i:i+TIME_STEPS]]) 24 | y.append([seq[i+TIME_STEPS]]) 25 | return np.array(X,dtype=np.float32),np.array(y,dtype=np.float32) 26 | 27 | #s=[i for i in range(30)] 28 | #X,y=generate(s) 29 | #print(X) 30 | #print(y) 31 | 32 | seq_train=np.sin(np.linspace(start=0,stop=100,num=TRAIN_EXAMPLES,dtype=np.float32)) 33 | seq_test=np.sin(np.linspace(start=100,stop=110,num=TEST_EXAMPLES,dtype=np.float32)) 34 | 35 | #plt.plot(np.linspace(start=0,stop=100,num=10000,dtype=np.float32),seq_train) 36 | 37 | #plt.plot(np.linspace(start=100,stop=110,num=1000,dtype=np.float32),seq_test) 38 | #plt.show() 39 | 40 | X_train,y_train=generate(seq_train) 41 | #print(X_train.shape,y_train.shape) 42 | X_test,y_test=generate(seq_test) 43 | 44 | #reshape to (batch,time_steps,input_size) 45 | X_train=np.reshape(X_train,newshape=(-1,TIME_STEPS,1)) 46 | X_test=np.reshape(X_test,newshape=(-1,TIME_STEPS,1)) 47 | 48 | #draw y_test 49 | plt.plot(range(1000),y_test[:1000,0],"r*") 50 | #print(X_train.shape) 51 | #print(X_test.shape) 52 | 53 | 54 | #-----------------------------------------------------------------------------------------------------# 55 | 56 | 57 | #--------------------------------------Define Graph---------------------------------------------------# 58 | graph=tf.Graph() 59 | with graph.as_default(): 60 | 61 | #------------------------------------construct LSTM------------------------------------------# 62 | #place hoder 63 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,1),name="input_placeholder") 64 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,1),name="pred_placeholder") 65 | 66 | #lstm instance 67 | lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 68 | 69 | #initialize to zero 70 | init_state=lstm_cell.zero_state(batch_size=BATCH_SIZE,dtype=tf.float32) 71 | 72 | #dynamic rnn 73 | outputs,states=tf.nn.dynamic_rnn(cell=lstm_cell,inputs=X_p,initial_state=init_state,dtype=tf.float32) 74 | #print(outputs.shape) 75 | h=outputs[:,-1,:] 76 | #print(h.shape) 77 | #--------------------------------------------------------------------------------------------# 78 | 79 | #---------------------------------define loss and optimizer----------------------------------# 80 | mse=tf.losses.mean_squared_error(labels=y_p,predictions=h) 81 | #print(loss.shape) 82 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=mse) 83 | 84 | 85 | init=tf.global_variables_initializer() 86 | 87 | 88 | #-------------------------------------------Define Session---------------------------------------# 89 | with tf.Session(graph=graph) as sess: 90 | sess.run(init) 91 | for epoch in range(1,EPOCH+1): 92 | results = np.zeros(shape=(TEST_EXAMPLES, 1)) 93 | train_losses=[] 94 | test_losses=[] 95 | print("epoch:",epoch) 96 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 97 | _,train_loss=sess.run( 98 | fetches=(optimizer,mse), 99 | feed_dict={ 100 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 101 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 102 | } 103 | ) 104 | train_losses.append(train_loss) 105 | print("average training loss:", sum(train_losses) / len(train_losses)) 106 | 107 | 108 | for j in range(TEST_EXAMPLES//BATCH_SIZE): 109 | result,test_loss=sess.run( 110 | fetches=(h,mse), 111 | feed_dict={ 112 | X_p:X_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 113 | y_p:y_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 114 | } 115 | ) 116 | results[j*BATCH_SIZE:(j+1)*BATCH_SIZE]=result 117 | test_losses.append(test_loss) 118 | print("average test loss:", sum(test_losses) / len(test_losses)) 119 | plt.plot(range(1000),results[:1000,0]) 120 | plt.show() 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/lstm_sin_prediction_2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow.contrib.rnn as rnn 4 | import matplotlib.pyplot as plt 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=10 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=1 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=11000 16 | TEST_EXAMPLES=1100 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | def generate(seq): 21 | X=[] 22 | y=[] 23 | for i in range(len(seq)-TIME_STEPS): 24 | X.append([seq[i:i+TIME_STEPS]]) 25 | y.append([seq[i+TIME_STEPS]]) 26 | return np.array(X,dtype=np.float32),np.array(y,dtype=np.float32) 27 | 28 | #s=[i for i in range(30)] 29 | #X,y=generate(s) 30 | #print(X) 31 | #print(y) 32 | 33 | seq_train=np.sin(np.linspace(start=0,stop=100,num=TRAIN_EXAMPLES,dtype=np.float32)) 34 | seq_test=np.sin(np.linspace(start=100,stop=110,num=TEST_EXAMPLES,dtype=np.float32)) 35 | 36 | #plt.plot(np.linspace(start=0,stop=100,num=10000,dtype=np.float32),seq_train) 37 | 38 | #plt.plot(np.linspace(start=100,stop=110,num=1000,dtype=np.float32),seq_test) 39 | #plt.show() 40 | 41 | X_train,y_train=generate(seq_train) 42 | #print(X_train.shape,y_train.shape) 43 | X_test,y_test=generate(seq_test) 44 | 45 | #reshape to (batch,time_steps,input_size) 46 | X_train=np.reshape(X_train,newshape=(-1,TIME_STEPS,1)) 47 | X_test=np.reshape(X_test,newshape=(-1,TIME_STEPS,1)) 48 | 49 | #draw y_test 50 | plt.plot(range(1000),y_test[:1000,0],"r*") 51 | #print(X_train.shape) 52 | #print(X_test.shape) 53 | 54 | 55 | #-----------------------------------------------------------------------------------------------------# 56 | 57 | 58 | #--------------------------------------Define Graph---------------------------------------------------# 59 | graph=tf.Graph() 60 | with graph.as_default(): 61 | 62 | #------------------------------------construct LSTM------------------------------------------# 63 | #place hoder 64 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,1),name="input_placeholder") 65 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,1),name="pred_placeholder") 66 | 67 | #lstm instance 68 | lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 69 | lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 70 | 71 | multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell]) 72 | 73 | #initialize to zero 74 | init_state=multi_lstm.zero_state(batch_size=BATCH_SIZE,dtype=tf.float32) 75 | 76 | #dynamic rnn 77 | outputs,states=tf.nn.dynamic_rnn(cell=multi_lstm,inputs=X_p,initial_state=init_state,dtype=tf.float32) 78 | #print(outputs.shape) 79 | h=outputs[:,-1,:] 80 | #print(h.shape) 81 | #--------------------------------------------------------------------------------------------# 82 | 83 | #---------------------------------define loss and optimizer----------------------------------# 84 | mse=tf.losses.mean_squared_error(labels=y_p,predictions=h) 85 | #print(loss.shape) 86 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=mse) 87 | 88 | 89 | init=tf.global_variables_initializer() 90 | 91 | 92 | #-------------------------------------------Define Session---------------------------------------# 93 | with tf.Session(graph=graph) as sess: 94 | sess.run(init) 95 | for epoch in range(1,EPOCH+1): 96 | results = np.zeros(shape=(TEST_EXAMPLES, 1)) 97 | train_losses=[] 98 | test_losses=[] 99 | print("epoch:",epoch) 100 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 101 | _,train_loss=sess.run( 102 | fetches=(optimizer,mse), 103 | feed_dict={ 104 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 105 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 106 | } 107 | ) 108 | train_losses.append(train_loss) 109 | print("average training loss:", sum(train_losses) / len(train_losses)) 110 | 111 | 112 | for j in range(TEST_EXAMPLES//BATCH_SIZE): 113 | result,test_loss=sess.run( 114 | fetches=(h,mse), 115 | feed_dict={ 116 | X_p:X_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 117 | y_p:y_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 118 | } 119 | ) 120 | results[j*BATCH_SIZE:(j+1)*BATCH_SIZE]=result 121 | test_losses.append(test_loss) 122 | print("average test loss:", sum(test_losses) / len(test_losses)) 123 | plt.plot(range(1000),results[:1000,0]) 124 | plt.show() -------------------------------------------------------------------------------- /14.RNN/BasicLSTM/lstm_sin_prediction_3.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow.contrib.rnn as rnn 4 | import matplotlib.pyplot as plt 5 | 6 | TIME_STEPS=10 7 | BATCH_SIZE=128 8 | HIDDEN_UNITS1=30 9 | HIDDEN_UNITS=1 10 | LEARNING_RATE=0.001 11 | EPOCH=50 12 | 13 | TRAIN_EXAMPLES=11000 14 | TEST_EXAMPLES=1100 15 | 16 | #------------------------------------Generate Data-----------------------------------------------# 17 | #generate data 18 | def generate(seq): 19 | X=[] 20 | y=[] 21 | for i in range(len(seq)-TIME_STEPS): 22 | X.append([seq[i:i+TIME_STEPS]]) 23 | y.append([seq[i+TIME_STEPS]]) 24 | return np.array(X,dtype=np.float32),np.array(y,dtype=np.float32) 25 | 26 | #s=[i for i in range(30)] 27 | #X,y=generate(s) 28 | #print(X) 29 | #print(y) 30 | 31 | seq_train=np.sin(np.linspace(start=0,stop=100,num=TRAIN_EXAMPLES,dtype=np.float32)) 32 | seq_test=np.sin(np.linspace(start=100,stop=110,num=TEST_EXAMPLES,dtype=np.float32)) 33 | 34 | #plt.plot(np.linspace(start=0,stop=100,num=10000,dtype=np.float32),seq_train) 35 | 36 | #plt.plot(np.linspace(start=100,stop=110,num=1000,dtype=np.float32),seq_test) 37 | #plt.show() 38 | 39 | X_train,y_train=generate(seq_train) 40 | #print(X_train.shape,y_train.shape) 41 | X_test,y_test=generate(seq_test) 42 | 43 | #reshape to (batch,time_steps,input_size) 44 | X_train=np.reshape(X_train,newshape=(-1,TIME_STEPS,1)) 45 | X_test=np.reshape(X_test,newshape=(-1,TIME_STEPS,1)) 46 | 47 | #draw y_test 48 | plt.plot(range(1000),y_test[:1000,0],"r*") 49 | #print(X_train.shape) 50 | #print(X_test.shape) 51 | 52 | 53 | #-----------------------------------------------------------------------------------------------------# 54 | 55 | 56 | #--------------------------------------Define Graph---------------------------------------------------# 57 | graph=tf.Graph() 58 | with graph.as_default(): 59 | 60 | #------------------------------------construct LSTM------------------------------------------# 61 | #place hoder 62 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,1),name="input_placeholder") 63 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,1),name="pred_placeholder") 64 | 65 | #lstm instance 66 | lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 67 | lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS) 68 | 69 | multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell]) 70 | 71 | 72 | #自己初始化state 73 | #第一层state 74 | lstm_layer1_c=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1)) 75 | lstm_layer1_h=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1)) 76 | layer1_state=rnn.LSTMStateTuple(c=lstm_layer1_c,h=lstm_layer1_h) 77 | 78 | #第二层state 79 | lstm_layer2_c = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS)) 80 | lstm_layer2_h = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS)) 81 | layer2_state = rnn.LSTMStateTuple(c=lstm_layer2_c, h=lstm_layer2_h) 82 | 83 | init_state=(layer1_state,layer2_state) 84 | print(init_state) 85 | 86 | 87 | #自己展开RNN计算 88 | outputs = list() #用来接收存储每步的结果 89 | state = init_state 90 | with tf.variable_scope('RNN'): 91 | for timestep in range(TIME_STEPS): 92 | if timestep > 0: 93 | tf.get_variable_scope().reuse_variables() 94 | # 这里的state保存了每一层 LSTM 的状态 95 | (cell_output, state) = multi_lstm(X_p[:, timestep, :], state) 96 | outputs.append(cell_output) 97 | h = outputs[-1] 98 | 99 | 100 | #---------------------------------define loss and optimizer----------------------------------# 101 | mse=tf.losses.mean_squared_error(labels=y_p,predictions=h) 102 | #print(loss.shape) 103 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=mse) 104 | 105 | init=tf.global_variables_initializer() 106 | 107 | 108 | #-------------------------------------------Define Session---------------------------------------# 109 | with tf.Session(graph=graph) as sess: 110 | sess.run(init) 111 | for epoch in range(1,EPOCH+1): 112 | results = np.zeros(shape=(TEST_EXAMPLES, 1)) 113 | train_losses=[] 114 | test_losses=[] 115 | print("epoch:",epoch) 116 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 117 | _,train_loss=sess.run( 118 | fetches=(optimizer,mse), 119 | feed_dict={ 120 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 121 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 122 | } 123 | ) 124 | train_losses.append(train_loss) 125 | print("average training loss:", sum(train_losses) / len(train_losses)) 126 | 127 | 128 | for j in range(TEST_EXAMPLES//BATCH_SIZE): 129 | result,test_loss=sess.run( 130 | fetches=(h,mse), 131 | feed_dict={ 132 | X_p:X_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 133 | y_p:y_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 134 | } 135 | ) 136 | results[j*BATCH_SIZE:(j+1)*BATCH_SIZE]=result 137 | test_losses.append(test_loss) 138 | print("average test loss:", sum(test_losses) / len(test_losses)) 139 | plt.plot(range(1000),results[:1000,0]) 140 | plt.show() 141 | -------------------------------------------------------------------------------- /14.RNN/GRU/1.basic.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | def test1(): 5 | inputs=tf.ones(shape=(3,5,10)) 6 | gru=tf.keras.layers.GRU(units=20,return_sequences=True,return_state=True) 7 | outputs,states=gru(inputs=inputs) 8 | print("outputs:\n",outputs) 9 | print("states:\n",states) 10 | 11 | 12 | def test2(): 13 | inputs = tf.ones(shape=(3, 5, 10)) 14 | bi_gru=tf.keras.layers.Bidirectional( 15 | layer=tf.keras.layers.GRU(units=20, return_sequences=True, return_state=False) 16 | ) 17 | outputs = bi_gru(inputs=inputs) 18 | print("outputs:\n", outputs) 19 | 20 | 21 | 22 | def test3(): 23 | inputs = tf.ones(shape=(3, 5, 10)) 24 | bi_gru = tf.keras.layers.Bidirectional( 25 | layer=tf.keras.layers.GRU(units=20, return_sequences=True, return_state=True) 26 | ) 27 | outputs,states_forward,states_backward = bi_gru(inputs=inputs) 28 | print("outputs:\n", outputs) 29 | print("states_forward:\n", states_forward) 30 | print("states_backward:\n", states_backward) 31 | 32 | 33 | 34 | 35 | 36 | if __name__=="__main__": 37 | #test1() 38 | test3() -------------------------------------------------------------------------------- /14.RNN/GRU/bigru_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | from tensorflow.contrib.layers.python.layers import initializers 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | TIME_STEPS=28 10 | BATCH_SIZE=128 11 | HIDDEN_UNITS1=30 12 | HIDDEN_UNITS=10 13 | LEARNING_RATE=0.001 14 | EPOCH=10 15 | 16 | TRAIN_EXAMPLES=42000 17 | TEST_EXAMPLES=28000 18 | 19 | #------------------------------------Generate Data-----------------------------------------------# 20 | #generate data 21 | train_frame = pd.read_csv("../../Mnist/train.csv") 22 | test_frame = pd.read_csv("../../Mnist/test.csv") 23 | 24 | # pop the labels and one-hot coding 25 | train_labels_frame = train_frame.pop("label") 26 | 27 | # get values 28 | # one-hot on labels 29 | X_train = train_frame.astype(np.float32).values 30 | y_train=pd.get_dummies(data=train_labels_frame).values 31 | X_test = test_frame.astype(np.float32).values 32 | 33 | #trans the shape to (batch,time_steps,input_size) 34 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 35 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 36 | #print(X_train.shape) 37 | #print(y_dummy.shape) 38 | #print(X_test.shape) 39 | 40 | #-----------------------------------------------------------------------------------------------------# 41 | 42 | 43 | #--------------------------------------Define Graph---------------------------------------------------# 44 | graph=tf.Graph() 45 | with graph.as_default(): 46 | 47 | #------------------------------------construct LSTM------------------------------------------# 48 | #place hoder 49 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 50 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 51 | 52 | #gru instance 53 | gru_forward_1=tf.nn.rnn_cell.GRUCell( 54 | num_units=HIDDEN_UNITS1, 55 | kernel_initializer=initializers.xavier_initializer(), 56 | bias_initializer=tf.initializers.random_normal() 57 | ) 58 | 59 | gru_forward_1 = rnn.AttentionCellWrapper(cell=gru_forward_1, attn_length=5) 60 | 61 | gru_forward_2=tf.nn.rnn_cell.GRUCell( 62 | num_units=HIDDEN_UNITS, 63 | kernel_initializer=initializers.xavier_initializer(), 64 | bias_initializer=tf.initializers.random_normal() 65 | ) 66 | gru_forward_2 = rnn.AttentionCellWrapper(cell=gru_forward_2, attn_length=5) 67 | gru_forward=rnn.MultiRNNCell(cells=[gru_forward_1,gru_forward_2]) 68 | 69 | gru_backward_1=tf.nn.rnn_cell.GRUCell( 70 | num_units=HIDDEN_UNITS1, 71 | kernel_initializer=initializers.xavier_initializer(), 72 | bias_initializer=tf.initializers.random_normal() 73 | ) 74 | 75 | gru_backward_1 = rnn.AttentionCellWrapper(cell=gru_backward_1, attn_length=5) 76 | gru_backward_2=tf.nn.rnn_cell.GRUCell( 77 | num_units=HIDDEN_UNITS, 78 | kernel_initializer=initializers.xavier_initializer(), 79 | bias_initializer=tf.initializers.random_normal() 80 | ) 81 | gru_backward_2 = rnn.AttentionCellWrapper(cell=gru_backward_2, attn_length=5) 82 | gru_backward=rnn.MultiRNNCell(cells=[gru_backward_1,gru_backward_2]) 83 | 84 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 85 | cell_fw=gru_forward, 86 | cell_bw=gru_backward, 87 | inputs=X_p, 88 | dtype=tf.float32 89 | ) 90 | 91 | outputs_fw=outputs[0] 92 | outputs_bw = outputs[1] 93 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 94 | # print(h.shape) 95 | #---------------------------------------;-----------------------------------------------------# 96 | 97 | #---------------------------------define loss and optimizer----------------------------------# 98 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 99 | #print(loss.shape) 100 | 101 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 102 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 103 | 104 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 105 | 106 | init=tf.global_variables_initializer() 107 | 108 | 109 | #-------------------------------------------Define Session---------------------------------------# 110 | with tf.Session(graph=graph) as sess: 111 | sess.run(init) 112 | for epoch in range(1,EPOCH+1): 113 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 114 | train_losses=[] 115 | accus=[] 116 | #test_losses=[] 117 | print("epoch:",epoch) 118 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 119 | _,train_loss,accu=sess.run( 120 | fetches=(optimizer,cross_loss,accuracy), 121 | feed_dict={ 122 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 123 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 124 | } 125 | ) 126 | train_losses.append(train_loss) 127 | accus.append(accu) 128 | print("average training loss:", sum(train_losses) / len(train_losses)) 129 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/GRU/gru.py: -------------------------------------------------------------------------------- 1 | ''' 2 | tf.contrib.rnn.GRUCell 3 | 4 | __init__( 5 | num_units, 6 | activation=None, 7 | reuse=None, 8 | kernel_initializer=None, 9 | bias_initializer=None, 10 | name=None, 11 | dtype=None 12 | ) 13 | 14 | Args: 15 | num_units: int, GRU cell 中的units数量 16 | activation: Nonlinearity to use. Default: tanh. 17 | reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not True, and the existing scope already has the given variables, an error is raised. 18 | kernel_initializer: (optional) The initializer to use for the weight and projection matrices. 19 | bias_initializer: (optional) The initializer to use for the bias. 20 | name: String, the name of the layer. Layers with the same name will share weights, but to avoid mistakes we require reuse=True in such cases. 21 | dtype: Default dtype of the layer (default of None means use the type of the first input). Required when build is called before call. 22 | 23 | ''' 24 | 25 | 26 | import numpy as np 27 | import tensorflow as tf 28 | from tensorflow.contrib.layers.python.layers import initializers 29 | 30 | gru_cell=tf.nn.rnn_cell.GRUCell( 31 | num_units=128, 32 | kernel_initializer=initializers.xavier_initializer(), 33 | bias_initializer=tf.initializers.truncated_normal() 34 | ) 35 | 36 | 37 | #gru_cell=tf.nn.rnn_cell.LSTMCell( 38 | # num_units=128, 39 | # use_peepholes=True, 40 | # initializer=initializers.xavier_initializer(), 41 | # num_proj=64, 42 | # name="LSTM_CELL" 43 | #) 44 | 45 | print("output_size:",gru_cell.output_size) #output_size: 128 46 | print("state_size:",gru_cell.state_size) #state_size: 128 -------------------------------------------------------------------------------- /14.RNN/IndyLSTM/IndyLSTM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | __init__ 3 | __init__( 4 | num_units, 5 | forget_bias=1.0, 6 | activation=None, 7 | reuse=None, 8 | kernel_initializer=None, 9 | bias_initializer=None, 10 | name=None, 11 | dtype=None 12 | ) 13 | Initialize the IndyLSTM cell. 14 | 15 | Args: 16 | num_units: int, The number of units in the LSTM cell. 17 | forget_bias: float, The bias added to forget gates (see above). Must set to 0.0 manually when restoring from CudnnLSTM-trained checkpoints. 18 | activation: Activation function of the inner states. Default: tanh. 19 | reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not True, and the existing scope already has the given variables, an error is raised. 20 | kernel_initializer: (optional) The initializer to use for the weight matrix applied to the inputs. 21 | bias_initializer: (optional) The initializer to use for the bias. 22 | name: String, the name of the layer. Layers with the same name will share weights, but to avoid mistakes we require reuse=True in such cases. 23 | dtype: Default dtype of the layer (default of None means use the type of the first input). Required when build is called before call. 24 | 25 | ''' 26 | 27 | import numpy as np 28 | import tensorflow as tf 29 | from tensorflow.contrib.layers.python.layers import initializers 30 | import tensorflow.contrib.rnn as rnn 31 | 32 | indy_lstm_cell=rnn.IndyLSTMCell(num_units=128) 33 | 34 | 35 | #lstm_cell=tf.nn.rnn_cell.LSTMCell( 36 | # num_units=128, 37 | # use_peepholes=True, 38 | # initializer=initializers.xavier_initializer(), 39 | # num_proj=64, 40 | # name="LSTM_CELL" 41 | #) 42 | 43 | print("output_size:",indy_lstm_cell.output_size) 44 | print("state_size:",indy_lstm_cell.state_size) 45 | print(indy_lstm_cell.state_size.h) 46 | print(indy_lstm_cell.state_size.c) -------------------------------------------------------------------------------- /14.RNN/IndyLSTM/Indylstm_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | from tensorflow.contrib.layers.python.layers import initializers 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | TIME_STEPS=28 10 | BATCH_SIZE=128 11 | HIDDEN_UNITS1=30 12 | HIDDEN_UNITS=10 13 | LEARNING_RATE=0.001 14 | EPOCH=50 15 | 16 | TRAIN_EXAMPLES=42000 17 | TEST_EXAMPLES=28000 18 | 19 | #------------------------------------Generate Data-----------------------------------------------# 20 | #generate data 21 | train_frame = pd.read_csv("../../Mnist/train.csv") 22 | test_frame = pd.read_csv("../../Mnist/test.csv") 23 | 24 | # pop the labels and one-hot coding 25 | train_labels_frame = train_frame.pop("label") 26 | 27 | # get values 28 | # one-hot on labels 29 | X_train = train_frame.astype(np.float32).values 30 | y_train=pd.get_dummies(data=train_labels_frame).values 31 | X_test = test_frame.astype(np.float32).values 32 | 33 | #trans the shape to (batch,time_steps,input_size) 34 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 35 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 36 | #print(X_train.shape) 37 | #print(y_dummy.shape) 38 | #print(X_test.shape) 39 | 40 | #-----------------------------------------------------------------------------------------------------# 41 | 42 | #--------------------------------------Define Graph---------------------------------------------------# 43 | graph=tf.Graph() 44 | with graph.as_default(): 45 | 46 | #------------------------------------construct LSTM------------------------------------------# 47 | #place hoder 48 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 49 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 50 | 51 | #lstm instance 52 | #lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1) 53 | lstm_cell=rnn.IndyLSTMCell( 54 | num_units=HIDDEN_UNITS, 55 | kernel_initializer=initializers.xavier_initializer(), 56 | bias_initializer=tf.initializers.random_normal() 57 | ) 58 | #lstm_cell=tf.nn.rnn_cell.LSTMCell( 59 | # num_units=HIDDEN_UNITS, 60 | # use_peepholes=True, 61 | # initializer=initializers.xavier_initializer(), 62 | # num_proj=HIDDEN_UNITS 63 | #) 64 | 65 | #multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell]) 66 | 67 | #initialize to zero 68 | init_state=lstm_cell.zero_state(batch_size=BATCH_SIZE,dtype=tf.float32) 69 | 70 | ''' 71 | #dynamic rnn 72 | outputs,states=tf.nn.dynamic_rnn(cell=multi_lstm,inputs=X_p,initial_state=init_state,dtype=tf.float32) 73 | #print(outputs.shape) 74 | h=outputs[:,-1,:] 75 | #print(h.shape) 76 | ''' 77 | 78 | outputs=[] 79 | state=init_state 80 | with tf.variable_scope("RNN"): 81 | for time_step in range(TIME_STEPS): 82 | if time_step>0: 83 | tf.get_variable_scope().reuse_variables() 84 | #shape of output is [batch_size,units_size],input:[batch_size,input_size] 85 | output,state=lstm_cell(inputs=X_p[:,time_step,:],state=state) 86 | #print(output.shape) 87 | outputs.append(output) 88 | #print(len(outputs)) 89 | h=outputs[-1] 90 | #print(h.shape) 91 | #--------------------------------------------------------------------------------------------# 92 | 93 | #---------------------------------define loss and optimizer----------------------------------# 94 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 95 | #print(loss.shape) 96 | 97 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 98 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 99 | 100 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 101 | 102 | init=tf.global_variables_initializer() 103 | 104 | 105 | #-------------------------------------------Define Session---------------------------------------# 106 | with tf.Session(graph=graph) as sess: 107 | sess.run(init) 108 | for epoch in range(1,EPOCH+1): 109 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 110 | train_losses=[] 111 | accus=[] 112 | #test_losses=[] 113 | print("epoch:",epoch) 114 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 115 | _,train_loss,accu=sess.run( 116 | fetches=(optimizer,cross_loss,accuracy), 117 | feed_dict={ 118 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 119 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 120 | } 121 | ) 122 | train_losses.append(train_loss) 123 | accus.append(accu) 124 | print("average training loss:", sum(train_losses) / len(train_losses)) 125 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/IndyLSTM/Indylstm_mnist2.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | from tensorflow.contrib.layers.python.layers.initializers import xavier_initializer 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | TIME_STEPS=28 10 | BATCH_SIZE=128 11 | HIDDEN_UNITS1=30 12 | HIDDEN_UNITS=10 13 | LEARNING_RATE=0.01 14 | EPOCH=50 15 | 16 | TRAIN_EXAMPLES=42000 17 | TEST_EXAMPLES=28000 18 | 19 | #------------------------------------Generate Data-----------------------------------------------# 20 | #generate data 21 | train_frame = pd.read_csv("../../Mnist/train.csv") 22 | test_frame = pd.read_csv("../../Mnist/test.csv") 23 | 24 | # pop the labels and one-hot coding 25 | train_labels_frame = train_frame.pop("label") 26 | 27 | # get values 28 | # one-hot on labels 29 | X_train = train_frame.astype(np.float32).values 30 | y_train=pd.get_dummies(data=train_labels_frame).values 31 | X_test = test_frame.astype(np.float32).values 32 | 33 | #trans the shape to (batch,time_steps,input_size) 34 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 35 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 36 | #print(X_train.shape) 37 | #print(y_dummy.shape) 38 | #print(X_test.shape) 39 | 40 | #-----------------------------------------------------------------------------------------------------# 41 | 42 | 43 | #--------------------------------------Define Graph---------------------------------------------------# 44 | graph=tf.Graph() 45 | with graph.as_default(): 46 | 47 | #------------------------------------construct LSTM------------------------------------------# 48 | #place hoder 49 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 50 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 51 | 52 | #lstm instance 53 | lstm_forward_1=rnn.IndyLSTMCell( 54 | num_units=HIDDEN_UNITS1, 55 | activation=tf.nn.leaky_relu, 56 | kernel_initializer=xavier_initializer(), 57 | bias_initializer=tf.initializers.random_normal() 58 | ) 59 | 60 | 61 | #加attention(这里的attention和encoder-decoder架构的attention稍有不同,可以看做简化版本的attention) 62 | lstm_forward_1=rnn.AttentionCellWrapper(cell=lstm_forward_1,attn_length=5) 63 | 64 | lstm_forward_2 = rnn.IndyLSTMCell( 65 | num_units=HIDDEN_UNITS, 66 | activation=tf.nn.leaky_relu, 67 | kernel_initializer=tf.initializers.ones(), 68 | bias_initializer=tf.initializers.random_normal() 69 | ) 70 | 71 | # 加attention 72 | lstm_forward_2 = rnn.AttentionCellWrapper(cell=lstm_forward_2, attn_length=5) 73 | lstm_forward=tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_forward_1,lstm_forward_2]) 74 | 75 | lstm_backward_1 = rnn.IndyLSTMCell( 76 | num_units=HIDDEN_UNITS1, 77 | activation=tf.nn.leaky_relu, 78 | kernel_initializer=xavier_initializer(), 79 | bias_initializer=tf.initializers.random_normal() 80 | ) 81 | #加attention 82 | lstm_backward_1 = rnn.AttentionCellWrapper(cell=lstm_backward_1, attn_length=5) 83 | 84 | lstm_backward_2 = rnn.IndyLSTMCell( 85 | num_units=HIDDEN_UNITS, 86 | activation=tf.nn.leaky_relu, 87 | kernel_initializer=tf.initializers.ones(), 88 | bias_initializer=tf.initializers.random_normal() 89 | ) 90 | lstm_backward_2 = rnn.AttentionCellWrapper(cell=lstm_backward_2, attn_length=5) 91 | lstm_backward=tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_backward_1,lstm_backward_2]) 92 | 93 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 94 | cell_fw=lstm_forward, 95 | cell_bw=lstm_backward, 96 | inputs=X_p, 97 | dtype=tf.float32 98 | ) 99 | 100 | outputs_fw=outputs[0] 101 | outputs_bw = outputs[1] 102 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 103 | # print(h.shape) 104 | #--------------------------------------------------------------------------------------------# 105 | 106 | #---------------------------------define loss and optimizer----------------------------------# 107 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 108 | #print(loss.shape) 109 | 110 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 111 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 112 | 113 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 114 | 115 | init=tf.global_variables_initializer() 116 | 117 | 118 | #-------------------------------------------Define Session---------------------------------------# 119 | with tf.Session(graph=graph) as sess: 120 | sess.run(init) 121 | for epoch in range(1,EPOCH+1): 122 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 123 | train_losses=[] 124 | accus=[] 125 | #test_losses=[] 126 | print("epoch:",epoch) 127 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 128 | _,train_loss,accu=sess.run( 129 | fetches=(optimizer,cross_loss,accuracy), 130 | feed_dict={ 131 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 132 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 133 | } 134 | ) 135 | train_losses.append(train_loss) 136 | accus.append(accu) 137 | print("average training loss:", sum(train_losses) / len(train_losses)) 138 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/LSTM/lstm_cell.py: -------------------------------------------------------------------------------- 1 | ''' 2 | tf.keras.layers.LSTMCell 3 | 4 | __init__( 5 | units, 6 | activation='tanh', 7 | recurrent_activation='hard_sigmoid', 8 | use_bias=True, 9 | kernel_initializer='glorot_uniform', 10 | recurrent_initializer='orthogonal', 11 | bias_initializer='zeros', 12 | unit_forget_bias=True, 13 | kernel_regularizer=None, 14 | recurrent_regularizer=None, 15 | bias_regularizer=None, 16 | kernel_constraint=None, 17 | recurrent_constraint=None, 18 | bias_constraint=None, 19 | dropout=0.0, 20 | recurrent_dropout=0.0, 21 | implementation=1, 22 | **kwargs 23 | ) 24 | 25 | 参数: 26 | - units: 正整数，表示输出维度 27 | - activation: 激活函数，默认是tanh，要是传入为None，那么不会有任何激活函数被使用 28 | - recurrent_activation: 在递归步(recurrent step)使用的激活函数，默认是hard_sigmoid. 要是传入为None，那么不会有任何激活函数被使用 29 | - use_bias: 布尔值，表示是否使用bias 30 | - kernel_initializer: input的线性变换这里的kernel权重矩阵的初始化方法 31 | - recurrent_initializer: recurent这里的线性变换这里的kernel权重矩阵的初始化方法 32 | - bias_initializer: bias初始化方法 33 | - unit_forget_bias: Boolean. If True, add 1 to the bias of the forget gate at initialization. Setting it to true will also force bias_initializer="zeros". This is recommended in Jozefowicz et al. 34 | - kernel_regularizer: Regularizer function applied to the kernel weights matrix. 35 | - recurrent_regularizer: Regularizer function applied to the recurrent_kernel weights matrix. 36 | - bias_regularizer: Regularizer function applied to the bias vector. 37 | - kernel_constraint: Constraint function applied to the kernel weights matrix. 38 | - recurrent_constraint: Constraint function applied to the recurrent_kernel weights matrix. 39 | - bias_constraint: Constraint function applied to the bias vector. 40 | - dropout: inputs线性变换时候的dropout比率 41 | - recurrent_dropout: recurrent state线性变换时候的dropout比率 42 | - implementation: Implementation mode, either 1 or 2. Mode 1 will structure its operations as a larger number of smaller dot products and additions, whereas mode 2 will batch them into fewer, larger operations. These modes will have different performance profiles on different hardware and for different applications. 43 | 44 | 45 | 46 | 调用时参数: 47 | - inputs: A 2D tensor. 48 | - states: List of state tensors corresponding to the previous timestep. 49 | - training: 布尔值，表示这层是否表现为训练模式或者是前向计算模式。在dropout或者recurrent_dropout被使用的时候会有区别 50 | 51 | ''' 52 | 53 | 54 | -------------------------------------------------------------------------------- /14.RNN/LSTM/lstm_cell_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | from tensorflow.contrib.layers.python.layers.initializers import xavier_initializer 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | TIME_STEPS=28 10 | BATCH_SIZE=128 11 | HIDDEN_UNITS1=30 12 | HIDDEN_UNITS=10 13 | LEARNING_RATE=0.001 14 | EPOCH=50 15 | 16 | TRAIN_EXAMPLES=42000 17 | TEST_EXAMPLES=28000 18 | 19 | #------------------------------------Generate Data-----------------------------------------------# 20 | #generate data 21 | train_frame = pd.read_csv("../../Mnist/train.csv") 22 | test_frame = pd.read_csv("../../Mnist/test.csv") 23 | 24 | # pop the labels and one-hot coding 25 | train_labels_frame = train_frame.pop("label") 26 | 27 | # get values 28 | # one-hot on labels 29 | X_train = train_frame.astype(np.float32).values 30 | y_train=pd.get_dummies(data=train_labels_frame).values 31 | X_test = test_frame.astype(np.float32).values 32 | 33 | #trans the shape to (batch,time_steps,input_size) 34 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 35 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 36 | #print(X_train.shape) 37 | #print(y_dummy.shape) 38 | #print(X_test.shape) 39 | 40 | #-----------------------------------------------------------------------------------------------------# 41 | 42 | 43 | #--------------------------------------Define Graph---------------------------------------------------# 44 | graph=tf.Graph() 45 | with graph.as_default(): 46 | 47 | #------------------------------------construct LSTM------------------------------------------# 48 | #place hoder 49 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 50 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 51 | 52 | #lstm instance 53 | lstm_forward_1=tf.nn.rnn_cell.LSTMCell( 54 | num_units=HIDDEN_UNITS1, 55 | use_peepholes=True, 56 | initializer=xavier_initializer(), 57 | #num_proj=HIDDEN_UNITS1 58 | ) 59 | 60 | #加attention(这里的attention和encoder-decoder架构的attention稍有不同,可以看做简化版本的attention) 61 | lstm_forward_1=rnn.AttentionCellWrapper(cell=lstm_forward_1,attn_length=5) 62 | 63 | lstm_forward_2 = tf.nn.rnn_cell.LSTMCell( 64 | num_units=HIDDEN_UNITS, 65 | use_peepholes=True, 66 | initializer=xavier_initializer(), 67 | #num_proj=HIDDEN_UNITS 68 | ) 69 | 70 | # 加attention 71 | lstm_forward_2 = rnn.AttentionCellWrapper(cell=lstm_forward_2, attn_length=5) 72 | lstm_forward=tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_forward_1,lstm_forward_2]) 73 | 74 | lstm_backward_1 = tf.nn.rnn_cell.LSTMCell( 75 | num_units=HIDDEN_UNITS1, 76 | use_peepholes=True, 77 | initializer=xavier_initializer(), 78 | #num_proj=HIDDEN_UNITS1 79 | ) 80 | #加attention 81 | lstm_backward_1 = rnn.AttentionCellWrapper(cell=lstm_backward_1, attn_length=5) 82 | 83 | lstm_backward_2 = tf.nn.rnn_cell.LSTMCell( 84 | num_units=HIDDEN_UNITS, 85 | use_peepholes=True, 86 | initializer=xavier_initializer(), 87 | #num_proj=HIDDEN_UNITS 88 | ) 89 | lstm_backward_2 = rnn.AttentionCellWrapper(cell=lstm_backward_2, attn_length=5) 90 | lstm_backward=tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_backward_1,lstm_backward_2]) 91 | 92 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 93 | cell_fw=lstm_forward, 94 | cell_bw=lstm_backward, 95 | inputs=X_p, 96 | dtype=tf.float32 97 | ) 98 | 99 | outputs_fw=outputs[0] 100 | outputs_bw = outputs[1] 101 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 102 | # print(h.shape) 103 | #--------------------------------------------------------------------------------------------# 104 | 105 | #---------------------------------define loss and optimizer----------------------------------# 106 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 107 | #print(loss.shape) 108 | 109 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 110 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 111 | 112 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 113 | 114 | init=tf.global_variables_initializer() 115 | 116 | 117 | #-------------------------------------------Define Session---------------------------------------# 118 | with tf.Session(graph=graph) as sess: 119 | sess.run(init) 120 | for epoch in range(1,EPOCH+1): 121 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 122 | train_losses=[] 123 | accus=[] 124 | #test_losses=[] 125 | print("epoch:",epoch) 126 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 127 | _,train_loss,accu=sess.run( 128 | fetches=(optimizer,cross_loss,accuracy), 129 | feed_dict={ 130 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 131 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 132 | } 133 | ) 134 | train_losses.append(train_loss) 135 | accus.append(accu) 136 | print("average training loss:", sum(train_losses) / len(train_losses)) 137 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /14.RNN/LSTM/lstm_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | from tensorflow.keras import layers 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../../Mnist/train.csv") 21 | test_frame = pd.read_csv("../../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | print(X_train.shape) 36 | print(y_train.shape) 37 | print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | class LSTM_Model(tf.keras.Model): 41 | def __init__(self): 42 | super(LSTM_Model, self).__init__() 43 | self.lstm_1=layers.LSTM(units=HIDDEN_UNITS1,return_sequences=True) 44 | self.lstm_2=layers.LSTM(units=HIDDEN_UNITS,return_sequences=True) 45 | 46 | 47 | def __call__(self, inputs,training=True): 48 | h1=self.lstm_1(inputs=inputs,training=training) 49 | h2=self.lstm_2(inputs=h1,training=training)[:,-1,:] 50 | return h2 51 | 52 | def train(): 53 | model=LSTM_Model() 54 | optimizer=tf.keras.optimizers.Adam(LEARNING_RATE) 55 | cce=tf.keras.losses.CategoricalCrossentropy(from_logits=True) 56 | for epoch in range(1,EPOCH+1): 57 | print("epoch:", epoch) 58 | train_losses=[] 59 | accus=[] 60 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 61 | with tf.GradientTape() as tape: 62 | logits=model(inputs=X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE],training=True) 63 | #print("logits:\n",logits.shape) 64 | #loss 65 | loss=cce(y_true=y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE],y_pred=logits) 66 | #prediction 67 | correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 1)) 68 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 69 | # print("loss:",loss.numpy()) 70 | # print("accuracy:",accuracy.numpy()) 71 | 72 | #计算梯度 73 | gradient = tape.gradient(target=loss, sources=model.trainable_variables) 74 | #print("gradient:",gradient) 75 | #应用梯度 76 | optimizer.apply_gradients(zip(gradient, model.trainable_variables)) 77 | 78 | train_losses.append(loss) 79 | accus.append(accuracy) 80 | print("average training loss:", sum(train_losses) / len(train_losses)) 81 | print("accuracy:",sum(accus)/len(accus)) 82 | 83 | 84 | if __name__=="__main__": 85 | train() -------------------------------------------------------------------------------- /14.RNN/LSTM/lstm_sin_prediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/14.RNN/LSTM/lstm_sin_prediction.py -------------------------------------------------------------------------------- /14.RNN/RNN.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | 5 | ''' 6 | cell = tf.nn.rnn_cell.BasicRNNCell(num_units=128) # state_size = 128 7 | print(cell.state_size) # 128 8 | 9 | inputs = tf.placeholder(np.float32, shape=(32, 100)) # 32 是 batch_size 10 | h0 = cell.zero_state(32, np.float32) # 通过zero_state得到一个全0的初始状态，形状为(batch_size, state_size) 11 | output, h1 = cell.call(inputs, h0) #调用call函数 12 | 13 | print(h1.shape) # (32, 128) 14 | print(output.shape) 15 | 16 | lstm_cell_forward = tf.nn.rnn_cell.BasicLSTMCell(num_units=128) 17 | lstm_cell_backward = tf.nn.rnn_cell.BasicLSTMCell(num_units=128) 18 | inputs = tf.placeholder(np.float32, shape=(32, 100)) # 32 是 batch_size 19 | lstm_cell=tf.nn.bidirectional_dynamic_rnn( 20 | cell_fw=lstm_cell_forward, 21 | cell_bw=lstm_cell_backward, 22 | inputs=inputs, 23 | ) 24 | h0 = lstm_cell.zero_state(32, np.float32) # 通过zero_state得到一个全0的初始状态 25 | print("shape of h0:",h0.shape) 26 | output, h1 = lstm_cell.call(inputs, h0) 27 | 28 | print(h1.h) # shape=(32, 128) 29 | print(h1.c) # shape=(32, 128) 30 | 31 | ''' 32 | 33 | 34 | 35 | ''' 36 | inputs = tf.placeholder(np.float32, shape=(32, 100)) # 32 是 batch_size 37 | h0 = lstm_cell.zero_state(32, np.float32) # 通过zero_state得到一个全0的初始状态 38 | print("len of h0:",len(h0)) 39 | print("shape of h0.h:",h0.h.shape) 40 | print("shape of h0.c:",h0.c.shape) 41 | output, h1 = lstm_cell.call(inputs,h0) 42 | 43 | print(h1.h) # shape=(32, 128) 44 | print(h1.c) # shape=(32, 128) 45 | 46 | 47 | import tensorflow as tf 48 | import numpy as np 49 | lstm_cell_1 = tf.contrib.rnn.BasicLSTMCell(num_units=128) 50 | lstm_cell_2 = tf.contrib.rnn.BasicLSTMCell(num_units=256) 51 | lstm_cell_3 = tf.contrib.rnn.BasicLSTMCell(num_units=512) 52 | #多层lstm_cell 53 | lstm_cell=tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell_1,lstm_cell_2,lstm_cell_3]) 54 | 55 | print("output_size:",lstm_cell.output_size) 56 | print("state_size:",lstm_cell.state_size) 57 | #print(lstm_cell.state_size.h) 58 | #print(lstm_cell.state_size.c) 59 | ''' 60 | 61 | import tensorflow as tf 62 | import numpy as np 63 | 64 | inputs = tf.placeholder(np.float32, shape=(32,40,5)) # 32 是 batch_size 65 | lstm_cell_fw = tf.contrib.rnn.BasicLSTMCell(num_units=128) 66 | lstm_cell_bw = tf.contrib.rnn.BasicLSTMCell(num_units=128) 67 | 68 | #多层lstm_cell 69 | #lstm_cell=tf.contrib.rnn.MultiRNNCell(cells=[lstm_cell_1,lstm_cell_2,lstm_cell_3]) 70 | 71 | print("output_fw_size:",lstm_cell_fw.output_size) 72 | print("state_fw_size:",lstm_cell_fw.state_size) 73 | print("output_bw_size:",lstm_cell_bw.output_size) 74 | print("state_bw_size:",lstm_cell_bw.state_size) 75 | 76 | #print(lstm_cell.state_size.h) 77 | #print(lstm_cell.state_size.c) 78 | output,state=tf.nn.bidirectional_dynamic_rnn( 79 | cell_fw=lstm_cell_fw, 80 | cell_bw=lstm_cell_bw, 81 | inputs=inputs, 82 | dtype=tf.float32 83 | ) 84 | output_fw=output[0] 85 | output_bw=output[1] 86 | state_fw=state[0] 87 | state_bw=state[1] 88 | 89 | print("output_fw.shape:",output_fw.shape) 90 | print("output_bw.shape:",output_bw.shape) 91 | print("len of state tuple",len(state_fw)) 92 | print("state_fw:",state_fw) 93 | print("state_bw:",state_bw) 94 | #print("state.h.shape:",state.h.shape) 95 | #print("state.c.shape:",state.c.shape) 96 | 97 | #state_concat=tf.concat(values=[state_fw,state_fw],axis=1) 98 | #print(state_concat) 99 | state_h_concat=tf.concat(values=[state_fw.h,state_bw.h],axis=1) 100 | print("state_fw_h_concat.shape",state_h_concat.shape) 101 | 102 | state_c_concat=tf.concat(values=[state_fw.c,state_bw.c],axis=1) 103 | print("state_fw_h_concat.shape",state_c_concat.shape) 104 | 105 | state_concat=tf.contrib.rnn.LSTMStateTuple(c=state_c_concat,h=state_h_concat) 106 | print(state_concat) -------------------------------------------------------------------------------- /14.RNN/layerNorm_mnist.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import tensorflow.contrib.rnn as rnn 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | TIME_STEPS=28 9 | BATCH_SIZE=128 10 | HIDDEN_UNITS1=30 11 | HIDDEN_UNITS=10 12 | LEARNING_RATE=0.001 13 | EPOCH=50 14 | 15 | TRAIN_EXAMPLES=42000 16 | TEST_EXAMPLES=28000 17 | 18 | #------------------------------------Generate Data-----------------------------------------------# 19 | #generate data 20 | train_frame = pd.read_csv("../Mnist/train.csv") 21 | test_frame = pd.read_csv("../Mnist/test.csv") 22 | 23 | # pop the labels and one-hot coding 24 | train_labels_frame = train_frame.pop("label") 25 | 26 | # get values 27 | # one-hot on labels 28 | X_train = train_frame.astype(np.float32).values 29 | y_train=pd.get_dummies(data=train_labels_frame).values 30 | X_test = test_frame.astype(np.float32).values 31 | 32 | #trans the shape to (batch,time_steps,input_size) 33 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 34 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 35 | #print(X_train.shape) 36 | #print(y_dummy.shape) 37 | #print(X_test.shape) 38 | 39 | #-----------------------------------------------------------------------------------------------------# 40 | 41 | 42 | #--------------------------------------Define Graph---------------------------------------------------# 43 | graph=tf.Graph() 44 | with graph.as_default(): 45 | 46 | #------------------------------------construct LSTM------------------------------------------# 47 | #place hoder 48 | X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,28),name="input_placeholder") 49 | y_p=tf.placeholder(dtype=tf.float32,shape=(None,10),name="pred_placeholder") 50 | 51 | #gru instance 52 | ln_forward_1=rnn.LayerNormBasicLSTMCell(num_units=HIDDEN_UNITS1) 53 | ln_forward_2 = rnn.LayerNormBasicLSTMCell(num_units=HIDDEN_UNITS) 54 | ln_forward=rnn.MultiRNNCell(cells=[ln_forward_1,ln_forward_2]) 55 | 56 | ln_backward_1 = rnn.LayerNormBasicLSTMCell(num_units=HIDDEN_UNITS1) 57 | ln_backward_2 = rnn.LayerNormBasicLSTMCell(num_units=HIDDEN_UNITS) 58 | ln_backward = rnn.MultiRNNCell(cells=[ln_backward_1, ln_backward_2]) 59 | 60 | 61 | outputs,states=tf.nn.bidirectional_dynamic_rnn( 62 | cell_fw=ln_forward, 63 | cell_bw=ln_backward, 64 | inputs=X_p, 65 | dtype=tf.float32 66 | ) 67 | 68 | outputs_fw=outputs[0] 69 | outputs_bw = outputs[1] 70 | h=outputs_fw[:,-1,:]+outputs_bw[:,-1,:] 71 | # print(h.shape) 72 | #---------------------------------------;-----------------------------------------------------# 73 | 74 | #---------------------------------define loss and optimizer----------------------------------# 75 | cross_loss=tf.losses.softmax_cross_entropy(onehot_labels=y_p,logits=h) 76 | #print(loss.shape) 77 | 78 | correct_prediction = tf.equal(tf.argmax(h, 1), tf.argmax(y_p, 1)) 79 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 80 | 81 | optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=cross_loss) 82 | 83 | init=tf.global_variables_initializer() 84 | 85 | 86 | #-------------------------------------------Define Session---------------------------------------# 87 | with tf.Session(graph=graph) as sess: 88 | sess.run(init) 89 | for epoch in range(1,EPOCH+1): 90 | #results = np.zeros(shape=(TEST_EXAMPLES, 10)) 91 | train_losses=[] 92 | accus=[] 93 | #test_losses=[] 94 | print("epoch:",epoch) 95 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 96 | _,train_loss,accu=sess.run( 97 | fetches=(optimizer,cross_loss,accuracy), 98 | feed_dict={ 99 | X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 100 | y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE] 101 | } 102 | ) 103 | train_losses.append(train_loss) 104 | accus.append(accu) 105 | print("average training loss:", sum(train_losses) / len(train_losses)) 106 | print("accuracy:",sum(accus)/len(accus)) -------------------------------------------------------------------------------- /15.Tricks/argxxx.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | value=np.array([[0,1,2,3,4],[4,0,1,2,3],[3,4,0,1,1],[1,2,4,3,1]]) 5 | graph=tf.Graph() 6 | session=tf.Session(graph=graph) 7 | 8 | with graph.as_default(): 9 | y_pred=tf.Variable(initial_value=value,name="y_pred") 10 | print(y_pred.shape) 11 | 12 | y_arg=tf.argmax(y_pred, 1) 13 | print("y_arg.shape:",y_arg.shape) 14 | 15 | #flatten 16 | y_pred2=tf.reshape(y_pred, [-1]) 17 | print("y_pred2.shape:",y_pred2.shape) -------------------------------------------------------------------------------- /15.Tricks/sequence_mask.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | array=np.arange(start=0,stop=90) 5 | #print(a) 6 | a=array.reshape((10,9)) 7 | #print(a) 8 | 9 | b=array.reshape((10,3,3)) 10 | print(b) 11 | lens=np.array([2,2,2,2,2,2,2,2,2,2]) 12 | 13 | #lens_b=np.array([[2,3],[2,3],[2,3],[2,3],[2,3],[2,3],[2,3],[2,3],[2,3],[2,3]]) 14 | 15 | 16 | 17 | X_p=tf.placeholder(dtype=tf.int32,shape=(None,3,3)) 18 | sequence_len_p=tf.placeholder(dtype=tf.int32,shape=(None,)) 19 | mask=tf.sequence_mask(lengths=sequence_len_p,maxlen=3) 20 | x=X_p 21 | x_masked=tf.boolean_mask(tensor=X_p,mask=mask) 22 | 23 | 24 | with tf.Session() as sess: 25 | ''' 26 | sequence_len,mask_index_get,X,X_masked=sess.run( 27 | fetches=[sequence_len_p,mask,x,x_masked], 28 | feed_dict={ 29 | X_p:a[:5], 30 | sequence_len_p:lens[:5] 31 | } 32 | ) 33 | 34 | print("sequence_len:",sequence_len) 35 | print("mask_index_get:\n",mask_index_get) 36 | print("X:\n",X) 37 | print("X_masked:",X_masked) 38 | 39 | ''' 40 | 41 | sequence_len, mask_index_get, X,X_masked = sess.run( 42 | fetches=[sequence_len_p, mask, x,x_masked], 43 | feed_dict={ 44 | X_p: b[:5], 45 | sequence_len_p: lens[:5] 46 | } 47 | ) 48 | 49 | print("sequence_len:", sequence_len) 50 | print("mask_index_get:\n", mask_index_get.shape) 51 | print("X:\n", X) 52 | print("X_masked:\n", X_masked) 53 | -------------------------------------------------------------------------------- /16.DataPipelines/Preprocess/one-hot.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | a=np.array([1,2,3,4,5,6,7,8]) 5 | 6 | index=tf.constant(value=a,name="index") 7 | #print(index) 8 | one_hoted1=tf.one_hot(indices=index,depth=3) 9 | print(one_hoted1.shape) 10 | 11 | index2=tf.reshape(tensor=index,shape=[4,2],name="index2") 12 | one_hoted2=tf.one_hot(indices=index2,depth=9) 13 | print(one_hoted2.shape) 14 | 15 | with tf.Session() as sess: 16 | print(sess.run(index)) 17 | print(sess.run(one_hoted1)) 18 | print(sess.run(one_hoted2)) -------------------------------------------------------------------------------- /16.DataPipelines/dataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | dataset1=tf.data.Dataset.from_tensors(np.array([1.,2.,3.,4.,5.])) 5 | 6 | dataset2=tf.data.Dataset.from_tensor_slices(tensors=np.array([1.,2.,3.,4.,5.,6.])).repeat().shuffle(buffer_size=2).batch(2) 7 | 8 | print("element shape of dataset1:",dataset1.output_shapes) 9 | print("element shape of dataset2:",dataset2.output_shapes) 10 | print("element type of dataset2:",dataset2.output_types) 11 | 12 | iterator=dataset1.make_one_shot_iterator() 13 | iterator2=dataset2.make_one_shot_iterator() 14 | 15 | 16 | element=iterator.get_next() 17 | element2=iterator2.get_next() 18 | with tf.Session() as sess: 19 | print("elements of dataset1:") 20 | print(sess.run(element)) 21 | print("elements of dataset2:") 22 | for i in range(3): 23 | print("epoch:",i) 24 | for j in range(6//2): 25 | print("---mini_batch:",j,sess.run(element2)) 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /16.DataPipelines/dataset2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | dataset1=tf.data.Dataset.from_tensors(np.zeros(shape=(10,5,2),dtype=np.float32)) 5 | 6 | dataset2=tf.data.Dataset.from_tensor_slices(tensors=np.zeros(shape=(10,5,2),dtype=np.float32)) 7 | 8 | print("element shape of dataset1:",dataset1.output_shapes) 9 | print("element shape of dataset2:",dataset2.output_shapes) 10 | print("element type of dataset2:",dataset2.output_types) 11 | 12 | iterator=dataset1.make_one_shot_iterator() 13 | iterator2=dataset2.make_one_shot_iterator() 14 | 15 | element=iterator.get_next() 16 | element2=iterator2.get_next() 17 | with tf.Session() as sess: 18 | print("elements of dataset1:") 19 | print(sess.run(element)) 20 | 21 | print("elements of dataset2:") 22 | for i in range(5): 23 | print(sess.run(element2)) -------------------------------------------------------------------------------- /16.DataPipelines/example1.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import pandas as pd 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | 7 | 8 | #-------------------------------Generate Data--------------------------------------# 9 | #generate data 10 | train_frame = pd.read_csv("../Mnist/train.csv") 11 | test_frame = pd.read_csv("../Mnist/test.csv") 12 | 13 | # pop the labels and one-hot coding 14 | train_labels_frame = train_frame.pop("label") 15 | 16 | # get values 17 | # one-hot on labels 18 | X_train = train_frame.astype(np.float32).values 19 | y_train=pd.get_dummies(data=train_labels_frame).values 20 | X_test = test_frame.astype(np.float32).values 21 | 22 | #trans the shape to (batch,time_steps,input_size) 23 | X_train=np.reshape(X_train,newshape=(-1,28,28)) 24 | X_test=np.reshape(X_test,newshape=(-1,28,28)) 25 | print(X_train.shape) 26 | print(y_train.shape) 27 | #print(X_test.shape) 28 | 29 | #-----------------------------------------------------------------------------------# 30 | 31 | #create dataset 32 | dataset=tf.data.Dataset.from_tensor_slices(tensors=(X_train,y_train)) 33 | #print(dataset.output_shapes) 34 | #print(dataset[0]) 35 | 36 | #create iterator 37 | iterator=dataset.make_one_shot_iterator() 38 | elememt=iterator.get_next() 39 | x=elememt[0] 40 | label=elememt[1] 41 | 42 | 43 | with tf.Session() as sess: 44 | #ele=sess.run(elememt) 45 | for i in range(3): 46 | image=sess.run(x) 47 | #label_image=sess.run(label) 48 | print("image:\n",image) 49 | #print("label:\n",label_image) 50 | plt.imshow(image) 51 | plt.show() 52 | -------------------------------------------------------------------------------- /16.DataPipelines/multi_gpu.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 6 | 7 | N_GPU=2 8 | 9 | a=np.arange(start=0,stop=100) 10 | print("a:\n",a) 11 | 12 | dataset=tf.data.Dataset.from_tensor_slices(tensors=a) 13 | 14 | iterator=dataset.make_one_shot_iterator() 15 | batch=iterator.get_next() 16 | 17 | data_list=[] 18 | for i in range(N_GPU): 19 | with tf.device("/gpu:%d" % i): 20 | with tf.name_scope("gpu_%d" % i) as scope: 21 | data=batch 22 | data_list.append(data) 23 | 24 | with tf.Session() as sess: 25 | print("elements of dataset:") 26 | for i in range(2): 27 | #print(sess.run(data)) 28 | print(data_list) 29 | print(sess.run(data_list[0])) 30 | print(sess.run(data_list[1])) 31 | -------------------------------------------------------------------------------- /2.constant_and_Variable/Tensor.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function,division 2 | import tensorflow as tf 3 | 4 | #basic 5 | def basic_test(): 6 | print("基础测试") 7 | a=tf.constant([[1,2],[3,4]]) 8 | b=tf.constant([[1,1],[0,1]]) 9 | print("a:\n",a) 10 | print("b:\n",b) 11 | print("type of a:\n",type(a)) 12 | c=tf.matmul(a,b) 13 | print("c:\n",c) 14 | print("c.numpy:\n",c.numpy()) 15 | print("type of c.numpy():\n",type(c.numpy())) 16 | print("\n") 17 | 18 | # attribute 19 | print("device:", c.device) 20 | print("dtype:", c.dtype) 21 | print("shape:", type(c.shape)) 22 | 23 | # member function 24 | 25 | #索引测试 26 | def index_test(): 27 | print("索引测试") 28 | a = tf.constant([[1, 2], [3, 4]]) 29 | print("a:\n", a) 30 | print ("a[1,:]\n",a[1,:]) 31 | print("a[1,1]:\n",a[1,1]) 32 | 33 | 34 | 35 | 36 | def assign_value_test(): 37 | print("赋值测试") 38 | a = tf.constant([[1, 2], [3, 4]]) 39 | print("a:\n", a) 40 | # a[1, 1]=999 错误的赋值操作 41 | np_value=a.numpy() 42 | np_value[1,1]=999 43 | print("np_value:\n",np_value) 44 | print("a:\n", a) 45 | 46 | 47 | 48 | if __name__=="__main__": 49 | basic_test() 50 | index_test() 51 | assign_value_test() 52 | -------------------------------------------------------------------------------- /2.constant_and_Variable/Variable.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | #基本测试 4 | def basic_test(): 5 | print("----------------基本测试------------------") 6 | a = tf.Variable(initial_value=[[1, 2, 3, 4], [5, 6, 7, 8]]) 7 | print("a:\n", a) 8 | print("shape of a:", a.shape) 9 | print("type of a.shape:", type(a.shape)) 10 | print("a.shape[0]:", a.shape[0]) 11 | print("type of a.shape[0]:", type(a.shape[0])) 12 | print("a.numpy():\n", a.numpy()) 13 | print("type of a.numpy():", type(a.numpy())) 14 | 15 | 16 | #索引测试 17 | def index_test(): 18 | print("----------------索引测试--------------------") 19 | a = tf.Variable(initial_value=[[1, 2, 3, 4], [5, 6, 7, 8]]) 20 | print("a:\n", a) 21 | print("a[:1]:\n",a[:1]) 22 | print("a[1,:2]：\n",a[1,:2]) 23 | print("a[1,3]:\n",a[1,3]) 24 | 25 | 26 | #改变值测试 27 | def assin_value_test(): 28 | print("改变值的测试") 29 | a = tf.Variable(initial_value=[[1, 2, 3, 4], [5, 6, 7, 8]]) 30 | print("a:\n", a) 31 | 32 | #a[1,3]=15 错误的赋值操作 33 | 34 | #下面的操作并不会改变a的值 35 | a.numpy()[1,3]=15 36 | print("a:\n", a) 37 | 38 | #通过asign来改变a的值 39 | np_value=a.numpy() 40 | np_value[1,3]=15 41 | print("np_vapue:",np_value) 42 | a.assign(value=np_value) 43 | print("a:\n", a) 44 | 45 | 46 | #tf.keras.layers.Conv1D() 47 | if __name__=="__main__": 48 | basic_test() 49 | index_test() 50 | assin_value_test() 51 | 52 | -------------------------------------------------------------------------------- /21.Metrics/1.accuracy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | lables=np.array([1,2,3,4,1,1,1,3,4,1]) 6 | pred=np.array([1,2,3,4,1,1,1,3,4,0]) 7 | 8 | 9 | 10 | labels_t=tf.constant(value=lables) 11 | pred_t=tf.constant(value=pred) 12 | 13 | 14 | # correct_prediction 15 | correct_prediction = tf.equal(labels_t,pred_t) 16 | 17 | # accracy 18 | accuracy_pw = tf.reduce_mean( 19 | input_tensor=tf.cast(x=correct_prediction, dtype=tf.float32), 20 | name="accuracy_pw" 21 | ) 22 | 23 | accu=tf.metrics.accuracy(labels=labels_t,predictions=pred_t,name="accu")[1] 24 | local_init=tf.local_variables_initializer() 25 | global_init=tf.global_variables_initializer() 26 | with tf.Session() as sess: 27 | sess.run(global_init) 28 | sess.run(local_init) 29 | print(sess.run(accuracy_pw)) 30 | print(sess.run(accu)) -------------------------------------------------------------------------------- /21.Metrics/2.precision.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from sklearn.metrics import f1_score 4 | 5 | lables=np.array([0,1,0,1,1,0,1,0,0,0]) 6 | pred=np.array([0,1,0,0,1,0,0,0,0,0]) 7 | 8 | f_score=f1_score(y_true=lables,y_pred=pred) 9 | print("sklearn_based f1 score:",f_score) 10 | 11 | labels_t=tf.constant(value=lables) 12 | pred_t=tf.constant(value=pred) 13 | 14 | 15 | # correct_prediction 16 | #correct_prediction = tf.equal(labels_t,pred_t) 17 | 18 | # accracy 19 | #accuracy_pw = tf.reduce_mean( 20 | # input_tensor=tf.cast(x=correct_prediction, dtype=tf.float32), 21 | # name="accuracy_pw" 22 | #) 23 | 24 | p=tf.metrics.precision(labels=labels_t,predictions=pred_t,name="precision")[1] 25 | r=tf.metrics.recall(labels=labels_t,predictions=pred_t,name="recall")[1] 26 | f1=2*p*r/(r+p) 27 | local_init=tf.local_variables_initializer() 28 | global_init=tf.global_variables_initializer() 29 | with tf.Session() as sess: 30 | sess.run(global_init) 31 | sess.run(local_init) 32 | print("precision:",sess.run(p)) 33 | print("recall:",sess.run(r)) 34 | print("f1 score:",sess.run(f1)) 35 | #print(sess.run(accu)) -------------------------------------------------------------------------------- /21.Metrics/softmax.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | graph=tf.Graph() 5 | np_a=np.zeros(shape=(10,1)) 6 | np_b=np.ones(shape=(10,1)) 7 | np_c=np.full(shape=(10,1),fill_value=5) 8 | np_d=np.full(shape=(10,2,3),fill_value=1,dtype=np.float64) 9 | with graph.as_default(): 10 | e=[] 11 | a=tf.Variable(initial_value=(np_a)) 12 | e.append(a) 13 | b = tf.Variable(initial_value=(np_b)) 14 | e.append(b) 15 | print("e:",e) 16 | d = tf.Variable(initial_value=(np_d)) 17 | e=tf.concat(values=e,axis=1) 18 | print("e:",e) 19 | print("e.shape",e.shape) 20 | 21 | #softmax 22 | softed=tf.nn.softmax(logits=e,axis=1) 23 | print("softed.shape",softed.shape) 24 | print("softed[0].shape:",softed[0].shape) 25 | 26 | result=[] 27 | for i in range(10): 28 | matrix=tf.matmul(tf.reshape(tensor=softed[i],shape=(1,-1)),d[i]) 29 | result.append(matrix) 30 | 31 | print("result:",result) 32 | result=tf.concat(values=result,axis=0) 33 | print("result:",result) 34 | 35 | init=tf.global_variables_initializer() 36 | 37 | 38 | with tf.Session(graph=graph) as sess: 39 | sess.run(init) 40 | print("a:\n",sess.run(a)) 41 | print("b:\n", sess.run(b)) 42 | print("b:\n", sess.run(d)) 43 | print("e:\n", sess.run(e)) 44 | print("softed:\n",sess.run(softed)) 45 | print("result:\n",sess.run(result)) 46 | 47 | 48 | -------------------------------------------------------------------------------- /22.Muilti_GPU/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | class CNN(tf.keras.Model): 6 | def __init__(self,filters,kernel_size,strides,activation): 7 | super(CNN,self).__init__() 8 | self.conv1=tf.keras.layers.Conv2D(filters=filters,kernel_size=kernel_size,strides=strides,activation=activation) 9 | self.max_pool1=tf.keras.layers.MaxPooling2D() 10 | self.conv2=tf.keras.layers.Conv2D(filters=filters,kernel_size=kernel_size,strides=strides,activation=activation) 11 | self.max_pool2=tf.keras.layers.MaxPooling2D() 12 | self.flatten=tf.keras.layers.Flatten() 13 | self.fc1=tf.keras.layers.Dense(units=filters,activation=activation) 14 | self.fc2=tf.keras.layers.Dense(units=10,activation=None) 15 | 16 | def __call__(self,x): 17 | x=self.conv1(x) 18 | x=self.max_pool1(x) 19 | x=self.conv2(x) 20 | x=self.max_pool2(x) 21 | x=self.flatten(x) 22 | x=self.fc1(x) 23 | x=self.fc2(x) 24 | #print("x.shape",x) 25 | return x 26 | 27 | 28 | 29 | 30 | if __name__=="__main__": 31 | cnn=CNN(filters=64,kernel_size=3,strides=1,activation=tf.nn.relu) 32 | 33 | inputs=tf.ones(shape=(30,28,28,3),dtype=tf.float32) 34 | 35 | cnn(inputs) 36 | 37 | -------------------------------------------------------------------------------- /22.Muilti_GPU/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | import matplotlib.pyplot as plt 5 | 6 | import model 7 | 8 | 9 | #define strategy 10 | strategy=tf.distribute.MirroredStrategy() 11 | print("num devices:",strategy.num_replicas_in_sync) 12 | 13 | #parameters 14 | BATCH_SIZE_PER_REPLICA=64 15 | BATCH_SIZE=BATCH_SIZE_PER_REPLICA*strategy.num_replicas_in_sync 16 | print("batch_size_per_replica:",BATCH_SIZE_PER_REPLICA) 17 | print("batch_size:",BATCH_SIZE) 18 | 19 | CLASS_NUM=10 20 | EPOCHS=10 21 | 22 | #model save path 23 | CHECK_POINTS_PATH="./checkpoints/cnn" 24 | 25 | 26 | 27 | print(tf.__version__) 28 | 29 | (x_train, y_train), (x_test, y_test)=tf.keras.datasets.fashion_mnist.load_data() 30 | 31 | #expand dim to use convlution 2D 32 | x_train=np.expand_dims(a=x_train,axis=-1)/np.float32(255) 33 | print("train sample size:",len(x_train)) 34 | train_dataset=tf.data.Dataset.from_tensor_slices((x_train,y_train)).shuffle(buffer_size=len(x_train)).batch(BATCH_SIZE) 35 | # for records in train_dataset: 36 | # print("records:\n",records[0]) 37 | # print("records:\n",records[1]) 38 | 39 | 40 | x_test=np.expand_dims(a=x_test,axis=-1)/np.float32(255) 41 | test_dataset=tf.data.Dataset.from_tensor_slices((x_test,y_test)).batch(BATCH_SIZE) 42 | 43 | # 44 | # plt.imshow(X=x_train[1,:,:,0]) 45 | # plt.show() 46 | 47 | 48 | def train_step(): 49 | pass 50 | 51 | def train(): 52 | #trans dataset to distribute dataset 53 | with strategy.scope(): 54 | train_dist_dataset=strategy.experimental_distribute_dataset(train_dataset) 55 | test_dist_dataset=strategy.experimental_distribute_dataset(test_dataset) 56 | 57 | #define loss 58 | with strategy.scope(): 59 | SCC=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,reduction=tf.keras.losses.Reduction.NONE) 60 | def compute_loss(labels,predictions): 61 | device_losses=SCC(labels,predictions) 62 | #print("device_loss") 63 | device_loss=tf.nn.compute_average_loss(per_example_loss=device_losses,global_batch_size=BATCH_SIZE) 64 | return device_loss 65 | 66 | #define metrics 67 | with strategy.scope(): 68 | train_accuracy=tf.keras.metrics.SparseCategoricalAccuracy(name="train_loss") 69 | 70 | 71 | 72 | # model and optimizer must be created under `strategy.scope`. 73 | with strategy.scope(): 74 | cnn=model.CNN(filters=64,kernel_size=3,strides=1,activation=tf.nn.relu) 75 | #model = create_model() 76 | optimizer=tf.keras.optimizers.Adam(learning_rate=0.001) 77 | checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=cnn) 78 | 79 | #basic train step in one device 80 | with strategy.scope(): 81 | def train_step(inputs): 82 | x,y=inputs 83 | with tf.GradientTape() as tape: 84 | logits=cnn(x) 85 | loss=compute_loss(labels=y,predictions=logits) 86 | 87 | gradients=tape.gradient(target=loss,sources=cnn.trainable_variables) 88 | optimizer.apply_gradients(zip(gradients,cnn.trainable_variables)) 89 | 90 | train_accuracy.update_state(y,logits) 91 | return loss 92 | 93 | #distribute train_step use basic train step 94 | with strategy.scope(): 95 | def dist_train_step(dataset_inputs): 96 | replica_losses=strategy.experimental_run_v2(fn=train_step,args=(dataset_inputs,)) 97 | #print("replica_losses:\n",replica_losses) 98 | return strategy.reduce(reduce_op=tf.distribute.ReduceOp.SUM,value=replica_losses,axis=None) 99 | 100 | for epoch in range(EPOCHS): 101 | print("-----------EPOCH:",epoch) 102 | epoch_loss=0.0 103 | num_batchs=0 104 | for records in train_dist_dataset: 105 | epoch_loss+=dist_train_step(records) 106 | num_batchs+=1 107 | epoch_loss=epoch_loss/num_batchs 108 | 109 | print("epoch_loss:",epoch_loss.numpy()) 110 | print("epoch_accuracy:",train_accuracy.result().numpy()) 111 | 112 | #reset states 113 | train_accuracy.reset_states() 114 | 115 | #save model 116 | checkpoint.save(CHECK_POINTS_PATH) 117 | 118 | 119 | 120 | if __name__=="__main__": 121 | train() 122 | 123 | -------------------------------------------------------------------------------- /23.tf.layers/MNIST_TEST/model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import parameter 4 | from tensorflow.contrib.layers import xavier_initializer 5 | 6 | class Model(): 7 | def __init__(self): 8 | self.class_num=10 9 | 10 | def forward(self,input,regularizer): 11 | #---------------------------------------conv1--------------------------------------# 12 | logits_conv1=tf.layers.conv2d( 13 | inputs=input,filters=32,kernel_size=5,strides=1,padding="SAME",activation=None, 14 | kernel_initializer=xavier_initializer(),kernel_regularizer=regularizer,name="conv_1" 15 | ) 16 | print("shape of logits_conv1:",logits_conv1.shape) 17 | logits_conv1=tf.layers.max_pooling2d(inputs=logits_conv1,pool_size=2,strides=2,padding="SAME",name="padded_conv1") 18 | print("shape of logits_conv1:", logits_conv1.shape) 19 | 20 | # ---------------------------------------conv2--------------------------------------# 21 | logits_conv2 = tf.layers.conv2d( 22 | inputs=logits_conv1, filters=64, kernel_size=5, strides=1, padding="SAME", activation=None, 23 | kernel_initializer=xavier_initializer(),kernel_regularizer=regularizer, name="conv_2" 24 | ) 25 | print("shape of logits_conv2:", logits_conv2.shape) 26 | logits_conv2 = tf.layers.max_pooling2d(inputs=logits_conv2, pool_size=2, strides=2, padding="SAME",name="padded_conv2") 27 | print("shape of logits_conv2:", logits_conv2.shape) 28 | 29 | logits_flatten=tf.layers.flatten(inputs=logits_conv2,name="logits_flatten") 30 | print("shape of logits_flatten:", logits_flatten.shape) 31 | 32 | logits=tf.layers.dense(inputs=logits_flatten,units=10,activation=None, 33 | kernel_initializer=xavier_initializer(), 34 | kernel_regularizer=regularizer, 35 | name="logits") 36 | print("shape of logits:", logits.shape) 37 | return logits 38 | 39 | 40 | -------------------------------------------------------------------------------- /23.tf.layers/MNIST_TEST/parameter.py: -------------------------------------------------------------------------------- 1 | NUM_CLASS=10 2 | H=28 3 | W=28 -------------------------------------------------------------------------------- /23.tf.layers/MNIST_TEST/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/23.tf.layers/MNIST_TEST/test.py -------------------------------------------------------------------------------- /23.tf.layers/MNIST_TEST/train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import model 5 | import parameter 6 | 7 | 8 | BATCH_SIZE=128 9 | LEARNING_RATE=0.003 10 | EPOCH=50 11 | 12 | TRAIN_EXAMPLES=42000 13 | TEST_EXAMPLES=28000 14 | 15 | def train(train_set,train_lables): 16 | # place hoder 17 | X_p = tf.placeholder(dtype=tf.float32, shape=(None, 28, 28,1), name="input_placeholder") 18 | y_p = tf.placeholder(dtype=tf.float32, shape=(None, 10), name="pred_placeholder") 19 | 20 | m=model.Model() 21 | #regularizer = tf.contrib.layers.l2_regularizer(0.0001) 22 | regularizer = None 23 | logits=m.forward(input=X_p,regularizer=regularizer) 24 | 25 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y_p, logits=logits) 26 | # print(loss.shape) 27 | correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y_p, 1)) 28 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) 29 | optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=loss) 30 | init = tf.global_variables_initializer() 31 | 32 | # -------------------------------------------Define Session---------------------------------------# 33 | with tf.Session() as sess: 34 | sess.run(init) 35 | for epoch in range(1, EPOCH + 1): 36 | # results = np.zeros(shape=(TEST_EXAMPLES, 10)) 37 | train_losses = [] 38 | accus = [] 39 | # test_losses=[] 40 | print("epoch:", epoch) 41 | for j in range(TRAIN_EXAMPLES // BATCH_SIZE): 42 | _, train_loss, accu= sess.run( 43 | fetches=(optimizer, loss, accuracy), 44 | feed_dict={ 45 | X_p: X_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 46 | y_p: y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE] 47 | } 48 | ) 49 | train_losses.append(train_loss) 50 | accus.append(accu) 51 | # print("s_fw:",s_fw[0]) 52 | # print("o_fw:",o_fw[0]) 53 | print("train_losses.size",len(train_losses)) 54 | print("accus.size",len(accus)) 55 | print("average training loss:", sum(train_losses) / len(train_losses)) 56 | print("accuracy:", sum(accus) / len(accus)) 57 | 58 | 59 | 60 | 61 | 62 | if __name__=="__main__": 63 | # ------------------------------------Generate Data-----------------------------------------------# 64 | # generate data 65 | train_frame = pd.read_csv("../../Mnist/train.csv") 66 | test_frame = pd.read_csv("../../Mnist/test.csv") 67 | 68 | # pop the labels and one-hot coding 69 | train_labels_frame = train_frame.pop("label") 70 | 71 | # get values 72 | # one-hot on labels 73 | X_train = train_frame.astype(np.float32).values 74 | y_train = pd.get_dummies(data=train_labels_frame).values 75 | X_test = test_frame.astype(np.float32).values 76 | 77 | # trans the shape to (batch,time_steps,input_size) 78 | X_train = np.reshape(X_train, newshape=(-1, 28, 28, 1)) 79 | X_test = np.reshape(X_test, newshape=(-1, 28, 28, 1)) 80 | print(X_train.shape) 81 | print(y_train.shape) 82 | print(X_test.shape) 83 | 84 | # -----------------------------------------------------------------------------------------------------# 85 | train(X_train/255,y_train) 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /3.Graph_and_Session/graph.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | c=tf.constant(value=1) 5 | #print(assert c.graph is tf.get_default_graph()) 6 | print(c.graph) 7 | print(tf.get_default_graph()) 8 | 9 | g=tf.Graph() 10 | print("g:",g) 11 | with g.as_default(): 12 | d=tf.constant(value=2) 13 | print(d.graph) 14 | #print(g) 15 | 16 | g2=tf.Graph() 17 | print("g2:",g2) 18 | g2.as_default() 19 | e=tf.constant(value=15) 20 | print(e.graph) 21 | 22 | -------------------------------------------------------------------------------- /3.Graph_and_Session/graph_test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import tensorflow as tf\n", 12 | "import numpy as np" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 4, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "#create a Graph instance\n", 24 | "g=tf.Graph()" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 5, 30 | "metadata": { 31 | "collapsed": true 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "with g.as_default():\n", 36 | " a=tf.constant(value=1)\n", 37 | " b=tf.constant(value=2,name=\"b\")" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": { 44 | "collapsed": true 45 | }, 46 | "outputs": [], 47 | "source": [] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "# attribute test" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 6, 59 | "metadata": {}, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "text/plain": [ 64 | "False" 65 | ] 66 | }, 67 | "execution_count": 6, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "#building_function\n", 74 | "g.building_function" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 7, 80 | "metadata": {}, 81 | "outputs": [ 82 | { 83 | "data": { 84 | "text/plain": [ 85 | "False" 86 | ] 87 | }, 88 | "execution_count": 7, 89 | "metadata": {}, 90 | "output_type": "execute_result" 91 | } 92 | ], 93 | "source": [ 94 | "#finalized\n", 95 | "g.finalized" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 8, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "data": { 105 | "text/plain": [ 106 | "producer: 21" 107 | ] 108 | }, 109 | "execution_count": 8, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "#graph_def_versions\n", 116 | "g.graph_def_versions" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 9, 122 | "metadata": { 123 | "collapsed": true 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "#seed\n", 128 | "g.seed" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 10, 134 | "metadata": {}, 135 | "outputs": [ 136 | { 137 | "data": { 138 | "text/plain": [ 139 | "2" 140 | ] 141 | }, 142 | "execution_count": 10, 143 | "metadata": {}, 144 | "output_type": "execute_result" 145 | } 146 | ], 147 | "source": [ 148 | "#version\n", 149 | "g.version" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": { 156 | "collapsed": true 157 | }, 158 | "outputs": [], 159 | "source": [] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "# method test" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 11, 171 | "metadata": { 172 | "collapsed": true 173 | }, 174 | "outputs": [], 175 | "source": [ 176 | "#finalize()\n", 177 | "g.finalize()" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": 12, 183 | "metadata": {}, 184 | "outputs": [ 185 | { 186 | "data": { 187 | "text/plain": [ 188 | "True" 189 | ] 190 | }, 191 | "execution_count": 12, 192 | "metadata": {}, 193 | "output_type": "execute_result" 194 | } 195 | ], 196 | "source": [ 197 | "g.finalized" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 13, 203 | "metadata": {}, 204 | "outputs": [ 205 | { 206 | "data": { 207 | "text/plain": [ 208 | "[]" 209 | ] 210 | }, 211 | "execution_count": 13, 212 | "metadata": {}, 213 | "output_type": "execute_result" 214 | } 215 | ], 216 | "source": [ 217 | "#get_all_collection_keys()\n", 218 | "g.get_all_collection_keys()" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 15, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "data": { 228 | "text/plain": [ 229 | "" 230 | ] 231 | }, 232 | "execution_count": 15, 233 | "metadata": {}, 234 | "output_type": "execute_result" 235 | } 236 | ], 237 | "source": [ 238 | "#get_opration_by_name\n", 239 | "g.get_operation_by_name(\"b\")" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 17, 245 | "metadata": {}, 246 | "outputs": [ 247 | { 248 | "data": { 249 | "text/plain": [ 250 | "[, ]" 251 | ] 252 | }, 253 | "execution_count": 17, 254 | "metadata": {}, 255 | "output_type": "execute_result" 256 | } 257 | ], 258 | "source": [ 259 | "#get_operations()\n", 260 | "g.get_operations()" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": 20, 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "#get_tensor_by_name()" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 21, 275 | "metadata": {}, 276 | "outputs": [ 277 | { 278 | "data": { 279 | "text/plain": [ 280 | "True" 281 | ] 282 | }, 283 | "execution_count": 21, 284 | "metadata": {}, 285 | "output_type": "execute_result" 286 | } 287 | ], 288 | "source": [ 289 | "#is_feedable\n", 290 | "g.is_feedable(a)" 291 | ] 292 | }, 293 | { 294 | "cell_type": "code", 295 | "execution_count": null, 296 | "metadata": { 297 | "collapsed": true 298 | }, 299 | "outputs": [], 300 | "source": [] 301 | } 302 | ], 303 | "metadata": { 304 | "anaconda-cloud": {}, 305 | "kernelspec": { 306 | "display_name": "Python 3", 307 | "language": "python", 308 | "name": "python3" 309 | }, 310 | "language_info": { 311 | "codemirror_mode": { 312 | "name": "ipython", 313 | "version": 3 314 | }, 315 | "file_extension": ".py", 316 | "mimetype": "text/x-python", 317 | "name": "python", 318 | "nbconvert_exporter": "python", 319 | "pygments_lexer": "ipython3", 320 | "version": "3.5.2" 321 | } 322 | }, 323 | "nbformat": 4, 324 | "nbformat_minor": 1 325 | } 326 | -------------------------------------------------------------------------------- /3.Graph_and_Session/session.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | -------------------------------------------------------------------------------- /3.Graph_and_Session/session_test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import tensorflow as tf\n", 12 | "import numpy as np" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "#graph\n", 24 | "g=tf.Graph()\n", 25 | "with g.as_default():\n", 26 | " a=tf.constant(value=2,name=\"a\")\n", 27 | " b=tf.constant(value=5,name=\"b\")\n", 28 | " c=a*b" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 5, 34 | "metadata": { 35 | "collapsed": true 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "#Session\n", 40 | "with tf.Session(graph=g) as sess:\n", 41 | " result=sess.run(c)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 6, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "data": { 51 | "text/plain": [ 52 | "10" 53 | ] 54 | }, 55 | "execution_count": 6, 56 | "metadata": {}, 57 | "output_type": "execute_result" 58 | } 59 | ], 60 | "source": [ 61 | "result" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "collapsed": true 69 | }, 70 | "outputs": [], 71 | "source": [] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": { 77 | "collapsed": true 78 | }, 79 | "outputs": [], 80 | "source": [] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "# attribute test" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 7, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "data": { 96 | "text/plain": [ 97 | "" 98 | ] 99 | }, 100 | "execution_count": 7, 101 | "metadata": {}, 102 | "output_type": "execute_result" 103 | } 104 | ], 105 | "source": [ 106 | "#graph\n", 107 | "sess.graph" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 8, 113 | "metadata": {}, 114 | "outputs": [ 115 | { 116 | "data": { 117 | "text/plain": [ 118 | "node {\n", 119 | " name: \"a\"\n", 120 | " op: \"Const\"\n", 121 | " attr {\n", 122 | " key: \"dtype\"\n", 123 | " value {\n", 124 | " type: DT_INT32\n", 125 | " }\n", 126 | " }\n", 127 | " attr {\n", 128 | " key: \"value\"\n", 129 | " value {\n", 130 | " tensor {\n", 131 | " dtype: DT_INT32\n", 132 | " tensor_shape {\n", 133 | " }\n", 134 | " int_val: 2\n", 135 | " }\n", 136 | " }\n", 137 | " }\n", 138 | "}\n", 139 | "node {\n", 140 | " name: \"b\"\n", 141 | " op: \"Const\"\n", 142 | " attr {\n", 143 | " key: \"dtype\"\n", 144 | " value {\n", 145 | " type: DT_INT32\n", 146 | " }\n", 147 | " }\n", 148 | " attr {\n", 149 | " key: \"value\"\n", 150 | " value {\n", 151 | " tensor {\n", 152 | " dtype: DT_INT32\n", 153 | " tensor_shape {\n", 154 | " }\n", 155 | " int_val: 5\n", 156 | " }\n", 157 | " }\n", 158 | " }\n", 159 | "}\n", 160 | "node {\n", 161 | " name: \"mul\"\n", 162 | " op: \"Mul\"\n", 163 | " input: \"a\"\n", 164 | " input: \"b\"\n", 165 | " attr {\n", 166 | " key: \"T\"\n", 167 | " value {\n", 168 | " type: DT_INT32\n", 169 | " }\n", 170 | " }\n", 171 | "}\n", 172 | "versions {\n", 173 | " producer: 21\n", 174 | "}" 175 | ] 176 | }, 177 | "execution_count": 8, 178 | "metadata": {}, 179 | "output_type": "execute_result" 180 | } 181 | ], 182 | "source": [ 183 | "#graph_def\n", 184 | "sess.graph_def" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 9, 190 | "metadata": {}, 191 | "outputs": [ 192 | { 193 | "data": { 194 | "text/plain": [ 195 | "b''" 196 | ] 197 | }, 198 | "execution_count": 9, 199 | "metadata": {}, 200 | "output_type": "execute_result" 201 | } 202 | ], 203 | "source": [ 204 | "#sess_str\n", 205 | "sess.sess_str" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "metadata": {}, 211 | "source": [ 212 | "# method test" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "metadata": { 219 | "collapsed": true 220 | }, 221 | "outputs": [], 222 | "source": [] 223 | } 224 | ], 225 | "metadata": { 226 | "kernelspec": { 227 | "display_name": "Python 3", 228 | "language": "python", 229 | "name": "python3" 230 | }, 231 | "language_info": { 232 | "codemirror_mode": { 233 | "name": "ipython", 234 | "version": 3 235 | }, 236 | "file_extension": ".py", 237 | "mimetype": "text/x-python", 238 | "name": "python", 239 | "nbconvert_exporter": "python", 240 | "pygments_lexer": "ipython3", 241 | "version": "3.5.2" 242 | } 243 | }, 244 | "nbformat": 4, 245 | "nbformat_minor": 1 246 | } 247 | -------------------------------------------------------------------------------- /4.Optimizer&GradientTape/gradient_tape.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | def gradient_test(): 4 | #-------------------一元梯度案例--------------------------- 5 | print("一元梯度") 6 | x=tf.constant(value=3.0) 7 | with tf.GradientTape(persistent=True,watch_accessed_variables=True) as tape: 8 | tape.watch(x) 9 | y1=2*x 10 | y2=x*x+2 11 | y3=x*x+2*x 12 | #一阶导数 13 | dy1_dx=tape.gradient(target=y1,sources=x) 14 | dy2_dx = tape.gradient(target=y2, sources=x) 15 | dy3_dx = tape.gradient(target=y3, sources=x) 16 | print("dy1_dx:",dy1_dx) 17 | print("dy2_dx:", dy2_dx) 18 | print("dy3_dx:", dy3_dx) 19 | 20 | 21 | # # -------------------二元梯度案例--------------------------- 22 | print("二元梯度") 23 | x = tf.constant(value=3.0) 24 | y = tf.constant(value=2.0) 25 | with tf.GradientTape(persistent=True,watch_accessed_variables=True) as tape: 26 | tape.watch([x,y]) 27 | z1=x*x*y+x*y 28 | # 一阶导数 29 | dz1_dx=tape.gradient(target=z1,sources=x) 30 | dz1_dy = tape.gradient(target=z1, sources=y) 31 | dz1_d=tape.gradient(target=z1,sources=[x,y]) 32 | print("dz1_dx:", dz1_dx) 33 | print("dz1_dy:", dz1_dy) 34 | print("dz1_d:",dz1_d) 35 | print("type of dz1_d:",type(dz1_d)) 36 | 37 | 38 | def keras_test(): 39 | input=tf.ones(shape=(20,748)) 40 | a = tf.keras.layers.Dense(32) 41 | b = tf.keras.layers.Dense(32) 42 | print("a.variabels:",a.variables) 43 | 44 | with tf.GradientTape(watch_accessed_variables=False) as tape: 45 | tape.watch(a.variables) 46 | result=a(input) 47 | print("a.variabels:", a.variables) 48 | gradient=tape.gradient(result, a.variables) # The result of this computation will be 49 | print("gradient:",gradient) 50 | # a list of `None`s since a's variables 51 | # are not being watched. 52 | 53 | if __name__=="__main__": 54 | #gradient_test() 55 | keras_test() -------------------------------------------------------------------------------- /4.Optimizer&GradientTape/optimizer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | TRAIN_STEPS=20 6 | 7 | # Prepare train data 8 | train_X = np.linspace(-1, 1, 100) 9 | train_Y = 2 * train_X + np.random.randn(*train_X.shape) * 0.33 + 10 10 | 11 | print(train_X.shape) 12 | 13 | w=tf.Variable(initial_value=1.0) 14 | b=tf.Variable(initial_value=1.0) 15 | 16 | optimizer=tf.keras.optimizers.SGD(0.1) 17 | mse=tf.keras.losses.MeanSquaredError() 18 | 19 | for i in range(TRAIN_STEPS): 20 | print("epoch:",i) 21 | print("w:", w.numpy()) 22 | print("b:", b.numpy()) 23 | #计算和更新梯度 24 | with tf.GradientTape() as tape: 25 | logit = w * train_X + b 26 | loss=mse(train_Y,logit) 27 | gradients=tape.gradient(target=loss,sources=[w,b]) #计算梯度 28 | #print("gradients:",gradients) 29 | #print("zip:\n",list(zip(gradients,[w,b]))) 30 | optimizer.apply_gradients(zip(gradients,[w,b])) #更新梯度 31 | 32 | 33 | #draw 34 | plt.plot(train_X,train_Y,"+") 35 | plt.plot(train_X,w * train_X + b) 36 | plt.show() -------------------------------------------------------------------------------- /5.math_random_shape/math.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/5.math_random_shape/math.py -------------------------------------------------------------------------------- /5.math_random_shape/random.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/5.math_random_shape/random.py -------------------------------------------------------------------------------- /5.math_random_shape/shape.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | #基本形状操作 4 | def basic_shape(): 5 | a=tf.constant(value=[[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]) 6 | print("tf.shape:",tf.shape(a)) 7 | print("tf.size:",tf.size(a)) 8 | print("tf.rank:",tf.rank(a)) 9 | 10 | def reshape_ops(): 11 | a = tf.constant(value=[[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]) 12 | print("a:\n",a) 13 | b=tf.reshape(tensor=a,shape=(a.shape[0]*a.shape[1],-1)) 14 | print("b:\n",b) 15 | 16 | 17 | if __name__=="__main__": 18 | basic_shape() 19 | reshape_ops() -------------------------------------------------------------------------------- /6.NeuralNet_basic/mlp.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | EPOCH=20 6 | BATCH_SIZE=100 7 | TRAIN_EXAMPLES=42000 8 | LEARNING_RATE=0.01 9 | 10 | 11 | 12 | #------------------------Generate Data---------------------------# 13 | #generate data 14 | train_frame = pd.read_csv("../Mnist/train.csv") 15 | test_frame = pd.read_csv("../Mnist/test.csv") 16 | 17 | # pop the labels and one-hot coding 18 | train_labels_frame = train_frame.pop("label") 19 | 20 | # get values 21 | # one-hot on labels 22 | X_train = train_frame.astype(np.float32).values/255 23 | y_train=pd.get_dummies(data=train_labels_frame).values 24 | X_test = test_frame.astype(np.float32).values/255 25 | 26 | #trans the shape to (batch,time_steps,input_size) 27 | #X_train=np.reshape(X_train,newshape=(-1,28,28)) 28 | #X_test=np.reshape(X_test,newshape=(-1,28,28)) 29 | print(X_train.shape) 30 | print(y_train.shape) 31 | print(X_test.shape) 32 | 33 | #------------------------------------------------------------------# 34 | 35 | 36 | 37 | def train(): 38 | w_1=tf.Variable( 39 | initial_value=tf.random.normal(shape=(784,200)), 40 | name="w_1" 41 | ) 42 | b_1=tf.Variable( 43 | initial_value=tf.zeros(shape=(200,)), 44 | name="b_1" 45 | ) 46 | w_2 = tf.Variable( 47 | initial_value=tf.random.normal(shape=(200, 10)), 48 | name="w_2" 49 | ) 50 | b_2 = tf.Variable( 51 | initial_value=tf.zeros(shape=(10, )), 52 | name="b_2" 53 | ) 54 | optimizer=tf.keras.optimizers.SGD(LEARNING_RATE) 55 | for epoch in range(1,EPOCH+1): 56 | print("epoch:",epoch) 57 | train_losses = [] 58 | accus = [] 59 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 60 | with tf.GradientTape() as tape: 61 | logits_1=tf.matmul(X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE],w_1)+b_1 62 | logits_1=tf.nn.relu(logits_1) 63 | logits_2=tf.matmul(logits_1,w_2)+b_2 64 | entropy=tf.nn.softmax_cross_entropy_with_logits( 65 | labels=y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 66 | logits=logits_2 67 | ) 68 | loss=tf.math.reduce_mean(entropy) 69 | #print("loss:",loss) 70 | 71 | #计算梯度 72 | gradient=tape.gradient(target=loss,sources=[w_1,b_1,w_2,b_2]) 73 | #print("gradient:",gradient) 74 | #应用梯度 75 | optimizer.apply_gradients(zip(gradient,[w_1,b_1,w_2,b_2])) 76 | 77 | train_losses.append(loss.numpy()) 78 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 1)) 79 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")).numpy() 80 | accus.append(accuracy) 81 | 82 | print("average training loss:", sum(train_losses) / len(train_losses)) 83 | print("accuracy:",sum(accus)/len(accus)) 84 | 85 | 86 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 1)) 87 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")) 88 | 89 | 90 | 91 | if __name__=="__main__": 92 | train() 93 | 94 | 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /7.keras/1.basic.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | print("version of tensorflow:",tf.__version__) 5 | print("version of tensorflow.keras:\n",tf.keras.__version__) 6 | 7 | 8 | -------------------------------------------------------------------------------- /7.keras/2.layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/7.keras/2.layers.py -------------------------------------------------------------------------------- /7.keras/3.model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | def functional_api_test(): 5 | input_x=tf.keras.Input(shape=(784,)) 6 | logits_1=tf.keras.layers.Dense(units=200,activation=tf.nn.relu,use_bias=True)(input_x) 7 | logits_2=tf.keras.layers.Dense(units=100,activation=None,use_bias=True)(logits_1) 8 | pred=tf.keras.layers.Dense(units=10,activation=tf.nn.softmax,use_bias=True)(logits_2) 9 | 10 | model=tf.keras.Model(inputs=input_x,outputs=pred) 11 | 12 | #compile你想要的训练配置 13 | model.compile( 14 | optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), 15 | loss=tf.keras.losses.categorical_crossentropy, 16 | metrics=["accuracy"] 17 | ) 18 | #keras.layers.Dense() 19 | 20 | 21 | 22 | def subclass_test(): 23 | pass 24 | 25 | 26 | 27 | 28 | if __name__=="__main__": 29 | functional_api_test() -------------------------------------------------------------------------------- /8.Template/1.module.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | class MLP(tf.Module): 6 | def __init__(self, hidden_dim): 7 | super(MLP, self).__init__() 8 | self.w_1 = tf.Variable(initial_value=tf.random.normal(shape=(784, hidden_dim)), name="w_1") 9 | self.b_1 = tf.Variable(initial_value=tf.zeros(shape=(hidden_dim,)), name="b_1") 10 | self.w_2 = tf.Variable(initial_value=tf.random.normal(shape=(hidden_dim, 10)), name="w_2") 11 | self.b_2 = tf.Variable(initial_value=tf.zeros(shape=(10,)), name="b_2") 12 | 13 | def __call__(self, inputs): 14 | logits_1 = tf.matmul(inputs, self.w_1) + self.b_1 15 | logits_1 = tf.nn.relu(logits_1) 16 | logits_2 = tf.matmul(logits_1, self.w_2) + self.b_2 17 | return logits_2 18 | 19 | 20 | class MLP2(tf.Module): 21 | def __init__(self, hidden_dim): 22 | super(MLP2, self).__init__() 23 | self.linear1 = tf.keras.layers.Dense(units=hidden_dim, activation=tf.nn.relu, use_bias=True) 24 | self.linear2=tf.keras.layers.Dense(units=10,activation=tf.nn.relu,use_bias=True) 25 | 26 | 27 | def __call__(self, inputs): 28 | logits_1 = self.linear1(inputs) 29 | logits_2 = self.linear2(logits_1) 30 | return logits_2 31 | 32 | 33 | class MLP3(tf.keras.Model): 34 | def __init__(self, hidden_dim): 35 | super(MLP3, self).__init__() 36 | 37 | self.linear1 = tf.keras.layers.Dense(units=hidden_dim, activation=tf.nn.relu, use_bias=True) 38 | self.linear2=tf.keras.layers.Dense(units=10,activation=tf.nn.relu,use_bias=True) 39 | 40 | 41 | def __call__(self, inputs): 42 | logits_1 = self.linear1(inputs) 43 | logits_2 = self.linear2(logits_1) 44 | return logits_2 45 | 46 | 47 | 48 | 49 | 50 | 51 | # class MLP2(tf.keras.Model): 52 | # def __init__(self,hidden_dim): 53 | # super(MLP2,self).__init__() 54 | # self.linear1=tf.keras.layers.Dense(units=hidden_dim,activation=tf.nn.relu,use_bias=True) 55 | # self.linear2=tf.keras.layers.Dense(units=10,activation=tf.nn.relu,use_bias=True) 56 | # # self.model=tf.keras.Sequential() 57 | # # self.model.add(tf.keras.layers.Dense(units=hidden_dim,activation=tf.nn.relu,use_bias=True)) 58 | # # self.model.add(tf.keras.layers.Dense(units=10,activation=None,use_bias=True)) 59 | # 60 | # def __call__(self,inputs): 61 | # # logits_1 = self.linear1(inputs) 62 | # # logits_2 = self.linear2(logits_1) 63 | # # return logits_2 64 | # pass 65 | # 66 | # def linear(inputs,hidden_dim,name_scope): 67 | # with tf.name_scope("name_scope"): 68 | # w_1 = tf.Variable(initial_value=tf.random.normal(shape=(784, hidden_dim)), name="w_1") 69 | # b_1 = tf.Variable(initial_value=tf.zeros(shape=(hidden_dim,)), name="b_1") 70 | # logits=tf.matmul(inputs, w_1) + b_1 71 | # return logits 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | def basic_test(): 85 | #mlp=MLP(hidden_dim=200) 86 | #print("trainable_Variables:",mlp.trainable_variables) 87 | #print("variabels:",mlp.variables) 88 | 89 | # mlp2 = MLP2(hidden_dim=200) 90 | # result=mlp2(inputs=tf.random.normal(shape=(20,784))) 91 | # print("trainable_Variables:", mlp2.linear1.variables) 92 | # print(mlp2.linear2.variables) 93 | # print(mlp2.submodules) 94 | #print("variabels:", mlp2.variables) 95 | # print("type of mlp2:",type(mlp2)) 96 | 97 | mlp3 = MLP3(hidden_dim=200) 98 | result = mlp3(inputs=tf.random.normal(shape=(20, 784))) 99 | print(mlp3.variables) 100 | #print("trainable_Variables:", mlp3.linear1.variables) 101 | #print(mlp3.linear2.variables) 102 | #print(mlp3.submodules) 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | if __name__=="__main__": 114 | basic_test() -------------------------------------------------------------------------------- /8.Template/example.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | EPOCH=20 6 | BATCH_SIZE=100 7 | TRAIN_EXAMPLES=42000 8 | LEARNING_RATE=0.01 9 | 10 | #------------------------Generate Data---------------------------# 11 | #generate data 12 | train_frame = pd.read_csv("../Mnist/train.csv") 13 | test_frame = pd.read_csv("../Mnist/test.csv") 14 | 15 | # pop the labels and one-hot coding 16 | train_labels_frame = train_frame.pop("label") 17 | 18 | # get values 19 | # one-hot on labels 20 | X_train = train_frame.astype(np.float32).values/255 21 | y_train=pd.get_dummies(data=train_labels_frame).values 22 | X_test = test_frame.astype(np.float32).values/255 23 | 24 | #trans the shape to (batch,time_steps,input_size) 25 | #X_train=np.reshape(X_train,newshape=(-1,28,28)) 26 | #X_test=np.reshape(X_test,newshape=(-1,28,28)) 27 | print(X_train.shape) 28 | print(y_train.shape) 29 | print(X_test.shape) 30 | 31 | #------------------------------------------------------------------# 32 | 33 | class MLP(tf.keras.Model): 34 | def __init__(self,hidden_dim): 35 | super(MLP,self).__init__() 36 | self.w_1=tf.Variable(initial_value=tf.random.normal(shape=(784,hidden_dim)),name="w_1") 37 | self.b_1=tf.Variable(initial_value=tf.zeros(shape=(hidden_dim,)),name="b_1") 38 | self.w_2 = tf.Variable(initial_value=tf.random.normal(shape=(hidden_dim, 10)),name="w_2") 39 | self.b_2 = tf.Variable(initial_value=tf.zeros(shape=(10, )),name="b_2") 40 | 41 | 42 | def __call__(self,inputs): 43 | logits_1 = tf.matmul(inputs, self.w_1) + self.b_1 44 | logits_1=tf.nn.relu(logits_1) 45 | logits_2 = tf.matmul(logits_1, self.w_2) + self.b_2 46 | return logits_2 47 | 48 | def train(): 49 | mlp=MLP(hidden_dim=200) 50 | optimizer=tf.keras.optimizers.SGD(LEARNING_RATE) 51 | for epoch in range(1,EPOCH+1): 52 | print("epoch:",epoch) 53 | train_losses = [] 54 | accus = [] 55 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 56 | with tf.GradientTape() as tape: 57 | logits_2=mlp(X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE]) 58 | entropy=tf.nn.softmax_cross_entropy_with_logits( 59 | labels=y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 60 | logits=logits_2 61 | ) 62 | loss=tf.math.reduce_mean(entropy) 63 | #print("loss:",loss) 64 | 65 | #计算梯度 66 | gradient=tape.gradient(target=loss,sources=mlp.trainable_variables) 67 | #print("gradient:",gradient) 68 | #应用梯度 69 | optimizer.apply_gradients(zip(gradient,mlp.trainable_variables)) 70 | 71 | train_losses.append(loss.numpy()) 72 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 1)) 73 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")).numpy() 74 | accus.append(accuracy) 75 | 76 | print("average training loss:", sum(train_losses) / len(train_losses)) 77 | print("accuracy:",sum(accus)/len(accus)) 78 | 79 | 80 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 1)) 81 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")) 82 | 83 | 84 | 85 | if __name__=="__main__": 86 | train() 87 | 88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /8.Template/example2.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | EPOCH=20 6 | BATCH_SIZE=100 7 | TRAIN_EXAMPLES=42000 8 | LEARNING_RATE=0.01 9 | 10 | #------------------------Generate Data---------------------------# 11 | #generate data 12 | train_frame = pd.read_csv("../Mnist/train.csv") 13 | test_frame = pd.read_csv("../Mnist/test.csv") 14 | 15 | # pop the labels and one-hot coding 16 | train_labels_frame = train_frame.pop("label") 17 | 18 | # get values 19 | # one-hot on labels 20 | X_train = train_frame.astype(np.float32).values/255 21 | y_train=pd.get_dummies(data=train_labels_frame).values 22 | X_test = test_frame.astype(np.float32).values/255 23 | 24 | #trans the shape to (batch,time_steps,input_size) 25 | #X_train=np.reshape(X_train,newshape=(-1,28,28)) 26 | #X_test=np.reshape(X_test,newshape=(-1,28,28)) 27 | print(X_train.shape) 28 | print(y_train.shape) 29 | print(X_test.shape) 30 | 31 | #------------------------------------------------------------------# 32 | 33 | class MLP(tf.keras.Model): 34 | def __init__(self,hidden_dim): 35 | super(MLP,self).__init__() 36 | self.linear1=tf.keras.layers.Dense( 37 | units=hidden_dim, 38 | activation=tf.nn.relu, 39 | use_bias=True 40 | ) 41 | self.linear2=tf.keras.layers.Dense( 42 | units=10, 43 | activation=None, 44 | use_bias=True 45 | ) 46 | 47 | 48 | def __call__(self,inputs): 49 | logits_1 = self.linear1(inputs) 50 | logits_2 = self.linear2(logits_1) 51 | return logits_2 52 | 53 | def train(): 54 | mlp=MLP(hidden_dim=200) 55 | optimizer=tf.keras.optimizers.SGD(LEARNING_RATE) 56 | for epoch in range(1,EPOCH+1): 57 | print("epoch:",epoch) 58 | train_losses = [] 59 | accus = [] 60 | for j in range(TRAIN_EXAMPLES//BATCH_SIZE): 61 | with tf.GradientTape() as tape: 62 | logits_2=mlp(X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE]) 63 | entropy=tf.nn.softmax_cross_entropy_with_logits( 64 | labels=y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 65 | logits=logits_2 66 | ) 67 | loss=tf.math.reduce_mean(entropy) 68 | #print("loss:",loss) 69 | 70 | #计算梯度 71 | gradient=tape.gradient(target=loss,sources=mlp.trainable_variables) 72 | #print("gradient:",gradient) 73 | #应用梯度 74 | optimizer.apply_gradients(zip(gradient,mlp.trainable_variables)) 75 | 76 | train_losses.append(loss.numpy()) 77 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE], 1)) 78 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")).numpy() 79 | accus.append(accuracy) 80 | 81 | print("average training loss:", sum(train_losses) / len(train_losses)) 82 | print("accuracy:",sum(accus)/len(accus)) 83 | 84 | 85 | correct_prediction = tf.equal(tf.argmax(logits_2, 1), tf.argmax(y_train[j * BATCH_SIZE:(j + 1) * BATCH_SIZE], 1)) 86 | accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, "float")) 87 | 88 | 89 | 90 | if __name__=="__main__": 91 | train() 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /9.CNN/conv2d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | x=np.array([[[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]], 5 | [[0,0,0],[0,1,2],[1,1,0],[1,1,2],[2,2,0],[2,0,2],[0,0,0]], 6 | [[0,0,0],[0,0,0],[1,2,0],[1,1,1],[0,1,2],[0,2,1],[0,0,0]], 7 | [[0,0,0],[1,1,1],[1,2,0],[0,0,2],[1,0,2],[0,2,1],[0,0,0]], 8 | [[0,0,0],[1,0,2],[0,2,0],[1,1,2],[1,2,0],[1,1,0],[0,0,0]], 9 | [[0,0,0],[0,2,0],[2,0,0],[0,1,1],[1,2,1],[0,0,2],[0,0,0]], 10 | [[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0],[0,0,0]]]) 11 | 12 | 13 | W=np.array([[[[1,-1,0],[1,0,1],[-1,-1,0]], 14 | [[-1,0,1],[0,0,0],[1,-1,1]], 15 | [[-1,1,0],[-1,-1,-1],[0,0,1]]], 16 | 17 | [[[-1,1,-1],[-1,-1,0],[0,0,1]], 18 | [[-1,-1,1],[1,0,0],[0,-1,1]], 19 | [[-1,-1,0],[1,0,-1],[0,0,0]]]]) 20 | 21 | print("input:\n") 22 | print("x[:,:,0]:\n",x[:,:,0]) 23 | print("x[:,:,1]:\n",x[:,:,1]) 24 | print("x[:,:,2]:\n",x[:,:,2]) 25 | 26 | print("filter:") 27 | print("W[0][:,:,0]:\n",W[0][:,:,0]) 28 | print("W[0][:,:,1]:\n",W[0][:,:,1]) 29 | print("W[0][:,:,2]:\n",W[0][:,:,2]) 30 | print("W[1][:,:,0]:\n",W[1][:,:,0]) 31 | print("W[1][:,:,1]:\n",W[1][:,:,1]) 32 | print("W[1][:,:,2]:\n",W[1][:,:,2]) 33 | 34 | #this 35 | x=np.reshape(a=x,newshape=(1,7,7,3)) 36 | W=np.transpose(W,axes=(1,2,3,0)) #weights,[height,width,in_channels,out_channels] 37 | print(W.shape) 38 | b=np.array([1,0]) #bias 39 | 40 | input = tf.constant(value=x, dtype=tf.float32, name="input") 41 | filter = tf.constant(value=W, dtype=tf.float32, name="filter") 42 | bias = tf.constant(value=b, dtype=tf.float32, name="bias") 43 | out=tf.nn.conv2d(input=input,filters=filter,strides=2,padding="VALID",name="conv2d")+bias 44 | 45 | print(out[0][:,:,0]) 46 | print(out[0][:,:,1]) 47 | -------------------------------------------------------------------------------- /9.CNN/example_basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/9.CNN/example_basic.py -------------------------------------------------------------------------------- /9.CNN/example_keras.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/9.CNN/example_keras.py -------------------------------------------------------------------------------- /9.CNN/max_pool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/9.CNN/max_pool.py -------------------------------------------------------------------------------- /Project/DeepDream/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/Project/DeepDream/README.md -------------------------------------------------------------------------------- /Project/ImageCaptioning/README.md: -------------------------------------------------------------------------------- 1 | ## 1.Download coco 2 | 3 | http://images.cocodataset.org/annotations/annotations_trainval2014.zip 4 | 5 | http://images.cocodataset.org/zips/train2014.zip 6 | 7 | -------------------------------------------------------------------------------- /Project/ImageCaptioning/model/model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import time 4 | from PIL import Image 5 | import tensorflow as tf 6 | 7 | 8 | inceptionv3 = tf.keras.applications.InceptionV3(include_top=False,weights='imagenet') 9 | new_input = inceptionv3.input 10 | hidden_layer = inceptionv3.layers[-1].output 11 | 12 | image_features_extract_model = tf.keras.Model(new_input, hidden_layer) 13 | 14 | 15 | -------------------------------------------------------------------------------- /Project/ImageCaptioning/model/parameter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/Project/ImageCaptioning/model/parameter.py -------------------------------------------------------------------------------- /Project/ImageCaptioning/model/preprocess.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy as np 3 | import os 4 | import time 5 | import json 6 | from glob import glob 7 | from PIL import Image 8 | import pickle 9 | from sklearn.utils import shuffle 10 | import tensorflow as tf 11 | 12 | 13 | annotation_file="/data3/xiekun/DataSets/coco/annotations/captions_train2014.json" 14 | image_dir="/data3/xiekun/DataSets/coco/train2014/" 15 | 16 | num_examples = 30000 17 | 18 | def load_image(image_path): 19 | img = tf.io.read_file(image_path) 20 | img = tf.image.decode_jpeg(img, channels=3) 21 | img = tf.image.resize(img, (299, 299)) 22 | img = tf.keras.applications.inception_v3.preprocess_input(img) 23 | return img, image_path 24 | 25 | def get_image_annotation(): 26 | # Read the json file 27 | with open(annotation_file, 'r') as f: 28 | annotations = json.load(f) 29 | 30 | # print("annotations:\n",annotations) 31 | 32 | # Store captions and image names in vectors 33 | all_captions = [] 34 | all_img_names = [] 35 | 36 | for annot in annotations['annotations']: 37 | caption = ' ' + annot['caption'] + ' ' 38 | image_id = annot['image_id'] 39 | full_coco_image_path = image_dir + 'COCO_train2014_' + '%012d.jpg' % (image_id) 40 | 41 | all_img_names.append(full_coco_image_path) 42 | all_captions.append(caption) 43 | 44 | print("all_captions:\n", all_captions[0], len(all_captions)) 45 | print("all_img_names\n", all_img_names[0], len(all_img_names)) 46 | 47 | all_captions, all_img_names = shuffle(all_captions, all_img_names, random_state=1) 48 | print("all_captions:\n", all_captions[0], len(all_captions)) 49 | print("all_img_names\n", all_img_names[0], len(all_img_names)) 50 | 51 | train_captions = all_captions[:num_examples] 52 | train_img_names = all_img_names[:num_examples] 53 | return train_captions,train_img_names 54 | 55 | if __name__=="__main__": 56 | train_captions, train_img_names=get_image_annotation() 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /Project/ImageCaptioning/model/train.py: -------------------------------------------------------------------------------- 1 | import re 2 | import numpy as np 3 | import os 4 | import time 5 | import json 6 | from glob import glob 7 | from PIL import Image 8 | import pickle 9 | from sklearn.utils import shuffle 10 | import tensorflow as tf 11 | 12 | import model 13 | import preprocess 14 | 15 | 16 | def extract_img_features(img_names,extractor): 17 | # Get unique images 18 | encode_train = sorted(set(img_names)) 19 | print("encode_train:\n",encode_train) 20 | 21 | # # Feel free to change batch_size according to your system configuration 22 | # image_dataset = tf.data.Dataset.from_tensor_slices(encode_train) 23 | # image_dataset = image_dataset.map( 24 | # load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16) 25 | # 26 | # for img, path in image_dataset: 27 | # batch_features = image_features_extract_model(img) 28 | # batch_features = tf.reshape(batch_features, 29 | # (batch_features.shape[0], -1, batch_features.shape[3])) 30 | # 31 | # for bf, p in zip(batch_features, path): 32 | # path_of_feature = p.numpy().decode("utf-8") 33 | # np.save(path_of_feature, bf.numpy()) 34 | 35 | if __name__=="__main__": 36 | train_captions, train_img_names = preprocess.get_image_annotation() 37 | extract_img_features(img_names=train_img_names,extractor=model.image_features_extract_model) 38 | -------------------------------------------------------------------------------- /Project/LanguageModel/README.md: -------------------------------------------------------------------------------- 1 | ## 语言模型 2 | 3 | tensorflow 2.x实现的简单语言模型案例，通过这个简单案例，旨在对于**语言模型**，**数据预处理**，**tfrecords**， 4 | **LSTM**等等一些基础API和思想有一个简单了解。 5 | 6 | 7 | ## 使用步骤 8 | - 运行utils下面的`generate_index.py`生成索引表，思想就是把所有的词汇指定一个索引值。 9 | - 把原始corpus制作为更方便的tfrecords 10 | - 训练模型和测试模型 11 | 12 | -------------------------------------------------------------------------------- /Project/LanguageModel/dataset/ptb_process.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | 5 | MODE="test" 6 | 7 | IN_FILE="../ptb_corpus/ptb."+MODE+".txt" 8 | OUT_FILE="./"+MODE+".tfrecords" 9 | 10 | write=True 11 | 12 | def getWordsMapper(IndexFile): 13 | df_words_ids = pd.read_csv(filepath_or_buffer=IndexFile, encoding="utf-8") 14 | words2id = pd.Series(data=df_words_ids["id"].values, index=df_words_ids["word"].values) 15 | id2words = pd.Series(data=df_words_ids["word"].values, index=df_words_ids["id"].values) 16 | # print("word2id:\n",words2id) 17 | # print("id2words:\n",id2words) 18 | # print("words2id.shape",words2id.shape) 19 | return words2id,id2words 20 | 21 | 22 | 23 | def index(word_list,mapper): 24 | ''' 25 | :param word_list: 26 | :param mapper: 27 | :return: 28 | ''' 29 | #print("word_list:",word_list) 30 | x=[] 31 | y=[] 32 | for i in range(len(word_list)): 33 | if i==len(word_list)-1: 34 | x.append(mapper[word_list[i]]) 35 | y.append(mapper[""]) 36 | break 37 | else: 38 | x.append(mapper[word_list[i]]) 39 | y.append(mapper[word_list[i+1]]) 40 | seq_len = len(x) 41 | # print("x:\n",x) 42 | # print("y:\n",y) 43 | # print("\n\n") 44 | return x,y,seq_len 45 | 46 | def preprocess(infile,outfile): 47 | ''' 48 | 49 | :param infile: 50 | :param outfile: 51 | :return: 52 | ''' 53 | words2id, id2words=getWordsMapper("../index_files/words_ids.csv") 54 | writer=tf.io.TFRecordWriter(path=outfile) 55 | #print("word2id:",words2id) 56 | with open(file=infile,encoding="utf-8",errors="ignore") as in_file: 57 | lines=in_file.readlines() 58 | # print("lines:\n",lines) 59 | for line in lines: 60 | word_list=line.strip().split(sep=" ") 61 | #print("word_list:",word_list) 62 | x,y,seq_len=index(word_list=word_list,mapper=words2id) 63 | # 写入到tfrecords 64 | example = tf.train.Example( 65 | features=tf.train.Features( 66 | feature={ 67 | "word": tf.train.Feature(int64_list=tf.train.Int64List(value=x)), 68 | "label": tf.train.Feature(int64_list=tf.train.Int64List(value=y)), 69 | "seq_len": tf.train.Feature(int64_list=tf.train.Int64List(value=[seq_len])) 70 | } 71 | ) 72 | ) 73 | writer.write(record=example.SerializeToString()) 74 | writer.close() 75 | 76 | 77 | 78 | def _parse_data(example_proto): 79 | ''' 80 | 定义tfrecords解析和预处理函数 81 | :param example_proto: 82 | :return: 83 | ''' 84 | parsed_features = tf.io.parse_single_example( 85 | serialized=example_proto, 86 | features={ 87 | "word":tf.io.VarLenFeature(dtype=tf.int64), 88 | "label":tf.io.VarLenFeature(dtype=tf.int64), 89 | "seq_len":tf.io.FixedLenFeature(shape=[], dtype=tf.int64) 90 | } 91 | ) 92 | #get data,变长Feature会被处理为SparseTensor 93 | word=tf.cast(x=parsed_features["word"],dtype=tf.int32) 94 | label=tf.cast(x=parsed_features["label"],dtype=tf.int32) 95 | seq_len=tf.cast(x=parsed_features["seq_len"],dtype=tf.int32) 96 | return word,label,seq_len 97 | 98 | 99 | def readTFRecords(tfrecords_file_list): 100 | ''' 101 | 读取tfrecords中的内容 102 | :param inFile: tfrecords 103 | :return: 104 | ''' 105 | # ----------------------------------------data set API----------------------------------------- 106 | # 创建dataset对象 107 | dataset = tf.data.TFRecordDataset(filenames=tfrecords_file_list) 108 | print("dataset:",dataset) 109 | # 使用map处理得到新的dataset 110 | parsed_dataset = dataset.map(map_func=_parse_data) 111 | # dataset = dataset.map(map_func=_parse_data) 112 | parsed_dataset = parsed_dataset.batch(2) 113 | print("parsed_dataset:", parsed_dataset) 114 | #for parsed_record in parsed_dataset.take(1): 115 | # print("parsed_records:",parsed_record[0]) 116 | for parsed_record in parsed_dataset: 117 | print("parsed_records:", parsed_record[1]) 118 | 119 | 120 | if __name__=="__main__": 121 | print(IN_FILE) 122 | print(OUT_FILE) 123 | if write: 124 | preprocess(infile=IN_FILE,outfile=OUT_FILE) 125 | else: 126 | readTFRecords(tfrecords_file_list=["train.tfrecords"]) 127 | 128 | -------------------------------------------------------------------------------- /Project/LanguageModel/model/lstm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.keras import layers 4 | import parameter 5 | 6 | 7 | class LSTM_Model(tf.keras.Model): 8 | def __init__(self): 9 | super(LSTM_Model,self).__init__() 10 | self.word_embeddings = tf.Variable( 11 | initial_value=tf.random.truncated_normal(shape=(parameter.VOCAB_SIZE, parameter.EMBEDDINGS_DIM)), 12 | trainable=True 13 | ) 14 | self.lstm_1=layers.Bidirectional( 15 | layer=layers.LSTM(units=parameter.HIDDEN_UNITS,return_sequences=True), 16 | merge_mode="concat" 17 | ) 18 | self.lstm_2=layers.Bidirectional( 19 | layer=layers.LSTM(units=parameter.HIDDEN_UNITS, return_sequences=True), 20 | merge_mode="concat" 21 | ) 22 | self.lstm_3 = layers.Bidirectional( 23 | layer=layers.LSTM(units=parameter.HIDDEN_UNITS, return_sequences=True), 24 | merge_mode="concat" 25 | ) 26 | self.linear=layers.Dense(units=parameter.CLASS_NUM) 27 | 28 | 29 | def __call__(self,word_ids,mask,training=True): 30 | #embeddings look up 31 | inputs=tf.nn.embedding_lookup(params=self.word_embeddings,ids=word_ids) 32 | h1=self.lstm_1(inputs=inputs,mask=mask,training=training) 33 | h2=self.lstm_2(inputs=h1,mask=mask,training=training) 34 | h3 = self.lstm_3(inputs=h2,mask=mask,training=training) 35 | logits=self.linear(h3) 36 | return logits 37 | -------------------------------------------------------------------------------- /Project/LanguageModel/model/parameter.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | EMBEDDINGS_DIM=200 4 | VOCAB_SIZE=10001+1 5 | CLASS_NUM=10001+1 6 | DROP_OUT_RATE=0.2 7 | HIDDEN_UNITS=128 8 | 9 | MAX_EPOCH=10 10 | BATCH_SIZE=30 11 | LEARNING_RATE=0.001 12 | 13 | 14 | 15 | TRAIN_FILE_LIST=["../dataset/train.tfrecords"] 16 | 17 | #TRAIN_SIZE=statistic.getTFRecordsListAmount(tfFileList=TRAIN_FILE_LIST) 18 | 19 | #测试集样本数目 20 | TEST_FILE_LIST=["../dataset/test.tfrecords"] 21 | #TEST_SIZE=statistic.getTFRecordsListAmount(tfFileList=TEST_FILE_LIST) 22 | 23 | 24 | #模型存放地址 25 | CHECKPOINT_PATH="./checkpoints/lstm" -------------------------------------------------------------------------------- /Project/LanguageModel/model/test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import lstm 4 | import parameter 5 | 6 | 7 | def _parse_data(example_proto): 8 | ''' 9 | 定义tfrecords解析和预处理函数 10 | :param example_proto: 11 | :return: 12 | ''' 13 | parsed_features = tf.io.parse_single_example( 14 | serialized=example_proto, 15 | features={ 16 | "word":tf.io.VarLenFeature(dtype=tf.int64), 17 | "label":tf.io.VarLenFeature(dtype=tf.int64), 18 | "seq_len":tf.io.FixedLenFeature(shape=[], dtype=tf.int64) 19 | } 20 | ) 21 | #get data,变长Feature会被处理为SparseTensor 22 | word=tf.cast(x=parsed_features["word"],dtype=tf.int32) 23 | label=tf.cast(x=parsed_features["label"],dtype=tf.int32) 24 | seq_len=tf.cast(x=parsed_features["seq_len"],dtype=tf.int32) 25 | return word,label,seq_len 26 | 27 | 28 | 29 | def test(tfrecords_file_list): 30 | ''' 31 | :param file_list: 32 | :return: 33 | ''' 34 | model=lstm.LSTM_Model() 35 | 36 | #optimizer = tf.keras.optimizers.Adam(parameter.LEARNING_RATE) 37 | cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True) 38 | checkpoint=tf.train.Checkpoint(model=model) 39 | 40 | checkpoint.restore(save_path=parameter.CHECKPOINT_PATH+"-1") 41 | 42 | 43 | # ----------------------------------------data set API----------------------------------------- 44 | # 创建dataset对象 45 | dataset = tf.data.TFRecordDataset(filenames=tfrecords_file_list) 46 | print("dataset:", dataset) 47 | # 使用map处理得到新的dataset 48 | parsed_dataset = dataset.map(map_func=_parse_data) 49 | parsed_dataset = parsed_dataset.batch(parameter.BATCH_SIZE).repeat(count=1) 50 | print("parsed_dataset:", parsed_dataset) 51 | # ---------------------------------------------------------------------------------------------- 52 | iter_num=0 53 | for parsed_record in parsed_dataset: 54 | with tf.GradientTape() as tape: 55 | seq_len = parsed_record[2] 56 | mask = tf.sequence_mask(lengths=seq_len) 57 | 58 | # print("mask:\n",mask) 59 | word_dense=tf.sparse.to_dense(sp_input=parsed_record[0]) 60 | #print("word_dense:\n",word_dense) 61 | label_dense=tf.sparse.to_dense(sp_input=parsed_record[1]) 62 | label_onehot=tf.one_hot(indices=label_dense,depth=parameter.CLASS_NUM) 63 | label_onehot_masked=tf.boolean_mask(tensor=label_onehot,mask=mask,axis=0) 64 | #print("label_onehot_masked.shape:\n",label_onehot_masked.shape) 65 | 66 | logits=model(word_ids=word_dense,mask=mask,training=True) 67 | #print("logits.shape:\n",logits) 68 | 69 | logits_masked=tf.boolean_mask(tensor=logits,mask=mask,axis=0) 70 | #print("logits_mask.shape:\n",logits_masked.shape) 71 | 72 | loss=cce(y_true=label_onehot_masked,y_pred=logits_masked) 73 | 74 | print("loss:",loss.numpy()) 75 | print("perplexity:",np.exp(loss.numpy())) 76 | 77 | # 计算梯度 78 | #gradient = tape.gradient(target=loss, sources=model.trainable_variables) 79 | # print("gradient:",gradient) 80 | # 应用梯度 81 | #optimizer.apply_gradients(zip(gradient, model.trainable_variables)) 82 | 83 | iter_num+=1 84 | 85 | #save checkpoints every 200 iterations 86 | #if iter_num%500==0: 87 | # checkpoint.save(file_prefix=parameter.CHECKPOINT_PATH) 88 | 89 | 90 | 91 | 92 | if __name__=="__main__": 93 | test(tfrecords_file_list=parameter.TEST_FILE_LIST) -------------------------------------------------------------------------------- /Project/LanguageModel/model/train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import lstm 4 | import parameter 5 | 6 | 7 | def _parse_data(example_proto): 8 | ''' 9 | 定义tfrecords解析和预处理函数 10 | :param example_proto: 11 | :return: 12 | ''' 13 | parsed_features = tf.io.parse_single_example( 14 | serialized=example_proto, 15 | features={ 16 | "word":tf.io.VarLenFeature(dtype=tf.int64), 17 | "label":tf.io.VarLenFeature(dtype=tf.int64), 18 | "seq_len":tf.io.FixedLenFeature(shape=[], dtype=tf.int64) 19 | } 20 | ) 21 | #get data,变长Feature会被处理为SparseTensor 22 | word=tf.cast(x=parsed_features["word"],dtype=tf.int32) 23 | label=tf.cast(x=parsed_features["label"],dtype=tf.int32) 24 | seq_len=tf.cast(x=parsed_features["seq_len"],dtype=tf.int32) 25 | return word,label,seq_len 26 | 27 | 28 | 29 | def train(tfrecords_file_list): 30 | ''' 31 | :param file_list: 32 | :return: 33 | ''' 34 | model=lstm.LSTM_Model() 35 | optimizer = tf.keras.optimizers.Adam(parameter.LEARNING_RATE) 36 | cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True) 37 | checkpoint=tf.train.Checkpoint(model=model) 38 | 39 | # ----------------------------------------data set API----------------------------------------- 40 | # 创建dataset对象 41 | dataset = tf.data.TFRecordDataset(filenames=tfrecords_file_list) 42 | print("dataset:", dataset) 43 | # 使用map处理得到新的dataset 44 | parsed_dataset = dataset.map(map_func=_parse_data) 45 | parsed_dataset = parsed_dataset.shuffle(buffer_size=1000).batch(parameter.BATCH_SIZE).repeat(count=parameter.MAX_EPOCH) 46 | print("parsed_dataset:", parsed_dataset) 47 | # ---------------------------------------------------------------------------------------------- 48 | iter_num=0 49 | for parsed_record in parsed_dataset: 50 | with tf.GradientTape() as tape: 51 | seq_len = parsed_record[2] 52 | mask = tf.sequence_mask(lengths=seq_len) 53 | 54 | # print("mask:\n",mask) 55 | word_dense=tf.sparse.to_dense(sp_input=parsed_record[0]) 56 | #print("word_dense:\n",word_dense) 57 | label_dense=tf.sparse.to_dense(sp_input=parsed_record[1]) 58 | label_onehot=tf.one_hot(indices=label_dense,depth=parameter.CLASS_NUM) 59 | label_onehot_masked=tf.boolean_mask(tensor=label_onehot,mask=mask,axis=0) 60 | #print("label_onehot_masked.shape:\n",label_onehot_masked.shape) 61 | 62 | logits=model(word_ids=word_dense,mask=mask,training=True) 63 | #print("logits.shape:\n",logits) 64 | 65 | logits_masked=tf.boolean_mask(tensor=logits,mask=mask,axis=0) 66 | #print("logits_mask.shape:\n",logits_masked.shape) 67 | 68 | loss=cce(y_true=label_onehot_masked,y_pred=logits_masked) 69 | 70 | print("loss:",loss.numpy()) 71 | print("perplexity:",np.exp(loss.numpy())) 72 | 73 | # 计算梯度 74 | gradient = tape.gradient(target=loss, sources=model.trainable_variables) 75 | # print("gradient:",gradient) 76 | # 应用梯度 77 | optimizer.apply_gradients(zip(gradient, model.trainable_variables)) 78 | 79 | iter_num+=1 80 | 81 | #save checkpoints every 200 iterations 82 | if iter_num%500==0: 83 | checkpoint.save(file_prefix=parameter.CHECKPOINT_PATH) 84 | 85 | 86 | 87 | 88 | if __name__=="__main__": 89 | train(tfrecords_file_list=parameter.TRAIN_FILE_LIST) -------------------------------------------------------------------------------- /Project/LanguageModel/ptb_corpus/README.md: -------------------------------------------------------------------------------- 1 | PTB Corpus 2 | 3 | [**下载地址**](http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz) 4 | 5 | 下载下来之后，把`data`文件夹下面`ptb.test.txt`,`ptb.train.txt`,`ptb.valid.txt`放到这个文件夹就行。 6 | -------------------------------------------------------------------------------- /Project/LanguageModel/utils/generate_index.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import Counter 3 | import pandas as pd 4 | 5 | OUT_FILE="chars_ids.csv" 6 | IN_FILE_LIST=["../ptb_corpus/ptb.test.txt","../ptb_corpus/ptb.train.txt","../ptb_corpus/ptb.valid.txt"] 7 | EXTRA_CHARS=["","",""] 8 | 9 | def generate(file_list): 10 | counter=Counter() 11 | print("counter:\n",counter) 12 | for file in file_list: 13 | with open(file=file,encoding="utf-8",errors="ignore") as in_file: 14 | lines=in_file.readlines() 15 | #print("lines:\n",lines) 16 | for line in lines: 17 | word_list=line.strip().split(sep=" ") 18 | #print("word_list:",word_list) 19 | for word in word_list: 20 | counter[word]+=1 21 | #print("counter:\n",counter) 22 | #print("counter size:",len(counter)) 23 | counter.pop("") 24 | #print("counter:\n", counter) 25 | #print("counter size:", len(counter)) 26 | 27 | all_word=EXTRA_CHARS+list(counter.keys()) 28 | #print("all_word:", len(all_word)) 29 | ids=[i for i in range(1,len(all_word)+1)] 30 | #print("ids:",ids) 31 | pd.DataFrame(data={"id": ids, "word": all_word}). \ 32 | to_csv(path_or_buf="../index_files/words_ids.csv", index=False, encoding="utf_8") 33 | 34 | if __name__=="__main__": 35 | generate(file_list=IN_FILE_LIST) 36 | 37 | 38 | -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/README.md: -------------------------------------------------------------------------------- 1 | ## 神经翻译简单教程 2 | 3 | 参考： 4 | 5 | - https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/text/nmt_with_attention.ipynb 6 | 7 | - https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/summary -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/corpus/README.md: -------------------------------------------------------------------------------- 1 | # Corpus 2 | you can download the corpus from: http://www.manythings.org/anki/ -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/model/model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | class Encoder(tf.keras.Model): 5 | def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz): 6 | super(Encoder, self).__init__() 7 | self.batch_sz = batch_sz 8 | self.enc_units = enc_units 9 | self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim) 10 | self.gru = tf.keras.layers.GRU( 11 | self.enc_units, 12 | return_sequences=True, 13 | return_state=True, 14 | recurrent_initializer='glorot_uniform' 15 | ) 16 | 17 | def call(self, x, hidden): 18 | x = self.embedding(x) 19 | output, state = self.gru(x, initial_state = hidden) 20 | return output, state 21 | 22 | def initialize_hidden_state(self): 23 | return tf.zeros((self.batch_sz, self.enc_units)) 24 | 25 | 26 | class BahdanauAttention(tf.keras.Model): 27 | def __init__(self,attention_dim): 28 | ''' 29 | BahdanuAttention初始化函数 30 | :param hidden_units: attention里面的全连接的隐藏层节点数 31 | ''' 32 | super(BahdanauAttention,self).__init__() 33 | self.dense_w1=tf.keras.layers.Dense(units=attention_dim) 34 | self.dense_w2=tf.keras.layers.Dense(units=attention_dim) 35 | self.dense_v=tf.keras.layers.Dense(units=1) 36 | 37 | def __call__(self,query,keys,values): 38 | ''' 39 | 进行Bahdanau Attention操作 40 | :param query: 一个时刻t的query,我们一般使用解码器的某个时刻的hidden state，形状为[batch_size,query_dim] 41 | :param keys: 和query比较的keys, 一般使用编码器的全部输出，形状为[batch_size,max_time,keys_dim] 42 | :param values: 需要被attention的values,一般和key相同，你也可以使用自定的values，形状为[batch_size,max_time,values_dim] 43 | :return: 返回一个形状为[batch_size,max_time,1]的attention权重矩阵和形状为[batch_size,values_dim]的注意力向量 44 | ''' 45 | query=tf.expand_dims(input=query,axis=1) #[batch_size,1,query_dim] 46 | logits_query=self.dense_w1(query) #[batch_size,1,attention_dim] 47 | logits_keys=self.dense_w2(keys) #[batch_size,max_time,attention_dim] 48 | logits=tf.nn.tanh(x=logits_query+logits_keys) #broad casting,[batch_size,max_time,hidden_units] 49 | logits_v=self.dense_v(logits) #[batch_size,max_time,1] 50 | attention_weights=tf.nn.softmax(logits=logits_v,axis=1) #[batch_size,max_time,1] 51 | context_vector=tf.reduce_sum(input_tensor=attention_weights*values,axis=1) #[batch_size,values_dim] 52 | #print("context_vector:", context_vector) 53 | return attention_weights,context_vector 54 | 55 | 56 | class Decoder(tf.keras.Model): 57 | def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz): 58 | super(Decoder, self).__init__() 59 | self.batch_sz = batch_sz 60 | self.dec_units = dec_units 61 | self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim) 62 | self.gru = tf.keras.layers.GRU(self.dec_units, 63 | return_sequences=True, 64 | return_state=True, 65 | recurrent_initializer='glorot_uniform') 66 | self.fc = tf.keras.layers.Dense(vocab_size) 67 | 68 | # used for attention 69 | self.attention = BahdanauAttention(self.dec_units) 70 | 71 | def call(self, x, hidden, enc_output): 72 | # enc_output shape == (batch_size, max_length, hidden_size) 73 | attention_weights,context_vector= self.attention(hidden, enc_output,enc_output) 74 | # x shape after passing through embedding == (batch_size, 1, embedding_dim) 75 | x = self.embedding(x) 76 | # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size) 77 | x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1) 78 | # passing the concatenated vector to the GRU 79 | output, state = self.gru(x) 80 | # print("outpouts:\n",output) 81 | # print("state:\n",state) 82 | # output shape == (batch_size * 1, hidden_size) 83 | output = tf.reshape(output, (-1, output.shape[2])) 84 | # output shape == (batch_size, vocab) 85 | x = self.fc(output) 86 | return x, state, attention_weights 87 | 88 | 89 | 90 | 91 | 92 | if __name__=="__main__": 93 | # src_word_ids=np.array([[9,26,7,40],[7,24,6,100],[5,4,200,300],[5,4,200,300]]) 94 | 95 | encoder=Encoder(vocab_size=25216,embedding_dim=256,enc_units=1024,batch_sz=64) 96 | sample_hidden = encoder.initialize_hidden_state() 97 | example_input_batch=tf.ones(shape=(64,88),dtype=tf.int32) 98 | sample_output, sample_hidden = encoder(example_input_batch, sample_hidden) 99 | print('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape)) 100 | print('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape)) 101 | 102 | attention_layer = BahdanauAttention(128) 103 | attention_weights,attention_result= attention_layer(sample_hidden, sample_output,sample_output) 104 | print("Attention result shape: (batch size, units) {}".format(attention_result.shape)) 105 | print("Attention weights shape: (batch_size, sequence_length, 1) {}".format(attention_weights.shape)) 106 | 107 | decoder = Decoder(13053, 256, 1024, 64) 108 | 109 | sample_decoder_output, _, _ = decoder(tf.random.uniform((64, 1)),sample_hidden, sample_output) 110 | 111 | print('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape)) 112 | 113 | 114 | 115 | # encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE) 116 | 117 | # target_word_ids = np.array([[500, 26, 7, 40,10], [7, 24, 6, 100,0], [5, 4, 200, 300,80], [5, 4, 200, 300,90]]) 118 | 119 | # encoder=Encoder(vocab_size=5230,embeddings_dim=200,units=128,batch_size=30) 120 | # en_outputs,en_states=encoder(word_ids=src_word_ids,mask=None,training=True) 121 | 122 | 123 | # decoder=Decoder(vocab_size=62054,embeddings_dim=200,units=128,batch_size=30) 124 | # word_ids_one_step=target_word_ids[:,0] 125 | # print("word_ids_one_step:\n",word_ids_one_step) 126 | # word_ids_one_step=np.expand_dims(a=word_ids_one_step,axis=-1) 127 | # print("word_ids_one_step:\n",word_ids_one_step) 128 | # 129 | # 130 | # decoder(word_ids=word_ids_one_step,pre_states=en_states,encoder_outputs=en_outputs) -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/model/parameter.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | EMBEDDINGS_DIM=256 6 | 7 | SRC_VOCAB_SIZE=25216 8 | TARGET_VOCAB_SIZE=13053 9 | # CLASS_NUM=TARGET_VOCAB_SIZE 10 | 11 | DROP_OUT_RATE=0.2 12 | HIDDEN_UNITS=1024 13 | 14 | MAX_EPOCH=20 15 | DATASET_SIZE=123376 16 | BATCH_SIZE=64 17 | STEPS_PER_EPOCH=DATASET_SIZE//BATCH_SIZE 18 | LEARNING_RATE=0.001 19 | 20 | 21 | 22 | #corpus存放地址 23 | CORPUS_PATH="../corpus/spa.txt" 24 | 25 | #模型存放地址 26 | CHECKPOINT_PATH="./checkpoints/lstm" 27 | 28 | #可视化log存放地址 29 | LOG_DIR="./log/" 30 | -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/model/process.py: -------------------------------------------------------------------------------- 1 | import os 2 | import io 3 | import re 4 | import time 5 | import unicodedata 6 | import numpy as np 7 | import tensorflow as tf 8 | 9 | # Converts the unicode file to ascii 10 | def unicode_to_ascii(s): 11 | return ''.join(c for c in unicodedata.normalize('NFD', s) 12 | if unicodedata.category(c) != 'Mn') 13 | 14 | def preprocess_sentence(w): 15 | w = unicode_to_ascii(w.lower().strip()) 16 | 17 | # creating a space between a word and the punctuation following it 18 | # eg: "he is a boy." => "he is a boy ." 19 | # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation 20 | w = re.sub(r"([?.!,¿])", r" \1 ", w) 21 | w = re.sub(r'[" "]+', " ", w) 22 | 23 | # replacing everything with space except (a-z, A-Z, ".", "?", "!", ",") 24 | w = re.sub(r"[^a-zA-Z?.!,¿]+", " ", w) 25 | 26 | w = w.rstrip().strip() 27 | 28 | # adding a start and an end token to the sentence 29 | # so that the model know when to start and stop predicting. 30 | w = ' ' + w + ' ' 31 | return w 32 | 33 | # 1. Remove the accents 34 | # 2. Clean the sentences 35 | # 3. Return word pairs in the format: [ENGLISH, SPANISH] 36 | def create_dataset(path, num_examples): 37 | lines = io.open(path, encoding='UTF-8').read().strip().split('\n') 38 | word_pairs = [[preprocess_sentence(w) for w in l.split('\t')] for l in lines[:num_examples]] 39 | return zip(*word_pairs) 40 | 41 | 42 | def max_length(tensor): 43 | return max(len(t) for t in tensor) 44 | 45 | 46 | def tokenize(lang): 47 | lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='') 48 | lang_tokenizer.fit_on_texts(lang) 49 | tensor = lang_tokenizer.texts_to_sequences(lang) 50 | tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,padding='post') 51 | return tensor, lang_tokenizer 52 | 53 | 54 | 55 | def load_dataset(path, num_examples=None): 56 | # creating cleaned input, output pairs 57 | targ_lang, inp_lang = create_dataset(path, num_examples) 58 | input_tensor, inp_lang_tokenizer = tokenize(inp_lang) 59 | target_tensor, targ_lang_tokenizer = tokenize(targ_lang) 60 | return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer 61 | 62 | 63 | def convert(lang, tensor): 64 | for t in tensor: 65 | if t!=0: 66 | print ("%d ----> %s" % (t, lang.index_word[t])) 67 | 68 | 69 | if __name__=="__main__": 70 | # en_sentence = u"May I borrow this book?" 71 | # sp_sentence = u"¿Puedo tomar prestado este libro?" 72 | # print(preprocess_sentence(en_sentence)) 73 | # print(preprocess_sentence(sp_sentence).encode('utf-8')) 74 | # 75 | # en, sp = create_dataset(path="../corpus/spa.txt", num_examples=None) 76 | # print(en) 77 | #print(sp) 78 | 79 | input_tensor, target_tensor, inp_lang, targ_lang = load_dataset(path="../corpus/spa.txt", num_examples=None) 80 | print("input_tensor:\n",input_tensor) 81 | print("target_tensor:\n",target_tensor) 82 | 83 | max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor) 84 | print("max_length_targ:",max_length_targ) 85 | print("max_length_inp:",max_length_inp) 86 | 87 | print("Input Language; index to word mapping") 88 | convert(inp_lang, input_tensor[0]) 89 | print() 90 | print("Target Language; index to word mapping") 91 | convert(targ_lang, target_tensor[0]) 92 | 93 | vocab_inp_size = len(inp_lang.word_index) + 1 94 | print("vocab_inp_size:",vocab_inp_size) 95 | vocab_tar_size = len(targ_lang.word_index) + 1 96 | print("vocab_tar_size:",vocab_tar_size) -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/model/test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import tensorflow as tf 4 | import model 5 | import parameter 6 | 7 | MAX_TIME_TARGET=80 8 | 9 | 10 | def _parse_data(example_proto): 11 | ''' 12 | 定义tfrecords解析和预处理函数 13 | :param example_proto: 14 | :return: 15 | ''' 16 | parsed_features = tf.io.parse_single_example( 17 | serialized=example_proto, 18 | features={ 19 | "src_word":tf.io.VarLenFeature(dtype=tf.int64), 20 | "src_len": tf.io.FixedLenFeature(shape=[], dtype=tf.int64), 21 | "target_word_input": tf.io.VarLenFeature(dtype=tf.int64), 22 | "target_word_output": tf.io.VarLenFeature(dtype=tf.int64), 23 | "target_len":tf.io.FixedLenFeature(shape=[], dtype=tf.int64) 24 | } 25 | ) 26 | 27 | #变长Feature会被处理为SparseTensor 28 | src_word=tf.cast(x=parsed_features["src_word"],dtype=tf.int32) 29 | src_len = tf.cast(x=parsed_features["src_len"], dtype=tf.int32) 30 | target_word_input=tf.cast(x=parsed_features["target_word_input"],dtype=tf.int32) 31 | target_word_output = tf.cast(x=parsed_features["target_word_output"], dtype=tf.int32) 32 | target_len=tf.cast(x=parsed_features["target_len"],dtype=tf.int32) 33 | return src_word,src_len,target_word_input,target_word_output,target_len 34 | 35 | def getWordsMapper(IndexFile): 36 | df_words_ids = pd.read_csv(filepath_or_buffer=IndexFile, encoding="utf-8") 37 | words2id = pd.Series(data=df_words_ids["id"].values, index=df_words_ids["word"].values) 38 | id2words = pd.Series(data=df_words_ids["word"].values, index=df_words_ids["id"].values) 39 | # print("word2id:\n",words2id) 40 | # print("id2words:\n",id2words) 41 | # print("words2id.shape",words2id.shape) 42 | return words2id,id2words 43 | 44 | 45 | def loss_func(loss_obj,real,pred): 46 | ''' 47 | 精细的损失函数 48 | :param loss_obj:损失函数对象tf.keras.losses.xxx 49 | :param real: 真实标签，形状为[batch_size,] 50 | :param pred: 预测标签，这里可以是logits，形状为[batch_size,class_num] 51 | :return: 52 | ''' 53 | #print("real:\n",real) 54 | mask=tf.math.logical_not(tf.math.equal(x=real,y=0)) 55 | #print("mask:\n",mask) 56 | 57 | loss=loss_obj(real,pred) 58 | #print("loss:",loss) 59 | 60 | mask=tf.cast(x=mask,dtype=loss.dtype) 61 | loss*=mask 62 | #print("loss:", loss) 63 | 64 | loss=tf.reduce_mean(loss) 65 | #print("loss:", loss) 66 | #print("\n\n") 67 | 68 | return loss 69 | 70 | 71 | 72 | def test(tfrecords_file_list): 73 | ''' 74 | :param file_list: 75 | :return: 76 | ''' 77 | 78 | words2id_zh, id2words_zh = getWordsMapper("../index_files/zh_ids.csv") 79 | words2id_en, id2words_en = getWordsMapper("../index_files/en_ids.csv") 80 | encoder=model.Encoder( 81 | vocab_size=parameter.SRC_VOCAB_SIZE, 82 | embeddings_dim=parameter.EMBEDDINGS_DIM, 83 | units=512, 84 | batch_size=parameter.BATCH_SIZE 85 | ) 86 | 87 | decoder=model.Decoder( 88 | vocab_size=parameter.TARGET_VOCAB_SIZE, 89 | embeddings_dim=parameter.EMBEDDINGS_DIM, 90 | units=512, 91 | batch_size=parameter.BATCH_SIZE 92 | ) 93 | 94 | optimizer = tf.keras.optimizers.Adam(parameter.LEARNING_RATE) 95 | cce=tf.keras.losses.SparseCategoricalCrossentropy( 96 | from_logits=True, 97 | reduction=tf.keras.losses.Reduction.NONE 98 | ) 99 | checkpoint = tf.train.Checkpoint(optimizer=optimizer,encoder=encoder, decoder=decoder) 100 | 101 | #restore 102 | checkpoint.restore(save_path=parameter.CHECKPOINT_PATH+"-1") 103 | 104 | # ----------------------------------------data set API----------------------------------------- 105 | # 创建dataset对象 106 | dataset = tf.data.TFRecordDataset(filenames=tfrecords_file_list) 107 | print("dataset:", dataset) 108 | # 使用map处理得到新的dataset 109 | parsed_dataset = dataset.map(map_func=_parse_data) 110 | parsed_dataset = parsed_dataset.batch(1).repeat(1) #batch_size只为1 111 | print("parsed_dataset:", parsed_dataset) 112 | # ---------------------------------------------------------------------------------------------- 113 | iter_num=0 #迭代次数 114 | for parsed_record in parsed_dataset: #一次一个mini_batch 115 | #准备数据,这里只取源序列，忽略目标序列 116 | src_word=tf.sparse.to_dense(parsed_record[0]) 117 | #print("src_word:",src_word[0].numpy()) 118 | src_len=parsed_record[1] 119 | #target_word_input=tf.sparse.to_dense(parsed_record[2]) 120 | #target_word_output=tf.sparse.to_dense(parsed_record[3]) 121 | #target_len=parsed_record[4] 122 | 123 | src_mask = tf.sequence_mask(lengths=src_len) 124 | #print("src_mask:\n",src_mask) 125 | 126 | #encode 127 | en_outputs, en_states = encoder(word_ids=src_word, mask=src_mask, training=False) 128 | # print("en_outputs:\n",en_outputs) 129 | # print("en_states:\n",en_states) 130 | 131 | #decode 132 | pre_states=en_states #decoder的第一个state设置为encoder输出的那个states 133 | 134 | target_word_input_one_step=tf.expand_dims(input=[1],axis=0) #start flag 135 | #print("target_word_input_one_step:", target_word_input_one_step) 136 | 137 | #receive result 138 | predict_ids=[] 139 | for time in range(MAX_TIME_TARGET): 140 | de_outputs,de_states,attention_weights=decoder( 141 | word_ids=target_word_input_one_step, 142 | pre_states=pre_states, 143 | encoder_outputs=en_outputs 144 | ) 145 | #loss+=loss_func(loss_obj=cce,real=target_word_output_one_step,pred=de_outputs) 146 | pre_states=de_states #重新赋值states 147 | 148 | predict_id=tf.argmax(input=de_outputs[0]).numpy() 149 | 150 | if predict_id==2: 151 | break 152 | predict_ids.append(predict_id) 153 | recover(src_ids=src_word[0].numpy(),pred_ids=predict_ids,src_mapper=id2words_zh,target_mapper=id2words_en) 154 | 155 | 156 | def recover(src_ids,pred_ids,src_mapper,target_mapper): 157 | ''' 158 | 从id序列中恢复为文本 159 | :param src_ids: 源单词id序列 160 | :param pred_ids: 预测结果id序列 161 | :param src_mapper: 源索引mapper 162 | :param target_mapper: 预测索引mapper 163 | :return: 164 | ''' 165 | src_text="" 166 | pred_text="" 167 | 168 | for src_id in src_ids: 169 | src_text+=src_mapper[src_id]+" " 170 | 171 | for pred_id in pred_ids: 172 | pred_text+=target_mapper[pred_id]+" " 173 | 174 | print("src_text:",src_text) 175 | print("pred_text:",pred_text) 176 | print("\n\n") 177 | 178 | 179 | 180 | if __name__=="__main__": 181 | test(tfrecords_file_list=parameter.TRAIN_FILE_LIST) -------------------------------------------------------------------------------- /Project/NeuralMachineTranslation/model/train.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import model 4 | import parameter 5 | import process 6 | import time 7 | 8 | def train(): 9 | ''' 10 | :param file_list: 11 | :return: 12 | ''' 13 | encoder = model.Encoder( 14 | vocab_size=parameter.SRC_VOCAB_SIZE, 15 | embedding_dim=parameter.EMBEDDINGS_DIM, 16 | enc_units=parameter.HIDDEN_UNITS, 17 | batch_sz=parameter.BATCH_SIZE 18 | ) 19 | 20 | decoder=model.Decoder( 21 | vocab_size=parameter.TARGET_VOCAB_SIZE, 22 | embedding_dim=parameter.EMBEDDINGS_DIM, 23 | dec_units=parameter.HIDDEN_UNITS, 24 | batch_sz=parameter.BATCH_SIZE 25 | ) 26 | 27 | optimizer = tf.keras.optimizers.Adam(parameter.LEARNING_RATE) 28 | cce=tf.keras.losses.SparseCategoricalCrossentropy( 29 | from_logits=True, 30 | reduction=tf.keras.losses.Reduction.NONE 31 | ) 32 | 33 | def loss_function(real, pred): 34 | mask = tf.math.logical_not(tf.math.equal(real, 0)) 35 | loss_ = cce(real, pred) 36 | mask = tf.cast(mask, dtype=loss_.dtype) 37 | loss_ *= mask 38 | return tf.reduce_mean(loss_) 39 | 40 | checkpoint = tf.train.Checkpoint(optimizer=optimizer, encoder=encoder, decoder=decoder) 41 | 42 | #log 43 | file_writer=tf.summary.create_file_writer(logdir=parameter.LOG_DIR) 44 | 45 | # data set API 46 | input_tensor, target_tensor, inp_lang, targ_lang = process.load_dataset( 47 | path=parameter.CORPUS_PATH, 48 | num_examples=None 49 | ) 50 | dataset = tf.data.Dataset.from_tensor_slices((input_tensor, target_tensor)).shuffle(parameter.DATASET_SIZE) 51 | dataset = dataset.batch(parameter.BATCH_SIZE, drop_remainder=True) 52 | 53 | 54 | def train_step(inp,targ,init_enc_hidden): 55 | loss=0.0 56 | with tf.GradientTape() as tape: 57 | #encoder 58 | enc_output, enc_hidden = encoder(inp, init_enc_hidden) 59 | 60 | #decoder 61 | dec_hidden=enc_hidden 62 | dec_input = tf.expand_dims([targ_lang.word_index['']] * parameter.BATCH_SIZE, 1) 63 | #print("dec_input:\n",dec_input) 64 | 65 | # Teacher forcing - feeding the target as the next input 66 | for t in range(1, targ.shape[1]): 67 | predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output) 68 | print("predictions:\n",predictions) 69 | 70 | loss += loss_function(targ[:, t], predictions) 71 | # using teacher forcing 72 | dec_input = tf.expand_dims(targ[:, t], 1) 73 | 74 | batch_loss = (loss / int(targ.shape[1])) 75 | variables = encoder.trainable_variables + decoder.trainable_variables 76 | gradients = tape.gradient(loss, variables) 77 | optimizer.apply_gradients(zip(gradients, variables)) 78 | return batch_loss 79 | 80 | 81 | for epoch in range(parameter.MAX_EPOCH): 82 | total_loss=0 83 | init_enc_hidden=encoder.initialize_hidden_state() 84 | 85 | for (batch, (inp, targ)) in enumerate(dataset.take(parameter.STEPS_PER_EPOCH)): 86 | batch_loss = train_step(inp, targ, init_enc_hidden) 87 | total_loss += batch_loss 88 | if batch % 100 == 0: 89 | print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,batch,batch_loss.numpy())) 90 | # saving (checkpoint) the model every 2 epochs 91 | if (epoch + 1) % 2 == 0: 92 | checkpoint.save(file_prefix=parameter.CHECKPOINT_PATH) 93 | print('Epoch {} Loss {:.4f}'.format(epoch + 1,total_loss / parameter.STEPS_PER_EPOCH)) 94 | 95 | 96 | if __name__=="__main__": 97 | train() 98 | 99 | 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /Project/README.md: -------------------------------------------------------------------------------- 1 | ## Project Practice 2 | 3 | -------------------------------------------------------------------------------- /Project/StyleTransfer/model/image_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import tensorflow as tf 6 | 7 | def load_img(path_to_img): 8 | max_dim = 512 9 | img = tf.io.read_file(path_to_img) 10 | img = tf.image.decode_image(img, channels=3) 11 | img = tf.image.convert_image_dtype(img, tf.float32) 12 | 13 | shape = tf.cast(tf.shape(img)[:-1], tf.float32) 14 | long_dim = max(shape) 15 | scale = max_dim / long_dim 16 | 17 | new_shape = tf.cast(shape * scale, tf.int32) 18 | 19 | img = tf.image.resize(img, new_shape) 20 | img = img[tf.newaxis, :] 21 | return img 22 | 23 | def imshow(image, title=None): 24 | if len(image.shape) > 3: 25 | image = tf.squeeze(image, axis=0) 26 | 27 | plt.imshow(image) 28 | if title: 29 | plt.title(title) 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /Project/StyleTransfer/model/parameter.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import tensorflow as tf 6 | 7 | 8 | content_path = tf.keras.utils.get_file( 9 | 'turtle.jpg', 10 | 'https://storage.googleapis.com/download.tensorflow.org/example_images/Green_Sea_Turtle_grazing_seagrass.jpg' 11 | ) 12 | style_path = tf.keras.utils.get_file( 13 | 'kandinsky.jpg', 14 | 'https://storage.googleapis.com/download.tensorflow.org/example_images/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg' 15 | ) 16 | 17 | # Content layer where will pull our feature maps 18 | content_layers = ['block5_conv2'] 19 | 20 | # Style layer of interest 21 | style_layers = ['block1_conv1', 22 | 'block2_conv1', 23 | 'block3_conv1', 24 | 'block4_conv1', 25 | 'block5_conv1'] 26 | 27 | num_content_layers = len(content_layers) 28 | num_style_layers = len(style_layers) 29 | 30 | -------------------------------------------------------------------------------- /Project/StyleTransfer/model/style_transfer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import tensorflow as tf 6 | 7 | def gram_matrix(input_tensor): 8 | result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor) 9 | input_shape = tf.shape(input_tensor) 10 | num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32) 11 | return result/(num_locations) 12 | 13 | 14 | def vgg_layers(layer_names): 15 | """ 16 | Creates a vgg model that returns a list of intermediate output values. 17 | """ 18 | # Load our model. Load pretrained VGG, trained on imagenet data 19 | vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet') 20 | vgg.trainable = False 21 | outputs = [vgg.get_layer(name).output for name in layer_names] 22 | model = tf.keras.Model([vgg.input], outputs) 23 | return model 24 | 25 | 26 | class StyleContentModel(tf.keras.models.Model): 27 | def __init__(self, style_layers, content_layers): 28 | super(StyleContentModel, self).__init__() 29 | self.vgg = vgg_layers(style_layers + content_layers) 30 | self.style_layers = style_layers 31 | self.content_layers = content_layers 32 | self.num_style_layers = len(style_layers) 33 | self.vgg.trainable = False 34 | 35 | def call(self, inputs): 36 | "Expects float input in [0,1]" 37 | inputs = inputs * 255.0 38 | preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs) 39 | outputs = self.vgg(preprocessed_input) 40 | 41 | style_outputs, content_outputs = (outputs[:self.num_style_layers],outputs[self.num_style_layers:]) 42 | style_outputs = [gram_matrix(style_output) for style_output in style_outputs] 43 | #content and style dict 44 | content_dict = {content_name: value for content_name, value in zip(self.content_layers, content_outputs)} 45 | style_dict = {style_name: value for style_name, value in zip(self.style_layers, style_outputs)} 46 | return {'content': content_dict, 'style': style_dict} 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /Project/StyleTransfer/model/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XierHacker/LearningTensorFlow/c1c9be585efc524e06cca7f970bf7fe3f46cda4a/Project/StyleTransfer/model/test.py -------------------------------------------------------------------------------- /Project/StyleTransfer/model/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append("../") 4 | sys.path.append("../../") 5 | import time 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.image as img 9 | import tensorflow as tf 10 | 11 | import parameter 12 | import image_utils 13 | from style_transfer import StyleContentModel 14 | 15 | 16 | def clip_0_1(image): 17 | return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0) 18 | 19 | 20 | def train(content_image,style_image): 21 | extractor = StyleContentModel(parameter.style_layers, parameter.content_layers) 22 | 23 | style_targets = extractor(style_image)['style'] 24 | content_targets = extractor(content_image)['content'] 25 | 26 | image = tf.Variable(content_image) 27 | 28 | opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1) 29 | 30 | style_weight = 1e-2 31 | content_weight = 1e4 32 | 33 | def style_content_loss(outputs): 34 | style_outputs = outputs['style'] 35 | content_outputs = outputs['content'] 36 | style_loss = tf.add_n([tf.reduce_mean((style_outputs[name] - style_targets[name]) ** 2) 37 | for name in style_outputs.keys()]) 38 | style_loss *= style_weight / parameter.num_style_layers 39 | 40 | content_loss = tf.add_n([tf.reduce_mean((content_outputs[name] - content_targets[name]) ** 2) 41 | for name in content_outputs.keys()]) 42 | content_loss *= content_weight / parameter.num_content_layers 43 | loss = style_loss + content_loss 44 | return loss 45 | 46 | for i in range(1000): 47 | start_time=time.time() 48 | print("Step:",i) 49 | with tf.GradientTape() as tape: 50 | outputs = extractor(image) 51 | loss = style_content_loss(outputs) 52 | print("---loss:",loss.numpy()) 53 | grad = tape.gradient(loss, image) 54 | opt.apply_gradients([(grad, image)]) 55 | image.assign(clip_0_1(image)) 56 | end_time=time.time() 57 | print("---spend:",end_time-start_time," seconds") 58 | 59 | 60 | #save result 61 | img.imsave("./result.png",image.numpy()[0]) 62 | 63 | #plt.imshow(image.read_value()[0]) 64 | #plt.show() 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | if __name__=="__main__": 73 | content_image = image_utils.load_img("./6.jpeg") 74 | style_image = image_utils.load_img("./4.jpeg") 75 | train(content_image=content_image,style_image=style_image) 76 | # 77 | # plt.subplot(1, 2, 1) 78 | # image_utils.imshow(content_image, 'Content Image') 79 | # 80 | # plt.subplot(1, 2, 2) 81 | # image_utils.imshow(style_image, 'Style Image') 82 | # 83 | # plt.show() 84 | 85 | # x = tf.keras.applications.vgg19.preprocess_input(content_image*255) 86 | # #print("x:\n",x) 87 | # x = tf.image.resize(x, (224, 224)) 88 | # #print("x:\n", x) 89 | # vgg = tf.keras.applications.VGG19(include_top=True, weights='imagenet') 90 | # prediction_probabilities = vgg(x) 91 | # print("pred:\n",prediction_probabilities.shape) 92 | # 93 | # predicted_top_5 = tf.keras.applications.vgg19.decode_predictions(prediction_probabilities.numpy())[0] 94 | # result=[(class_name, prob) for (number, class_name, prob) in predicted_top_5] 95 | # print("result:\n",result) 96 | 97 | # style_outputs = style_extractor(style_image * 255) 98 | # for name, output in zip(style_layers, style_outputs): 99 | # print(name) 100 | # print(" shape: ", output.numpy().shape) 101 | # print(" min: ", output.numpy().min()) 102 | # print(" max: ", output.numpy().max()) 103 | # print(" mean: ", output.numpy().mean()) 104 | # print() 105 | 106 | 107 | # results = extractor(tf.constant(content_image)) 108 | # 109 | # style_results = results['style'] 110 | # 111 | # print('Styles:') 112 | # for name, output in sorted(results['style'].items()): 113 | # print(" ", name) 114 | # print(" shape: ", output.numpy().shape) 115 | # print(" min: ", output.numpy().min()) 116 | # print(" max: ", output.numpy().max()) 117 | # print(" mean: ", output.numpy().mean()) 118 | # print() 119 | # 120 | # print("Contents:") 121 | # for name, output in sorted(results['content'].items()): 122 | # print(" ", name) 123 | # print(" shape: ", output.numpy().shape) 124 | # print(" min: ", output.numpy().min()) 125 | # print(" max: ", output.numpy().max()) 126 | # print(" mean: ", output.numpy().mean()) 127 | -------------------------------------------------------------------------------- /Project/Transformer/dataset/process.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow_datasets as tfds 4 | 5 | #parameters 6 | MAX_LENGTH = 40 7 | 8 | def get_dataset(): 9 | dataset,info=tfds.load(name="ted_hrlr_translate/pt_to_en",with_info=True,as_supervised=True) 10 | #print("dataset:\n",dataset) 11 | # print("info:\n",info) 12 | 13 | train_dataset=dataset["train"] 14 | val_dataset=dataset["validation"] 15 | 16 | #load subword and restore tokenizer 17 | tokenizer_en=tfds.features.text.SubwordTextEncoder.load_from_file(filename_prefix="../index_files/en") 18 | tokenizer_pt=tfds.features.text.SubwordTextEncoder.load_from_file(filename_prefix="../index_files/pt") 19 | 20 | # sample_string = 'Transformer is awesome.' 21 | # tokenized_string = loaded_tokenizer_en.encode(sample_string) 22 | # print ('Tokenized string is {}'.format(tokenized_string)) 23 | 24 | # for ts in tokenized_string: 25 | # print(ts,"---->",loaded_tokenizer_en.decode([ts])) 26 | 27 | # sample_string = 'este é o primeiro livro que eu fiz.' 28 | # tokenized_string = loaded_tokenizer_pt.encode(sample_string) 29 | # print ('Tokenized string is {}'.format(tokenized_string)) 30 | 31 | # for ts in tokenized_string: 32 | # print(ts,"---->",loaded_tokenizer_pt.decode([ts])) 33 | 34 | def encode(lang1,lang2): 35 | ''' 36 | Add a start and end token to the input and target. 37 | ''' 38 | lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode( 39 | lang1.numpy()) + [tokenizer_pt.vocab_size+1] 40 | lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode( 41 | lang2.numpy()) + [tokenizer_en.vocab_size+1] 42 | return lang1, lang2 43 | 44 | def filter_max_length(x, y, max_length=MAX_LENGTH): 45 | return tf.logical_and(tf.size(x) <= max_length,tf.size(y) <= max_length) 46 | 47 | def tf_encode(pt, en): 48 | return tf.py_function(encode, [pt, en], [tf.int64, tf.int64]) 49 | 50 | 51 | train_dataset = train_dataset.map(tf_encode) 52 | train_dataset = train_dataset.filter(filter_max_length) 53 | 54 | 55 | # val_dataset = val_examples.map(tf_encode) 56 | # val_dataset = val_dataset.filter(filter_max_length).padded_batch(BATCH_SIZE, padded_shapes=([-1], [-1])) 57 | return train_dataset 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /Project/Transformer/model/test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | sys.path.append("../") 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import tensorflow as tf 8 | import tensorflow_datasets as tfds 9 | from transformer import * 10 | from dataset import process 11 | 12 | 13 | MODEL_PATH="" 14 | 15 | 16 | -------------------------------------------------------------------------------- /Project/Transformer/utils/display.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | def draw_positional_encodings(PE): 7 | plt.pcolormesh(PE[0], cmap='RdBu') 8 | plt.xlabel('Depth') 9 | plt.xlim((0, 512)) 10 | plt.ylabel('Position') 11 | plt.colorbar() 12 | plt.show() 13 | 14 | 15 | def draw_multihead_attention(attention_weights,src_seq,tar_seq,layer): 16 | fig=plt.figure(figsize=(16,8)) 17 | 18 | #sentence = tokenizer_pt.encode(sentence) 19 | attention_weights = tf.squeeze(attention_weights[layer], axis=0) 20 | 21 | for head in range(attention_weights.shape[0]): 22 | ax = fig.add_subplot(2, 4, head+1) 23 | 24 | # plot the attention weights 25 | ax.matshow(attention_weights[head][:-1, :], cmap='viridis') 26 | fontdict = {'fontsize': 10} 27 | ax.set_xticks(range(len(src_seq)+2)) 28 | ax.set_yticks(range(len(tar_seq))) 29 | 30 | ax.set_ylim(len(tar_seq)-1.5, -0.5) 31 | 32 | ax.set_xticklabels(['']+[i for i in src_seq]+[''], fontdict=fontdict, rotation=90) 33 | 34 | ax.set_yticklabels([i for i in tar_seq], fontdict=fontdict) 35 | 36 | ax.set_xlabel('Head {}'.format(head+1)) 37 | 38 | plt.tight_layout() 39 | plt.show() 40 | -------------------------------------------------------------------------------- /Project/Transformer/utils/generate_index.py: -------------------------------------------------------------------------------- 1 | ''' 2 | generate subword index files in `../index_files/` 3 | ''' 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | import tensorflow_datasets as tfds 8 | 9 | dataset,info=tfds.load(name="ted_hrlr_translate/pt_to_en",with_info=True,as_supervised=True) 10 | print("dataset:\n",dataset) 11 | # print("info:\n",info) 12 | 13 | train_dataset=dataset["train"] 14 | val_dataset=dataset["validation"] 15 | 16 | #generate english subword files 17 | tokenizer_en=tfds.features.text.SubwordTextEncoder.build_from_corpus( 18 | corpus_generator=(en.numpy() for pt,en in train_dataset), 19 | target_vocab_size=2**13 20 | ) 21 | tokenizer_en.save_to_file(filename_prefix="../index_files/en") 22 | 23 | #generate Portugese subword files 24 | tokenizer_pt=tfds.features.text.SubwordTextEncoder.build_from_corpus( 25 | corpus_generator=(pt.numpy() for pt,en in train_dataset), 26 | target_vocab_size=2**13 27 | ) 28 | tokenizer_pt.save_to_file(filename_prefix="../index_files/pt") 29 | 30 | 31 | 32 | # loaded_tokenizer_en=tfds.features.text.SubwordTextEncoder.load_from_file(filename_prefix="../index_files/en") 33 | # sample_string = 'Transformer is awesome.' 34 | 35 | # tokenized_string = loaded_tokenizer_en.encode(sample_string) 36 | # print ('Tokenized string is {}'.format(tokenized_string)) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LearningTensorFlow 2 | 3 | 注意： 4 | - 教程已经全部迁移到**tensorflow 2.0** 5 | 6 | 7 | 8 | ## [谢小小XH的博客](http://blog.csdn.net/xierhacker/article/category/6511974) 9 | 10 | ### [**TensorFlow学习（一）：感受一下**](http://blog.csdn.net/xierhacker/article/details/53102355) 11 | 12 | ### [**TensorFlow学习（二）：变量常量类型**](http://blog.csdn.net/xierhacker/article/details/53103979) 13 | 14 | ### [TensorFlow学习（三）：Graph和Session](http://blog.csdn.net/xierhacker/article/details/53860379) 15 | 16 | ### [TensorFlow学习（四）：优化器Optimizer](http://blog.csdn.net/xierhacker/article/details/53174558) 17 | 18 | ### [TensorFlow学习（五）：数学与随机值](http://blog.csdn.net/xierhacker/article/details/53462070) 19 | 20 | ### [TensorFlow学习（六）：形状相关操作](http://blog.csdn.net/xierhacker/article/details/53462072) 21 | 22 | ### [TensorFlow学习（七）：基本神经网络"组件"](http://blog.csdn.net/xierhacker/article/details/53174579) 23 | 24 | ### [TensorFlow学习（八）：tensorborad可视化](http://blog.csdn.net/xierhacker/article/details/53697515) 25 | 26 | ### [TensorFlow学习（九）：卷积网络CNN](http://blog.csdn.net/xierhacker/article/details/53174594) 27 | 28 | ### [TensorFlow学习（十）：图像预处理](http://blog.csdn.net/xierhacker/article/details/72385422) 29 | 30 | ### [TensorFlow学习（十一）：保存TFRecord文件](http://blog.csdn.net/xierhacker/article/details/72357651) 31 | 32 | ### [TensorFlow学习（十二）：模型的保存与恢复](http://blog.csdn.net/xierhacker/article/details/58637829) 33 | 34 | ### [TensorFlow学习（十三）：构造LSTM最简明教程](http://blog.csdn.net/xierhacker/article/details/78772560) 35 | 36 | ### [TensorFlow学习（十四）：条件随机场CRF](https://blog.csdn.net/xierhacker/article/details/78923758) 37 | 38 | ### [TensorFlow学习（十五）：使用tf.data来创建输入流(上)](https://blog.csdn.net/xierhacker/article/details/79002902) --------------------------------------------------------------------------------