├── .gitignore
├── LICENSE
├── README.md
├── clean.sh
├── contributors.txt
├── data
    ├── create_examples_list.py
    ├── git.dir
    ├── readme.md
    └── relation_tag_to_id.py
├── datasets.py
├── debug
    └── git.dir
├── debug_test
    └── git.dir
├── model.py
├── predict.py
├── settings.py
├── slim
    ├── BUILD
    ├── README.md
    ├── __init__.py
    ├── collections_test.py
    ├── inception_model.py
    ├── inception_test.py
    ├── losses.py
    ├── losses_test.py
    ├── ops.py
    ├── ops_test.py
    ├── scopes.py
    ├── scopes_test.py
    ├── slim.py
    ├── variables.py
    └── variables_test.py
├── train.sh
├── train_operation.py
└── trainer.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.png
3 | *.jpg
4 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2016 Masahiro Imai, Yixuan Hu (yeephycho)
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # InceptionV3_TensorFlow #
 2 | InceptionV3_TensorFlow is an implementation of inception v3 using tensorflow and slim according to our guidline.
 3 | 
 4 | 
 5 | ## Dependencies ##
 6 | - TensorFlow (>= 0.12)
 7 | 
 8 | 
 9 | ## Features ##
10 | - train
11 | - predict
12 | - save checkpoint
13 | - real time data augumentation
14 | 
15 | ## Quick start ##
16 | If you want a quick start to run training of Inception_v3, you can simply do:
17 | ``` bash
18 | ./train.sh
19 | ```
20 | The above script has passed test under Ubuntu15.10, CentOS and macOS.
21 | 
22 | If you want to go through the train process step by step, please take the following content as example.
23 | 
24 | ### Setup ###
25 | 1. download data in data/readme.md
26 | 2. execute "data/create_examples_list.py"
27 | 3. execute "data/relation_tag_to_id.py"
28 | 4. you can see train_csv.txt and test_csv.txt
29 | 
30 | ### Start to train##
31 | ```
32 | python trainer.py
33 | ```
34 | Pass test under Ubuntu15.10 and CentOS
35 | 
36 | ### How to use your own data sets ###
37 | - create train_csv.txt and test_csv.txt in data directory.
38 | 
39 | ### datalist format ###
40 | 
41 | ```
42 | <image path>,<label number>  
43 | ...
44 | ```
45 | - change num_classes in settings.py
46 | 
47 | ### Fine tune ###
48 | - change fine_tune in settings.py
49 | 
50 | ## TensorBoard ##
51 | ```
52 | tensorboard --logdir /to/your/path/train_dir --port=6006
53 | ```
54 | ---
55 | 
56 | Copyright (c) 2016 Masahiro Imai, Yixuan Hu (yeephycho)
57 | Released under the MIT license
58 | 


--------------------------------------------------------------------------------
/clean.sh:
--------------------------------------------------------------------------------
 1 | #########################################################################
 2 | # File Name: clean.sh
 3 | # Author: yeephycho
 4 | # mail: huyixuanhyx@gmail.com
 5 | # Created Time: Tuesday, December 06, 2016 PM05:09:51 HKT
 6 | #########################################################################
 7 | #!/bin/bash
 8 | 
 9 | rm -rf train
10 | rm -rf ./data/*.txt
11 | rm -rf ./debug/*
12 | rm -rf ./debug_test/*
13 | 


--------------------------------------------------------------------------------
/contributors.txt:
--------------------------------------------------------------------------------
1 | Masahiro Imai
2 | Hu Yixuan
3 | 


--------------------------------------------------------------------------------
/data/create_examples_list.py:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | import glob
 3 | import os
 4 | 
 5 | def create_examples(directory):
 6 |     print directory
 7 |     directories = glob.glob(directory)
 8 |     for category_path in directories:
 9 |         category_name = os.path.basename(category_path) 
10 |         print("category: %s" % (category_name))
11 |        
12 |         images = glob.glob("%s/%s" % (category_path, "*"))
13 |         for image in images:
14 |             with open('101Caltech_examples.txt', 'a') as f:
15 |                 f.write(image)
16 |                 f.write(',')
17 |                 f.write(category_name)
18 |                 f.write("\n")
19 | 
20 | if __name__ == '__main__':
21 |     current_directory = os.getcwd()
22 |     path = os.path.join(current_directory + "/", "101_ObjectCategories/*")
23 |     create_examples(path) 
24 | 


--------------------------------------------------------------------------------
/data/git.dir:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MasazI/InceptionV3_TensorFlow/90cce1d735589137e234fc1a38f410adfb145fbd/data/git.dir


--------------------------------------------------------------------------------
/data/readme.md:
--------------------------------------------------------------------------------
 1 | inputs data format: csv  
 2 | each raw: <imagepath>,<label number>  
 3 | 
 4 | samples: Caltech101  
 5 | https://s3-us-west-2.amazonaws.com/deep-learning.datasection.co.jp/datasets/101_ObjectCategories.tar.gz  
 6 | unzip the file, you can get 101_ObjectCategories directory.  
 7 |   
 8 | use script:  
 9 | create_examples_list.py --> image path,category list  
10 | relation_tag_to_id.py --> image path,label list to train  
11 | 


--------------------------------------------------------------------------------
/data/relation_tag_to_id.py:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | 
 3 | import numpy as np
 4 | import random
 5 | 
 6 | def main(filepath):
 7 |     data = np.loadtxt(filepath, delimiter=",", dtype=np.str)
 8 |     id_index = -1
 9 |     classes = set()
10 |     datasets = {}
11 |     testsets = {}
12 |     for line in data:
13 |         image = line[0]
14 |         label = line[1]
15 |         if not label in classes:
16 |             classes.add(label)
17 |             id_index += 1
18 |             with open('classes.txt', 'a') as f:
19 |                 f.write(label)
20 |                 f.write(',')
21 |                 f.write(str(id_index))
22 |                 f.write('\n')
23 | 
24 |         if random.random() < 0.15:
25 |             testsets[image] = id_index
26 |         else:
27 |             datasets[image] = id_index
28 | 
29 |     with open('train_csv.txt', 'w') as f:
30 |         for key, value in datasets.iteritems():
31 |             f.write(key)
32 |             f.write(',')
33 |             f.write(str(value))
34 |             f.write("\n")
35 | 
36 |     with open('test_csv.txt', 'w') as f:
37 |         for key, value in testsets.iteritems():
38 |             f.write(key)
39 |             f.write(',')
40 |             f.write(str(value))
41 |             f.write("\n")
42 | 
43 | if __name__ == '__main__':
44 |     main('101Caltech_examples.txt')
45 | 


--------------------------------------------------------------------------------
/datasets.py:
--------------------------------------------------------------------------------
  1 | # encoding: utf-8
  2 | 
  3 | import tensorflow as tf
  4 | import settings
  5 | FLAGS = settings.FLAGS
  6 | import os
  7 | import numpy as np
  8 | from PIL import Image
  9 | 
 10 | import csv
 11 | 
 12 | class DataSet:
 13 |     def __init__(self):
 14 |         pass
 15 | 
 16 |     def distort_color(self, image, thread_id=0):
 17 |         """Distort the color of the image.
 18 | 
 19 |         Each color distortion is non-commutative and thus ordering of the color ops
 20 |         matters. Ideally we would randomly permute the ordering of the color ops.
 21 |         Rather then adding that level of complication, we select a distinct ordering
 22 |         of color ops for each preprocessing thread.
 23 | 
 24 |         Args:
 25 |           image: Tensor containing single image.
 26 |           thread_id: preprocessing thread ID.
 27 |           scope: Optional scope for op_scope.
 28 |         Returns:
 29 |           color-distorted image
 30 |         """
 31 |         with tf.op_scope([image], 'distort_color'):
 32 |             color_ordering = thread_id % 2
 33 | 
 34 |             if color_ordering == 0:
 35 |                 image = tf.image.random_brightness(image, max_delta=32. / 255.)
 36 |                 image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
 37 |                 image = tf.image.random_hue(image, max_delta=0.2)
 38 |                 image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
 39 |             elif color_ordering == 1:
 40 |                 image = tf.image.random_brightness(image, max_delta=32. / 255.)
 41 |                 image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
 42 |                 image = tf.image.random_saturation(image, lower=0.5, upper=1.5)
 43 |                 image = tf.image.random_hue(image, max_delta=0.2)
 44 | 
 45 |             # The random_* ops do not necessarily clamp.
 46 |             image = tf.clip_by_value(image, 0.0, 1.0)
 47 |         return image
 48 | 
 49 |     def _generate_image_and_label_batch(self, image, label, min_queue_examples, batch_size, shuffle=True):
 50 |         '''
 51 |         imageとlabelのmini batchを生成
 52 |         '''
 53 |         num_preprocess_threads = FLAGS.num_threads
 54 |         if shuffle:
 55 |             images, labels = tf.train.shuffle_batch(
 56 |                 [image, label],
 57 |                 batch_size=batch_size,
 58 |                 num_threads=num_preprocess_threads,
 59 |                 capacity=min_queue_examples + 3 * FLAGS.batch_size,
 60 |                 min_after_dequeue=min_queue_examples
 61 |             )
 62 |             # Display the training images in the visualizer
 63 |             tf.summary.image('images', images, max_outputs=batch_size)
 64 |         else:
 65 |             images, labels = tf.train.batch(
 66 |                 [image, label],
 67 |                 batch_size=batch_size,
 68 |                 num_threads=num_preprocess_threads,
 69 |                 capacity=min_queue_examples + 3 * FLAGS.batch_size
 70 |             )
 71 | 
 72 |         return images, labels
 73 | 
 74 |     def cnt_samples(self, filepath):
 75 |         return sum(1 for line in open(filepath))
 76 | 
 77 |     def test_inputs(self, csv, batch_size, verbose=False):
 78 |         print("input csv file path: %s, batch size: %d" % (csv, batch_size))
 79 |         filename_queue = tf.train.string_input_producer([csv], shuffle=False)
 80 |         reader = tf.TextLineReader()
 81 |         _, serialized_example = reader.read(filename_queue)
 82 |         filename, label = tf.decode_csv(serialized_example, [["path"], [0]])
 83 | 
 84 |         label = tf.cast(label, tf.int32)
 85 |         jpg = tf.read_file(filename)
 86 |         image = tf.image.decode_jpeg(jpg, channels=3)
 87 |         image = tf.cast(image, tf.float32)
 88 |         if verbose:
 89 |             print "original image shape:"
 90 |             print image.get_shape()
 91 | 
 92 |         # resize to distort
 93 |         dist = tf.image.resize_images(image, (FLAGS.scale_h, FLAGS.scale_w))
 94 |         # random crop
 95 |         dist = tf.image.resize_image_with_crop_or_pad(dist, FLAGS.input_h, FLAGS.input_w)
 96 | 
 97 |         min_fraction_of_examples_in_queue = 0.4
 98 |         min_queue_examples = int(FLAGS.num_examples_per_epoch_for_train * min_fraction_of_examples_in_queue)
 99 |         print (
100 |         'filling queue with %d train images before starting to train.  This will take a few minutes.' % min_queue_examples)
101 | 
102 |         return self._generate_image_and_label_batch(dist, label, min_queue_examples, batch_size, shuffle=False)
103 | 
104 | 
105 |     def csv_inputs(self, csv, batch_size, distorted=False, verbose=False):
106 |         print("input csv file path: %s, batch size: %d" % (csv, batch_size))
107 |         filename_queue = tf.train.string_input_producer([csv], shuffle=True)
108 |         reader = tf.TextLineReader()
109 |         _, serialized_example = reader.read(filename_queue)
110 |         filename, label = tf.decode_csv(serialized_example, [["path"], [0]])
111 | 
112 |         label = tf.cast(label, tf.int32)
113 |         jpg = tf.read_file(filename)
114 |         image = tf.image.decode_jpeg(jpg, channels=3)
115 |         image = tf.cast(image, tf.float32)
116 |         if verbose:
117 |             print "original image shape:"
118 |             print image.get_shape()
119 | 
120 |         if distorted:
121 |             # resize to distort
122 |             dist = tf.image.resize_images(image, (FLAGS.scale_h, FLAGS.scale_w))
123 | 
124 |             # random crop
125 |             dist = tf.image.resize_image_with_crop_or_pad(dist, FLAGS.input_h, FLAGS.input_w)
126 | 
127 |             # random flip
128 |             dist = tf.image.random_flip_left_right(dist)
129 | 
130 |             # color constancy
131 |             #dist = self.distort_color(dist)
132 |         else:
133 |             # resize to input
134 |             dist = tf.image.resize_images(image, FLAGS.input_h, FLAGS.input_w)
135 | 
136 |         if verbose:
137 |             print "dist image shape:"
138 |             print dist.get_shape()
139 | 
140 |         min_fraction_of_examples_in_queue = 0.4
141 |         min_queue_examples = int(FLAGS.num_examples_per_epoch_for_train * min_fraction_of_examples_in_queue)
142 |         print ('filling queue with %d train images before starting to train.  This will take a few minutes.' % min_queue_examples)
143 | 
144 |         return self._generate_image_and_label_batch(dist, label, min_queue_examples, batch_size)
145 | 
146 |     def output_images(self, images, output_dir, name):
147 |         for i, image in enumerate(images):
148 |             pilimg = Image.fromarray(np.uint8(image))
149 |             image_name = "%s/%05d_%s.png" % (output_dir, i, name)
150 |             pilimg.save(image_name)
151 | 
152 | 
153 |     def load_csv(self, path):
154 |         print("load csv: %s" % (path))
155 |         images = []
156 |         with open(path, 'r') as f:
157 |             rows = csv.reader(f)
158 |             for row in rows:
159 |                 print row
160 |                 images.append(row)
161 |         return images
162 | 
163 | 
164 | def debug(data):
165 |     return data
166 | 
167 | if __name__ == "__main__":
168 |     dataset = DataSet()
169 |     images, labels = dataset.csv_inputs(FLAGS.traincsv, FLAGS.batch_size)
170 | 
171 |     images_eval = debug(images)
172 |     labels_eval = debug(labels)
173 | 
174 |     # initialization
175 |     init = tf.initialize_all_variables()
176 |     sess = tf.Session()
177 |     coord = tf.train.Coordinator()
178 |     threads = tf.train.start_queue_runners(sess=sess, coord=coord)
179 |     images_val, labels_val = sess.run([images_eval, labels])
180 | 
181 | 


--------------------------------------------------------------------------------
/debug/git.dir:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MasazI/InceptionV3_TensorFlow/90cce1d735589137e234fc1a38f410adfb145fbd/debug/git.dir


--------------------------------------------------------------------------------
/debug_test/git.dir:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MasazI/InceptionV3_TensorFlow/90cce1d735589137e234fc1a38f410adfb145fbd/debug_test/git.dir


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | # encoding: utf-8
  2 | 
  3 | import re
  4 | import tensorflow as tf
  5 | from slim import slim
  6 | 
  7 | import settings
  8 | FLAGS = settings.FLAGS
  9 | 
 10 | TOWER_NAME = 'tower'
 11 | 
 12 | def _activation_summary(x):
 13 |   """Helper to create summaries for activations.
 14 | 
 15 |   Creates a summary that provides a histogram of activations.
 16 |   Creates a summary that measure the sparsity of activations.
 17 | 
 18 |   Args:
 19 |     x: Tensor
 20 |   """
 21 |   # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
 22 |   # session. This helps the clarity of presentation on tensorboard.
 23 |   tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
 24 |   tf.histogram_summary(tensor_name + '/activations', x)
 25 |   tf.scalar_summary(tensor_name + '/sparsity', tf.nn.zero_fraction(x))
 26 | 
 27 | 
 28 | def _activation_summaries(endpoints):
 29 |   with tf.name_scope('summaries'):
 30 |     for act in endpoints.values():
 31 |       _activation_summary(act)
 32 | 
 33 | def inference(images, num_classes, for_training=False, restore_logits=True, scope=None, reuse=False, dropout_keep_prob=0.8, verbose=False):
 34 |     batch_norm_params = {
 35 |         # Decay for the moving averages.
 36 |         'decay': FLAGS.batchnorm_moving_average_decay,
 37 |         # epsilon to prevent 0s in variance.
 38 |         'epsilon': 0.001,
 39 |     }
 40 | 
 41 |     with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
 42 |         with slim.arg_scope([slim.ops.conv2d],
 43 |                             stddev=0.1,
 44 |                             activation=tf.nn.relu,
 45 |                             batch_norm_params=batch_norm_params):
 46 |             # endpoints = {"logits", "predictions"}, predictions is softmax of logits
 47 |             logits, endpoints = slim.inception.inception_v3(
 48 |                 images,
 49 |                 dropout_keep_prob=dropout_keep_prob,
 50 |                 num_classes=num_classes,
 51 |                 is_training=for_training,
 52 |                 restore_logits=restore_logits,
 53 |                 scope=scope,
 54 |                 reuse=reuse)
 55 | 
 56 |         # Add summaries for viewing model statistics on TensorBoard.
 57 |         # _activation_summaries(endpoints)
 58 | 
 59 |         # Grab the logits associated with the side head. Employed during training.
 60 |         auxiliary_logits = endpoints['aux_logits']
 61 | 
 62 |     softmax = endpoints['predictions']
 63 | 
 64 |     if verbose:
 65 |         print logits.get_shape()
 66 |         print softmax.get_shape()
 67 | 
 68 |     # logits: output of final layer, auxliary_logits: output of hidden layer, softmax: predictions
 69 |     return logits, auxiliary_logits, softmax
 70 | 
 71 | def loss_regression(logits, targets, batch_size=None):
 72 |     if not batch_size:
 73 |         batch_size = FLAGS.batch_size
 74 | 
 75 |     print "-" * 10
 76 |     print "predicts:"
 77 |     print type(logits)
 78 |     print len(logits)
 79 |     print logits[0].get_shape()
 80 |     print logits[1].get_shape()
 81 |     print "targets:"
 82 |     print type(targets)
 83 |     print len(targets)
 84 |     print targets.get_shape()
 85 |     print "-" * 10
 86 | 
 87 |     # Square loss for the main regression head.
 88 |     slim.losses.square_loss(logits[0], targets, label_smoothing=0.1, weight=1.0)
 89 | 
 90 |     # Square loss for the auxiliary regression head.
 91 |     slim.losses.square_loss(logits[1], targets, label_smoothing=0.1, weight=1.0, scope='aux_loss')
 92 | 
 93 | def loss_test(logits, labels, batch_size=None, verbose=False):
 94 |     # Reshape the labels into a dense Tensor of
 95 |     # shape [FLAGS.batch_size, num_classes].
 96 |     sparse_labels = tf.reshape(labels, [batch_size, 1])
 97 |     indices = tf.reshape(tf.range(batch_size), [batch_size, 1])
 98 |     sparse_labels = tf.cast(sparse_labels, tf.int32)
 99 |     concated = tf.concat(1, [indices, sparse_labels])
100 |     num_classes = logits[0].get_shape()[-1].value
101 |     dense_labels = tf.sparse_to_dense(concated,
102 |                                     [batch_size, num_classes],
103 |                                     1.0, 0.0)
104 | 
105 |     if verbose:
106 |         print "-"*10
107 |         print type(logits)
108 |         print len(logits)
109 |         print logits[0].get_shape()
110 |         print logits[1].get_shape()
111 |         print "-"*10
112 | 
113 |     # Cross entropy loss for the main softmax prediction.
114 |     loss = slim.losses.cross_entropy_loss_without_collection(logits[0],
115 |                                  dense_labels,
116 |                                  label_smoothing=0.1,
117 |                                  weight=1.0)
118 | 
119 |     # Cross entropy loss for the auxiliary softmax head.
120 |     aux_loss = slim.losses.cross_entropy_loss_without_collection(logits[1],
121 |                                  dense_labels,
122 |                                  label_smoothing=0.1,
123 |                                  weight=0.4,
124 |                                  scope='aux_loss')
125 |     return loss, aux_loss
126 | 
127 | 
128 | 
129 | def loss(logits, labels, batch_size=None, verbose=False):
130 |     if not batch_size:
131 |         batch_size = FLAGS.batch_size
132 | 
133 |     # Reshape the labels into a dense Tensor of
134 |     # shape [FLAGS.batch_size, num_classes].
135 |     sparse_labels = tf.reshape(labels, [batch_size, 1])
136 |     indices = tf.reshape(tf.range(batch_size), [batch_size, 1])
137 |     sparse_labels = tf.cast(sparse_labels, tf.int32)
138 |     concated = tf.concat(1, [indices, sparse_labels])
139 |     num_classes = logits[0].get_shape()[-1].value
140 |     dense_labels = tf.sparse_to_dense(concated,
141 |                                     [batch_size, num_classes],
142 |                                     1.0, 0.0)
143 | 
144 |     if verbose:
145 |         print "-"*10
146 |         print type(logits)
147 |         print len(logits)
148 |         print logits[0].get_shape()
149 |         print logits[1].get_shape()
150 |         print "-"*10
151 | 
152 |     # Cross entropy loss for the main softmax prediction.
153 |     slim.losses.cross_entropy_loss(logits[0],
154 |                                  dense_labels,
155 |                                  label_smoothing=0.1,
156 |                                  weight=1.0)
157 | 
158 |     # Cross entropy loss for the auxiliary softmax head.
159 |     slim.losses.cross_entropy_loss(logits[1],
160 |                                  dense_labels,
161 |                                  label_smoothing=0.1,
162 |                                  weight=0.4,
163 |                                  scope='aux_loss')
164 | 


--------------------------------------------------------------------------------
/predict.py:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | import tensorflow as tf
 3 | import settings
 4 | FLAGS = settings.FLAGS
 5 | 
 6 | import os
 7 | import re
 8 | import copy
 9 | from datetime import datetime
10 | import time
11 | from datasets import DataSet
12 | import datasets
13 | 
14 | from PIL import Image
15 | 
16 | import model
17 | import train_operation
18 | import slim.slim
19 | import numpy as np
20 | 
21 | def train():
22 |     with tf.Graph().as_default():
23 |         # global step number
24 |         global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
25 |         dataset = DataSet()
26 | 
27 |         # get training set
28 |         print("The number of training images is: %d" % (dataset.cnt_samples(FLAGS.predictcsv)))
29 |         csv_predict = FLAGS.predictcsv
30 |         lines = dataset.load_csv(csv_predict)
31 |         lines.sort()
32 | 
33 |         images_ph = tf.placeholder(tf.float32, [1, 229, 229, 3])
34 | 
35 |         num_classes = FLAGS.num_classes
36 |         restore_logits = not FLAGS.fine_tune
37 | 
38 |         # inference
39 |         logits = model.inference(images_ph, num_classes, for_training=False, restore_logits=restore_logits)
40 | 
41 | 
42 |         # Retain the summaries from the final tower.
43 |         batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION)
44 | 
45 |         # saver
46 |         saver = tf.train.Saver(tf.all_variables())
47 | 
48 |         # Build the summary operation from the last tower summaries.
49 |         summary_op = tf.merge_all_summaries()
50 | 
51 |         # initialization
52 |         init = tf.initialize_all_variables()
53 | 
54 |         # session
55 |         sess = tf.Session(config=tf.ConfigProto(
56 |             allow_soft_placement=True,
57 |             log_device_placement=FLAGS.log_device_placement))
58 |         sess.run(init)
59 | 
60 |         coord = tf.train.Coordinator()
61 |         threads = tf.train.start_queue_runners(sess=sess, coord=coord)
62 | 
63 |         ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
64 |         if ckpt and ckpt.model_checkpoint_path:
65 |             print("load: checkpoint %s" % (ckpt.model_checkpoint_path))
66 |             saver.restore(sess, ckpt.model_checkpoint_path)
67 |         
68 |         print("start to predict.")
69 |         for step, line in enumerate(lines):
70 |             pil_img = Image.open(line[0])
71 |             pil_img = pil_img.resize((250, 250))
72 |             img_array_r = np.asarray(pil_img)
73 |             img_array_r = img_array_r[15:244,15:244,:]
74 |             img_array = img_array_r[None, ...]
75 |             softmax_eval = sess.run([logits[2]], feed_dict={images_ph: img_array})
76 |             print("%s,%s,%s" % (line[0], line[1], np.argmax(softmax_eval)))
77 |         print("finish to predict.")
78 |         coord.request_stop()
79 |         coord.join(threads)
80 |         sess.close()
81 | 
82 | 
83 | def test():
84 |     # load settings file
85 |     print(FLAGS.tfcsv)
86 | 
87 | 
88 | def main(argv=None):
89 |     if tf.gfile.Exists(FLAGS.train_dir) and not FLAGS.fine_tune:
90 |         print("Caution: train dir is already exists.")
91 |     if not tf.gfile.Exists(FLAGS.train_dir):
92 |         tf.gfile.MakeDirs(FLAGS.train_dir)
93 |     train()
94 | 
95 | 
96 | if __name__ == '__main__':
97 |     tf.app.run()
98 | 


--------------------------------------------------------------------------------
/settings.py:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | 
 3 | import tensorflow as tf
 4 | 
 5 | flags = tf.app.flags
 6 | FLAGS = flags.FLAGS
 7 | 
 8 | # Flags train settings
 9 | flags.DEFINE_integer('batch_size', 100, 'the number of images in a batch.')
10 | flags.DEFINE_string('tfcsv', 'data/train_csv.txt', 'path to tf csv file for training.')
11 | flags.DEFINE_string('testcsv', 'data/test_csv.txt', 'path to tf csv file for testing.')
12 | flags.DEFINE_string('predictcsv', 'data/predict_csv.txt', 'path to tf csv file to predict.')
13 | tf.app.flags.DEFINE_string('train_dir', 'train', "Directory where to write event logs and checkpoint")
14 | tf.app.flags.DEFINE_integer('num_classes', 101, "Number of classes")
15 | tf.app.flags.DEFINE_integer('max_steps', 10000000, "Number of batches to run.")
16 | tf.app.flags.DEFINE_integer('num_threads', 4, "Number of threads")
17 | tf.app.flags.DEFINE_string('subset', 'train', "Either 'train' or 'validation'.")
18 | tf.app.flags.DEFINE_float('batchnorm_moving_average_decay', 0.9997, "decay rate of batchnorm.")
19 | tf.app.flags.DEFINE_float('moving_average_decay', 0.9999, "decay rate of movieng average.")
20 | tf.app.flags.DEFINE_integer('num_examples_per_epoch_for_train', 500, "the number of examples per epoch train.")
21 | 
22 | # FLags train images settings
23 | tf.app.flags.DEFINE_integer('image_h_org', 64, "original image height")
24 | tf.app.flags.DEFINE_integer('image_w_org', 64, "original image weight")
25 | tf.app.flags.DEFINE_integer('image_c_org', 3, "original image weight")
26 | 
27 | # FLags train inputs settings
28 | tf.app.flags.DEFINE_integer('scale_h', 250, "scale image height")
29 | tf.app.flags.DEFINE_integer('scale_w', 250, "scale image weight")
30 | tf.app.flags.DEFINE_integer('input_h', 229, "input image height")
31 | tf.app.flags.DEFINE_integer('input_w', 229, "input image weight")
32 | tf.app.flags.DEFINE_integer('input_c', 3, "input image weight")
33 | tf.app.flags.DEFINE_integer('crop_h', 200, "crop image height")
34 | tf.app.flags.DEFINE_integer('crop_w', 200, "crop image weight")
35 | 
36 | # Flags governing the hardware employed for running TensorFlow.
37 | tf.app.flags.DEFINE_integer('num_gpus', 1, "How many GPUs to use.")
38 | tf.app.flags.DEFINE_boolean('log_device_placement', False, "Whether to log device placement.")
39 | 
40 | # Flags governing the type of training.
41 | tf.app.flags.DEFINE_boolean('fine_tune', False, "is fine tune")
42 | tf.app.flags.DEFINE_string('pretrained_model_checkpoint_path', '', "pretrained model's checkpoint path")
43 | tf.app.flags.DEFINE_float('initial_learning_rate', 0.001, "Initial learning rate.")
44 | tf.app.flags.DEFINE_float('num_epochs_per_decay', 30.0, "Epochs after which learning rate decays.")
45 | tf.app.flags.DEFINE_float('learning_rate_decay_factor', 0.16, "Learning rate decay factor.")
46 | 
47 | # GPU
48 | tf.app.flags.DEFINE_float('gpu_memory_fraction', 0.7, "gpu memory fraction.")
49 | 


--------------------------------------------------------------------------------
/slim/BUILD:
--------------------------------------------------------------------------------
  1 | # Description:
  2 | #   Contains the operations and nets for building TensorFlow-Slim models.
  3 | 
  4 | package(default_visibility = ["//inception:internal"])
  5 | 
  6 | licenses(["notice"])  # Apache 2.0
  7 | 
  8 | exports_files(["LICENSE"])
  9 | 
 10 | py_library(
 11 |     name = "scopes",
 12 |     srcs = ["scopes.py"],
 13 | )
 14 | 
 15 | py_test(
 16 |     name = "scopes_test",
 17 |     size = "small",
 18 |     srcs = ["scopes_test.py"],
 19 |     deps = [
 20 |         ":scopes",
 21 |     ],
 22 | )
 23 | 
 24 | py_library(
 25 |     name = "variables",
 26 |     srcs = ["variables.py"],
 27 |     deps = [
 28 |         ":scopes",
 29 |     ],
 30 | )
 31 | 
 32 | py_test(
 33 |     name = "variables_test",
 34 |     size = "small",
 35 |     srcs = ["variables_test.py"],
 36 |     deps = [
 37 |         ":variables",
 38 |     ],
 39 | )
 40 | 
 41 | py_library(
 42 |     name = "losses",
 43 |     srcs = ["losses.py"],
 44 | )
 45 | 
 46 | py_test(
 47 |     name = "losses_test",
 48 |     size = "small",
 49 |     srcs = ["losses_test.py"],
 50 |     deps = [
 51 |         ":losses",
 52 |     ],
 53 | )
 54 | 
 55 | py_library(
 56 |     name = "ops",
 57 |     srcs = ["ops.py"],
 58 |     deps = [
 59 |         ":losses",
 60 |         ":scopes",
 61 |         ":variables",
 62 |     ],
 63 | )
 64 | 
 65 | py_test(
 66 |     name = "ops_test",
 67 |     size = "small",
 68 |     srcs = ["ops_test.py"],
 69 |     deps = [
 70 |         ":ops",
 71 |         ":variables",
 72 |     ],
 73 | )
 74 | 
 75 | py_library(
 76 |     name = "inception",
 77 |     srcs = ["inception_model.py"],
 78 |     deps = [
 79 |         ":ops",
 80 |         ":scopes",
 81 |     ],
 82 | )
 83 | 
 84 | py_test(
 85 |     name = "inception_test",
 86 |     size = "medium",
 87 |     srcs = ["inception_test.py"],
 88 |     deps = [
 89 |         ":inception",
 90 |     ],
 91 | )
 92 | 
 93 | py_library(
 94 |     name = "slim",
 95 |     srcs = ["slim.py"],
 96 |     deps = [
 97 |         ":inception",
 98 |         ":losses",
 99 |         ":ops",
100 |         ":scopes",
101 |         ":variables",
102 |     ],
103 | )
104 | 
105 | py_test(
106 |     name = "collections_test",
107 |     size = "small",
108 |     srcs = ["collections_test.py"],
109 |     deps = [
110 |         ":slim",
111 |     ],
112 | )
113 | 


--------------------------------------------------------------------------------
/slim/README.md:
--------------------------------------------------------------------------------
  1 | # TensorFlow-Slim
  2 | 
  3 | TF-Slim is a lightweight library for defining, training and evaluating models in
  4 | TensorFlow. It enables defining complex networks quickly and concisely while
  5 | keeping a model's architecture transparent and its hyperparameters explicit.
  6 | 
  7 | [TOC]
  8 | 
  9 | ## Teaser
 10 | 
 11 | As a demonstration of the simplicity of using TF-Slim, compare the simplicity of
 12 | the code necessary for defining the entire [VGG]
 13 | (http://www.robots.ox.ac.uk/~vgg/research/very_deep/) network using TF-Slim to
 14 | the lengthy and verbose nature of defining just the first three layers (out of
 15 | 16) using native tensorflow:
 16 | 
 17 | ```python{.good}
 18 | # VGG16 in TF-Slim.
 19 | def vgg16(inputs):
 20 |   with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
 21 |     net = slim.ops.repeat_op(2, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
 22 |     net = slim.ops.max_pool(net, [2, 2], scope='pool1')
 23 |     net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
 24 |     net = slim.ops.max_pool(net, [2, 2], scope='pool2')
 25 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
 26 |     net = slim.ops.max_pool(net, [2, 2], scope='pool3')
 27 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
 28 |     net = slim.ops.max_pool(net, [2, 2], scope='pool4')
 29 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
 30 |     net = slim.ops.max_pool(net, [2, 2], scope='pool5')
 31 |     net = slim.ops.flatten(net, scope='flatten5')
 32 |     net = slim.ops.fc(net, 4096, scope='fc6')
 33 |     net = slim.ops.dropout(net, 0.5, scope='dropout6')
 34 |     net = slim.ops.fc(net, 4096, scope='fc7')
 35 |     net = slim.ops.dropout(net, 0.5, scope='dropout7')
 36 |     net = slim.ops.fc(net, 1000, activation=None, scope='fc8')
 37 |   return net
 38 | ```
 39 | 
 40 | ```python{.bad}
 41 | # Layers 1-3 (out of 16) of VGG16 in native tensorflow.
 42 | def vgg16(inputs):
 43 |   with tf.name_scope('conv1_1') as scope:
 44 |     kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32, stddev=1e-1), name='weights')
 45 |     conv = tf.nn.conv2d(inputs, kernel, [1, 1, 1, 1], padding='SAME')
 46 |     biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases')
 47 |     bias = tf.nn.bias_add(conv, biases)
 48 |     conv1 = tf.nn.relu(bias, name=scope)
 49 |   with tf.name_scope('conv1_2') as scope:
 50 |     kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32, stddev=1e-1), name='weights')
 51 |     conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
 52 |     biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases')
 53 |     bias = tf.nn.bias_add(conv, biases)
 54 |     conv1 = tf.nn.relu(bias, name=scope)
 55 |   with tf.name_scope('pool1')
 56 |     pool1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID', name='pool1')
 57 | ```
 58 | 
 59 | ## Why TF-Slim?
 60 | 
 61 | TF-Slim offers several advantages over just the built-in tensorflow libraries:
 62 | 
 63 | *   Allows one to define models much more compactly by eliminating boilerplate
 64 |     code. This is accomplished through the use of [argument scoping](scopes.py)
 65 |     and numerous high level [operations](ops.py). These tools increase
 66 |     readability and maintainability, reduce the likelihood of an error from
 67 |     copy-and-pasting hyperparameter values and simplifies hyperparameter tuning.
 68 | *   Makes developing models simple by providing commonly used [loss functions]
 69 |     (losses.py)
 70 | *   Provides a concise [definition](inception.py) of [Inception v3]
 71 |     (http://arxiv.org/abs/1512.00567) network architecture ready to be used
 72 |     out-of-the-box or subsumed into new models.
 73 | 
 74 | Additionally TF-Slim was designed with several principles in mind:
 75 | 
 76 | *   The various modules of TF-Slim (scopes, variables, ops, losses) are
 77 |     independent. This flexibility allows users to pick and choose components of
 78 |     TF-Slim completely à la carte.
 79 | *   TF-Slim is written using a Functional Programming style. That means it's
 80 |     super-lightweight and can be used right alongside any of TensorFlow's native
 81 |     operations.
 82 | *   Makes re-using network architectures easy. This allows users to build new
 83 |     networks on top of existing ones as well as fine-tuning pre-trained models
 84 |     on new tasks.
 85 | 
 86 | ## What are the various components of TF-Slim?
 87 | 
 88 | TF-Slim is composed of several parts which were designed to exist independently.
 89 | These include:
 90 | 
 91 | *   [scopes.py](./scopes.py): provides a new scope named `arg_scope` that allows
 92 |     a user to define default arguments for specific operations within that
 93 |     scope.
 94 | *   [variables.py](./variables.py): provides convenience wrappers for variable
 95 |     creation and manipulation.
 96 | *   [ops.py](./ops.py): provides high level operations for building models using
 97 |     tensorflow.
 98 | *   [losses.py](./losses.py): contains commonly used loss functions.
 99 | 
100 | ## Defining Models
101 | 
102 | Models can be succinctly defined using TF-Slim by combining its variables,
103 | operations and scopes. Each of these elements are defined below.
104 | 
105 | ### Variables
106 | 
107 | Creating [`Variables`](https://www.tensorflow.org/how_tos/variables/index.html)
108 | in native tensorflow requires either a predefined value or an initialization
109 | mechanism (random, normally distributed). Furthermore, if a variable needs to be
110 | created on a specific device, such as a GPU, the specification must be [made
111 | explicit](https://www.tensorflow.org/how_tos/using_gpu/index.html). To alleviate
112 | the code required for variable creation, TF-Slim provides a set of thin wrapper
113 | functions in [variables.py](./variables.py) which allow callers to easily define
114 | variables.
115 | 
116 | For example, to create a `weight` variable, initialize it using a truncated
117 | normal distribution, regularize it with an `l2_loss` and place it on the `CPU`,
118 | one need only declare the following:
119 | 
120 | ```python
121 | weights = variables.variable('weights',
122 |                              shape=[10, 10, 3 , 3],
123 |                              initializer=tf.truncated_normal_initializer(stddev=0.1),
124 |                              regularizer=lambda t: losses.l2_loss(t, weight=0.05),
125 |                              device='/cpu:0')
126 | ```
127 | 
128 | In addition to the functionality provided by `tf.Variable`, `slim.variables`
129 | keeps track of the variables created by `slim.ops` to define a model, which
130 | allows one to distinguish variables that belong to the model versus other
131 | variables.
132 | 
133 | ```python
134 | # Get all the variables defined by the model.
135 | model_variables = slim.variables.get_variables()
136 | 
137 | # Get all the variables with the same given name, i.e. 'weights', 'biases'.
138 | weights = slim.variables.get_variables_by_name('weights')
139 | biases = slim.variables.get_variables_by_name('biases')
140 | 
141 | # Get all the variables in VARIABLES_TO_RESTORE collection.
142 | variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
143 | 
144 | 
145 | weights = variables.variable('weights',
146 |                              shape=[10, 10, 3 , 3],
147 |                              initializer=tf.truncated_normal_initializer(stddev=0.1),
148 |                              regularizer=lambda t: losses.l2_loss(t, weight=0.05),
149 |                              device='/cpu:0')
150 | ```
151 | 
152 | ### Operations (Layers)
153 | 
154 | While the set of TensorFlow operations is quite extensive, builders of neural
155 | networks typically think of models in terms of "layers". A layer, such as a
156 | Convolutional Layer, a Fully Connected Layer or a BatchNorm Layer are more
157 | abstract than a single TensorFlow operation and typically involve many such
158 | operations. For example, a Convolutional Layer in a neural network is built
159 | using several steps:
160 | 
161 | 1.  Creating the weight variables
162 | 2.  Creating the bias variables
163 | 3.  Convolving the weights with the input from the previous layer
164 | 4.  Adding the biases to the result of the convolution.
165 | 
166 | In python code this can be rather laborious:
167 | 
168 | ```python
169 | input = ...
170 | with tf.name_scope('conv1_1') as scope:
171 |   kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
172 |                                            stddev=1e-1), name='weights')
173 |   conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME')
174 |   biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
175 |                        trainable=True, name='biases')
176 |   bias = tf.nn.bias_add(conv, biases)
177 |   conv1 = tf.nn.relu(bias, name=scope)
178 | ```
179 | 
180 | To alleviate the need to duplicate this code repeatedly, TF-Slim provides a
181 | number of convenient operations defined at the (more abstract) level of neural
182 | network layers. For example, compare the code above to an invocation of the
183 | TF-Slim code:
184 | 
185 | ```python
186 | input = ...
187 | net = slim.ops.conv2d(input, [3, 3], 128, scope='conv1_1')
188 | ```
189 | 
190 | TF-Slim provides numerous operations used in building neural networks which
191 | roughly correspond to such layers. These include:
192 | 
193 | Layer                 | TF-Slim Op
194 | --------------------- | ------------------------
195 | Convolutional Layer   | [ops.conv2d](ops.py)
196 | Fully Connected Layer | [ops.fc](ops.py)
197 | BatchNorm layer       | [ops.batch_norm](ops.py)
198 | Max Pooling Layer     | [ops.max_pool](ops.py)
199 | Avg Pooling Layer     | [ops.avg_pool](ops.py)
200 | Dropout Layer         | [ops.dropout](ops.py)
201 | 
202 | [ops.py](./ops.py) also includes operations that are not really "layers" per se,
203 | but are often used to manipulate hidden unit representations during inference:
204 | 
205 | Operation | TF-Slim Op
206 | --------- | ---------------------
207 | Flatten   | [ops.flatten](ops.py)
208 | 
209 | TF-Slim also provides a meta-operation called `repeat_op` that allows one to
210 | repeatedly perform the same operation. Consider the following snippet from the
211 | [VGG](https://www.robots.ox.ac.uk/~vgg/research/very_deep/) network whose layers
212 | perform several convolutions in a row between pooling layers:
213 | 
214 | ```python
215 | net = ...
216 | net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_1')
217 | net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_2')
218 | net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_3')
219 | net = slim.ops.max_pool(net, [2, 2], scope='pool3')
220 | ```
221 | 
222 | This clear duplication of code can be removed via a standard loop:
223 | 
224 | ```python
225 | net = ...
226 | for i in range(3):
227 |   net = slim.ops.conv2d(net, 256, [3, 3], scope='conv3_' % (i+1))
228 | net = slim.ops.max_pool(net, [2, 2], scope='pool3')
229 | ```
230 | 
231 | While this does reduce the amount of duplication, it can be made even cleaner by
232 | using the `RepeatOp`:
233 | 
234 | ```python
235 | net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
236 | net = slim.ops.max_pool(net, [2, 2], scope='pool2')
237 | ```
238 | 
239 | Notice that the RepeatOp not only applies the same argument in-line, it also is
240 | smart enough to unroll the scopes such that the scopes assigned to each
241 | subsequent call of `ops.conv2d` is appended with an underscore and iteration
242 | number. More concretely, the scopes in the example above would be 'conv3_1',
243 | 'conv3_2' and 'conv3_3'.
244 | 
245 | ### Scopes
246 | 
247 | In addition to the types of scope mechanisms in TensorFlow ([name_scope]
248 | (https://www.tensorflow.org/api_docs/python/framework.html#name_scope),
249 | [op_scope](https://www.tensorflow.org/api_docs/python/framework.html#op_scope),
250 | [variable_scope]
251 | (https://www.tensorflow.org/api_docs/python/state_ops.html#variable_scope),
252 | [variable_op_scope]
253 | (https://www.tensorflow.org/api_docs/python/state_ops.html#variable_op_scope)),
254 | TF-Slim adds a new scoping mechanism called "argument scope" or [arg_scope]
255 | (scopes.py). This new scope allows a user to specify one or more operations and
256 | a set of arguments which will be passed to each of the operations defined in the
257 | `arg_scope`. This functionality is best illustrated by example. Consider the
258 | following code snippet:
259 | 
260 | ```python
261 | net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding='SAME', stddev=0.01, weight_decay=0.0005, scope='conv1')
262 | net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', stddev=0.01, weight_decay=0.0005, scope='conv2')
263 | net = slim.ops.conv2d(net, 256, [11, 11], padding='SAME', stddev=0.01, weight_decay=0.0005, scope='conv3')
264 | ```
265 | 
266 | It should be clear that these three Convolution layers share many of the same
267 | hyperparameters. Two have the same padding, all three have the same weight_decay
268 | and standard deviation of its weights. Not only do the duplicated values make
269 | the code more difficult to read, it also adds the addition burder to the writer
270 | of needing to doublecheck that all of the values are identical in each step. One
271 | solution would be to specify default values using variables:
272 | 
273 | ```python
274 | padding='SAME'
275 | stddev=0.01
276 | weight_decay=0.0005
277 | net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding=padding, stddev=stddev, weight_decay=weight_decay, scope='conv1')
278 | net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', stddev=stddev, weight_decay=weight_decay, scope='conv2')
279 | net = slim.ops.conv2d(net, 256, [11, 11], padding=padding, stddev=stddev, weight_decay=weight_decay, scope='conv3')
280 | 
281 | ```
282 | 
283 | This solution ensures that all three convolutions share the exact same variable
284 | values but doesn't reduce the code clutter. By using an `arg_scope`, we can both
285 | ensure that each layer uses the same values and simplify the code:
286 | 
287 | ```python
288 |   with slim.arg_scope([slim.ops.conv2d], padding='SAME', stddev=0.01, weight_decay=0.0005):
289 |     net = slim.ops.conv2d(inputs, 64, [11, 11], scope='conv1')
290 |     net = slim.ops.conv2d(net, 128, [11, 11], padding='VALID', scope='conv2')
291 |     net = slim.ops.conv2d(net, 256, [11, 11], scope='conv3')
292 | ```
293 | 
294 | As the example illustrates, the use of arg_scope makes the code cleaner, simpler
295 | and easier to maintain. Notice that while argument values are specifed in the
296 | arg_scope, they can be overwritten locally. In particular, while the padding
297 | argument has been set to 'SAME', the second convolution overrides it with the
298 | value of 'VALID'.
299 | 
300 | One can also nest `arg_scope`s and use multiple operations in the same scope.
301 | For example:
302 | 
303 | ```python
304 | with arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
305 |   with arg_scope([slim.ops.conv2d], padding='SAME'), slim.arg_scope([slim.ops.fc], bias=1.0):
306 |     net = slim.ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1')
307 |     net = slim.ops.conv2d(net, 256, [5, 5], stddev=0.03, scope='conv2')
308 |     net = slim.ops.flatten(net)
309 |     net = slim.ops.fc(net, 1000, activation=None, scope='fc')
310 | ```
311 | 
312 | In this example, the first `arg_scope` applies the same `stddev` and
313 | `weight_decay` arguments to the `conv2d` and `fc` ops in its scope. In the
314 | second `arg_scope`, additional default arguments to `conv2d` only are specified.
315 | 
316 | In addition to `arg_scope`, TF-Slim provides several decorators that wrap the
317 | use of tensorflow arg scopes. These include `@AddArgScope`, `@AddNameScope`,
318 | `@AddVariableScope`, `@AddOpScope` and `@AddVariableOpScope`. To illustrate
319 | their use, consider the following example.
320 | 
321 | ```python
322 | def MyNewOp(inputs):
323 |   varA = ...
324 |   varB = ...
325 |   outputs = tf.mul(varA, inputs) + varB
326 |   return outputs
327 | 
328 | ```
329 | 
330 | In this example, the user has created a new op which creates two variables. To
331 | ensure that these variables exist within a certain variable scope (to avoid
332 | collisions with variables with the same name), in standard TF, the op must be
333 | called within a variable scope:
334 | 
335 | ```python
336 | inputs = ...
337 | with tf.variable_scope('layer1'):
338 |   outputs = MyNewOp(inputs)
339 | ```
340 | 
341 | As an alternative, one can use TF-Slim's decorators to decorate the function and
342 | simplify the call:
343 | 
344 | ```python
345 | @AddVariableScope
346 | def MyNewOp(inputs):
347 |   ...
348 |   return outputs
349 | 
350 | 
351 | inputs = ...
352 | outputs = MyNewOp('layer1')
353 | ```
354 | 
355 | The `@AddVariableScope` decorater simply applies the `tf.variable_scope` scoping
356 | to the called function taking "layer1" as its argument. This allows the code to
357 | be written more concisely.
358 | 
359 | ### Losses
360 | 
361 | The loss function defines a quantity that we want to minimize. For
362 | classification problems, this is typically the cross entropy between the true
363 | (one-hot) distribution and the predicted probability distribution across
364 | classes. For regression problems, this is often the sum-of-squares differences
365 | between the predicted and true values.
366 | 
367 | Certain models, such as multi-task learning models, require the use of multiple
368 | loss functions simultaneously. In other words, the loss function ultimatey being
369 | minimized is the sum of various other loss functions. For example, consider a
370 | model that predicts both the type of scene in an image as well as the depth from
371 | the camera of each pixel. This model's loss function would be the sum of the
372 | classification loss and depth prediction loss.
373 | 
374 | TF-Slim provides an easy-to-use mechanism for defining and keeping track of loss
375 | functions via the [losses.py](./losses.py) module. Consider the simple case
376 | where we want to train the VGG network:
377 | 
378 | ```python
379 | # Load the images and labels.
380 | images, labels = ...
381 | 
382 | # Create the model.
383 | predictions =  ...
384 | 
385 | # Define the loss functions and get the total loss.
386 | loss = losses.ClassificationLoss(predictions, labels)
387 | ```
388 | 
389 | In this example, we start by creating the model (using TF-Slim's VGG
390 | implementation), and add the standard classification loss. Now, lets turn to the
391 | case where we have a multi-task model that produces multiple outputs:
392 | 
393 | ```python
394 | # Load the images and labels.
395 | images, scene_labels, depth_labels = ...
396 | 
397 | # Create the model.
398 | scene_predictions, depth_predictions = CreateMultiTaskModel(images)
399 | 
400 | # Define the loss functions and get the total loss.
401 | classification_loss = slim.losses.ClassificationLoss(scene_predictions, scene_labels)
402 | sum_of_squares_loss = slim.losses.SumOfSquaresLoss(depth_predictions, depth_labels)
403 | 
404 | # The following two lines have the same effect:
405 | total_loss1 = classification_loss + sum_of_squares_loss
406 | total_loss2 = tf.get_collection(slim.losses.LOSSES_COLLECTION)
407 | ```
408 | 
409 | In this example, we have two losses which we add by calling
410 | `losses.ClassificationLoss` and `losses.SumOfSquaresLoss`. We can obtain the
411 | total loss by adding them together (`total_loss1`) or by calling
412 | `losses.GetTotalLoss()`. How did this work? When you create a loss function via
413 | TF-Slim, TF-Slim adds the loss to a special TensorFlow collection of loss
414 | functions. This enables you to either manage the total loss manually, or allow
415 | TF-Slim to manage them for you.
416 | 
417 | What if you want to let TF-Slim manage the losses for you but have a custom loss
418 | function? [losses.py](./losses.py) also has a function that adds this loss to
419 | TF-Slims collection. For example:
420 | 
421 | ```python
422 | # Load the images and labels.
423 | images, scene_labels, depth_labels, pose_labels = ...
424 | 
425 | # Create the model.
426 | scene_predictions, depth_predictions, pose_predictions = CreateMultiTaskModel(images)
427 | 
428 | # Define the loss functions and get the total loss.
429 | classification_loss = slim.losses.ClassificationLoss(scene_predictions, scene_labels)
430 | sum_of_squares_loss = slim.losses.SumOfSquaresLoss(depth_predictions, depth_labels)
431 | pose_loss = MyCustomLossFunction(pose_predictions, pose_labels)
432 | tf.add_to_collection(slim.losses.LOSSES_COLLECTION, pose_loss) # Letting TF-Slim know about the additional loss.
433 | 
434 | # The following two lines have the same effect:
435 | total_loss1 = classification_loss + sum_of_squares_loss + pose_loss
436 | total_loss2 = losses.GetTotalLoss()
437 | ```
438 | 
439 | In this example, we can again either produce the total loss function manually or
440 | let TF-Slim know about the additional loss and let TF-Slim handle the losses.
441 | 
442 | ## Putting the Pieces Together
443 | 
444 | By combining TF-Slim Variables, Operations and scopes, we can write a normally
445 | very complex network with very few lines of code. For example, the entire [VGG]
446 | (https://www.robots.ox.ac.uk/~vgg/research/very_deep/) architecture can be
447 | defined with just the following snippet:
448 | 
449 | ```python
450 | with arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
451 |   net = slim.ops.repeat_op(1, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
452 |   net = slim.ops.max_pool(net, [2, 2], scope='pool1')
453 |   net = slim.ops.repeat_op(1, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
454 |   net = slim.ops.max_pool(net, [2, 2], scope='pool2')
455 |   net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
456 |   net = slim.ops.max_pool(net, [2, 2], scope='pool3')
457 |   net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
458 |   net = slim.ops.max_pool(net, [2, 2], scope='pool4')
459 |   net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
460 |   net = slim.ops.max_pool(net, [2, 2], scope='pool5')
461 |   net = slim.ops.flatten(net, scope='flatten5')
462 |   net = slim.ops.fc(net, 4096, scope='fc6')
463 |   net = slim.ops.dropout(net, 0.5, scope='dropout6')
464 |   net = slim.ops.fc(net, 4096, scope='fc7')
465 |   net = slim.ops.dropout(net, 0.5, scope='dropout7')
466 |   net = slim.ops.fc(net, 1000, activation=None, scope='fc8')
467 | return net
468 | ```
469 | 
470 | ## Re-using previously defined network architectures and pre-trained models.
471 | 
472 | ### Brief Recap on Restoring Variables from a Checkpoint
473 | 
474 | After a model has been trained, it can be restored using `tf.train.Saver()`
475 | which restores `Variables` from a given checkpoint. For many cases,
476 | `tf.train.Saver()` provides a simple mechanism to restore all or just a few
477 | variables.
478 | 
479 | ```python
480 | # Create some variables.
481 | v1 = tf.Variable(..., name="v1")
482 | v2 = tf.Variable(..., name="v2")
483 | ...
484 | # Add ops to restore all the variables.
485 | restorer = tf.train.Saver()
486 | 
487 | # Add ops to restore some variables.
488 | restorer = tf.train.Saver([v1, v2])
489 | 
490 | # Later, launch the model, use the saver to restore variables from disk, and
491 | # do some work with the model.
492 | with tf.Session() as sess:
493 |   # Restore variables from disk.
494 |   restorer.restore(sess, "/tmp/model.ckpt")
495 |   print("Model restored.")
496 |   # Do some work with the model
497 |   ...
498 | ```
499 | 
500 | See [Restoring Variables]
501 | (https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html#restoring-variables)
502 | and [Choosing which Variables to Save and Restore]
503 | (https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html#choosing-which-variables-to-save-and-restore)
504 | sections of the [Variables]
505 | (https://www.tensorflow.org/versions/r0.7/how_tos/variables/index.html) page for
506 | more details.
507 | 
508 | ### Using slim.variables to Track which Variables need to be Restored
509 | 
510 | It is often desirable to fine-tune a pre-trained model on an entirely new
511 | dataset or even a new task. In these situations, one must specify which layers
512 | of the model should be reused (and consequently loaded from a checkpoint) and
513 | which layers are new. Indicating which variables or layers should be restored is
514 | a process that quickly becomes cumbersome when done manually.
515 | 
516 | To help keep track of which variables to restore, `slim.variables` provides a
517 | `restore` argument when creating each Variable. By default, all variables are
518 | marked as `restore=True`, which results in all variables defined by the model
519 | being restored.
520 | 
521 | ```python
522 | # Create some variables.
523 | v1 = slim.variables.variable(name="v1", ..., restore=False)
524 | v2 = slim.variables.variable(name="v2", ...) # By default restore=True
525 | ...
526 | # Get list of variables to restore (which contains only 'v2')
527 | variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
528 | restorer = tf.train.Saver(variables_to_restore)
529 | with tf.Session() as sess:
530 |   # Restore variables from disk.
531 |   restorer.restore(sess, "/tmp/model.ckpt")
532 |   print("Model restored.")
533 |   # Do some work with the model
534 |   ...
535 | ```
536 | 
537 | Additionally, every layer in `slim.ops` that creates slim.variables (such as
538 | `slim.ops.conv2d`, `slim.ops.fc`, `slim.ops.batch_norm`) also has a `restore`
539 | argument which controls whether the variables created by that layer should be
540 | restored or not.
541 | 
542 | ```python
543 | # Create a small network.
544 | net = slim.ops.conv2d(images, 32, [7, 7], stride=2, scope='conv1')
545 | net = slim.ops.conv2d(net, 64, [3, 3], scope='conv2')
546 | net = slim.ops.conv2d(net, 128, [3, 3], scope='conv3')
547 | net = slim.ops.max_pool(net, [3, 3], stride=2, scope='pool3')
548 | net = slim.ops.flatten(net)
549 | net = slim.ops.fc(net, 10, scope='logits', restore=False)
550 | ...
551 | 
552 | # VARIABLES_TO_RESTORE would contain the 'weights' and 'bias' defined by 'conv1'
553 | # 'conv2' and 'conv3' but not the ones defined by 'logits'
554 | variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
555 | 
556 | # Create a restorer that would restore only the needed variables.
557 | restorer = tf.train.Saver(variables_to_restore)
558 | 
559 | # Create a saver that would save all the variables (including 'logits').
560 | saver = tf.train.Saver()
561 | with tf.Session() as sess:
562 |   # Restore variables from disk.
563 |   restorer.restore(sess, "/tmp/model.ckpt")
564 |   print("Model restored.")
565 | 
566 |   # Do some work with the model
567 |   ...
568 |   saver.save(sess, "/tmp/new_model.ckpt")
569 | ```
570 | 
571 | Note: When restoring variables from a checkpoint, the `Saver` locates the
572 | variable names in a checkpoint file and maps them to variables in the current
573 | graph. Above, we created a saver by passing to it a list of variables. In this
574 | case, the names of the variables to locate in the checkpoint file were
575 | implicitly obtained from each provided variable's `var.op.name`.
576 | 
577 | This works well when the variable names in the checkpoint file match those in
578 | the graph. However, sometimes, we want to restore a model from a checkpoint
579 | whose variables have different names those in the current graph. In this case,
580 | we must provide the `Saver` a dictionary that maps from each checkpoint variable
581 | name to each graph variable. Consider the following example where the checkpoint
582 | variables names are obtained via a simple function:
583 | 
584 | ```python
585 | # Assuming than 'conv1/weights' should be restored from 'vgg16/conv1/weights'
586 | def name_in_checkpoint(var):
587 |   return 'vgg16/' + var.op.name
588 | 
589 | # Assuming than 'conv1/weights' and 'conv1/bias' should be restored from 'conv1/params1' and 'conv1/params2'
590 | def name_in_checkpoint(var):
591 |   if "weights" in var.op.name:
592 |     return var.op.name.replace("weights", "params1")
593 |   if "bias" in var.op.name:
594 |     return var.op.name.replace("bias", "params2")
595 | 
596 | variables_to_restore = tf.get_collection(slim.variables.VARIABLES_TO_RESTORE)
597 | variables_to_restore = {name_in_checkpoint(var):var for var in variables_to_restore}
598 | restorer = tf.train.Saver(variables_to_restore)
599 | with tf.Session() as sess:
600 |   # Restore variables from disk.
601 |   restorer.restore(sess, "/tmp/model.ckpt")
602 | ```
603 | 
604 | ### Reusing the VGG16 network defined in TF-Slim on a different task, i.e. PASCAL-VOC.
605 | 
606 | Assuming one have already a pre-trained VGG16 model, one just need to replace
607 | the last layer `fc8` with a new layer `fc8_pascal` and use `restore=False`.
608 | 
609 | ```python
610 | def vgg16_pascal(inputs):
611 |   with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], stddev=0.01, weight_decay=0.0005):
612 |     net = slim.ops.repeat_op(2, inputs, slim.ops.conv2d, 64, [3, 3], scope='conv1')
613 |     net = slim.ops.max_pool(net, [2, 2], scope='pool1')
614 |     net = slim.ops.repeat_op(2, net, slim.ops.conv2d, 128, [3, 3], scope='conv2')
615 |     net = slim.ops.max_pool(net, [2, 2], scope='pool2')
616 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 256, [3, 3], scope='conv3')
617 |     net = slim.ops.max_pool(net, [2, 2], scope='pool3')
618 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv4')
619 |     net = slim.ops.max_pool(net, [2, 2], scope='pool4')
620 |     net = slim.ops.repeat_op(3, net, slim.ops.conv2d, 512, [3, 3], scope='conv5')
621 |     net = slim.ops.max_pool(net, [2, 2], scope='pool5')
622 |     net = slim.ops.flatten(net, scope='flatten5')
623 |     net = slim.ops.fc(net, 4096, scope='fc6')
624 |     net = slim.ops.dropout(net, 0.5, scope='dropout6')
625 |     net = slim.ops.fc(net, 4096, scope='fc7')
626 |     net = slim.ops.dropout(net, 0.5, scope='dropout7')
627 |     # To reuse vgg16 on PASCAL-VOC, just change the last layer.
628 |     net = slim.ops.fc(net, 21, activation=None, scope='fc8_pascal', restore=False)
629 |   return net
630 | ```
631 | 
632 | ## Authors
633 | 
634 | Sergio Guadarrama and Nathan Silberman
635 | 


--------------------------------------------------------------------------------
/slim/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MasazI/InceptionV3_TensorFlow/90cce1d735589137e234fc1a38f410adfb145fbd/slim/__init__.py


--------------------------------------------------------------------------------
/slim/collections_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests for inception."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | import tensorflow as tf
 21 | 
 22 | from inception.slim import slim
 23 | 
 24 | 
 25 | def get_variables(scope=None):
 26 |   return slim.variables.get_variables(scope)
 27 | 
 28 | 
 29 | def get_variables_by_name(name):
 30 |   return slim.variables.get_variables_by_name(name)
 31 | 
 32 | 
 33 | class CollectionsTest(tf.test.TestCase):
 34 | 
 35 |   def testVariables(self):
 36 |     batch_size = 5
 37 |     height, width = 299, 299
 38 |     with self.test_session():
 39 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 40 |       with slim.arg_scope([slim.ops.conv2d],
 41 |                           batch_norm_params={'decay': 0.9997}):
 42 |         slim.inception.inception_v3(inputs)
 43 |       self.assertEqual(len(get_variables()), 388)
 44 |       self.assertEqual(len(get_variables_by_name('weights')), 98)
 45 |       self.assertEqual(len(get_variables_by_name('biases')), 2)
 46 |       self.assertEqual(len(get_variables_by_name('beta')), 96)
 47 |       self.assertEqual(len(get_variables_by_name('gamma')), 0)
 48 |       self.assertEqual(len(get_variables_by_name('moving_mean')), 96)
 49 |       self.assertEqual(len(get_variables_by_name('moving_variance')), 96)
 50 | 
 51 |   def testVariablesWithoutBatchNorm(self):
 52 |     batch_size = 5
 53 |     height, width = 299, 299
 54 |     with self.test_session():
 55 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 56 |       with slim.arg_scope([slim.ops.conv2d],
 57 |                           batch_norm_params=None):
 58 |         slim.inception.inception_v3(inputs)
 59 |       self.assertEqual(len(get_variables()), 196)
 60 |       self.assertEqual(len(get_variables_by_name('weights')), 98)
 61 |       self.assertEqual(len(get_variables_by_name('biases')), 98)
 62 |       self.assertEqual(len(get_variables_by_name('beta')), 0)
 63 |       self.assertEqual(len(get_variables_by_name('gamma')), 0)
 64 |       self.assertEqual(len(get_variables_by_name('moving_mean')), 0)
 65 |       self.assertEqual(len(get_variables_by_name('moving_variance')), 0)
 66 | 
 67 |   def testVariablesByLayer(self):
 68 |     batch_size = 5
 69 |     height, width = 299, 299
 70 |     with self.test_session():
 71 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 72 |       with slim.arg_scope([slim.ops.conv2d],
 73 |                           batch_norm_params={'decay': 0.9997}):
 74 |         slim.inception.inception_v3(inputs)
 75 |       self.assertEqual(len(get_variables()), 388)
 76 |       self.assertEqual(len(get_variables('conv0')), 4)
 77 |       self.assertEqual(len(get_variables('conv1')), 4)
 78 |       self.assertEqual(len(get_variables('conv2')), 4)
 79 |       self.assertEqual(len(get_variables('conv3')), 4)
 80 |       self.assertEqual(len(get_variables('conv4')), 4)
 81 |       self.assertEqual(len(get_variables('mixed_35x35x256a')), 28)
 82 |       self.assertEqual(len(get_variables('mixed_35x35x288a')), 28)
 83 |       self.assertEqual(len(get_variables('mixed_35x35x288b')), 28)
 84 |       self.assertEqual(len(get_variables('mixed_17x17x768a')), 16)
 85 |       self.assertEqual(len(get_variables('mixed_17x17x768b')), 40)
 86 |       self.assertEqual(len(get_variables('mixed_17x17x768c')), 40)
 87 |       self.assertEqual(len(get_variables('mixed_17x17x768d')), 40)
 88 |       self.assertEqual(len(get_variables('mixed_17x17x768e')), 40)
 89 |       self.assertEqual(len(get_variables('mixed_8x8x2048a')), 36)
 90 |       self.assertEqual(len(get_variables('mixed_8x8x2048b')), 36)
 91 |       self.assertEqual(len(get_variables('logits')), 2)
 92 |       self.assertEqual(len(get_variables('aux_logits')), 10)
 93 | 
 94 |   def testVariablesToRestore(self):
 95 |     batch_size = 5
 96 |     height, width = 299, 299
 97 |     with self.test_session():
 98 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 99 |       with slim.arg_scope([slim.ops.conv2d],
100 |                           batch_norm_params={'decay': 0.9997}):
101 |         slim.inception.inception_v3(inputs)
102 |       variables_to_restore = tf.get_collection(
103 |           slim.variables.VARIABLES_TO_RESTORE)
104 |       self.assertEqual(len(variables_to_restore), 388)
105 |       self.assertListEqual(variables_to_restore, get_variables())
106 | 
107 |   def testVariablesToRestoreWithoutLogits(self):
108 |     batch_size = 5
109 |     height, width = 299, 299
110 |     with self.test_session():
111 |       inputs = tf.random_uniform((batch_size, height, width, 3))
112 |       with slim.arg_scope([slim.ops.conv2d],
113 |                           batch_norm_params={'decay': 0.9997}):
114 |         slim.inception.inception_v3(inputs, restore_logits=False)
115 |       variables_to_restore = tf.get_collection(
116 |           slim.variables.VARIABLES_TO_RESTORE)
117 |       self.assertEqual(len(variables_to_restore), 384)
118 | 
119 |   def testRegularizationLosses(self):
120 |     batch_size = 5
121 |     height, width = 299, 299
122 |     with self.test_session():
123 |       inputs = tf.random_uniform((batch_size, height, width, 3))
124 |       with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
125 |         slim.inception.inception_v3(inputs)
126 |       losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
127 |       self.assertEqual(len(losses), len(get_variables_by_name('weights')))
128 | 
129 |   def testTotalLossWithoutRegularization(self):
130 |     batch_size = 5
131 |     height, width = 299, 299
132 |     num_classes = 1001
133 |     with self.test_session():
134 |       inputs = tf.random_uniform((batch_size, height, width, 3))
135 |       dense_labels = tf.random_uniform((batch_size, num_classes))
136 |       with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0):
137 |         logits, end_points = slim.inception.inception_v3(
138 |             inputs,
139 |             num_classes=num_classes)
140 |         # Cross entropy loss for the main softmax prediction.
141 |         slim.losses.cross_entropy_loss(logits,
142 |                                        dense_labels,
143 |                                        label_smoothing=0.1,
144 |                                        weight=1.0)
145 |         # Cross entropy loss for the auxiliary softmax head.
146 |         slim.losses.cross_entropy_loss(end_points['aux_logits'],
147 |                                        dense_labels,
148 |                                        label_smoothing=0.1,
149 |                                        weight=0.4,
150 |                                        scope='aux_loss')
151 |       losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
152 |       self.assertEqual(len(losses), 2)
153 | 
154 |   def testTotalLossWithRegularization(self):
155 |     batch_size = 5
156 |     height, width = 299, 299
157 |     num_classes = 1000
158 |     with self.test_session():
159 |       inputs = tf.random_uniform((batch_size, height, width, 3))
160 |       dense_labels = tf.random_uniform((batch_size, num_classes))
161 |       with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
162 |         logits, end_points = slim.inception.inception_v3(inputs, num_classes)
163 |         # Cross entropy loss for the main softmax prediction.
164 |         slim.losses.cross_entropy_loss(logits,
165 |                                        dense_labels,
166 |                                        label_smoothing=0.1,
167 |                                        weight=1.0)
168 |         # Cross entropy loss for the auxiliary softmax head.
169 |         slim.losses.cross_entropy_loss(end_points['aux_logits'],
170 |                                        dense_labels,
171 |                                        label_smoothing=0.1,
172 |                                        weight=0.4,
173 |                                        scope='aux_loss')
174 |       losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
175 |       self.assertEqual(len(losses), 2)
176 |       reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
177 |       self.assertEqual(len(reg_losses), 98)
178 | 
179 | 
180 | if __name__ == '__main__':
181 |   tf.test.main()
182 | 


--------------------------------------------------------------------------------
/slim/inception_model.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Inception-v3 expressed in TensorFlow-Slim.
 16 | 
 17 |   Usage:
 18 | 
 19 |   # Parameters for BatchNorm.
 20 |   batch_norm_params = {
 21 |       # Decay for the batch_norm moving averages.
 22 |       'decay': BATCHNORM_MOVING_AVERAGE_DECAY,
 23 |       # epsilon to prevent 0s in variance.
 24 |       'epsilon': 0.001,
 25 |   }
 26 |   # Set weight_decay for weights in Conv and FC layers.
 27 |   with slim.arg_scope([slim.ops.conv2d, slim.ops.fc], weight_decay=0.00004):
 28 |     with slim.arg_scope([slim.ops.conv2d],
 29 |                         stddev=0.1,
 30 |                         activation=tf.nn.relu,
 31 |                         batch_norm_params=batch_norm_params):
 32 |       # Force all Variables to reside on the CPU.
 33 |       with slim.arg_scope([slim.variables.variable], device='/cpu:0'):
 34 |         logits, endpoints = slim.inception.inception_v3(
 35 |             images,
 36 |             dropout_keep_prob=0.8,
 37 |             num_classes=num_classes,
 38 |             is_training=for_training,
 39 |             restore_logits=restore_logits,
 40 |             scope=scope)
 41 | """
 42 | from __future__ import division
 43 | from __future__ import print_function
 44 | 
 45 | import tensorflow as tf
 46 | 
 47 | import ops
 48 | import scopes
 49 | 
 50 | 
 51 | def inception_v3(inputs,
 52 |                  dropout_keep_prob=0.8,
 53 |                  num_classes=1000,
 54 |                  is_training=True,
 55 |                  restore_logits=True,
 56 |                  scope='',
 57 |                  reuse=False):
 58 |   """Latest Inception from http://arxiv.org/abs/1512.00567.
 59 | 
 60 |     "Rethinking the Inception Architecture for Computer Vision"
 61 | 
 62 |     Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens,
 63 |     Zbigniew Wojna
 64 | 
 65 |   Args:
 66 |     inputs: a tensor of size [batch_size, height, width, channels].
 67 |     dropout_keep_prob: dropout keep_prob.
 68 |     num_classes: number of predicted classes.
 69 |     is_training: whether is training or not.
 70 |     restore_logits: whether or not the logits layers should be restored.
 71 |       Useful for fine-tuning a model with different num_classes.
 72 |     scope: Optional scope for name_scope.
 73 | 
 74 |   Returns:
 75 |     a list containing 'logits', 'aux_logits' Tensors.
 76 |   """
 77 |   # end_points will collect relevant activations for external use, for example
 78 |   # summaries or losses.
 79 |   end_points = {}
 80 |   with tf.name_scope(scope, 'inception_v3', [inputs]):
 81 |     with scopes.arg_scope([ops.conv2d, ops.fc, ops.batch_norm, ops.dropout],
 82 |                           is_training=is_training):
 83 |       with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
 84 |                             stride=1, padding='VALID') as scope:
 85 |         # 299 x 299 x 3
 86 |         end_points['conv0'] = ops.conv2d(inputs, 32, [3, 3], stride=2,
 87 |                                          scope='conv0', reuse=reuse)
 88 |         # 149 x 149 x 32
 89 |         end_points['conv1'] = ops.conv2d(end_points['conv0'], 32, [3, 3],
 90 |                                          scope='conv1', reuse=reuse)
 91 |         # 147 x 147 x 32
 92 |         end_points['conv2'] = ops.conv2d(end_points['conv1'], 64, [3, 3],
 93 |                                          padding='SAME', scope='conv2', reuse=reuse)
 94 |         # 147 x 147 x 64
 95 |         end_points['pool1'] = ops.max_pool(end_points['conv2'], [3, 3],
 96 |                                            stride=2, scope='pool1')
 97 |         # 73 x 73 x 64
 98 |         end_points['conv3'] = ops.conv2d(end_points['pool1'], 80, [1, 1],
 99 |                                          scope='conv3', reuse=reuse)
100 |         # 73 x 73 x 80.
101 |         end_points['conv4'] = ops.conv2d(end_points['conv3'], 192, [3, 3],
102 |                                          scope='conv4', reuse=reuse)
103 |         # 71 x 71 x 192.
104 |         end_points['pool2'] = ops.max_pool(end_points['conv4'], [3, 3],
105 |                                            stride=2, scope='pool2')
106 |         # 35 x 35 x 192.
107 |         net = end_points['pool2']
108 |       # Inception blocks
109 |       with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
110 |                             stride=1, padding='SAME'):
111 |         # mixed: 35 x 35 x 256.
112 |         with tf.variable_scope('mixed_35x35x256a'):
113 |           with tf.variable_scope('branch1x1') as scope:
114 |             if reuse:
115 |                 scope.reuse_variables()
116 |             branch1x1 = ops.conv2d(net, 64, [1, 1])
117 |           with tf.variable_scope('branch5x5') as scope:
118 |             if reuse:
119 |                 scope.reuse_variables()
120 |             branch5x5 = ops.conv2d(net, 48, [1, 1])
121 |             branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
122 |           with tf.variable_scope('branch3x3dbl') as scope:
123 |             if reuse:
124 |                 scope.reuse_variables()
125 |             branch3x3dbl = ops.conv2d(net, 64, [1, 1])
126 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
127 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
128 |           with tf.variable_scope('branch_pool') as scope:
129 |             if reuse:
130 |                 scope.reuse_variables()
131 |             branch_pool = ops.avg_pool(net, [3, 3])
132 |             branch_pool = ops.conv2d(branch_pool, 32, [1, 1])
133 |           net = tf.concat(3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
134 |           end_points['mixed_35x35x256a'] = net
135 |         # mixed_1: 35 x 35 x 288.
136 |         with tf.variable_scope('mixed_35x35x288a'):
137 |           with tf.variable_scope('branch1x1') as scope:
138 |             if reuse:
139 |               scope.reuse_variables()
140 |             branch1x1 = ops.conv2d(net, 64, [1, 1])
141 |           with tf.variable_scope('branch5x5') as scope:
142 |             if reuse:
143 |               scope.reuse_variables()
144 |             branch5x5 = ops.conv2d(net, 48, [1, 1])
145 |             branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
146 |           with tf.variable_scope('branch3x3dbl') as scope:
147 |             if reuse:
148 |               scope.reuse_variables()
149 |             branch3x3dbl = ops.conv2d(net, 64, [1, 1])
150 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
151 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
152 |           with tf.variable_scope('branch_pool') as scope:
153 |             if reuse:
154 |               scope.reuse_variables()
155 |             branch_pool = ops.avg_pool(net, [3, 3])
156 |             branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
157 |           net = tf.concat(3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
158 |           end_points['mixed_35x35x288a'] = net
159 |         # mixed_2: 35 x 35 x 288.
160 |         with tf.variable_scope('mixed_35x35x288b'):
161 |           with tf.variable_scope('branch1x1') as scope:
162 |             if reuse:
163 |               scope.reuse_variables()
164 |             branch1x1 = ops.conv2d(net, 64, [1, 1])
165 |           with tf.variable_scope('branch5x5') as scope:
166 |             if reuse:
167 |               scope.reuse_variables()
168 |             branch5x5 = ops.conv2d(net, 48, [1, 1])
169 |             branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
170 |           with tf.variable_scope('branch3x3dbl') as scope:
171 |             if reuse:
172 |               scope.reuse_variables()
173 |             branch3x3dbl = ops.conv2d(net, 64, [1, 1])
174 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
175 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
176 |           with tf.variable_scope('branch_pool') as scope:
177 |             if reuse:
178 |               scope.reuse_variables()
179 |             branch_pool = ops.avg_pool(net, [3, 3])
180 |             branch_pool = ops.conv2d(branch_pool, 64, [1, 1])
181 |           net = tf.concat(3, [branch1x1, branch5x5, branch3x3dbl, branch_pool])
182 |           end_points['mixed_35x35x288b'] = net
183 |         # mixed_3: 17 x 17 x 768.
184 |         with tf.variable_scope('mixed_17x17x768a'):
185 |           with tf.variable_scope('branch3x3') as scope:
186 |             if reuse:
187 |               scope.reuse_variables()
188 |             branch3x3 = ops.conv2d(net, 384, [3, 3], stride=2, padding='VALID')
189 |           with tf.variable_scope('branch3x3dbl') as scope:
190 |             if reuse:
191 |               scope.reuse_variables()
192 |             branch3x3dbl = ops.conv2d(net, 64, [1, 1])
193 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3])
194 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 96, [3, 3],
195 |                                       stride=2, padding='VALID')
196 |           with tf.variable_scope('branch_pool') as scope:
197 |             if reuse:
198 |               scope.reuse_variables()
199 |             branch_pool = ops.max_pool(net, [3, 3], stride=2, padding='VALID')
200 |           net = tf.concat(3, [branch3x3, branch3x3dbl, branch_pool])
201 |           end_points['mixed_17x17x768a'] = net
202 |         # mixed4: 17 x 17 x 768.
203 |         with tf.variable_scope('mixed_17x17x768b') as scope:
204 |           if reuse:
205 |             scope.reuse_variables()
206 |           with tf.variable_scope('branch1x1'):
207 |             branch1x1 = ops.conv2d(net, 192, [1, 1])
208 |           with tf.variable_scope('branch7x7'):
209 |             branch7x7 = ops.conv2d(net, 128, [1, 1])
210 |             branch7x7 = ops.conv2d(branch7x7, 128, [1, 7])
211 |             branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
212 |           with tf.variable_scope('branch7x7dbl'):
213 |             branch7x7dbl = ops.conv2d(net, 128, [1, 1])
214 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
215 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [1, 7])
216 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 128, [7, 1])
217 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
218 |           with tf.variable_scope('branch_pool'):
219 |             branch_pool = ops.avg_pool(net, [3, 3])
220 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
221 |           net = tf.concat(3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
222 |           end_points['mixed_17x17x768b'] = net
223 |         # mixed_5: 17 x 17 x 768.
224 |         with tf.variable_scope('mixed_17x17x768c') as scope:
225 |           if reuse:
226 |             scope.reuse_variables()
227 |           with tf.variable_scope('branch1x1'):
228 |             branch1x1 = ops.conv2d(net, 192, [1, 1])
229 |           with tf.variable_scope('branch7x7'):
230 |             branch7x7 = ops.conv2d(net, 160, [1, 1])
231 |             branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
232 |             branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
233 |           with tf.variable_scope('branch7x7dbl'):
234 |             branch7x7dbl = ops.conv2d(net, 160, [1, 1])
235 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
236 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
237 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
238 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
239 |           with tf.variable_scope('branch_pool'):
240 |             branch_pool = ops.avg_pool(net, [3, 3])
241 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
242 |           net = tf.concat(3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
243 |           end_points['mixed_17x17x768c'] = net
244 |         # mixed_6: 17 x 17 x 768.
245 |         with tf.variable_scope('mixed_17x17x768d') as scope:
246 |           if reuse:
247 |             scope.reuse_variables()
248 |           with tf.variable_scope('branch1x1'):
249 |             branch1x1 = ops.conv2d(net, 192, [1, 1])
250 |           with tf.variable_scope('branch7x7'):
251 |             branch7x7 = ops.conv2d(net, 160, [1, 1])
252 |             branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
253 |             branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
254 |           with tf.variable_scope('branch7x7dbl'):
255 |             branch7x7dbl = ops.conv2d(net, 160, [1, 1])
256 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
257 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [1, 7])
258 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 160, [7, 1])
259 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
260 |           with tf.variable_scope('branch_pool'):
261 |             branch_pool = ops.avg_pool(net, [3, 3])
262 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
263 |           net = tf.concat(3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
264 |           end_points['mixed_17x17x768d'] = net
265 |         # mixed_7: 17 x 17 x 768.
266 |         with tf.variable_scope('mixed_17x17x768e') as scope:
267 |           if reuse:
268 |             scope.reuse_variables()
269 |           with tf.variable_scope('branch1x1'):
270 |             branch1x1 = ops.conv2d(net, 192, [1, 1])
271 |           with tf.variable_scope('branch7x7'):
272 |             branch7x7 = ops.conv2d(net, 192, [1, 1])
273 |             branch7x7 = ops.conv2d(branch7x7, 192, [1, 7])
274 |             branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
275 |           with tf.variable_scope('branch7x7dbl'):
276 |             branch7x7dbl = ops.conv2d(net, 192, [1, 1])
277 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
278 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
279 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [7, 1])
280 |             branch7x7dbl = ops.conv2d(branch7x7dbl, 192, [1, 7])
281 |           with tf.variable_scope('branch_pool'):
282 |             branch_pool = ops.avg_pool(net, [3, 3])
283 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
284 |           net = tf.concat(3, [branch1x1, branch7x7, branch7x7dbl, branch_pool])
285 |           end_points['mixed_17x17x768e'] = net
286 |         # Auxiliary Head logits
287 |         aux_logits = tf.identity(end_points['mixed_17x17x768e'])
288 |         with tf.variable_scope('aux_logits') as scope:
289 |           if reuse:
290 |             scope.reuse_variables()
291 |           aux_logits = ops.avg_pool(aux_logits, [5, 5], stride=3,
292 |                                     padding='VALID')
293 |           aux_logits = ops.conv2d(aux_logits, 128, [1, 1], scope='proj')
294 |           # Shape of feature map before the final layer.
295 |           shape = aux_logits.get_shape()
296 |           aux_logits = ops.conv2d(aux_logits, 768, shape[1:3], stddev=0.01,
297 |                                   padding='VALID')
298 |           aux_logits = ops.flatten(aux_logits)
299 |           aux_logits = ops.fc(aux_logits, num_classes, activation=None,
300 |                               stddev=0.001, restore=restore_logits)
301 |           end_points['aux_logits'] = aux_logits
302 |         # mixed_8: 8 x 8 x 1280.
303 |         # Note that the scope below is not changed to not void previous
304 |         # checkpoints.
305 |         # (TODO) Fix the scope when appropriate.
306 |         with tf.variable_scope('mixed_17x17x1280a') as scope:
307 |           if reuse:
308 |             scope.reuse_variables()
309 |           with tf.variable_scope('branch3x3'):
310 |             branch3x3 = ops.conv2d(net, 192, [1, 1])
311 |             branch3x3 = ops.conv2d(branch3x3, 320, [3, 3], stride=2,
312 |                                    padding='VALID')
313 |           with tf.variable_scope('branch7x7x3'):
314 |             branch7x7x3 = ops.conv2d(net, 192, [1, 1])
315 |             branch7x7x3 = ops.conv2d(branch7x7x3, 192, [1, 7])
316 |             branch7x7x3 = ops.conv2d(branch7x7x3, 192, [7, 1])
317 |             branch7x7x3 = ops.conv2d(branch7x7x3, 192, [3, 3],
318 |                                      stride=2, padding='VALID')
319 |           with tf.variable_scope('branch_pool'):
320 |             branch_pool = ops.max_pool(net, [3, 3], stride=2, padding='VALID')
321 |           net = tf.concat(3, [branch3x3, branch7x7x3, branch_pool])
322 |           end_points['mixed_17x17x1280a'] = net
323 |         # mixed_9: 8 x 8 x 2048.
324 |         with tf.variable_scope('mixed_8x8x2048a') as scope:
325 |           if reuse:
326 |             scope.reuse_variables()
327 |           with tf.variable_scope('branch1x1'):
328 |             branch1x1 = ops.conv2d(net, 320, [1, 1])
329 |           with tf.variable_scope('branch3x3'):
330 |             branch3x3 = ops.conv2d(net, 384, [1, 1])
331 |             branch3x3 = tf.concat(3, [ops.conv2d(branch3x3, 384, [1, 3]),
332 |                                       ops.conv2d(branch3x3, 384, [3, 1])])
333 |           with tf.variable_scope('branch3x3dbl'):
334 |             branch3x3dbl = ops.conv2d(net, 448, [1, 1])
335 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
336 |             branch3x3dbl = tf.concat(3, [ops.conv2d(branch3x3dbl, 384, [1, 3]),
337 |                                          ops.conv2d(branch3x3dbl, 384, [3, 1])])
338 |           with tf.variable_scope('branch_pool'):
339 |             branch_pool = ops.avg_pool(net, [3, 3])
340 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
341 |           net = tf.concat(3, [branch1x1, branch3x3, branch3x3dbl, branch_pool])
342 |           end_points['mixed_8x8x2048a'] = net
343 |         # mixed_10: 8 x 8 x 2048.
344 |         with tf.variable_scope('mixed_8x8x2048b') as scope:
345 |           if reuse:
346 |             scope.reuse_variables()
347 |           with tf.variable_scope('branch1x1'):
348 |             branch1x1 = ops.conv2d(net, 320, [1, 1])
349 |           with tf.variable_scope('branch3x3'):
350 |             branch3x3 = ops.conv2d(net, 384, [1, 1])
351 |             branch3x3 = tf.concat(3, [ops.conv2d(branch3x3, 384, [1, 3]),
352 |                                       ops.conv2d(branch3x3, 384, [3, 1])])
353 |           with tf.variable_scope('branch3x3dbl'):
354 |             branch3x3dbl = ops.conv2d(net, 448, [1, 1])
355 |             branch3x3dbl = ops.conv2d(branch3x3dbl, 384, [3, 3])
356 |             branch3x3dbl = tf.concat(3, [ops.conv2d(branch3x3dbl, 384, [1, 3]),
357 |                                          ops.conv2d(branch3x3dbl, 384, [3, 1])])
358 |           with tf.variable_scope('branch_pool'):
359 |             branch_pool = ops.avg_pool(net, [3, 3])
360 |             branch_pool = ops.conv2d(branch_pool, 192, [1, 1])
361 |           net = tf.concat(3, [branch1x1, branch3x3, branch3x3dbl, branch_pool])
362 |           end_points['mixed_8x8x2048b'] = net
363 |         # Final pooling and prediction
364 |         with tf.variable_scope('logits'):
365 |           shape = net.get_shape()
366 |           net = ops.avg_pool(net, shape[1:3], padding='VALID', scope='pool')
367 |           # 1 x 1 x 2048
368 |           net = ops.dropout(net, dropout_keep_prob, scope='dropout')
369 |           net = ops.flatten(net, scope='flatten')
370 |           # 2048
371 |           logits = ops.fc(net, num_classes, activation=None, scope='logits',
372 |                           restore=restore_logits, reuse=reuse)
373 |           # 1000
374 |           end_points['logits'] = logits
375 |           end_points['predictions'] = tf.nn.softmax(logits, name='predictions')
376 | 
377 |       # logits, endpoints["aux_logits"]
378 |       return logits, end_points
379 | 
380 | 
381 | def inception_v3_parameters(weight_decay=0.00004, stddev=0.1,
382 |                             batch_norm_decay=0.9997, batch_norm_epsilon=0.001):
383 |   """Yields the scope with the default parameters for inception_v3.
384 | 
385 |   Args:
386 |     weight_decay: the weight decay for weights variables.
387 |     stddev: standard deviation of the truncated guassian weight distribution.
388 |     batch_norm_decay: decay for the moving average of batch_norm momentums.
389 |     batch_norm_epsilon: small float added to variance to avoid dividing by zero.
390 | 
391 |   Yields:
392 |     a arg_scope with the parameters needed for inception_v3.
393 |   """
394 |   # Set weight_decay for weights in Conv and FC layers.
395 |   with scopes.arg_scope([ops.conv2d, ops.fc],
396 |                         weight_decay=weight_decay):
397 |     # Set stddev, activation and parameters for batch_norm.
398 |     with scopes.arg_scope([ops.conv2d],
399 |                           stddev=stddev,
400 |                           activation=tf.nn.relu,
401 |                           batch_norm_params={
402 |                               'decay': batch_norm_decay,
403 |                               'epsilon': batch_norm_epsilon}) as arg_scope:
404 |       yield arg_scope
405 | 


--------------------------------------------------------------------------------
/slim/inception_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests for slim.inception."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | import tensorflow as tf
 21 | 
 22 | from inception.slim import inception_model as inception
 23 | 
 24 | 
 25 | class InceptionTest(tf.test.TestCase):
 26 | 
 27 |   def testBuildLogits(self):
 28 |     batch_size = 5
 29 |     height, width = 299, 299
 30 |     num_classes = 1000
 31 |     with self.test_session():
 32 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 33 |       logits, _ = inception.inception_v3(inputs, num_classes)
 34 |       self.assertTrue(logits.op.name.startswith('logits'))
 35 |       self.assertListEqual(logits.get_shape().as_list(),
 36 |                            [batch_size, num_classes])
 37 | 
 38 |   def testBuildEndPoints(self):
 39 |     batch_size = 5
 40 |     height, width = 299, 299
 41 |     num_classes = 1000
 42 |     with self.test_session():
 43 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 44 |       _, end_points = inception.inception_v3(inputs, num_classes)
 45 |       self.assertTrue('logits' in end_points)
 46 |       logits = end_points['logits']
 47 |       self.assertListEqual(logits.get_shape().as_list(),
 48 |                            [batch_size, num_classes])
 49 |       self.assertTrue('aux_logits' in end_points)
 50 |       aux_logits = end_points['aux_logits']
 51 |       self.assertListEqual(aux_logits.get_shape().as_list(),
 52 |                            [batch_size, num_classes])
 53 |       pre_pool = end_points['mixed_8x8x2048b']
 54 |       self.assertListEqual(pre_pool.get_shape().as_list(),
 55 |                            [batch_size, 8, 8, 2048])
 56 | 
 57 |   def testVariablesSetDevice(self):
 58 |     batch_size = 5
 59 |     height, width = 299, 299
 60 |     num_classes = 1000
 61 |     with self.test_session():
 62 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 63 |       # Force all Variables to reside on the device.
 64 |       with tf.variable_scope('on_cpu'), tf.device('/cpu:0'):
 65 |         inception.inception_v3(inputs, num_classes)
 66 |       with tf.variable_scope('on_gpu'), tf.device('/gpu:0'):
 67 |         inception.inception_v3(inputs, num_classes)
 68 |       for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_cpu'):
 69 |         self.assertDeviceEqual(v.device, '/cpu:0')
 70 |       for v in tf.get_collection(tf.GraphKeys.VARIABLES, scope='on_gpu'):
 71 |         self.assertDeviceEqual(v.device, '/gpu:0')
 72 | 
 73 |   def testHalfSizeImages(self):
 74 |     batch_size = 5
 75 |     height, width = 150, 150
 76 |     num_classes = 1000
 77 |     with self.test_session():
 78 |       inputs = tf.random_uniform((batch_size, height, width, 3))
 79 |       logits, end_points = inception.inception_v3(inputs, num_classes)
 80 |       self.assertTrue(logits.op.name.startswith('logits'))
 81 |       self.assertListEqual(logits.get_shape().as_list(),
 82 |                            [batch_size, num_classes])
 83 |       pre_pool = end_points['mixed_8x8x2048b']
 84 |       self.assertListEqual(pre_pool.get_shape().as_list(),
 85 |                            [batch_size, 3, 3, 2048])
 86 | 
 87 |   def testUnknowBatchSize(self):
 88 |     batch_size = 1
 89 |     height, width = 299, 299
 90 |     num_classes = 1000
 91 |     with self.test_session() as sess:
 92 |       inputs = tf.placeholder(tf.float32, (None, height, width, 3))
 93 |       logits, _ = inception.inception_v3(inputs, num_classes)
 94 |       self.assertTrue(logits.op.name.startswith('logits'))
 95 |       self.assertListEqual(logits.get_shape().as_list(),
 96 |                            [None, num_classes])
 97 |       images = tf.random_uniform((batch_size, height, width, 3))
 98 |       sess.run(tf.initialize_all_variables())
 99 |       output = sess.run(logits, {inputs: images.eval()})
100 |       self.assertEquals(output.shape, (batch_size, num_classes))
101 | 
102 |   def testEvaluation(self):
103 |     batch_size = 2
104 |     height, width = 299, 299
105 |     num_classes = 1000
106 |     with self.test_session() as sess:
107 |       eval_inputs = tf.random_uniform((batch_size, height, width, 3))
108 |       logits, _ = inception.inception_v3(eval_inputs, num_classes,
109 |                                          is_training=False)
110 |       predictions = tf.argmax(logits, 1)
111 |       sess.run(tf.initialize_all_variables())
112 |       output = sess.run(predictions)
113 |       self.assertEquals(output.shape, (batch_size,))
114 | 
115 |   def testTrainEvalWithReuse(self):
116 |     train_batch_size = 5
117 |     eval_batch_size = 2
118 |     height, width = 150, 150
119 |     num_classes = 1000
120 |     with self.test_session() as sess:
121 |       train_inputs = tf.random_uniform((train_batch_size, height, width, 3))
122 |       inception.inception_v3(train_inputs, num_classes)
123 |       tf.get_variable_scope().reuse_variables()
124 |       eval_inputs = tf.random_uniform((eval_batch_size, height, width, 3))
125 |       logits, _ = inception.inception_v3(eval_inputs, num_classes,
126 |                                          is_training=False)
127 |       predictions = tf.argmax(logits, 1)
128 |       sess.run(tf.initialize_all_variables())
129 |       output = sess.run(predictions)
130 |       self.assertEquals(output.shape, (eval_batch_size,))
131 | 
132 | 
133 | if __name__ == '__main__':
134 |   tf.test.main()
135 | 


--------------------------------------------------------------------------------
/slim/losses.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Contains convenience wrappers for various Neural Network TensorFlow losses.
 16 | 
 17 |   All the losses defined here add themselves to the LOSSES_COLLECTION
 18 |   collection.
 19 | 
 20 |   l1_loss: Define a L1 Loss, useful for regularization, i.e. lasso.
 21 |   l2_loss: Define a L2 Loss, useful for regularization, i.e. weight decay.
 22 |   cross_entropy_loss: Define a cross entropy loss using
 23 |     softmax_cross_entropy_with_logits. Useful for classification.
 24 | """
 25 | from __future__ import absolute_import
 26 | from __future__ import division
 27 | from __future__ import print_function
 28 | 
 29 | import tensorflow as tf
 30 | 
 31 | # In order to gather all losses in a network, the user should use this
 32 | # key for get_collection, i.e:
 33 | #   losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
 34 | LOSSES_COLLECTION = '_losses'
 35 | LOSSES_COLLECTION_TEST = '_losses_test'
 36 | 
 37 | def l1_regularizer(weight=1.0, scope=None):
 38 |   """Define a L1 regularizer.
 39 | 
 40 |   Args:
 41 |     weight: scale the loss by this factor.
 42 |     scope: Optional scope for name_scope.
 43 | 
 44 |   Returns:
 45 |     a regularizer function.
 46 |   """
 47 |   def regularizer(tensor):
 48 |     with tf.name_scope(scope, 'L1Regularizer', [tensor]):
 49 |       l1_weight = tf.convert_to_tensor(weight,
 50 |                                        dtype=tensor.dtype.base_dtype,
 51 |                                        name='weight')
 52 |       return tf.mul(l1_weight, tf.reduce_sum(tf.abs(tensor)), name='value')
 53 |   return regularizer
 54 | 
 55 | 
 56 | def l2_regularizer(weight=1.0, scope=None):
 57 |   """Define a L2 regularizer.
 58 | 
 59 |   Args:
 60 |     weight: scale the loss by this factor.
 61 |     scope: Optional scope for name_scope.
 62 | 
 63 |   Returns:
 64 |     a regularizer function.
 65 |   """
 66 |   def regularizer(tensor):
 67 |     with tf.name_scope(scope, 'L2Regularizer', [tensor]):
 68 |       l2_weight = tf.convert_to_tensor(weight,
 69 |                                        dtype=tensor.dtype.base_dtype,
 70 |                                        name='weight')
 71 |       return tf.mul(l2_weight, tf.nn.l2_loss(tensor), name='value')
 72 |   return regularizer
 73 | 
 74 | 
 75 | def l1_l2_regularizer(weight_l1=1.0, weight_l2=1.0, scope=None):
 76 |   """Define a L1L2 regularizer.
 77 | 
 78 |   Args:
 79 |     weight_l1: scale the L1 loss by this factor.
 80 |     weight_l2: scale the L2 loss by this factor.
 81 |     scope: Optional scope for name_scope.
 82 | 
 83 |   Returns:
 84 |     a regularizer function.
 85 |   """
 86 |   def regularizer(tensor):
 87 |     with tf.name_scope(scope, 'L1L2Regularizer', [tensor]):
 88 |       weight_l1_t = tf.convert_to_tensor(weight_l1,
 89 |                                          dtype=tensor.dtype.base_dtype,
 90 |                                          name='weight_l1')
 91 |       weight_l2_t = tf.convert_to_tensor(weight_l2,
 92 |                                          dtype=tensor.dtype.base_dtype,
 93 |                                          name='weight_l2')
 94 |       reg_l1 = tf.mul(weight_l1_t, tf.reduce_sum(tf.abs(tensor)),
 95 |                       name='value_l1')
 96 |       reg_l2 = tf.mul(weight_l2_t, tf.nn.l2_loss(tensor),
 97 |                       name='value_l2')
 98 |       return tf.add(reg_l1, reg_l2, name='value')
 99 |   return regularizer
100 | 
101 | 
102 | def l1_loss(tensor, weight=1.0, scope=None):
103 |   """Define a L1Loss, useful for regularize, i.e. lasso.
104 | 
105 |   Args:
106 |     tensor: tensor to regularize.
107 |     weight: scale the loss by this factor.
108 |     scope: Optional scope for name_scope.
109 | 
110 |   Returns:
111 |     the L1 loss op.
112 |   """
113 |   with tf.name_scope(scope, 'L1Loss', [tensor]):
114 |     weight = tf.convert_to_tensor(weight,
115 |                                   dtype=tensor.dtype.base_dtype,
116 |                                   name='loss_weight')
117 |     loss = tf.mul(weight, tf.reduce_sum(tf.abs(tensor)), name='value')
118 |     tf.add_to_collection(LOSSES_COLLECTION, loss)
119 |     return loss
120 | 
121 | 
122 | def l2_loss(tensor, weight=1.0, scope=None):
123 |   """Define a L2Loss, useful for regularize, i.e. weight decay.
124 | 
125 |   Args:
126 |     tensor: tensor to regularize.
127 |     weight: an optional weight to modulate the loss.
128 |     scope: Optional scope for name_scope.
129 | 
130 |   Returns:
131 |     the L2 loss op.
132 |   """
133 |   with tf.name_scope(scope, 'L2Loss', [tensor]):
134 |     weight = tf.convert_to_tensor(weight,
135 |                                   dtype=tensor.dtype.base_dtype,
136 |                                   name='loss_weight')
137 |     loss = tf.mul(weight, tf.nn.l2_loss(tensor), name='value')
138 |     tf.add_to_collection(LOSSES_COLLECTION, loss)
139 |     return loss
140 | 
141 | def cross_entropy_loss_without_collection(logits, one_hot_labels, label_smoothing=0,
142 |                        weight=1.0, scope=None):
143 |     logits.get_shape().assert_is_compatible_with(one_hot_labels.get_shape())
144 |     with tf.name_scope(scope, 'CrossEntropyLossTest', [logits, one_hot_labels]):
145 |         num_classes = one_hot_labels.get_shape()[-1].value
146 |         one_hot_labels = tf.cast(one_hot_labels, logits.dtype)
147 |         if label_smoothing > 0:
148 |             smooth_positives = 1.0 - label_smoothing
149 |             smooth_negatives = label_smoothing / num_classes
150 |             one_hot_labels = one_hot_labels * smooth_positives + smooth_negatives
151 |         cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,
152 |                                                                 one_hot_labels,
153 |                                                                 name='xentropy_test')
154 |         weight = tf.convert_to_tensor(weight,
155 |                                       dtype=logits.dtype.base_dtype,
156 |                                       name='loss_weight_test')
157 |         loss = tf.mul(weight, tf.reduce_mean(cross_entropy), name='value_test')
158 |         tf.add_to_collection(LOSSES_COLLECTION_TEST, loss)
159 |         return loss
160 | 
161 | 
162 | def cross_entropy_loss(logits, one_hot_labels, label_smoothing=0,
163 |                        weight=1.0, scope=None):
164 |   logits.get_shape().assert_is_compatible_with(one_hot_labels.get_shape())
165 |   with tf.name_scope(scope, 'CrossEntropyLoss', [logits, one_hot_labels]):
166 |     num_classes = one_hot_labels.get_shape()[-1].value
167 |     one_hot_labels = tf.cast(one_hot_labels, logits.dtype)
168 |     if label_smoothing > 0:
169 |       smooth_positives = 1.0 - label_smoothing
170 |       smooth_negatives = label_smoothing / num_classes
171 |       one_hot_labels = one_hot_labels * smooth_positives + smooth_negatives
172 |     cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,
173 |                                                             one_hot_labels,
174 |                                                             name='xentropy')
175 |     weight = tf.convert_to_tensor(weight,
176 |                                   dtype=logits.dtype.base_dtype,
177 |                                   name='loss_weight')
178 |     loss = tf.mul(weight, tf.reduce_mean(cross_entropy), name='value')
179 |     tf.add_to_collection(LOSSES_COLLECTION, loss)
180 |     return loss
181 | 
182 | 
183 | def square_loss(logits, targets, weight=1.0, scope=None):
184 |     with tf.name_scope(scope, 'SquareLoss', [logits, targets]):
185 |         weight = tf.convert_to_tensor(weight,
186 |                                       dtype=logits.dtype.base_dtype,
187 |                                       name='loss_weight')
188 |         square = tf.square(logits, targets)
189 |         loss = tf.mul(weight, tf.reduce_mean(square), name='value')
190 |         tf.add_to_collection(LOSSES_COLLECTION, loss)
191 |         return loss
192 | 
193 | 


--------------------------------------------------------------------------------
/slim/losses_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests for slim.losses."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | 
 21 | import tensorflow as tf
 22 | 
 23 | from inception.slim import losses
 24 | 
 25 | 
 26 | class LossesTest(tf.test.TestCase):
 27 | 
 28 |   def testL1Loss(self):
 29 |     with self.test_session():
 30 |       shape = [5, 5, 5]
 31 |       num_elem = 5 * 5 * 5
 32 |       weights = tf.constant(1.0, shape=shape)
 33 |       wd = 0.01
 34 |       loss = losses.l1_loss(weights, wd)
 35 |       self.assertEquals(loss.op.name, 'L1Loss/value')
 36 |       self.assertAlmostEqual(loss.eval(), num_elem * wd, 5)
 37 | 
 38 |   def testL2Loss(self):
 39 |     with self.test_session():
 40 |       shape = [5, 5, 5]
 41 |       num_elem = 5 * 5 * 5
 42 |       weights = tf.constant(1.0, shape=shape)
 43 |       wd = 0.01
 44 |       loss = losses.l2_loss(weights, wd)
 45 |       self.assertEquals(loss.op.name, 'L2Loss/value')
 46 |       self.assertAlmostEqual(loss.eval(), num_elem * wd / 2, 5)
 47 | 
 48 | 
 49 | class RegularizersTest(tf.test.TestCase):
 50 | 
 51 |   def testL1Regularizer(self):
 52 |     with self.test_session():
 53 |       shape = [5, 5, 5]
 54 |       num_elem = 5 * 5 * 5
 55 |       tensor = tf.constant(1.0, shape=shape)
 56 |       loss = losses.l1_regularizer()(tensor)
 57 |       self.assertEquals(loss.op.name, 'L1Regularizer/value')
 58 |       self.assertAlmostEqual(loss.eval(), num_elem, 5)
 59 | 
 60 |   def testL1RegularizerWithScope(self):
 61 |     with self.test_session():
 62 |       shape = [5, 5, 5]
 63 |       num_elem = 5 * 5 * 5
 64 |       tensor = tf.constant(1.0, shape=shape)
 65 |       loss = losses.l1_regularizer(scope='L1')(tensor)
 66 |       self.assertEquals(loss.op.name, 'L1/value')
 67 |       self.assertAlmostEqual(loss.eval(), num_elem, 5)
 68 | 
 69 |   def testL1RegularizerWithWeight(self):
 70 |     with self.test_session():
 71 |       shape = [5, 5, 5]
 72 |       num_elem = 5 * 5 * 5
 73 |       tensor = tf.constant(1.0, shape=shape)
 74 |       weight = 0.01
 75 |       loss = losses.l1_regularizer(weight)(tensor)
 76 |       self.assertEquals(loss.op.name, 'L1Regularizer/value')
 77 |       self.assertAlmostEqual(loss.eval(), num_elem * weight, 5)
 78 | 
 79 |   def testL2Regularizer(self):
 80 |     with self.test_session():
 81 |       shape = [5, 5, 5]
 82 |       num_elem = 5 * 5 * 5
 83 |       tensor = tf.constant(1.0, shape=shape)
 84 |       loss = losses.l2_regularizer()(tensor)
 85 |       self.assertEquals(loss.op.name, 'L2Regularizer/value')
 86 |       self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
 87 | 
 88 |   def testL2RegularizerWithScope(self):
 89 |     with self.test_session():
 90 |       shape = [5, 5, 5]
 91 |       num_elem = 5 * 5 * 5
 92 |       tensor = tf.constant(1.0, shape=shape)
 93 |       loss = losses.l2_regularizer(scope='L2')(tensor)
 94 |       self.assertEquals(loss.op.name, 'L2/value')
 95 |       self.assertAlmostEqual(loss.eval(), num_elem / 2, 5)
 96 | 
 97 |   def testL2RegularizerWithWeight(self):
 98 |     with self.test_session():
 99 |       shape = [5, 5, 5]
100 |       num_elem = 5 * 5 * 5
101 |       tensor = tf.constant(1.0, shape=shape)
102 |       weight = 0.01
103 |       loss = losses.l2_regularizer(weight)(tensor)
104 |       self.assertEquals(loss.op.name, 'L2Regularizer/value')
105 |       self.assertAlmostEqual(loss.eval(), num_elem * weight / 2, 5)
106 | 
107 |   def testL1L2Regularizer(self):
108 |     with self.test_session():
109 |       shape = [5, 5, 5]
110 |       num_elem = 5 * 5 * 5
111 |       tensor = tf.constant(1.0, shape=shape)
112 |       loss = losses.l1_l2_regularizer()(tensor)
113 |       self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
114 |       self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
115 | 
116 |   def testL1L2RegularizerWithScope(self):
117 |     with self.test_session():
118 |       shape = [5, 5, 5]
119 |       num_elem = 5 * 5 * 5
120 |       tensor = tf.constant(1.0, shape=shape)
121 |       loss = losses.l1_l2_regularizer(scope='L1L2')(tensor)
122 |       self.assertEquals(loss.op.name, 'L1L2/value')
123 |       self.assertAlmostEqual(loss.eval(), num_elem + num_elem / 2, 5)
124 | 
125 |   def testL1L2RegularizerWithWeights(self):
126 |     with self.test_session():
127 |       shape = [5, 5, 5]
128 |       num_elem = 5 * 5 * 5
129 |       tensor = tf.constant(1.0, shape=shape)
130 |       weight_l1 = 0.01
131 |       weight_l2 = 0.05
132 |       loss = losses.l1_l2_regularizer(weight_l1, weight_l2)(tensor)
133 |       self.assertEquals(loss.op.name, 'L1L2Regularizer/value')
134 |       self.assertAlmostEqual(loss.eval(),
135 |                              num_elem * weight_l1 + num_elem * weight_l2 / 2, 5)
136 | 
137 | 
138 | class CrossEntropyLossTest(tf.test.TestCase):
139 | 
140 |   def testCrossEntropyLossAllCorrect(self):
141 |     with self.test_session():
142 |       logits = tf.constant([[10.0, 0.0, 0.0],
143 |                             [0.0, 10.0, 0.0],
144 |                             [0.0, 0.0, 10.0]])
145 |       labels = tf.constant([[1, 0, 0],
146 |                             [0, 1, 0],
147 |                             [0, 0, 1]])
148 |       loss = losses.cross_entropy_loss(logits, labels)
149 |       self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
150 |       self.assertAlmostEqual(loss.eval(), 0.0, 3)
151 | 
152 |   def testCrossEntropyLossAllWrong(self):
153 |     with self.test_session():
154 |       logits = tf.constant([[10.0, 0.0, 0.0],
155 |                             [0.0, 10.0, 0.0],
156 |                             [0.0, 0.0, 10.0]])
157 |       labels = tf.constant([[0, 0, 1],
158 |                             [1, 0, 0],
159 |                             [0, 1, 0]])
160 |       loss = losses.cross_entropy_loss(logits, labels)
161 |       self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
162 |       self.assertAlmostEqual(loss.eval(), 10.0, 3)
163 | 
164 |   def testCrossEntropyLossAllWrongWithWeight(self):
165 |     with self.test_session():
166 |       logits = tf.constant([[10.0, 0.0, 0.0],
167 |                             [0.0, 10.0, 0.0],
168 |                             [0.0, 0.0, 10.0]])
169 |       labels = tf.constant([[0, 0, 1],
170 |                             [1, 0, 0],
171 |                             [0, 1, 0]])
172 |       loss = losses.cross_entropy_loss(logits, labels, weight=0.5)
173 |       self.assertEquals(loss.op.name, 'CrossEntropyLoss/value')
174 |       self.assertAlmostEqual(loss.eval(), 5.0, 3)
175 | 
176 | if __name__ == '__main__':
177 |   tf.test.main()
178 | 


--------------------------------------------------------------------------------
/slim/ops.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Contains convenience wrappers for typical Neural Network TensorFlow layers.
 16 | 
 17 |    Additionally it maintains a collection with update_ops that need to be
 18 |    updated after the ops have been computed, for exmaple to update moving means
 19 |    and moving variances of batch_norm.
 20 | 
 21 |    Ops that have different behavior during training or eval have an is_training
 22 |    parameter. Additionally Ops that contain variables.variable have a trainable
 23 |    parameter, which control if the ops variables are trainable or not.
 24 | """
 25 | from __future__ import division
 26 | from __future__ import print_function
 27 | 
 28 | 
 29 | import tensorflow as tf
 30 | 
 31 | from tensorflow.python.training import moving_averages
 32 | 
 33 | import losses
 34 | import scopes
 35 | import variables
 36 | 
 37 | # Used to keep the update ops done by batch_norm.
 38 | UPDATE_OPS_COLLECTION = '_update_ops_'
 39 | 
 40 | 
 41 | @scopes.add_arg_scope
 42 | def batch_norm(inputs,
 43 |                decay=0.999,
 44 |                center=True,
 45 |                scale=False,
 46 |                epsilon=0.001,
 47 |                moving_vars='moving_vars',
 48 |                activation=None,
 49 |                is_training=True,
 50 |                trainable=True,
 51 |                restore=True,
 52 |                scope=None,
 53 |                reuse=None):
 54 |   """Adds a Batch Normalization layer.
 55 | 
 56 |   Args:
 57 |     inputs: a tensor of size [batch_size, height, width, channels]
 58 |             or [batch_size, channels].
 59 |     decay: decay for the moving average.
 60 |     center: If True, subtract beta. If False, beta is not created and ignored.
 61 |     scale: If True, multiply by gamma. If False, gamma is
 62 |       not used. When the next layer is linear (also e.g. ReLU), this can be
 63 |       disabled since the scaling can be done by the next layer.
 64 |     epsilon: small float added to variance to avoid dividing by zero.
 65 |     moving_vars: collection to store the moving_mean and moving_variance.
 66 |     activation: activation function.
 67 |     is_training: whether or not the model is in training mode.
 68 |     trainable: whether or not the variables should be trainable or not.
 69 |     restore: whether or not the variables should be marked for restore.
 70 |     scope: Optional scope for variable_scope.
 71 |     reuse: whether or not the layer and its variables should be reused. To be
 72 |       able to reuse the layer scope must be given.
 73 | 
 74 |   Returns:
 75 |     a tensor representing the output of the operation.
 76 | 
 77 |   """
 78 |   inputs_shape = inputs.get_shape()
 79 |   with tf.variable_scope(scope, 'BatchNorm', [inputs], reuse=reuse):
 80 |     axis = list(range(len(inputs_shape) - 1))
 81 |     params_shape = inputs_shape[-1:]
 82 |     # Allocate parameters for the beta and gamma of the normalization.
 83 |     beta, gamma = None, None
 84 |     if center:
 85 |       beta = variables.variable('beta',
 86 |                                 params_shape,
 87 |                                 initializer=tf.zeros_initializer,
 88 |                                 trainable=trainable,
 89 |                                 restore=restore)
 90 |     if scale:
 91 |       gamma = variables.variable('gamma',
 92 |                                  params_shape,
 93 |                                  initializer=tf.ones_initializer(),
 94 |                                  trainable=trainable,
 95 |                                  restore=restore)
 96 |     # Create moving_mean and moving_variance add them to
 97 |     # GraphKeys.MOVING_AVERAGE_VARIABLES collections.
 98 |     moving_collections = [moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES]
 99 |     moving_mean = variables.variable('moving_mean',
100 |                                      params_shape,
101 |                                      initializer=tf.zeros_initializer,
102 |                                      trainable=False,
103 |                                      restore=restore,
104 |                                      collections=moving_collections)
105 |     moving_variance = variables.variable('moving_variance',
106 |                                          params_shape,
107 |                                          initializer=tf.ones_initializer(),
108 |                                          trainable=False,
109 |                                          restore=restore,
110 |                                          collections=moving_collections)
111 |     if is_training:
112 |       # Calculate the moments based on the individual batch.
113 |       mean, variance = tf.nn.moments(inputs, axis)
114 | 
115 |       update_moving_mean = moving_averages.assign_moving_average(
116 |           moving_mean, mean, decay)
117 |       tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
118 |       update_moving_variance = moving_averages.assign_moving_average(
119 |           moving_variance, variance, decay)
120 |       tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
121 |     else:
122 |       # Just use the moving_mean and moving_variance.
123 |       mean = moving_mean
124 |       variance = moving_variance
125 |     # Normalize the activations.
126 |     outputs = tf.nn.batch_normalization(
127 |         inputs, mean, variance, beta, gamma, epsilon)
128 |     outputs.set_shape(inputs.get_shape())
129 |     if activation:
130 |       outputs = activation(outputs)
131 |     return outputs
132 | 
133 | 
134 | def _two_element_tuple(int_or_tuple):
135 |   """Converts `int_or_tuple` to height, width.
136 | 
137 |   Several of the functions that follow accept arguments as either
138 |   a tuple of 2 integers or a single integer.  A single integer
139 |   indicates that the 2 values of the tuple are the same.
140 | 
141 |   This functions normalizes the input value by always returning a tuple.
142 | 
143 |   Args:
144 |     int_or_tuple: A list of 2 ints, a single int or a tf.TensorShape.
145 | 
146 |   Returns:
147 |     A tuple with 2 values.
148 | 
149 |   Raises:
150 |     ValueError: If `int_or_tuple` it not well formed.
151 |   """
152 |   if isinstance(int_or_tuple, (list, tuple)):
153 |     if len(int_or_tuple) != 2:
154 |       raise ValueError('Must be a list with 2 elements: %s' % int_or_tuple)
155 |     return int(int_or_tuple[0]), int(int_or_tuple[1])
156 |   if isinstance(int_or_tuple, int):
157 |     return int(int_or_tuple), int(int_or_tuple)
158 |   if isinstance(int_or_tuple, tf.TensorShape):
159 |     if len(int_or_tuple) == 2:
160 |       return int_or_tuple[0], int_or_tuple[1]
161 |   raise ValueError('Must be an int, a list with 2 elements or a TensorShape of '
162 |                    'length 2')
163 | 
164 | 
165 | @scopes.add_arg_scope
166 | def conv2d(inputs,
167 |            num_filters_out,
168 |            kernel_size,
169 |            stride=1,
170 |            padding='SAME',
171 |            activation=tf.nn.relu,
172 |            stddev=0.01,
173 |            bias=0.0,
174 |            weight_decay=0,
175 |            batch_norm_params=None,
176 |            is_training=True,
177 |            trainable=True,
178 |            restore=True,
179 |            scope=None,
180 |            reuse=None):
181 |   """Adds a 2D convolution followed by an optional batch_norm layer.
182 | 
183 |   conv2d creates a variable called 'weights', representing the convolutional
184 |   kernel, that is convolved with the input. If `batch_norm_params` is None, a
185 |   second variable called 'biases' is added to the result of the convolution
186 |   operation.
187 | 
188 |   Args:
189 |     inputs: a tensor of size [batch_size, height, width, channels].
190 |     num_filters_out: the number of output filters.
191 |     kernel_size: a list of length 2: [kernel_height, kernel_width] of
192 |       of the filters. Can be an int if both values are the same.
193 |     stride: a list of length 2: [stride_height, stride_width].
194 |       Can be an int if both strides are the same.  Note that presently
195 |       both strides must have the same value.
196 |     padding: one of 'VALID' or 'SAME'.
197 |     activation: activation function.
198 |     stddev: standard deviation of the truncated guassian weight distribution.
199 |     bias: the initial value of the biases.
200 |     weight_decay: the weight decay.
201 |     batch_norm_params: parameters for the batch_norm. If is None don't use it.
202 |     is_training: whether or not the model is in training mode.
203 |     trainable: whether or not the variables should be trainable or not.
204 |     restore: whether or not the variables should be marked for restore.
205 |     scope: Optional scope for variable_scope.
206 |     reuse: whether or not the layer and its variables should be reused. To be
207 |       able to reuse the layer scope must be given.
208 |   Returns:
209 |     a tensor representing the output of the operation.
210 | 
211 |   """
212 |   with tf.variable_scope(scope, 'Conv', [inputs], reuse=reuse):
213 |     kernel_h, kernel_w = _two_element_tuple(kernel_size)
214 |     stride_h, stride_w = _two_element_tuple(stride)
215 |     num_filters_in = inputs.get_shape()[-1]
216 |     weights_shape = [kernel_h, kernel_w,
217 |                      num_filters_in, num_filters_out]
218 |     weights_initializer = tf.truncated_normal_initializer(stddev=stddev)
219 |     l2_regularizer = None
220 |     if weight_decay and weight_decay > 0:
221 |       l2_regularizer = losses.l2_regularizer(weight_decay)
222 |     weights = variables.variable('weights',
223 |                                  shape=weights_shape,
224 |                                  initializer=weights_initializer,
225 |                                  regularizer=l2_regularizer,
226 |                                  trainable=trainable,
227 |                                  restore=restore)
228 |     conv = tf.nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1],
229 |                         padding=padding)
230 |     if batch_norm_params is not None:
231 |       with scopes.arg_scope([batch_norm], is_training=is_training,
232 |                             trainable=trainable, restore=restore):
233 |         outputs = batch_norm(conv, **batch_norm_params)
234 |     else:
235 |       bias_shape = [num_filters_out,]
236 |       bias_initializer = tf.constant_initializer(bias)
237 |       biases = variables.variable('biases',
238 |                                   shape=bias_shape,
239 |                                   initializer=bias_initializer,
240 |                                   trainable=trainable,
241 |                                   restore=restore)
242 |       outputs = tf.nn.bias_add(conv, biases)
243 |     if activation:
244 |       outputs = activation(outputs)
245 |     return outputs
246 | 
247 | 
248 | @scopes.add_arg_scope
249 | def fc(inputs,
250 |        num_units_out,
251 |        activation=tf.nn.relu,
252 |        stddev=0.01,
253 |        bias=0.0,
254 |        weight_decay=0,
255 |        batch_norm_params=None,
256 |        is_training=True,
257 |        trainable=True,
258 |        restore=True,
259 |        scope=None,
260 |        reuse=None):
261 |   """Adds a fully connected layer followed by an optional batch_norm layer.
262 | 
263 |   FC creates a variable called 'weights', representing the fully connected
264 |   weight matrix, that is multiplied by the input. If `batch_norm` is None, a
265 |   second variable called 'biases' is added to the result of the initial
266 |   vector-matrix multiplication.
267 | 
268 |   Args:
269 |     inputs: a [B x N] tensor where B is the batch size and N is the number of
270 |             input units in the layer.
271 |     num_units_out: the number of output units in the layer.
272 |     activation: activation function.
273 |     stddev: the standard deviation for the weights.
274 |     bias: the initial value of the biases.
275 |     weight_decay: the weight decay.
276 |     batch_norm_params: parameters for the batch_norm. If is None don't use it.
277 |     is_training: whether or not the model is in training mode.
278 |     trainable: whether or not the variables should be trainable or not.
279 |     restore: whether or not the variables should be marked for restore.
280 |     scope: Optional scope for variable_scope.
281 |     reuse: whether or not the layer and its variables should be reused. To be
282 |       able to reuse the layer scope must be given.
283 | 
284 |   Returns:
285 |      the tensor variable representing the result of the series of operations.
286 |   """
287 |   with tf.variable_scope(scope, 'FC', [inputs], reuse=reuse):
288 |     num_units_in = inputs.get_shape()[1]
289 |     weights_shape = [num_units_in, num_units_out]
290 |     weights_initializer = tf.truncated_normal_initializer(stddev=stddev)
291 |     l2_regularizer = None
292 |     if weight_decay and weight_decay > 0:
293 |       l2_regularizer = losses.l2_regularizer(weight_decay)
294 |     weights = variables.variable('weights',
295 |                                  shape=weights_shape,
296 |                                  initializer=weights_initializer,
297 |                                  regularizer=l2_regularizer,
298 |                                  trainable=trainable,
299 |                                  restore=restore)
300 |     if batch_norm_params is not None:
301 |       outputs = tf.matmul(inputs, weights)
302 |       with scopes.arg_scope([batch_norm], is_training=is_training,
303 |                             trainable=trainable, restore=restore):
304 |         outputs = batch_norm(outputs, **batch_norm_params)
305 |     else:
306 |       bias_shape = [num_units_out,]
307 |       bias_initializer = tf.constant_initializer(bias)
308 |       biases = variables.variable('biases',
309 |                                   shape=bias_shape,
310 |                                   initializer=bias_initializer,
311 |                                   trainable=trainable,
312 |                                   restore=restore)
313 |       outputs = tf.nn.xw_plus_b(inputs, weights, biases)
314 |     if activation:
315 |       outputs = activation(outputs)
316 |     return outputs
317 | 
318 | 
319 | def one_hot_encoding(labels, num_classes, scope=None):
320 |   """Transform numeric labels into onehot_labels.
321 | 
322 |   Args:
323 |     labels: [batch_size] target labels.
324 |     num_classes: total number of classes.
325 |     scope: Optional scope for name_scope.
326 |   Returns:
327 |     one hot encoding of the labels.
328 |   """
329 |   with tf.name_scope(scope, 'OneHotEncoding', [labels]):
330 |     batch_size = labels.get_shape()[0]
331 |     indices = tf.expand_dims(tf.range(0, batch_size), 1)
332 |     labels = tf.cast(tf.expand_dims(labels, 1), indices.dtype)
333 |     concated = tf.concat(1, [indices, labels])
334 |     onehot_labels = tf.sparse_to_dense(
335 |         concated, tf.pack([batch_size, num_classes]), 1.0, 0.0)
336 |     onehot_labels.set_shape([batch_size, num_classes])
337 |     return onehot_labels
338 | 
339 | 
340 | @scopes.add_arg_scope
341 | def max_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None):
342 |   """Adds a Max Pooling layer.
343 | 
344 |   It is assumed by the wrapper that the pooling is only done per image and not
345 |   in depth or batch.
346 | 
347 |   Args:
348 |     inputs: a tensor of size [batch_size, height, width, depth].
349 |     kernel_size: a list of length 2: [kernel_height, kernel_width] of the
350 |       pooling kernel over which the op is computed. Can be an int if both
351 |       values are the same.
352 |     stride: a list of length 2: [stride_height, stride_width].
353 |       Can be an int if both strides are the same.  Note that presently
354 |       both strides must have the same value.
355 |     padding: the padding method, either 'VALID' or 'SAME'.
356 |     scope: Optional scope for name_scope.
357 | 
358 |   Returns:
359 |     a tensor representing the results of the pooling operation.
360 |   Raises:
361 |     ValueError: if 'kernel_size' is not a 2-D list
362 |   """
363 |   with tf.name_scope(scope, 'MaxPool', [inputs]):
364 |     kernel_h, kernel_w = _two_element_tuple(kernel_size)
365 |     stride_h, stride_w = _two_element_tuple(stride)
366 |     return tf.nn.max_pool(inputs,
367 |                           ksize=[1, kernel_h, kernel_w, 1],
368 |                           strides=[1, stride_h, stride_w, 1],
369 |                           padding=padding)
370 | 
371 | 
372 | @scopes.add_arg_scope
373 | def avg_pool(inputs, kernel_size, stride=2, padding='VALID', scope=None):
374 |   """Adds a Avg Pooling layer.
375 | 
376 |   It is assumed by the wrapper that the pooling is only done per image and not
377 |   in depth or batch.
378 | 
379 |   Args:
380 |     inputs: a tensor of size [batch_size, height, width, depth].
381 |     kernel_size: a list of length 2: [kernel_height, kernel_width] of the
382 |       pooling kernel over which the op is computed. Can be an int if both
383 |       values are the same.
384 |     stride: a list of length 2: [stride_height, stride_width].
385 |       Can be an int if both strides are the same.  Note that presently
386 |       both strides must have the same value.
387 |     padding: the padding method, either 'VALID' or 'SAME'.
388 |     scope: Optional scope for name_scope.
389 | 
390 |   Returns:
391 |     a tensor representing the results of the pooling operation.
392 |   """
393 |   with tf.name_scope(scope, 'AvgPool', [inputs]):
394 |     kernel_h, kernel_w = _two_element_tuple(kernel_size)
395 |     stride_h, stride_w = _two_element_tuple(stride)
396 |     return tf.nn.avg_pool(inputs,
397 |                           ksize=[1, kernel_h, kernel_w, 1],
398 |                           strides=[1, stride_h, stride_w, 1],
399 |                           padding=padding)
400 | 
401 | 
402 | @scopes.add_arg_scope
403 | def dropout(inputs, keep_prob=0.5, is_training=True, scope=None):
404 |   """Returns a dropout layer applied to the input.
405 | 
406 |   Args:
407 |     inputs: the tensor to pass to the Dropout layer.
408 |     keep_prob: the probability of keeping each input unit.
409 |     is_training: whether or not the model is in training mode. If so, dropout is
410 |     applied and values scaled. Otherwise, inputs is returned.
411 |     scope: Optional scope for name_scope.
412 | 
413 |   Returns:
414 |     a tensor representing the output of the operation.
415 |   """
416 |   if is_training and keep_prob > 0:
417 |     with tf.name_scope(scope, 'Dropout', [inputs]):
418 |       return tf.nn.dropout(inputs, keep_prob)
419 |   else:
420 |     return inputs
421 | 
422 | 
423 | def flatten(inputs, scope=None):
424 |   """Flattens the input while maintaining the batch_size.
425 | 
426 |     Assumes that the first dimension represents the batch.
427 | 
428 |   Args:
429 |     inputs: a tensor of size [batch_size, ...].
430 |     scope: Optional scope for name_scope.
431 | 
432 |   Returns:
433 |     a flattened tensor with shape [batch_size, k].
434 |   Raises:
435 |     ValueError: if inputs.shape is wrong.
436 |   """
437 |   if len(inputs.get_shape()) < 2:
438 |     raise ValueError('Inputs must be have a least 2 dimensions')
439 |   dims = inputs.get_shape()[1:]
440 |   k = dims.num_elements()
441 |   with tf.name_scope(scope, 'Flatten', [inputs]):
442 |     return tf.reshape(inputs, [-1, k])
443 | 
444 | 
445 | def repeat_op(repetitions, inputs, op, *args, **kwargs):
446 |   """Build a sequential Tower starting from inputs by using an op repeatedly.
447 | 
448 |   It creates new scopes for each operation by increasing the counter.
449 |   Example: given repeat_op(3, _, ops.conv2d, 64, [3, 3], scope='conv1')
450 |     it will repeat the given op under the following variable_scopes:
451 |       conv1/Conv
452 |       conv1/Conv_1
453 |       conv1/Conv_2
454 | 
455 |   Args:
456 |     repetitions: number or repetitions.
457 |     inputs: a tensor of size [batch_size, height, width, channels].
458 |     op: an operation.
459 |     *args: args for the op.
460 |     **kwargs: kwargs for the op.
461 | 
462 |   Returns:
463 |     a tensor result of applying the operation op, num times.
464 |   Raises:
465 |     ValueError: if the op is unknown or wrong.
466 |   """
467 |   scope = kwargs.pop('scope', None)
468 |   with tf.variable_scope(scope, [inputs], 'RepeatOp'):
469 |     tower = inputs
470 |     for _ in range(repetitions):
471 |       tower = op(tower, *args, **kwargs)
472 |     return tower
473 | 


--------------------------------------------------------------------------------
/slim/ops_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests for slim.ops."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | 
 21 | import numpy as np
 22 | import tensorflow as tf
 23 | 
 24 | from tensorflow.python.ops import control_flow_ops
 25 | 
 26 | from inception.slim import ops
 27 | from inception.slim import scopes
 28 | from inception.slim import variables
 29 | 
 30 | 
 31 | class ConvTest(tf.test.TestCase):
 32 | 
 33 |   def testCreateConv(self):
 34 |     height, width = 3, 3
 35 |     with self.test_session():
 36 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 37 |       output = ops.conv2d(images, 32, [3, 3])
 38 |       self.assertEquals(output.op.name, 'Conv/Relu')
 39 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
 40 | 
 41 |   def testCreateSquareConv(self):
 42 |     height, width = 3, 3
 43 |     with self.test_session():
 44 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 45 |       output = ops.conv2d(images, 32, 3)
 46 |       self.assertEquals(output.op.name, 'Conv/Relu')
 47 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
 48 | 
 49 |   def testCreateConvWithTensorShape(self):
 50 |     height, width = 3, 3
 51 |     with self.test_session():
 52 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 53 |       output = ops.conv2d(images, 32, images.get_shape()[1:3])
 54 |       self.assertEquals(output.op.name, 'Conv/Relu')
 55 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 32])
 56 | 
 57 |   def testCreateFullyConv(self):
 58 |     height, width = 6, 6
 59 |     with self.test_session():
 60 |       images = tf.random_uniform((5, height, width, 32), seed=1)
 61 |       output = ops.conv2d(images, 64, images.get_shape()[1:3], padding='VALID')
 62 |       self.assertEquals(output.op.name, 'Conv/Relu')
 63 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 64])
 64 | 
 65 |   def testCreateVerticalConv(self):
 66 |     height, width = 3, 3
 67 |     with self.test_session():
 68 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 69 |       output = ops.conv2d(images, 32, [3, 1])
 70 |       self.assertEquals(output.op.name, 'Conv/Relu')
 71 |       self.assertListEqual(output.get_shape().as_list(),
 72 |                            [5, height, width, 32])
 73 | 
 74 |   def testCreateHorizontalConv(self):
 75 |     height, width = 3, 3
 76 |     with self.test_session():
 77 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 78 |       output = ops.conv2d(images, 32, [1, 3])
 79 |       self.assertEquals(output.op.name, 'Conv/Relu')
 80 |       self.assertListEqual(output.get_shape().as_list(),
 81 |                            [5, height, width, 32])
 82 | 
 83 |   def testCreateConvWithStride(self):
 84 |     height, width = 6, 6
 85 |     with self.test_session():
 86 |       images = tf.random_uniform((5, height, width, 3), seed=1)
 87 |       output = ops.conv2d(images, 32, [3, 3], stride=2)
 88 |       self.assertEquals(output.op.name, 'Conv/Relu')
 89 |       self.assertListEqual(output.get_shape().as_list(),
 90 |                            [5, height/2, width/2, 32])
 91 | 
 92 |   def testCreateConvCreatesWeightsAndBiasesVars(self):
 93 |     height, width = 3, 3
 94 |     images = tf.random_uniform((5, height, width, 3), seed=1)
 95 |     with self.test_session():
 96 |       self.assertFalse(variables.get_variables('conv1/weights'))
 97 |       self.assertFalse(variables.get_variables('conv1/biases'))
 98 |       ops.conv2d(images, 32, [3, 3], scope='conv1')
 99 |       self.assertTrue(variables.get_variables('conv1/weights'))
100 |       self.assertTrue(variables.get_variables('conv1/biases'))
101 | 
102 |   def testCreateConvWithScope(self):
103 |     height, width = 3, 3
104 |     with self.test_session():
105 |       images = tf.random_uniform((5, height, width, 3), seed=1)
106 |       output = ops.conv2d(images, 32, [3, 3], scope='conv1')
107 |       self.assertEquals(output.op.name, 'conv1/Relu')
108 | 
109 |   def testCreateConvWithoutActivation(self):
110 |     height, width = 3, 3
111 |     with self.test_session():
112 |       images = tf.random_uniform((5, height, width, 3), seed=1)
113 |       output = ops.conv2d(images, 32, [3, 3], activation=None)
114 |       self.assertEquals(output.op.name, 'Conv/BiasAdd')
115 | 
116 |   def testCreateConvValid(self):
117 |     height, width = 3, 3
118 |     with self.test_session():
119 |       images = tf.random_uniform((5, height, width, 3), seed=1)
120 |       output = ops.conv2d(images, 32, [3, 3], padding='VALID')
121 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 32])
122 | 
123 |   def testCreateConvWithWD(self):
124 |     height, width = 3, 3
125 |     with self.test_session() as sess:
126 |       images = tf.random_uniform((5, height, width, 3), seed=1)
127 |       ops.conv2d(images, 32, [3, 3], weight_decay=0.01)
128 |       wd = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)[0]
129 |       self.assertEquals(wd.op.name,
130 |                         'Conv/weights/Regularizer/L2Regularizer/value')
131 |       sess.run(tf.initialize_all_variables())
132 |       self.assertTrue(sess.run(wd) <= 0.01)
133 | 
134 |   def testCreateConvWithoutWD(self):
135 |     height, width = 3, 3
136 |     with self.test_session():
137 |       images = tf.random_uniform((5, height, width, 3), seed=1)
138 |       ops.conv2d(images, 32, [3, 3], weight_decay=0)
139 |       self.assertEquals(
140 |           tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), [])
141 | 
142 |   def testReuseVars(self):
143 |     height, width = 3, 3
144 |     with self.test_session():
145 |       images = tf.random_uniform((5, height, width, 3), seed=1)
146 |       ops.conv2d(images, 32, [3, 3], scope='conv1')
147 |       self.assertEquals(len(variables.get_variables()), 2)
148 |       ops.conv2d(images, 32, [3, 3], scope='conv1', reuse=True)
149 |       self.assertEquals(len(variables.get_variables()), 2)
150 | 
151 |   def testNonReuseVars(self):
152 |     height, width = 3, 3
153 |     with self.test_session():
154 |       images = tf.random_uniform((5, height, width, 3), seed=1)
155 |       ops.conv2d(images, 32, [3, 3])
156 |       self.assertEquals(len(variables.get_variables()), 2)
157 |       ops.conv2d(images, 32, [3, 3])
158 |       self.assertEquals(len(variables.get_variables()), 4)
159 | 
160 |   def testReuseConvWithWD(self):
161 |     height, width = 3, 3
162 |     with self.test_session():
163 |       images = tf.random_uniform((5, height, width, 3), seed=1)
164 |       ops.conv2d(images, 32, [3, 3], weight_decay=0.01, scope='conv1')
165 |       self.assertEquals(len(variables.get_variables()), 2)
166 |       self.assertEquals(
167 |           len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
168 |       ops.conv2d(images, 32, [3, 3], weight_decay=0.01, scope='conv1',
169 |                  reuse=True)
170 |       self.assertEquals(len(variables.get_variables()), 2)
171 |       self.assertEquals(
172 |           len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
173 | 
174 |   def testConvWithBatchNorm(self):
175 |     height, width = 3, 3
176 |     with self.test_session():
177 |       images = tf.random_uniform((5, height, width, 32), seed=1)
178 |       with scopes.arg_scope([ops.conv2d], batch_norm_params={'decay': 0.9}):
179 |         net = ops.conv2d(images, 32, [3, 3])
180 |         net = ops.conv2d(net, 32, [3, 3])
181 |       self.assertEquals(len(variables.get_variables()), 8)
182 |       self.assertEquals(len(variables.get_variables('Conv/BatchNorm')), 3)
183 |       self.assertEquals(len(variables.get_variables('Conv_1/BatchNorm')), 3)
184 | 
185 |   def testReuseConvWithBatchNorm(self):
186 |     height, width = 3, 3
187 |     with self.test_session():
188 |       images = tf.random_uniform((5, height, width, 32), seed=1)
189 |       with scopes.arg_scope([ops.conv2d], batch_norm_params={'decay': 0.9}):
190 |         net = ops.conv2d(images, 32, [3, 3], scope='Conv')
191 |         net = ops.conv2d(net, 32, [3, 3], scope='Conv', reuse=True)
192 |       self.assertEquals(len(variables.get_variables()), 4)
193 |       self.assertEquals(len(variables.get_variables('Conv/BatchNorm')), 3)
194 |       self.assertEquals(len(variables.get_variables('Conv_1/BatchNorm')), 0)
195 | 
196 | 
197 | class FCTest(tf.test.TestCase):
198 | 
199 |   def testCreateFC(self):
200 |     height, width = 3, 3
201 |     with self.test_session():
202 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
203 |       output = ops.fc(inputs, 32)
204 |       self.assertEquals(output.op.name, 'FC/Relu')
205 |       self.assertListEqual(output.get_shape().as_list(), [5, 32])
206 | 
207 |   def testCreateFCWithScope(self):
208 |     height, width = 3, 3
209 |     with self.test_session():
210 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
211 |       output = ops.fc(inputs, 32, scope='fc1')
212 |       self.assertEquals(output.op.name, 'fc1/Relu')
213 | 
214 |   def testCreateFcCreatesWeightsAndBiasesVars(self):
215 |     height, width = 3, 3
216 |     inputs = tf.random_uniform((5, height * width * 3), seed=1)
217 |     with self.test_session():
218 |       self.assertFalse(variables.get_variables('fc1/weights'))
219 |       self.assertFalse(variables.get_variables('fc1/biases'))
220 |       ops.fc(inputs, 32, scope='fc1')
221 |       self.assertTrue(variables.get_variables('fc1/weights'))
222 |       self.assertTrue(variables.get_variables('fc1/biases'))
223 | 
224 |   def testReuseVars(self):
225 |     height, width = 3, 3
226 |     inputs = tf.random_uniform((5, height * width * 3), seed=1)
227 |     with self.test_session():
228 |       ops.fc(inputs, 32, scope='fc1')
229 |       self.assertEquals(len(variables.get_variables('fc1')), 2)
230 |       ops.fc(inputs, 32, scope='fc1', reuse=True)
231 |       self.assertEquals(len(variables.get_variables('fc1')), 2)
232 | 
233 |   def testNonReuseVars(self):
234 |     height, width = 3, 3
235 |     inputs = tf.random_uniform((5, height * width * 3), seed=1)
236 |     with self.test_session():
237 |       ops.fc(inputs, 32)
238 |       self.assertEquals(len(variables.get_variables('FC')), 2)
239 |       ops.fc(inputs, 32)
240 |       self.assertEquals(len(variables.get_variables('FC')), 4)
241 | 
242 |   def testCreateFCWithoutActivation(self):
243 |     height, width = 3, 3
244 |     with self.test_session():
245 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
246 |       output = ops.fc(inputs, 32, activation=None)
247 |       self.assertEquals(output.op.name, 'FC/xw_plus_b')
248 | 
249 |   def testCreateFCWithWD(self):
250 |     height, width = 3, 3
251 |     with self.test_session() as sess:
252 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
253 |       ops.fc(inputs, 32, weight_decay=0.01)
254 |       wd = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)[0]
255 |       self.assertEquals(wd.op.name,
256 |                         'FC/weights/Regularizer/L2Regularizer/value')
257 |       sess.run(tf.initialize_all_variables())
258 |       self.assertTrue(sess.run(wd) <= 0.01)
259 | 
260 |   def testCreateFCWithoutWD(self):
261 |     height, width = 3, 3
262 |     with self.test_session():
263 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
264 |       ops.fc(inputs, 32, weight_decay=0)
265 |       self.assertEquals(
266 |           tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), [])
267 | 
268 |   def testReuseFCWithWD(self):
269 |     height, width = 3, 3
270 |     with self.test_session():
271 |       inputs = tf.random_uniform((5, height * width * 3), seed=1)
272 |       ops.fc(inputs, 32, weight_decay=0.01, scope='fc')
273 |       self.assertEquals(len(variables.get_variables()), 2)
274 |       self.assertEquals(
275 |           len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
276 |       ops.fc(inputs, 32, weight_decay=0.01, scope='fc', reuse=True)
277 |       self.assertEquals(len(variables.get_variables()), 2)
278 |       self.assertEquals(
279 |           len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
280 | 
281 |   def testFCWithBatchNorm(self):
282 |     height, width = 3, 3
283 |     with self.test_session():
284 |       images = tf.random_uniform((5, height * width * 3), seed=1)
285 |       with scopes.arg_scope([ops.fc], batch_norm_params={}):
286 |         net = ops.fc(images, 27)
287 |         net = ops.fc(net, 27)
288 |       self.assertEquals(len(variables.get_variables()), 8)
289 |       self.assertEquals(len(variables.get_variables('FC/BatchNorm')), 3)
290 |       self.assertEquals(len(variables.get_variables('FC_1/BatchNorm')), 3)
291 | 
292 |   def testReuseFCWithBatchNorm(self):
293 |     height, width = 3, 3
294 |     with self.test_session():
295 |       images = tf.random_uniform((5, height * width * 3), seed=1)
296 |       with scopes.arg_scope([ops.fc], batch_norm_params={'decay': 0.9}):
297 |         net = ops.fc(images, 27, scope='fc1')
298 |         net = ops.fc(net, 27, scope='fc1', reuse=True)
299 |       self.assertEquals(len(variables.get_variables()), 4)
300 |       self.assertEquals(len(variables.get_variables('fc1/BatchNorm')), 3)
301 | 
302 | 
303 | class MaxPoolTest(tf.test.TestCase):
304 | 
305 |   def testCreateMaxPool(self):
306 |     height, width = 3, 3
307 |     with self.test_session():
308 |       images = tf.random_uniform((5, height, width, 3), seed=1)
309 |       output = ops.max_pool(images, [3, 3])
310 |       self.assertEquals(output.op.name, 'MaxPool/MaxPool')
311 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
312 | 
313 |   def testCreateSquareMaxPool(self):
314 |     height, width = 3, 3
315 |     with self.test_session():
316 |       images = tf.random_uniform((5, height, width, 3), seed=1)
317 |       output = ops.max_pool(images, 3)
318 |       self.assertEquals(output.op.name, 'MaxPool/MaxPool')
319 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
320 | 
321 |   def testCreateMaxPoolWithScope(self):
322 |     height, width = 3, 3
323 |     with self.test_session():
324 |       images = tf.random_uniform((5, height, width, 3), seed=1)
325 |       output = ops.max_pool(images, [3, 3], scope='pool1')
326 |       self.assertEquals(output.op.name, 'pool1/MaxPool')
327 | 
328 |   def testCreateMaxPoolSAME(self):
329 |     height, width = 3, 3
330 |     with self.test_session():
331 |       images = tf.random_uniform((5, height, width, 3), seed=1)
332 |       output = ops.max_pool(images, [3, 3], padding='SAME')
333 |       self.assertListEqual(output.get_shape().as_list(), [5, 2, 2, 3])
334 | 
335 |   def testCreateMaxPoolStrideSAME(self):
336 |     height, width = 3, 3
337 |     with self.test_session():
338 |       images = tf.random_uniform((5, height, width, 3), seed=1)
339 |       output = ops.max_pool(images, [3, 3], stride=1, padding='SAME')
340 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
341 | 
342 |   def testGlobalMaxPool(self):
343 |     height, width = 3, 3
344 |     with self.test_session():
345 |       images = tf.random_uniform((5, height, width, 3), seed=1)
346 |       output = ops.max_pool(images, images.get_shape()[1:3], stride=1)
347 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
348 | 
349 | 
350 | class AvgPoolTest(tf.test.TestCase):
351 | 
352 |   def testCreateAvgPool(self):
353 |     height, width = 3, 3
354 |     with self.test_session():
355 |       images = tf.random_uniform((5, height, width, 3), seed=1)
356 |       output = ops.avg_pool(images, [3, 3])
357 |       self.assertEquals(output.op.name, 'AvgPool/AvgPool')
358 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
359 | 
360 |   def testCreateSquareAvgPool(self):
361 |     height, width = 3, 3
362 |     with self.test_session():
363 |       images = tf.random_uniform((5, height, width, 3), seed=1)
364 |       output = ops.avg_pool(images, 3)
365 |       self.assertEquals(output.op.name, 'AvgPool/AvgPool')
366 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
367 | 
368 |   def testCreateAvgPoolWithScope(self):
369 |     height, width = 3, 3
370 |     with self.test_session():
371 |       images = tf.random_uniform((5, height, width, 3), seed=1)
372 |       output = ops.avg_pool(images, [3, 3], scope='pool1')
373 |       self.assertEquals(output.op.name, 'pool1/AvgPool')
374 | 
375 |   def testCreateAvgPoolSAME(self):
376 |     height, width = 3, 3
377 |     with self.test_session():
378 |       images = tf.random_uniform((5, height, width, 3), seed=1)
379 |       output = ops.avg_pool(images, [3, 3], padding='SAME')
380 |       self.assertListEqual(output.get_shape().as_list(), [5, 2, 2, 3])
381 | 
382 |   def testCreateAvgPoolStrideSAME(self):
383 |     height, width = 3, 3
384 |     with self.test_session():
385 |       images = tf.random_uniform((5, height, width, 3), seed=1)
386 |       output = ops.avg_pool(images, [3, 3], stride=1, padding='SAME')
387 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
388 | 
389 |   def testGlobalAvgPool(self):
390 |     height, width = 3, 3
391 |     with self.test_session():
392 |       images = tf.random_uniform((5, height, width, 3), seed=1)
393 |       output = ops.avg_pool(images, images.get_shape()[1:3], stride=1)
394 |       self.assertListEqual(output.get_shape().as_list(), [5, 1, 1, 3])
395 | 
396 | 
397 | class OneHotEncodingTest(tf.test.TestCase):
398 | 
399 |   def testOneHotEncodingCreate(self):
400 |     with self.test_session():
401 |       labels = tf.constant([0, 1, 2])
402 |       output = ops.one_hot_encoding(labels, num_classes=3)
403 |       self.assertEquals(output.op.name, 'OneHotEncoding/SparseToDense')
404 |       self.assertListEqual(output.get_shape().as_list(), [3, 3])
405 | 
406 |   def testOneHotEncoding(self):
407 |     with self.test_session():
408 |       labels = tf.constant([0, 1, 2])
409 |       one_hot_labels = tf.constant([[1, 0, 0],
410 |                                     [0, 1, 0],
411 |                                     [0, 0, 1]])
412 |       output = ops.one_hot_encoding(labels, num_classes=3)
413 |       self.assertAllClose(output.eval(), one_hot_labels.eval())
414 | 
415 | 
416 | class DropoutTest(tf.test.TestCase):
417 | 
418 |   def testCreateDropout(self):
419 |     height, width = 3, 3
420 |     with self.test_session():
421 |       images = tf.random_uniform((5, height, width, 3), seed=1)
422 |       output = ops.dropout(images)
423 |       self.assertEquals(output.op.name, 'Dropout/dropout/mul_1')
424 |       output.get_shape().assert_is_compatible_with(images.get_shape())
425 | 
426 |   def testCreateDropoutNoTraining(self):
427 |     height, width = 3, 3
428 |     with self.test_session():
429 |       images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
430 |       output = ops.dropout(images, is_training=False)
431 |       self.assertEquals(output, images)
432 | 
433 | 
434 | class FlattenTest(tf.test.TestCase):
435 | 
436 |   def testFlatten4D(self):
437 |     height, width = 3, 3
438 |     with self.test_session():
439 |       images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
440 |       output = ops.flatten(images)
441 |       self.assertEquals(output.get_shape().num_elements(),
442 |                         images.get_shape().num_elements())
443 |       self.assertEqual(output.get_shape()[0], images.get_shape()[0])
444 | 
445 |   def testFlatten3D(self):
446 |     height, width = 3, 3
447 |     with self.test_session():
448 |       images = tf.random_uniform((5, height, width), seed=1, name='images')
449 |       output = ops.flatten(images)
450 |       self.assertEquals(output.get_shape().num_elements(),
451 |                         images.get_shape().num_elements())
452 |       self.assertEqual(output.get_shape()[0], images.get_shape()[0])
453 | 
454 |   def testFlattenBatchSize(self):
455 |     height, width = 3, 3
456 |     with self.test_session() as sess:
457 |       images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
458 |       inputs = tf.placeholder(tf.int32, (None, height, width, 3))
459 |       output = ops.flatten(inputs)
460 |       self.assertEquals(output.get_shape().as_list(),
461 |                         [None, height * width * 3])
462 |       output = sess.run(output, {inputs: images.eval()})
463 |       self.assertEquals(output.size,
464 |                         images.get_shape().num_elements())
465 |       self.assertEqual(output.shape[0], images.get_shape()[0])
466 | 
467 | 
468 | class BatchNormTest(tf.test.TestCase):
469 | 
470 |   def testCreateOp(self):
471 |     height, width = 3, 3
472 |     with self.test_session():
473 |       images = tf.random_uniform((5, height, width, 3), seed=1)
474 |       output = ops.batch_norm(images)
475 |       self.assertTrue(output.op.name.startswith('BatchNorm/batchnorm'))
476 |       self.assertListEqual(output.get_shape().as_list(), [5, height, width, 3])
477 | 
478 |   def testCreateVariables(self):
479 |     height, width = 3, 3
480 |     with self.test_session():
481 |       images = tf.random_uniform((5, height, width, 3), seed=1)
482 |       ops.batch_norm(images)
483 |       beta = variables.get_variables_by_name('beta')[0]
484 |       self.assertEquals(beta.op.name, 'BatchNorm/beta')
485 |       gamma = variables.get_variables_by_name('gamma')
486 |       self.assertEquals(gamma, [])
487 |       moving_mean = tf.moving_average_variables()[0]
488 |       moving_variance = tf.moving_average_variables()[1]
489 |       self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
490 |       self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
491 | 
492 |   def testCreateVariablesWithScale(self):
493 |     height, width = 3, 3
494 |     with self.test_session():
495 |       images = tf.random_uniform((5, height, width, 3), seed=1)
496 |       ops.batch_norm(images, scale=True)
497 |       beta = variables.get_variables_by_name('beta')[0]
498 |       gamma = variables.get_variables_by_name('gamma')[0]
499 |       self.assertEquals(beta.op.name, 'BatchNorm/beta')
500 |       self.assertEquals(gamma.op.name, 'BatchNorm/gamma')
501 |       moving_mean = tf.moving_average_variables()[0]
502 |       moving_variance = tf.moving_average_variables()[1]
503 |       self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
504 |       self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
505 | 
506 |   def testCreateVariablesWithoutCenterWithScale(self):
507 |     height, width = 3, 3
508 |     with self.test_session():
509 |       images = tf.random_uniform((5, height, width, 3), seed=1)
510 |       ops.batch_norm(images, center=False, scale=True)
511 |       beta = variables.get_variables_by_name('beta')
512 |       self.assertEquals(beta, [])
513 |       gamma = variables.get_variables_by_name('gamma')[0]
514 |       self.assertEquals(gamma.op.name, 'BatchNorm/gamma')
515 |       moving_mean = tf.moving_average_variables()[0]
516 |       moving_variance = tf.moving_average_variables()[1]
517 |       self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
518 |       self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
519 | 
520 |   def testCreateVariablesWithoutCenterWithoutScale(self):
521 |     height, width = 3, 3
522 |     with self.test_session():
523 |       images = tf.random_uniform((5, height, width, 3), seed=1)
524 |       ops.batch_norm(images, center=False, scale=False)
525 |       beta = variables.get_variables_by_name('beta')
526 |       self.assertEquals(beta, [])
527 |       gamma = variables.get_variables_by_name('gamma')
528 |       self.assertEquals(gamma, [])
529 |       moving_mean = tf.moving_average_variables()[0]
530 |       moving_variance = tf.moving_average_variables()[1]
531 |       self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
532 |       self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
533 | 
534 |   def testMovingAverageVariables(self):
535 |     height, width = 3, 3
536 |     with self.test_session():
537 |       images = tf.random_uniform((5, height, width, 3), seed=1)
538 |       ops.batch_norm(images, scale=True)
539 |       moving_mean = tf.moving_average_variables()[0]
540 |       moving_variance = tf.moving_average_variables()[1]
541 |       self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean')
542 |       self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
543 | 
544 |   def testUpdateOps(self):
545 |     height, width = 3, 3
546 |     with self.test_session():
547 |       images = tf.random_uniform((5, height, width, 3), seed=1)
548 |       ops.batch_norm(images)
549 |       update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
550 |       update_moving_mean = update_ops[0]
551 |       update_moving_variance = update_ops[1]
552 |       self.assertEquals(update_moving_mean.op.name,
553 |                         'BatchNorm/AssignMovingAvg')
554 |       self.assertEquals(update_moving_variance.op.name,
555 |                         'BatchNorm/AssignMovingAvg_1')
556 | 
557 |   def testReuseVariables(self):
558 |     height, width = 3, 3
559 |     with self.test_session():
560 |       images = tf.random_uniform((5, height, width, 3), seed=1)
561 |       ops.batch_norm(images, scale=True, scope='bn')
562 |       ops.batch_norm(images, scale=True, scope='bn', reuse=True)
563 |       beta = variables.get_variables_by_name('beta')
564 |       gamma = variables.get_variables_by_name('gamma')
565 |       self.assertEquals(len(beta), 1)
566 |       self.assertEquals(len(gamma), 1)
567 |       moving_vars = tf.get_collection('moving_vars')
568 |       self.assertEquals(len(moving_vars), 2)
569 | 
570 |   def testReuseUpdateOps(self):
571 |     height, width = 3, 3
572 |     with self.test_session():
573 |       images = tf.random_uniform((5, height, width, 3), seed=1)
574 |       ops.batch_norm(images, scope='bn')
575 |       self.assertEquals(len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 2)
576 |       ops.batch_norm(images, scope='bn', reuse=True)
577 |       self.assertEquals(len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 4)
578 | 
579 |   def testCreateMovingVars(self):
580 |     height, width = 3, 3
581 |     with self.test_session():
582 |       images = tf.random_uniform((5, height, width, 3), seed=1)
583 |       _ = ops.batch_norm(images, moving_vars='moving_vars')
584 |       moving_mean = tf.get_collection('moving_vars',
585 |                                       'BatchNorm/moving_mean')
586 |       self.assertEquals(len(moving_mean), 1)
587 |       self.assertEquals(moving_mean[0].op.name, 'BatchNorm/moving_mean')
588 |       moving_variance = tf.get_collection('moving_vars',
589 |                                           'BatchNorm/moving_variance')
590 |       self.assertEquals(len(moving_variance), 1)
591 |       self.assertEquals(moving_variance[0].op.name, 'BatchNorm/moving_variance')
592 | 
593 |   def testComputeMovingVars(self):
594 |     height, width = 3, 3
595 |     with self.test_session() as sess:
596 |       image_shape = (10, height, width, 3)
597 |       image_values = np.random.rand(*image_shape)
598 |       expected_mean = np.mean(image_values, axis=(0, 1, 2))
599 |       expected_var = np.var(image_values, axis=(0, 1, 2))
600 |       images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
601 |       output = ops.batch_norm(images, decay=0.1)
602 |       update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
603 |       with tf.control_dependencies(update_ops):
604 |         barrier = tf.no_op(name='gradient_barrier')
605 |         output = control_flow_ops.with_dependencies([barrier], output)
606 |       # Initialize all variables
607 |       sess.run(tf.initialize_all_variables())
608 |       moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
609 |       moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
610 |       mean, variance = sess.run([moving_mean, moving_variance])
611 |       # After initialization moving_mean == 0 and moving_variance == 1.
612 |       self.assertAllClose(mean, [0] * 3)
613 |       self.assertAllClose(variance, [1] * 3)
614 |       for _ in range(10):
615 |         sess.run([output])
616 |       mean = moving_mean.eval()
617 |       variance = moving_variance.eval()
618 |       # After 10 updates with decay 0.1 moving_mean == expected_mean and
619 |       # moving_variance == expected_var.
620 |       self.assertAllClose(mean, expected_mean)
621 |       self.assertAllClose(variance, expected_var)
622 | 
623 |   def testEvalMovingVars(self):
624 |     height, width = 3, 3
625 |     with self.test_session() as sess:
626 |       image_shape = (10, height, width, 3)
627 |       image_values = np.random.rand(*image_shape)
628 |       expected_mean = np.mean(image_values, axis=(0, 1, 2))
629 |       expected_var = np.var(image_values, axis=(0, 1, 2))
630 |       images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
631 |       output = ops.batch_norm(images, decay=0.1, is_training=False)
632 |       update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
633 |       with tf.control_dependencies(update_ops):
634 |         barrier = tf.no_op(name='gradient_barrier')
635 |         output = control_flow_ops.with_dependencies([barrier], output)
636 |       # Initialize all variables
637 |       sess.run(tf.initialize_all_variables())
638 |       moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
639 |       moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
640 |       mean, variance = sess.run([moving_mean, moving_variance])
641 |       # After initialization moving_mean == 0 and moving_variance == 1.
642 |       self.assertAllClose(mean, [0] * 3)
643 |       self.assertAllClose(variance, [1] * 3)
644 |       # Simulate assigment from saver restore.
645 |       init_assigns = [tf.assign(moving_mean, expected_mean),
646 |                       tf.assign(moving_variance, expected_var)]
647 |       sess.run(init_assigns)
648 |       for _ in range(10):
649 |         sess.run([output], {images: np.random.rand(*image_shape)})
650 |       mean = moving_mean.eval()
651 |       variance = moving_variance.eval()
652 |       # Although we feed different images, the moving_mean and moving_variance
653 |       # shouldn't change.
654 |       self.assertAllClose(mean, expected_mean)
655 |       self.assertAllClose(variance, expected_var)
656 | 
657 |   def testReuseVars(self):
658 |     height, width = 3, 3
659 |     with self.test_session() as sess:
660 |       image_shape = (10, height, width, 3)
661 |       image_values = np.random.rand(*image_shape)
662 |       expected_mean = np.mean(image_values, axis=(0, 1, 2))
663 |       expected_var = np.var(image_values, axis=(0, 1, 2))
664 |       images = tf.constant(image_values, shape=image_shape, dtype=tf.float32)
665 |       output = ops.batch_norm(images, decay=0.1, is_training=False)
666 |       update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION)
667 |       with tf.control_dependencies(update_ops):
668 |         barrier = tf.no_op(name='gradient_barrier')
669 |         output = control_flow_ops.with_dependencies([barrier], output)
670 |       # Initialize all variables
671 |       sess.run(tf.initialize_all_variables())
672 |       moving_mean = variables.get_variables('BatchNorm/moving_mean')[0]
673 |       moving_variance = variables.get_variables('BatchNorm/moving_variance')[0]
674 |       mean, variance = sess.run([moving_mean, moving_variance])
675 |       # After initialization moving_mean == 0 and moving_variance == 1.
676 |       self.assertAllClose(mean, [0] * 3)
677 |       self.assertAllClose(variance, [1] * 3)
678 |       # Simulate assigment from saver restore.
679 |       init_assigns = [tf.assign(moving_mean, expected_mean),
680 |                       tf.assign(moving_variance, expected_var)]
681 |       sess.run(init_assigns)
682 |       for _ in range(10):
683 |         sess.run([output], {images: np.random.rand(*image_shape)})
684 |       mean = moving_mean.eval()
685 |       variance = moving_variance.eval()
686 |       # Although we feed different images, the moving_mean and moving_variance
687 |       # shouldn't change.
688 |       self.assertAllClose(mean, expected_mean)
689 |       self.assertAllClose(variance, expected_var)
690 | 
691 | if __name__ == '__main__':
692 |   tf.test.main()
693 | 


--------------------------------------------------------------------------------
/slim/scopes.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Contains the new arg_scope used for TF-Slim ops.
 16 | 
 17 |   Allows one to define models much more compactly by eliminating boilerplate
 18 |   code. This is accomplished through the use of argument scoping (arg_scope).
 19 | 
 20 |   Example of how to use scopes.arg_scope:
 21 | 
 22 |   with scopes.arg_scope(ops.conv2d, padding='SAME',
 23 |                       stddev=0.01, weight_decay=0.0005):
 24 |     net = ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1')
 25 |     net = ops.conv2d(net, 256, [5, 5], scope='conv2')
 26 | 
 27 |   The first call to conv2d will use predefined args:
 28 |     ops.conv2d(inputs, 64, [11, 11], 4, padding='VALID',
 29 |               stddev=0.01, weight_decay=0.0005, scope='conv1')
 30 | 
 31 |   The second call to Conv will overwrite padding:
 32 |     ops.conv2d(inputs, 256, [5, 5], padding='SAME',
 33 |                stddev=0.01, weight_decay=0.0005, scope='conv2')
 34 | 
 35 |   Example of how to reuse an arg_scope:
 36 |   with scopes.arg_scope(ops.conv2d, padding='SAME',
 37 |                       stddev=0.01, weight_decay=0.0005) as conv2d_arg_scope:
 38 |     net = ops.conv2d(net, 256, [5, 5], scope='conv1')
 39 |     ....
 40 | 
 41 |   with scopes.arg_scope(conv2d_arg_scope):
 42 |     net = ops.conv2d(net, 256, [5, 5], scope='conv2')
 43 | 
 44 |   Example of how to use scopes.add_arg_scope:
 45 | 
 46 |   @scopes.add_arg_scope
 47 |   def conv2d(*args, **kwargs)
 48 | """
 49 | from __future__ import absolute_import
 50 | from __future__ import division
 51 | from __future__ import print_function
 52 | 
 53 | import contextlib
 54 | import functools
 55 | 
 56 | from tensorflow.python.framework import ops
 57 | 
 58 | _ARGSTACK_KEY = ("__arg_stack",)
 59 | 
 60 | _DECORATED_OPS = set()
 61 | 
 62 | 
 63 | def _get_arg_stack():
 64 |   stack = ops.get_collection(_ARGSTACK_KEY)
 65 |   if stack:
 66 |     return stack[0]
 67 |   else:
 68 |     stack = [{}]
 69 |     ops.add_to_collection(_ARGSTACK_KEY, stack)
 70 |     return stack
 71 | 
 72 | 
 73 | def _current_arg_scope():
 74 |   stack = _get_arg_stack()
 75 |   return stack[-1]
 76 | 
 77 | 
 78 | def _add_op(op):
 79 |   key_op = (op.__module__, op.__name__)
 80 |   if key_op not in _DECORATED_OPS:
 81 |     _DECORATED_OPS.add(key_op)
 82 | 
 83 | 
 84 | @contextlib.contextmanager
 85 | def arg_scope(list_ops_or_scope, **kwargs):
 86 |   """Stores the default arguments for the given set of list_ops.
 87 | 
 88 |   For usage, please see examples at top of the file.
 89 | 
 90 |   Args:
 91 |     list_ops_or_scope: List or tuple of operations to set argument scope for or
 92 |       a dictionary containg the current scope. When list_ops_or_scope is a dict,
 93 |       kwargs must be empty. When list_ops_or_scope is a list or tuple, then
 94 |       every op in it need to be decorated with @add_arg_scope to work.
 95 |     **kwargs: keyword=value that will define the defaults for each op in
 96 |               list_ops. All the ops need to accept the given set of arguments.
 97 | 
 98 |   Yields:
 99 |     the current_scope, which is a dictionary of {op: {arg: value}}
100 |   Raises:
101 |     TypeError: if list_ops is not a list or a tuple.
102 |     ValueError: if any op in list_ops has not be decorated with @add_arg_scope.
103 |   """
104 |   if isinstance(list_ops_or_scope, dict):
105 |     # Assumes that list_ops_or_scope is a scope that is being reused.
106 |     if kwargs:
107 |       raise ValueError("When attempting to re-use a scope by suppling a"
108 |                        "dictionary, kwargs must be empty.")
109 |     current_scope = list_ops_or_scope.copy()
110 |     try:
111 |       _get_arg_stack().append(current_scope)
112 |       yield current_scope
113 |     finally:
114 |       _get_arg_stack().pop()
115 |   else:
116 |     # Assumes that list_ops_or_scope is a list/tuple of ops with kwargs.
117 |     if not isinstance(list_ops_or_scope, (list, tuple)):
118 |       raise TypeError("list_ops_or_scope must either be a list/tuple or reused"
119 |                       "scope (i.e. dict)")
120 |     try:
121 |       current_scope = _current_arg_scope().copy()
122 |       for op in list_ops_or_scope:
123 |         key_op = (op.__module__, op.__name__)
124 |         if not has_arg_scope(op):
125 |           raise ValueError("%s is not decorated with @add_arg_scope", key_op)
126 |         if key_op in current_scope:
127 |           current_kwargs = current_scope[key_op].copy()
128 |           current_kwargs.update(kwargs)
129 |           current_scope[key_op] = current_kwargs
130 |         else:
131 |           current_scope[key_op] = kwargs.copy()
132 |       _get_arg_stack().append(current_scope)
133 |       yield current_scope
134 |     finally:
135 |       _get_arg_stack().pop()
136 | 
137 | 
138 | def add_arg_scope(func):
139 |   """Decorates a function with args so it can be used within an arg_scope.
140 | 
141 |   Args:
142 |     func: function to decorate.
143 | 
144 |   Returns:
145 |     A tuple with the decorated function func_with_args().
146 |   """
147 |   @functools.wraps(func)
148 |   def func_with_args(*args, **kwargs):
149 |     current_scope = _current_arg_scope()
150 |     current_args = kwargs
151 |     key_func = (func.__module__, func.__name__)
152 |     if key_func in current_scope:
153 |       current_args = current_scope[key_func].copy()
154 |       current_args.update(kwargs)
155 |     return func(*args, **current_args)
156 |   _add_op(func)
157 |   return func_with_args
158 | 
159 | 
160 | def has_arg_scope(func):
161 |   """Checks whether a func has been decorated with @add_arg_scope or not.
162 | 
163 |   Args:
164 |     func: function to check.
165 | 
166 |   Returns:
167 |     a boolean.
168 |   """
169 |   key_op = (func.__module__, func.__name__)
170 |   return key_op in _DECORATED_OPS
171 | 


--------------------------------------------------------------------------------
/slim/scopes_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests slim.scopes."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | 
 21 | import tensorflow as tf
 22 | from inception.slim import scopes
 23 | 
 24 | 
 25 | @scopes.add_arg_scope
 26 | def func1(*args, **kwargs):
 27 |   return (args, kwargs)
 28 | 
 29 | 
 30 | @scopes.add_arg_scope
 31 | def func2(*args, **kwargs):
 32 |   return (args, kwargs)
 33 | 
 34 | 
 35 | class ArgScopeTest(tf.test.TestCase):
 36 | 
 37 |   def testEmptyArgScope(self):
 38 |     with self.test_session():
 39 |       self.assertEqual(scopes._current_arg_scope(), {})
 40 | 
 41 |   def testCurrentArgScope(self):
 42 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 43 |     key_op = (func1.__module__, func1.__name__)
 44 |     current_scope = {key_op: func1_kwargs.copy()}
 45 |     with self.test_session():
 46 |       with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope:
 47 |         self.assertDictEqual(scope, current_scope)
 48 | 
 49 |   def testCurrentArgScopeNested(self):
 50 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 51 |     func2_kwargs = {'b': 2, 'd': [2]}
 52 |     key = lambda f: (f.__module__, f.__name__)
 53 |     current_scope = {key(func1): func1_kwargs.copy(),
 54 |                      key(func2): func2_kwargs.copy()}
 55 |     with self.test_session():
 56 |       with scopes.arg_scope([func1], a=1, b=None, c=[1]):
 57 |         with scopes.arg_scope([func2], b=2, d=[2]) as scope:
 58 |           self.assertDictEqual(scope, current_scope)
 59 | 
 60 |   def testReuseArgScope(self):
 61 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 62 |     key_op = (func1.__module__, func1.__name__)
 63 |     current_scope = {key_op: func1_kwargs.copy()}
 64 |     with self.test_session():
 65 |       with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope1:
 66 |         pass
 67 |       with scopes.arg_scope(scope1) as scope:
 68 |         self.assertDictEqual(scope, current_scope)
 69 | 
 70 |   def testReuseArgScopeNested(self):
 71 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 72 |     func2_kwargs = {'b': 2, 'd': [2]}
 73 |     key = lambda f: (f.__module__, f.__name__)
 74 |     current_scope1 = {key(func1): func1_kwargs.copy()}
 75 |     current_scope2 = {key(func1): func1_kwargs.copy(),
 76 |                       key(func2): func2_kwargs.copy()}
 77 |     with self.test_session():
 78 |       with scopes.arg_scope([func1], a=1, b=None, c=[1]) as scope1:
 79 |         with scopes.arg_scope([func2], b=2, d=[2]) as scope2:
 80 |           pass
 81 |       with scopes.arg_scope(scope1):
 82 |         self.assertDictEqual(scopes._current_arg_scope(), current_scope1)
 83 |       with scopes.arg_scope(scope2):
 84 |         self.assertDictEqual(scopes._current_arg_scope(), current_scope2)
 85 | 
 86 |   def testSimpleArgScope(self):
 87 |     func1_args = (0,)
 88 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 89 |     with self.test_session():
 90 |       with scopes.arg_scope([func1], a=1, b=None, c=[1]):
 91 |         args, kwargs = func1(0)
 92 |         self.assertTupleEqual(args, func1_args)
 93 |         self.assertDictEqual(kwargs, func1_kwargs)
 94 | 
 95 |   def testSimpleArgScopeWithTuple(self):
 96 |     func1_args = (0,)
 97 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
 98 |     with self.test_session():
 99 |       with scopes.arg_scope((func1,), a=1, b=None, c=[1]):
100 |         args, kwargs = func1(0)
101 |         self.assertTupleEqual(args, func1_args)
102 |         self.assertDictEqual(kwargs, func1_kwargs)
103 | 
104 |   def testOverwriteArgScope(self):
105 |     func1_args = (0,)
106 |     func1_kwargs = {'a': 1, 'b': 2, 'c': [1]}
107 |     with scopes.arg_scope([func1], a=1, b=None, c=[1]):
108 |       args, kwargs = func1(0, b=2)
109 |       self.assertTupleEqual(args, func1_args)
110 |       self.assertDictEqual(kwargs, func1_kwargs)
111 | 
112 |   def testNestedArgScope(self):
113 |     func1_args = (0,)
114 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
115 |     with scopes.arg_scope([func1], a=1, b=None, c=[1]):
116 |       args, kwargs = func1(0)
117 |       self.assertTupleEqual(args, func1_args)
118 |       self.assertDictEqual(kwargs, func1_kwargs)
119 |       func1_kwargs['b'] = 2
120 |       with scopes.arg_scope([func1], b=2):
121 |         args, kwargs = func1(0)
122 |         self.assertTupleEqual(args, func1_args)
123 |         self.assertDictEqual(kwargs, func1_kwargs)
124 | 
125 |   def testSharedArgScope(self):
126 |     func1_args = (0,)
127 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
128 |     with scopes.arg_scope([func1, func2], a=1, b=None, c=[1]):
129 |       args, kwargs = func1(0)
130 |       self.assertTupleEqual(args, func1_args)
131 |       self.assertDictEqual(kwargs, func1_kwargs)
132 |       args, kwargs = func2(0)
133 |       self.assertTupleEqual(args, func1_args)
134 |       self.assertDictEqual(kwargs, func1_kwargs)
135 | 
136 |   def testSharedArgScopeTuple(self):
137 |     func1_args = (0,)
138 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
139 |     with scopes.arg_scope((func1, func2), a=1, b=None, c=[1]):
140 |       args, kwargs = func1(0)
141 |       self.assertTupleEqual(args, func1_args)
142 |       self.assertDictEqual(kwargs, func1_kwargs)
143 |       args, kwargs = func2(0)
144 |       self.assertTupleEqual(args, func1_args)
145 |       self.assertDictEqual(kwargs, func1_kwargs)
146 | 
147 |   def testPartiallySharedArgScope(self):
148 |     func1_args = (0,)
149 |     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
150 |     func2_args = (1,)
151 |     func2_kwargs = {'a': 1, 'b': None, 'd': [2]}
152 |     with scopes.arg_scope([func1, func2], a=1, b=None):
153 |       with scopes.arg_scope([func1], c=[1]), scopes.arg_scope([func2], d=[2]):
154 |         args, kwargs = func1(0)
155 |         self.assertTupleEqual(args, func1_args)
156 |         self.assertDictEqual(kwargs, func1_kwargs)
157 |         args, kwargs = func2(1)
158 |         self.assertTupleEqual(args, func2_args)
159 |         self.assertDictEqual(kwargs, func2_kwargs)
160 | 
161 | if __name__ == '__main__':
162 |   tf.test.main()
163 | 


--------------------------------------------------------------------------------
/slim/slim.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2016 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | # http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 | """TF-Slim grouped API. Please see README.md for details and usage."""
16 | # pylint: disable=unused-import
17 | 
18 | # Collapse tf-slim into a single namespace.
19 | import inception_model as inception
20 | import losses
21 | import ops
22 | import scopes
23 | import variables
24 | from scopes import arg_scope
25 | 


--------------------------------------------------------------------------------
/slim/variables.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Contains convenience wrappers for creating variables in TF-Slim.
 16 | 
 17 | The variables module is typically used for defining model variables from the
 18 | ops routines (see slim.ops). Such variables are used for training, evaluation
 19 | and inference of models.
 20 | 
 21 | All the variables created through this module would be added to the
 22 | MODEL_VARIABLES collection, if you create a model variable outside slim, it can
 23 | be added with slim.variables.add_variable(external_variable, reuse).
 24 | 
 25 | Usage:
 26 |   weights_initializer = tf.truncated_normal_initializer(stddev=0.01)
 27 |   l2_regularizer = lambda t: losses.l2_loss(t, weight=0.0005)
 28 |   weights = variables.variable('weights',
 29 |                                shape=[100, 100],
 30 |                                initializer=weights_initializer,
 31 |                                regularizer=l2_regularizer,
 32 |                                device='/cpu:0')
 33 | 
 34 |   biases = variables.variable('biases',
 35 |                               shape=[100],
 36 |                               initializer=tf.zeros_initializer,
 37 |                               device='/cpu:0')
 38 | 
 39 |   # More complex example.
 40 | 
 41 |   net = slim.ops.conv2d(input, 32, [3, 3], scope='conv1')
 42 |   net = slim.ops.conv2d(net, 64, [3, 3], scope='conv2')
 43 |   with slim.arg_scope([variables.variable], restore=False):
 44 |     net = slim.ops.conv2d(net, 64, [3, 3], scope='conv3')
 45 | 
 46 |   # Get all model variables from all the layers.
 47 |   model_variables = slim.variables.get_variables()
 48 | 
 49 |   # Get all model variables from a specific the layer, i.e 'conv1'.
 50 |   conv1_variables = slim.variables.get_variables('conv1')
 51 | 
 52 |   # Get all weights from all the layers.
 53 |   weights = slim.variables.get_variables_by_name('weights')
 54 | 
 55 |   # Get all bias from all the layers.
 56 |   biases = slim.variables.get_variables_by_name('biases')
 57 | 
 58 |   # Get all variables to restore.
 59 |   # (i.e. only those created by 'conv1' and 'conv2')
 60 |   variables_to_restore = slim.variables.get_variables_to_restore()
 61 | 
 62 | ************************************************
 63 | * Initializing model variables from a checkpoint
 64 | ************************************************
 65 | 
 66 | # Create some variables.
 67 | v1 = slim.variables.variable(name="v1", ..., restore=False)
 68 | v2 = slim.variables.variable(name="v2", ...) # By default restore=True
 69 | ...
 70 | # The list of variables to restore should only contain 'v2'.
 71 | variables_to_restore = slim.variables.get_variables_to_restore()
 72 | restorer = tf.train.Saver(variables_to_restore)
 73 | with tf.Session() as sess:
 74 |   # Restore variables from disk.
 75 |   restorer.restore(sess, "/tmp/model.ckpt")
 76 |   print("Model restored.")
 77 |   # Do some work with the model
 78 |   ...
 79 | 
 80 | """
 81 | from __future__ import division
 82 | from __future__ import print_function
 83 | 
 84 | import tensorflow as tf
 85 | 
 86 | from tensorflow.core.framework import graph_pb2
 87 | import scopes
 88 | 
 89 | # Collection containing all the variables created using slim.variables
 90 | MODEL_VARIABLES = '_model_variables_'
 91 | 
 92 | # Collection containing the slim.variables that are created with restore=True.
 93 | VARIABLES_TO_RESTORE = '_variables_to_restore_'
 94 | 
 95 | 
 96 | def add_variable(var, restore=True):
 97 |   """Adds a variable to the MODEL_VARIABLES collection.
 98 | 
 99 |     Optionally it will add the variable to  the VARIABLES_TO_RESTORE collection.
100 |   Args:
101 |     var: a variable.
102 |     restore: whether the variable should be added to the
103 |       VARIABLES_TO_RESTORE collection.
104 | 
105 |   """
106 |   collections = [MODEL_VARIABLES]
107 |   if restore:
108 |     collections.append(VARIABLES_TO_RESTORE)
109 |   for collection in collections:
110 |     if var not in tf.get_collection(collection):
111 |       tf.add_to_collection(collection, var)
112 | 
113 | 
114 | def get_variables(scope=None, suffix=None):
115 |   """Gets the list of variables, filtered by scope and/or suffix.
116 | 
117 |   Args:
118 |     scope: an optional scope for filtering the variables to return.
119 |     suffix: an optional suffix for filtering the variables to return.
120 | 
121 |   Returns:
122 |     a copied list of variables with scope and suffix.
123 |   """
124 |   candidates = tf.get_collection(MODEL_VARIABLES, scope)[:]
125 |   if suffix is not None:
126 |     candidates = [var for var in candidates if var.op.name.endswith(suffix)]
127 |   return candidates
128 | 
129 | 
130 | def get_variables_to_restore():
131 |   """Gets the list of variables to restore.
132 | 
133 |   Returns:
134 |     a copied list of variables.
135 |   """
136 |   return tf.get_collection(VARIABLES_TO_RESTORE)[:]
137 | 
138 | 
139 | def get_variables_by_name(given_name, scope=None):
140 |   """Gets the list of variables that were given that name.
141 | 
142 |   Args:
143 |     given_name: name given to the variable without scope.
144 |     scope: an optional scope for filtering the variables to return.
145 | 
146 |   Returns:
147 |     a copied list of variables with the given name and prefix.
148 |   """
149 |   return get_variables(scope=scope, suffix=given_name)
150 | 
151 | 
152 | def get_unique_variable(name):
153 |   """Gets the variable uniquely identified by that name.
154 | 
155 |   Args:
156 |     name: a name that uniquely identifies the variable.
157 | 
158 |   Returns:
159 |     a tensorflow variable.
160 | 
161 |   Raises:
162 |     ValueError: if no variable uniquely identified by the name exists.
163 |   """
164 |   candidates = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, name)
165 |   if not candidates:
166 |     raise ValueError('Couldnt find variable %s' % name)
167 | 
168 |   for candidate in candidates:
169 |     if candidate.op.name == name:
170 |       return candidate
171 |   raise ValueError('Variable %s does not uniquely identify a variable', name)
172 | 
173 | 
174 | class VariableDeviceChooser(object):
175 |   """Slim device chooser for variables.
176 | 
177 |   When using a parameter server it will assign them in a round-robin fashion.
178 |   When not using a parameter server it allows GPU:0 placement otherwise CPU:0.
179 |   """
180 | 
181 |   def __init__(self,
182 |                num_parameter_servers=0,
183 |                ps_device='/job:ps',
184 |                placement='CPU:0'):
185 |     """Initialize VariableDeviceChooser.
186 | 
187 |     Args:
188 |       num_parameter_servers: number of parameter servers.
189 |       ps_device: string representing the parameter server device.
190 |       placement: string representing the placement of the variable either CPU:0
191 |         or GPU:0. When using parameter servers forced to CPU:0.
192 |     """
193 |     self._num_ps = num_parameter_servers
194 |     self._ps_device = ps_device
195 |     self._placement = placement if num_parameter_servers == 0 else 'CPU:0'
196 |     self._next_task_id = 0
197 | 
198 |   def __call__(self, op):
199 |     device_string = ''
200 |     if self._num_ps > 0:
201 |       task_id = self._next_task_id
202 |       self._next_task_id = (self._next_task_id + 1) % self._num_ps
203 |       device_string = '%s/task:%d' % (self._ps_device, task_id)
204 |     device_string += '/%s' % self._placement
205 |     return device_string
206 | 
207 | 
208 | # TODO(sguada) Remove once get_variable is able to colocate op.devices.
209 | def variable_device(device, name):
210 |   """Fix the variable device to colocate its ops."""
211 |   if callable(device):
212 |     var_name = tf.get_variable_scope().name + '/' + name
213 |     var_def = graph_pb2.NodeDef(name=var_name, op='Variable')
214 |     device = device(var_def)
215 |   if device is None:
216 |     device = ''
217 |   return device
218 | 
219 | 
220 | @scopes.add_arg_scope
221 | def global_step(device=''):
222 |   """Returns the global step variable.
223 | 
224 |   Args:
225 |     device: Optional device to place the variable. It can be an string or a
226 |       function that is called to get the device for the variable.
227 | 
228 |   Returns:
229 |     the tensor representing the global step variable.
230 |   """
231 |   global_step_ref = tf.get_collection(tf.GraphKeys.GLOBAL_STEP)
232 |   if global_step_ref:
233 |     return global_step_ref[0]
234 |   else:
235 |     collections = [
236 |         VARIABLES_TO_RESTORE,
237 |         tf.GraphKeys.GLOBAL_VARIABLES,
238 |         tf.GraphKeys.GLOBAL_STEP,
239 |     ]
240 |     # Get the device for the variable.
241 |     with tf.device(variable_device(device, 'global_step')):
242 |       return tf.get_variable('global_step', shape=[], dtype=tf.int64,
243 |                              initializer=tf.zeros_initializer,
244 |                              trainable=False, collections=collections)
245 | 
246 | 
247 | @scopes.add_arg_scope
248 | def variable(name, shape=None, dtype=tf.float32, initializer=None,
249 |              regularizer=None, trainable=True, collections=None, device='',
250 |              restore=True):
251 |   """Gets an existing variable with these parameters or creates a new one.
252 | 
253 |     It also add itself to a group with its name.
254 | 
255 |   Args:
256 |     name: the name of the new or existing variable.
257 |     shape: shape of the new or existing variable.
258 |     dtype: type of the new or existing variable (defaults to `DT_FLOAT`).
259 |     initializer: initializer for the variable if one is created.
260 |     regularizer: a (Tensor -> Tensor or None) function; the result of
261 |         applying it on a newly created variable will be added to the collection
262 |         GraphKeys.REGULARIZATION_LOSSES and can be used for regularization.
263 |     trainable: If `True` also add the variable to the graph collection
264 |       `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
265 |     collections: A list of collection names to which the Variable will be added.
266 |       Note that the variable is always also added to the tf.GraphKeys.GLOBAL_VARIABLES
267 |       and MODEL_VARIABLES collections.
268 |     device: Optional device to place the variable. It can be an string or a
269 |       function that is called to get the device for the variable.
270 |     restore: whether the variable should be added to the
271 |       VARIABLES_TO_RESTORE collection.
272 | 
273 |   Returns:
274 |     The created or existing variable.
275 |   """
276 |   collections = list(collections or [])
277 | 
278 |   # Make sure variables are added to tf.GraphKeys.GLOBAL_VARIABLES and MODEL_VARIABLES
279 |   collections += [tf.GraphKeys.GLOBAL_VARIABLES, MODEL_VARIABLES]
280 |   # Add to VARIABLES_TO_RESTORE if necessary
281 |   if restore:
282 |     collections.append(VARIABLES_TO_RESTORE)
283 |   # Remove duplicates
284 |   collections = set(collections)
285 |   # Get the device for the variable.
286 |   with tf.device(variable_device(device, name)):
287 |     return tf.get_variable(name, shape=shape, dtype=dtype,
288 |                            initializer=initializer, regularizer=regularizer,
289 |                            trainable=trainable, collections=collections)
290 | 


--------------------------------------------------------------------------------
/slim/variables_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2016 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | # http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | # ==============================================================================
 15 | """Tests for slim.variables."""
 16 | from __future__ import absolute_import
 17 | from __future__ import division
 18 | from __future__ import print_function
 19 | 
 20 | import tensorflow as tf
 21 | 
 22 | from inception.slim import scopes
 23 | from inception.slim import variables
 24 | 
 25 | 
 26 | class VariablesTest(tf.test.TestCase):
 27 | 
 28 |   def testCreateVariable(self):
 29 |     with self.test_session():
 30 |       with tf.variable_scope('A'):
 31 |         a = variables.variable('a', [5])
 32 |         self.assertEquals(a.op.name, 'A/a')
 33 |         self.assertListEqual(a.get_shape().as_list(), [5])
 34 | 
 35 |   def testGetVariables(self):
 36 |     with self.test_session():
 37 |       with tf.variable_scope('A'):
 38 |         a = variables.variable('a', [5])
 39 |       with tf.variable_scope('B'):
 40 |         b = variables.variable('a', [5])
 41 |       self.assertEquals([a, b], variables.get_variables())
 42 |       self.assertEquals([a], variables.get_variables('A'))
 43 |       self.assertEquals([b], variables.get_variables('B'))
 44 | 
 45 |   def testGetVariablesSuffix(self):
 46 |     with self.test_session():
 47 |       with tf.variable_scope('A'):
 48 |         a = variables.variable('a', [5])
 49 |       with tf.variable_scope('A'):
 50 |         b = variables.variable('b', [5])
 51 |       self.assertEquals([a], variables.get_variables(suffix='a'))
 52 |       self.assertEquals([b], variables.get_variables(suffix='b'))
 53 | 
 54 |   def testGetVariableWithSingleVar(self):
 55 |     with self.test_session():
 56 |       with tf.variable_scope('parent'):
 57 |         a = variables.variable('child', [5])
 58 |       self.assertEquals(a, variables.get_unique_variable('parent/child'))
 59 | 
 60 |   def testGetVariableWithDistractors(self):
 61 |     with self.test_session():
 62 |       with tf.variable_scope('parent'):
 63 |         a = variables.variable('child', [5])
 64 |         with tf.variable_scope('child'):
 65 |           variables.variable('grandchild1', [7])
 66 |           variables.variable('grandchild2', [9])
 67 |       self.assertEquals(a, variables.get_unique_variable('parent/child'))
 68 | 
 69 |   def testGetVariableThrowsExceptionWithNoMatch(self):
 70 |     var_name = 'cant_find_me'
 71 |     with self.test_session():
 72 |       with self.assertRaises(ValueError):
 73 |         variables.get_unique_variable(var_name)
 74 | 
 75 |   def testGetThrowsExceptionWithChildrenButNoMatch(self):
 76 |     var_name = 'parent/child'
 77 |     with self.test_session():
 78 |       with tf.variable_scope(var_name):
 79 |         variables.variable('grandchild1', [7])
 80 |         variables.variable('grandchild2', [9])
 81 |       with self.assertRaises(ValueError):
 82 |         variables.get_unique_variable(var_name)
 83 | 
 84 |   def testGetVariablesToRestore(self):
 85 |     with self.test_session():
 86 |       with tf.variable_scope('A'):
 87 |         a = variables.variable('a', [5])
 88 |       with tf.variable_scope('B'):
 89 |         b = variables.variable('a', [5])
 90 |       self.assertEquals([a, b], variables.get_variables_to_restore())
 91 | 
 92 |   def testNoneGetVariablesToRestore(self):
 93 |     with self.test_session():
 94 |       with tf.variable_scope('A'):
 95 |         a = variables.variable('a', [5], restore=False)
 96 |       with tf.variable_scope('B'):
 97 |         b = variables.variable('a', [5], restore=False)
 98 |       self.assertEquals([], variables.get_variables_to_restore())
 99 |       self.assertEquals([a, b], variables.get_variables())
100 | 
101 |   def testGetMixedVariablesToRestore(self):
102 |     with self.test_session():
103 |       with tf.variable_scope('A'):
104 |         a = variables.variable('a', [5])
105 |         b = variables.variable('b', [5], restore=False)
106 |       with tf.variable_scope('B'):
107 |         c = variables.variable('c', [5])
108 |         d = variables.variable('d', [5], restore=False)
109 |       self.assertEquals([a, b, c, d], variables.get_variables())
110 |       self.assertEquals([a, c], variables.get_variables_to_restore())
111 | 
112 |   def testReuseVariable(self):
113 |     with self.test_session():
114 |       with tf.variable_scope('A'):
115 |         a = variables.variable('a', [])
116 |       with tf.variable_scope('A', reuse=True):
117 |         b = variables.variable('a', [])
118 |       self.assertEquals(a, b)
119 |       self.assertListEqual([a], variables.get_variables())
120 | 
121 |   def testVariableWithDevice(self):
122 |     with self.test_session():
123 |       with tf.variable_scope('A'):
124 |         a = variables.variable('a', [], device='cpu:0')
125 |         b = variables.variable('b', [], device='cpu:1')
126 |       self.assertDeviceEqual(a.device, 'cpu:0')
127 |       self.assertDeviceEqual(b.device, 'cpu:1')
128 | 
129 |   def testVariableWithDeviceFromScope(self):
130 |     with self.test_session():
131 |       with tf.device('/cpu:0'):
132 |         a = variables.variable('a', [])
133 |         b = variables.variable('b', [], device='cpu:1')
134 |       self.assertDeviceEqual(a.device, 'cpu:0')
135 |       self.assertDeviceEqual(b.device, 'cpu:1')
136 | 
137 |   def testVariableWithDeviceFunction(self):
138 |     class DevFn(object):
139 | 
140 |       def __init__(self):
141 |         self.counter = -1
142 | 
143 |       def __call__(self, op):
144 |         self.counter += 1
145 |         return 'cpu:%d' % self.counter
146 | 
147 |     with self.test_session():
148 |       with scopes.arg_scope([variables.variable], device=DevFn()):
149 |         a = variables.variable('a', [])
150 |         b = variables.variable('b', [])
151 |         c = variables.variable('c', [], device='cpu:12')
152 |         d = variables.variable('d', [])
153 |         with tf.device('cpu:99'):
154 |           e_init = tf.constant(12)
155 |         e = variables.variable('e', initializer=e_init)
156 |       self.assertDeviceEqual(a.device, 'cpu:0')
157 |       self.assertDeviceEqual(a.initial_value.device, 'cpu:0')
158 |       self.assertDeviceEqual(b.device, 'cpu:1')
159 |       self.assertDeviceEqual(b.initial_value.device, 'cpu:1')
160 |       self.assertDeviceEqual(c.device, 'cpu:12')
161 |       self.assertDeviceEqual(c.initial_value.device, 'cpu:12')
162 |       self.assertDeviceEqual(d.device, 'cpu:2')
163 |       self.assertDeviceEqual(d.initial_value.device, 'cpu:2')
164 |       self.assertDeviceEqual(e.device, 'cpu:3')
165 |       self.assertDeviceEqual(e.initial_value.device, 'cpu:99')
166 | 
167 |   def testVariableWithReplicaDeviceSetter(self):
168 |     with self.test_session():
169 |       with tf.device(tf.train.replica_device_setter(ps_tasks=2)):
170 |         a = variables.variable('a', [])
171 |         b = variables.variable('b', [])
172 |         c = variables.variable('c', [], device='cpu:12')
173 |         d = variables.variable('d', [])
174 |         with tf.device('cpu:99'):
175 |           e_init = tf.constant(12)
176 |         e = variables.variable('e', initializer=e_init)
177 |       # The values below highlight how the replica_device_setter puts initial
178 |       # values on the worker job, and how it merges explicit devices.
179 |       self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0')
180 |       self.assertDeviceEqual(a.initial_value.device, '/job:worker/cpu:0')
181 |       self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0')
182 |       self.assertDeviceEqual(b.initial_value.device, '/job:worker/cpu:0')
183 |       self.assertDeviceEqual(c.device, '/job:ps/task:0/cpu:12')
184 |       self.assertDeviceEqual(c.initial_value.device, '/job:worker/cpu:12')
185 |       self.assertDeviceEqual(d.device, '/job:ps/task:1/cpu:0')
186 |       self.assertDeviceEqual(d.initial_value.device, '/job:worker/cpu:0')
187 |       self.assertDeviceEqual(e.device, '/job:ps/task:0/cpu:0')
188 |       self.assertDeviceEqual(e.initial_value.device, '/job:worker/cpu:99')
189 | 
190 |   def testVariableWithVariableDeviceChooser(self):
191 | 
192 |     with tf.Graph().as_default():
193 |       device_fn = variables.VariableDeviceChooser(num_parameter_servers=2)
194 |       with scopes.arg_scope([variables.variable], device=device_fn):
195 |         a = variables.variable('a', [])
196 |         b = variables.variable('b', [])
197 |         c = variables.variable('c', [], device='cpu:12')
198 |         d = variables.variable('d', [])
199 |         with tf.device('cpu:99'):
200 |           e_init = tf.constant(12)
201 |         e = variables.variable('e', initializer=e_init)
202 |       # The values below highlight how the VariableDeviceChooser puts initial
203 |       # values on the same device as the variable job.
204 |       self.assertDeviceEqual(a.device, '/job:ps/task:0/cpu:0')
205 |       self.assertDeviceEqual(a.initial_value.device, a.device)
206 |       self.assertDeviceEqual(b.device, '/job:ps/task:1/cpu:0')
207 |       self.assertDeviceEqual(b.initial_value.device, b.device)
208 |       self.assertDeviceEqual(c.device, '/cpu:12')
209 |       self.assertDeviceEqual(c.initial_value.device, c.device)
210 |       self.assertDeviceEqual(d.device, '/job:ps/task:0/cpu:0')
211 |       self.assertDeviceEqual(d.initial_value.device, d.device)
212 |       self.assertDeviceEqual(e.device, '/job:ps/task:1/cpu:0')
213 |       self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
214 | 
215 |   def testVariableGPUPlacement(self):
216 | 
217 |     with tf.Graph().as_default():
218 |       device_fn = variables.VariableDeviceChooser(placement='gpu:0')
219 |       with scopes.arg_scope([variables.variable], device=device_fn):
220 |         a = variables.variable('a', [])
221 |         b = variables.variable('b', [])
222 |         c = variables.variable('c', [], device='cpu:12')
223 |         d = variables.variable('d', [])
224 |         with tf.device('cpu:99'):
225 |           e_init = tf.constant(12)
226 |         e = variables.variable('e', initializer=e_init)
227 |       # The values below highlight how the VariableDeviceChooser puts initial
228 |       # values on the same device as the variable job.
229 |       self.assertDeviceEqual(a.device, '/gpu:0')
230 |       self.assertDeviceEqual(a.initial_value.device, a.device)
231 |       self.assertDeviceEqual(b.device, '/gpu:0')
232 |       self.assertDeviceEqual(b.initial_value.device, b.device)
233 |       self.assertDeviceEqual(c.device, '/cpu:12')
234 |       self.assertDeviceEqual(c.initial_value.device, c.device)
235 |       self.assertDeviceEqual(d.device, '/gpu:0')
236 |       self.assertDeviceEqual(d.initial_value.device, d.device)
237 |       self.assertDeviceEqual(e.device, '/gpu:0')
238 |       self.assertDeviceEqual(e.initial_value.device, '/cpu:99')
239 | 
240 |   def testVariableCollection(self):
241 |     with self.test_session():
242 |       a = variables.variable('a', [], collections='A')
243 |       b = variables.variable('b', [], collections='B')
244 |       self.assertEquals(a, tf.get_collection('A')[0])
245 |       self.assertEquals(b, tf.get_collection('B')[0])
246 | 
247 |   def testVariableCollections(self):
248 |     with self.test_session():
249 |       a = variables.variable('a', [], collections=['A', 'C'])
250 |       b = variables.variable('b', [], collections=['B', 'C'])
251 |       self.assertEquals(a, tf.get_collection('A')[0])
252 |       self.assertEquals(b, tf.get_collection('B')[0])
253 | 
254 |   def testVariableCollectionsWithArgScope(self):
255 |     with self.test_session():
256 |       with scopes.arg_scope([variables.variable], collections='A'):
257 |         a = variables.variable('a', [])
258 |         b = variables.variable('b', [])
259 |       self.assertListEqual([a, b], tf.get_collection('A'))
260 | 
261 |   def testVariableCollectionsWithArgScopeNested(self):
262 |     with self.test_session():
263 |       with scopes.arg_scope([variables.variable], collections='A'):
264 |         a = variables.variable('a', [])
265 |         with scopes.arg_scope([variables.variable], collections='B'):
266 |           b = variables.variable('b', [])
267 |       self.assertEquals(a, tf.get_collection('A')[0])
268 |       self.assertEquals(b, tf.get_collection('B')[0])
269 | 
270 |   def testVariableCollectionsWithArgScopeNonNested(self):
271 |     with self.test_session():
272 |       with scopes.arg_scope([variables.variable], collections='A'):
273 |         a = variables.variable('a', [])
274 |       with scopes.arg_scope([variables.variable], collections='B'):
275 |         b = variables.variable('b', [])
276 |       variables.variable('c', [])
277 |       self.assertListEqual([a], tf.get_collection('A'))
278 |       self.assertListEqual([b], tf.get_collection('B'))
279 | 
280 |   def testVariableRestoreWithArgScopeNested(self):
281 |     with self.test_session():
282 |       with scopes.arg_scope([variables.variable], restore=True):
283 |         a = variables.variable('a', [])
284 |         with scopes.arg_scope([variables.variable],
285 |                               trainable=False,
286 |                               collections=['A', 'B']):
287 |           b = variables.variable('b', [])
288 |         c = variables.variable('c', [])
289 |       self.assertListEqual([a, b, c], variables.get_variables_to_restore())
290 |       self.assertListEqual([a, c], tf.trainable_variables())
291 |       self.assertListEqual([b], tf.get_collection('A'))
292 |       self.assertListEqual([b], tf.get_collection('B'))
293 | 
294 | 
295 | class GetVariablesByNameTest(tf.test.TestCase):
296 | 
297 |   def testGetVariableGivenNameScoped(self):
298 |     with self.test_session():
299 |       with tf.variable_scope('A'):
300 |         a = variables.variable('a', [5])
301 |         b = variables.variable('b', [5])
302 |         self.assertEquals([a], variables.get_variables_by_name('a'))
303 |         self.assertEquals([b], variables.get_variables_by_name('b'))
304 | 
305 |   def testGetVariablesByNameReturnsByValueWithScope(self):
306 |     with self.test_session():
307 |       with tf.variable_scope('A'):
308 |         a = variables.variable('a', [5])
309 |         matched_variables = variables.get_variables_by_name('a')
310 | 
311 |         # If variables.get_variables_by_name returns the list by reference, the
312 |         # following append should persist, and be returned, in subsequent calls
313 |         # to variables.get_variables_by_name('a').
314 |         matched_variables.append(4)
315 | 
316 |         matched_variables = variables.get_variables_by_name('a')
317 |         self.assertEquals([a], matched_variables)
318 | 
319 |   def testGetVariablesByNameReturnsByValueWithoutScope(self):
320 |     with self.test_session():
321 |       a = variables.variable('a', [5])
322 |       matched_variables = variables.get_variables_by_name('a')
323 | 
324 |       # If variables.get_variables_by_name returns the list by reference, the
325 |       # following append should persist, and be returned, in subsequent calls
326 |       # to variables.get_variables_by_name('a').
327 |       matched_variables.append(4)
328 | 
329 |       matched_variables = variables.get_variables_by_name('a')
330 |       self.assertEquals([a], matched_variables)
331 | 
332 | 
333 | class GlobalStepTest(tf.test.TestCase):
334 | 
335 |   def testStable(self):
336 |     with tf.Graph().as_default():
337 |       gs = variables.global_step()
338 |       gs2 = variables.global_step()
339 |       self.assertTrue(gs is gs2)
340 | 
341 |   def testDevice(self):
342 |     with tf.Graph().as_default():
343 |       with scopes.arg_scope([variables.global_step], device='/gpu:0'):
344 |         gs = variables.global_step()
345 |       self.assertDeviceEqual(gs.device, '/gpu:0')
346 | 
347 |   def testDeviceFn(self):
348 |     class DevFn(object):
349 | 
350 |       def __init__(self):
351 |         self.counter = -1
352 | 
353 |       def __call__(self, op):
354 |         self.counter += 1
355 |         return '/cpu:%d' % self.counter
356 | 
357 |     with tf.Graph().as_default():
358 |       with scopes.arg_scope([variables.global_step], device=DevFn()):
359 |         gs = variables.global_step()
360 |         gs2 = variables.global_step()
361 |       self.assertDeviceEqual(gs.device, '/cpu:0')
362 |       self.assertEquals(gs, gs2)
363 |       self.assertDeviceEqual(gs2.device, '/cpu:0')
364 | 
365 |   def testReplicaDeviceSetter(self):
366 |     device_fn = tf.train.replica_device_setter(2)
367 |     with tf.Graph().as_default():
368 |       with scopes.arg_scope([variables.global_step], device=device_fn):
369 |         gs = variables.global_step()
370 |         gs2 = variables.global_step()
371 |         self.assertEquals(gs, gs2)
372 |         self.assertDeviceEqual(gs.device, '/job:ps/task:0')
373 |         self.assertDeviceEqual(gs.initial_value.device, '/job:ps/task:0')
374 |         self.assertDeviceEqual(gs2.device, '/job:ps/task:0')
375 |         self.assertDeviceEqual(gs2.initial_value.device, '/job:ps/task:0')
376 | 
377 |   def testVariableWithVariableDeviceChooser(self):
378 | 
379 |     with tf.Graph().as_default():
380 |       device_fn = variables.VariableDeviceChooser()
381 |       with scopes.arg_scope([variables.global_step], device=device_fn):
382 |         gs = variables.global_step()
383 |         gs2 = variables.global_step()
384 |         self.assertEquals(gs, gs2)
385 |         self.assertDeviceEqual(gs.device, 'cpu:0')
386 |         self.assertDeviceEqual(gs.initial_value.device, gs.device)
387 |         self.assertDeviceEqual(gs2.device, 'cpu:0')
388 |         self.assertDeviceEqual(gs2.initial_value.device, gs2.device)
389 | 
390 | 
391 | if __name__ == '__main__':
392 |   tf.test.main()
393 | 


--------------------------------------------------------------------------------
/train.sh:
--------------------------------------------------------------------------------
 1 | #########################################################################
 2 | # File Name: train.sh
 3 | # Author: yeephycho
 4 | # mail: huyixuanhyx@gmail.com
 5 | # Created Time: Tuesday, December 06, 2016 PM03:48:45 HKT
 6 | #########################################################################
 7 | #!/bin/bash
 8 | 
 9 | 
10 | # Download 101 object categories data set
11 | cd ./data
12 | 
13 | if [ ! -f "101_ObjectCategories.tar.gz" ]; then
14 |     echo "Download 101_ObjectCategories dataset"
15 |     wget https://s3-us-west-2.amazonaws.com/deep-learning.datasection.co.jp/datasets/101_ObjectCategories.tar.gz 
16 | fi
17 | 
18 | if [ ! -d "./101_ObjectCategories/" ]; then
19 |     PROJECT_DIR=`pwd`
20 |     echo "Unzip 101_ObjectCategories dataset to folder "$PROJECT_DIR"/data/101_ObjectCategories/"
21 |     tar zvxf 101_ObjectCategories.tar.gz
22 | fi
23 | 
24 | # Generate .txt file
25 | python create_examples_list.py
26 | python relation_tag_to_id.py
27 | 
28 | # Start training
29 | cd ../
30 | #python trainer.py
31 | 


--------------------------------------------------------------------------------
/train_operation.py:
--------------------------------------------------------------------------------
 1 | # encoding: utf-
 2 | # 8
 3 | import tensorflow as tf
 4 | 
 5 | import model
 6 | 
 7 | import settings
 8 | FLAGS = settings.FLAGS
 9 | 
10 | import numpy as np
11 | 
12 | RMSPROP_DECAY = 0.9                # Decay term for RMSProp.
13 | RMSPROP_MOMENTUM = 0.9             # Momentum in RMSProp.
14 | RMSPROP_EPSILON = 1.0              # Epsilon term for RMSProp.
15 | 
16 | def train(total_loss, global_step, summaries, batchnorm_updates):
17 |     # Calculate the learning rate schedule.
18 |     num_batches_per_epoch = (FLAGS.num_examples_per_epoch_for_train /
19 |                              FLAGS.batch_size)
20 |     decay_steps = int(num_batches_per_epoch * FLAGS.num_epochs_per_decay)
21 | 
22 |     # Decay the learning rate exponentially based on the number of steps.
23 |     lr = tf.train.exponential_decay(FLAGS.initial_learning_rate,
24 |                                     global_step,
25 |                                     decay_steps,
26 |                                     FLAGS.learning_rate_decay_factor,
27 |                                     staircase=True)
28 | 
29 |     # Create an optimizer that performs gradient descent.
30 |     #opt = tf.train.RMSPropOptimizer(lr, RMSPROP_DECAY,
31 |     #                                momentum=RMSPROP_MOMENTUM,
32 |     #                                epsilon=RMSPROP_EPSILON)
33 |     opt = tf.train.AdamOptimizer(lr)
34 |     grads = opt.compute_gradients(total_loss)
35 | 
36 |     # Add a summary to track the learning rate.
37 |     summaries.append(tf.summary.scalar('learning_rate', lr))
38 | 
39 |     # Add histograms for gradients.
40 |     for grad, var in grads:
41 |         if grad is not None:
42 |             summaries.append(
43 |                 tf.summary.histogram(var.op.name + '/gradients', grad))
44 | 
45 |     # Apply the gradients to adjust the shared variables.
46 |     apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
47 | 
48 |     # Track the moving averages of all trainable variables.
49 |     # Note that we maintain a "double-average" of the BatchNormalization
50 |     # global statistics. This is more complicated then need be but we employ
51 |     # this for backward-compatibility with our previous models.
52 |     variable_averages = tf.train.ExponentialMovingAverage(
53 |         FLAGS.moving_average_decay, global_step)
54 | 
55 |     # Another possiblility is to use tf.slim.get_variables().
56 |     variables_to_average = (tf.trainable_variables() +
57 |                             tf.moving_average_variables())
58 |     variables_averages_op = variable_averages.apply(variables_to_average)
59 | 
60 |     # Group all updates to into a single train op.
61 |     batchnorm_updates_op = tf.group(*batchnorm_updates)
62 |     train_op = tf.group(apply_gradient_op, variables_averages_op,
63 |                         batchnorm_updates_op)
64 | 
65 |     return train_op
66 | 


--------------------------------------------------------------------------------
/trainer.py:
--------------------------------------------------------------------------------
  1 | # encoding: utf-8
  2 | import tensorflow as tf
  3 | import settings
  4 | FLAGS = settings.FLAGS
  5 | 
  6 | import os
  7 | import re
  8 | import copy
  9 | from datetime import datetime
 10 | import time
 11 | from datasets import DataSet
 12 | import datasets
 13 | 
 14 | import model
 15 | import train_operation
 16 | import slim.slim
 17 | import numpy as np
 18 | 
 19 | def train(verbose=False):
 20 |     with tf.Graph().as_default():
 21 |         # global step number
 22 |         global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
 23 |         dataset = DataSet()
 24 | 
 25 |         # get training set
 26 |         print("The number of training images is: %d" % (dataset.cnt_samples(FLAGS.traincsv)))
 27 |         images, labels = dataset.csv_inputs(FLAGS.traincsv, FLAGS.batch_size, distorted=True)
 28 | 
 29 |         images_debug = datasets.debug(images)
 30 | 
 31 |         # get test set
 32 |         #test_cnt = dataset.cnt_samples(FLAGS.testcsv)
 33 |         test_cnt = 100
 34 | 
 35 |         images_test, labels_test = dataset.test_inputs(FLAGS.testcsv, test_cnt)
 36 | 
 37 |         images_test_debug = datasets.debug(images_test)
 38 | 
 39 |         input_summaries = copy.copy(tf.get_collection(tf.GraphKeys.SUMMARIES))
 40 | 
 41 |         num_classes = FLAGS.num_classes
 42 |         restore_logits = not FLAGS.fine_tune
 43 | 
 44 |         # inference
 45 |         # logits is tuple (logits, aux_liary_logits, predictions)
 46 |         # logits: output of final layer, auxliary_logits: output of hidden layer, softmax: predictions
 47 |         logits = model.inference(images, num_classes, for_training=True, restore_logits=restore_logits)
 48 |         logits_test = model.inference(images_test, num_classes, for_training=False, restore_logits=restore_logits, reuse=True, dropout_keep_prob=1.0)
 49 | 
 50 |         # loss
 51 |         model.loss(logits, labels, batch_size=FLAGS.batch_size)
 52 |         model.loss_test(logits_test, labels_test, batch_size=test_cnt)
 53 |         losses = tf.get_collection(slim.losses.LOSSES_COLLECTION)
 54 |         losses_test = tf.get_collection(slim.losses.LOSSES_COLLECTION_TEST)
 55 | 
 56 |         # Calculate the total loss for the current tower.
 57 |         regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
 58 |         total_loss = tf.add_n(losses + regularization_losses, name='total_loss')
 59 |         #total_loss = tf.add_n(losses, name='total_loss')
 60 |         total_loss_test = tf.add_n(losses_test, name='total_loss_test')
 61 | 
 62 |         # Compute the moving average of all individual losses and the total loss.
 63 |         loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
 64 |         loss_averages_op = loss_averages.apply(losses + [total_loss])
 65 |         loss_averages_test = tf.train.ExponentialMovingAverage(0.9, name='avg_test')
 66 |         loss_averages_op_test = loss_averages_test.apply(losses_test + [total_loss_test])
 67 | 
 68 |         if verbose:
 69 |             print "="*10
 70 |             print "loss length:"
 71 |             print len(losses)
 72 |             print len(losses_test)
 73 |             print "="*10
 74 | 
 75 |         # for l in losses + [total_loss]:
 76 |         #     # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
 77 |         #     # session. This helps the clarity of presentation on TensorBoard.
 78 |         #     loss_name = re.sub('%s_[0-9]*/' % model.TOWER_NAME, '', l.op.name)
 79 |         #     # Name each loss as '(raw)' and name the moving average version of the loss
 80 |         #     # as the original loss name.
 81 |         #     tf.scalar_summary(loss_name + ' (raw)', l)
 82 |         #     tf.scalar_summary(loss_name, loss_averages.average(l))
 83 | 
 84 |         # loss to calcurate gradients
 85 |         #
 86 |         with tf.control_dependencies([loss_averages_op]):
 87 |             total_loss = tf.identity(total_loss)
 88 |         tf.summary.scalar("loss", total_loss)
 89 | 
 90 |         with tf.control_dependencies([loss_averages_op_test]):
 91 |             total_loss_test = tf.identity(total_loss_test)
 92 |         tf.summary.scalar("loss_eval", total_loss_test)
 93 | 
 94 |         # Reuse variables for the next tower.
 95 |         #tf.get_variable_scope().reuse_variables()
 96 | 
 97 |         # Retain the summaries from the final tower.
 98 |         summaries = tf.get_collection(tf.GraphKeys.SUMMARIES)
 99 | 
100 |         # Retain the Batch Normalization updates operations only from the
101 |         # final tower. Ideally, we should grab the updates from all towers
102 |         # but these stats accumulate extremely fast so we can ignore the
103 |         # other stats from the other towers without significant detriment.
104 |         batchnorm_updates = tf.get_collection(slim.ops.UPDATE_OPS_COLLECTION)
105 | 
106 |         # add input summaries
107 |         # summaries.extend(input_summaries)
108 | 
109 |         # train_operation and operation summaries
110 |         train_op = train_operation.train(total_loss, global_step, summaries, batchnorm_updates)
111 | 
112 |         # trainable variables's summary
113 |         #for var in tf.trainable_variables():
114 |         #    summaries.append(tf.histogram_summary(var.op.name, var))
115 | 
116 |         # saver
117 |         saver = tf.train.Saver(tf.global_variables())
118 | 
119 |         # Build the summary operation from the last tower summaries.
120 |         #summary_op = tf.merge_summary(summaries)
121 |         summary_op = tf.summary.merge(summaries)
122 | 
123 |         # initialization
124 |         init = tf.global_variables_initializer()
125 | 
126 |         # session
127 |         gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
128 |         sess = tf.Session(config=tf.ConfigProto(
129 |             allow_soft_placement=True,
130 |             log_device_placement=FLAGS.log_device_placement,
131 |             gpu_options=gpu_options))
132 |         sess.run(init)
133 | 
134 |         coord = tf.train.Coordinator()
135 |         threads = tf.train.start_queue_runners(sess=sess, coord=coord)
136 | 
137 |         if FLAGS.fine_tune:
138 |             ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
139 |             if ckpt and ckpt.model_checkpoint_path:
140 |                 print('%s: Pre-trained model restored from %s' %
141 |                       (datetime.now(), ckpt.model_checkpoint_path))
142 |                 saver.restore(sess, ckpt.model_checkpoint_path)
143 | 
144 |             # if FLAGS.pretrained_model_checkpoint_path:
145 |             #     assert tf.gfile.Exists(FLAGS.pretrained_model_checkpoint_path)
146 |             #     variables_to_restore = tf.get_collection(
147 |             #         slim.variables.VARIABLES_TO_RESTORE)
148 |             #     restorer = tf.train.Saver(variables_to_restore)
149 |             #     restorer.restore(sess, FLAGS.pretrained_model_checkpoint_path)
150 | 
151 | 
152 |         summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
153 | 
154 |         for step in xrange(FLAGS.max_steps):
155 |             start_time = time.time()
156 |             _, logits_eval, loss_value, labels_eval, images_debug_eval = sess.run([train_op, logits[0], total_loss, labels, images_debug])
157 |             duration = time.time() - start_time
158 | 
159 |             dataset.output_images(images_debug_eval, "debug", "train")
160 | 
161 |             assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
162 | 
163 |             if step % 10 == 0:
164 |                 examples_per_sec = FLAGS.batch_size / float(duration)
165 |                 format_str = ('train %s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)')
166 |                 print(format_str % (datetime.now(), step, loss_value, examples_per_sec, duration))
167 | 
168 |             if step % 100 == 0:
169 |                 print("predict:")
170 |                 print type(logits_eval)
171 |                 print logits_eval.shape
172 |                 print logits_eval.argmax(1)
173 |                 print("target:")
174 |                 print labels_eval
175 |                 summary_str = sess.run(summary_op)
176 |                 summary_writer.add_summary(summary_str, step)
177 | 
178 |                 test_start_time = time.time()
179 |                 logits_test_eval, total_loss_test_val, labels_test_eval, images_test_debug_eval = sess.run([logits_test[0], total_loss_test, labels_test, images_test_debug])
180 |                 test_duration = time.time() - test_start_time
181 | 
182 |                 dataset.output_images(images_test_debug_eval, "debug_test", "test")
183 | 
184 |                 print("test predict:")
185 |                 print type(logits_test_eval)
186 |                 print logits_test_eval.shape
187 |                 print logits_test_eval.argmax(1)
188 |                 print("test target:")
189 |                 print labels_test_eval
190 |                 test_examples_per_sec = test_cnt / float(test_duration)
191 |                 format_str_test = ('test %s: step %d, loss = %.2f, (%.1f examples/sec; %.3f sec/batch)')
192 |                 print(format_str_test % (datetime.now(), step, total_loss_test_val, test_examples_per_sec, test_duration))
193 | 
194 |                 # Save the model checkpoint periodically.
195 |                 if step % 5000 == 0 or (step + 1) == FLAGS.max_steps:
196 |                     checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
197 |                 saver.save(sess, checkpoint_path, global_step=step)
198 | 
199 |         coord.request_stop()
200 |         coord.join(threads)
201 |         sess.close()
202 | 
203 | 
204 | def test():
205 |     # load settings file
206 |     print(FLAGS.tfcsv)
207 | 
208 | 
209 | def main(argv=None):
210 |     if tf.gfile.Exists(FLAGS.train_dir) and not FLAGS.fine_tune:
211 |         print("Caution: train dir is already exists.")
212 |     if not tf.gfile.Exists(FLAGS.train_dir):
213 |         tf.gfile.MakeDirs(FLAGS.train_dir)
214 |     train()
215 | 
216 | 
217 | if __name__ == '__main__':
218 |     tf.app.run()
219 | 


--------------------------------------------------------------------------------