├── data
    └── test
├── graph
    └── logs
    │   └── test
├── output
    ├── f_matte
    │   └── test
    └── gt_matte
    │   └── test
├── backup
    ├── checkpoint
    └── latest.index
├── README.md
├── layer.py
├── utils_vgg.py
├── tfrecord_utils.py
├── vgg16.py
├── train.py
└── model.py


/data/test:
--------------------------------------------------------------------------------
1 | test
2 | 


--------------------------------------------------------------------------------
/graph/logs/test:
--------------------------------------------------------------------------------
1 | test
2 | 


--------------------------------------------------------------------------------
/output/f_matte/test:
--------------------------------------------------------------------------------
1 | test
2 | 


--------------------------------------------------------------------------------
/output/gt_matte/test:
--------------------------------------------------------------------------------
1 | test
2 | 


--------------------------------------------------------------------------------
/backup/checkpoint:
--------------------------------------------------------------------------------
1 | model_checkpoint_path: "latest"
2 | all_model_checkpoint_paths: "latest"
3 | 


--------------------------------------------------------------------------------
/backup/latest.index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zylix666/DeshadowNet/HEAD/backup/latest.index


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # DeshadowNet
 2 | [CVPR2017] DeshadowNet: A Multi-context Embedding Deep Network for Shadow Removal<br />
 3 | TensorFlow implementation<br />
 4 | 
 5 | ## To Do
 6 | This project is on going<br />
 7 | Does not work yet
 8 | 
 9 | > weight decay<br />
10 | multi-stage training<br />
11 | multi-size training<br />
12 | data augmentation<br />
13 | data normalization
14 | 
15 | ### Prerequisites
16 | ```
17 | tensorflow
18 | tensorboard
19 | PIL
20 | numpy
21 | cv2
22 | ```
23 | 
24 | ## Forked code
25 | [vgg tensorflow](https://github.com/machrisaa/tensorflow-vgg)<br />
26 | [tfboard](https://github.com/chiphuyen/stanford-tensorflow-tutorials)<br />
27 | 


--------------------------------------------------------------------------------
/layer.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | 
 3 | def conv_layer(x, filtershape, stride, name):
 4 |     with tf.variable_scope(name):
 5 |         filters = tf.get_variable(
 6 |             name = 'weight',
 7 |             shape = filtershape,
 8 |             dtype = tf.float32,
 9 |             initializer = tf.random_normal_initializer(mean=0,stddev=0.001),
10 |             trainable = True)
11 |         conv = tf.nn.conv2d(x, filters, [1, stride, stride, 1], padding= 'SAME')
12 |         conv_biases = tf.Variable(tf.constant(0.0, shape = [filtershape[3]], dtype = tf.float32),
13 |                                 trainable=True, name ='bias')
14 |         bias = tf.nn.bias_add(conv, conv_biases)
15 |         output = prelu(bias)
16 |         #output = tf.nn.dropout(prelu, keep_prob=keep_prob)
17 |         img_filt = tf.reshape(filters[:,:,:,1], [-1,filtershape[0],filtershape[1],1])
18 |         tf.summary.image('conv_filter',img_filt)
19 |         return output
20 | 
21 | def deconv_layer(x, filtershape,output_shape, stride, name):
22 |     with tf.variable_scope(name):
23 |         filters = tf.get_variable(
24 |             name = 'weight',
25 |             shape = filtershape,
26 |             dtype = tf.float32,
27 |             initializer = tf.random_normal_initializer(mean=0,stddev=0.001),
28 |             trainable = True)
29 |         deconv = tf.nn.conv2d_transpose(x, filters, output_shape, [1, stride, stride, 1], padding ='SAME')
30 |         #deconv_biases = tf.Variable(tf.constant(0.0, shape = [filtershape[3]], dtype = tf.float32),
31 |         #                        trainable=True, name ='bias')
32 |         #bias = tf.nn.bias_add(deconv, deconv_biases)
33 |         #output = prelu(bias)
34 |         #output = tf.nn.dropout(prelu, keep_prob=keep_prob)
35 |         img_filt = tf.reshape(filters[:,:,:,1], [-1,filtershape[0],filtershape[1],1])
36 |         tf.summary.image('deconv_filter',img_filt)
37 |         return prelu(deconv)
38 | 
39 | def max_pool_layer(x,filtershape,stride,name):
40 |     return tf.nn.max_pool(x, filtershape, [1, stride, stride, 1], padding ='SAME',name = name)
41 | 
42 | def prelu(x):
43 |     with tf.variable_scope('prelu'):
44 |         alphas = tf.get_variable('alpha', x.get_shape()[-1],
45 |                        initializer=tf.constant_initializer(0.0),
46 |                         dtype=tf.float32)
47 |         pos = tf.nn.relu(x)
48 |         neg = alphas * (x - tf.abs(x)) * 0.5
49 |         return pos + neg


--------------------------------------------------------------------------------
/utils_vgg.py:
--------------------------------------------------------------------------------
 1 | # import skimage
 2 | # import skimage.io
 3 | # import skimage.transform
 4 | import numpy as np
 5 | import cv2
 6 | from PIL import Image
 7 | 
 8 | # ============================================================ #
 9 | # TODO
10 | # 1. replace skimage with image library from image completion
11 | # 2. test the vgg16_test code
12 | # 3. figure out how to import weight and model to other module
13 | # ============================================================ #
14 | 
15 | # synset = [l.strip() for l in open('synset.txt').readlines()]
16 | 
17 | # returns image of shape [224, 224, 3]
18 | # [height, width, depth]
19 | def load_image(path):
20 |     # load image
21 |     # img = skimage.io.imread(path)
22 |     # img = img / 255.0
23 |     img = Image.open(path)
24 |     img = np.asarray(img,np.float32)
25 |     img = img / 255.0
26 |     assert (0 <= img).all() and (img <= 1.0).all()
27 |     # print "Original Image Shape: ", img.shape
28 |     # we crop image from center
29 |     print('image shape: {}'.format(img.shape))
30 |     short_edge = min(img.shape[:2])
31 |     yy = int((img.shape[0] - short_edge) / 2)
32 |     xx = int((img.shape[1] - short_edge) / 2)
33 |     crop_img = img[yy: yy + short_edge, xx: xx + short_edge]
34 |     # resize to 224, 224
35 |     resized_img = cv2.resize(crop_img,(224,224),interpolation =cv2.INTER_CUBIC)
36 |     # resized_img = skimage.transform.resize(crop_img, (224, 224))
37 |     return resized_img
38 | 
39 | 
40 | # returns the top1 string
41 | def print_prob(prob, file_path):
42 |     synset = [l.strip() for l in open(file_path).readlines()]
43 | 
44 |     # print prob
45 |     pred = np.argsort(prob)[::-1]
46 | 
47 |     # Get top1 label
48 |     top1 = synset[pred[0]]
49 |     print(("Top1: ", top1, prob[pred[0]]))
50 |     # Get top5 label
51 |     top5 = [(synset[pred[i]], prob[pred[i]]) for i in range(5)]
52 |     print(("Top5: ", top5))
53 |     return top1
54 | 
55 | 
56 | # def load_image2(path, height=None, width=None):
57 | #     # load image
58 | #     img = skimage.io.imread(path)
59 | #     img = img / 255.0
60 | #     if height is not None and width is not None:
61 | #         ny = height
62 | #         nx = width
63 | #     elif height is not None:
64 | #         ny = height
65 | #         nx = img.shape[1] * ny / img.shape[0]
66 | #     elif width is not None:
67 | #         nx = width
68 | #         ny = img.shape[0] * nx / img.shape[1]
69 | #     else:
70 | #         ny = img.shape[0]
71 | #         nx = img.shape[1]
72 | #     return skimage.transform.resize(img, (ny, nx))
73 | 
74 | 
75 | def test():
76 |     img = skimage.io.imread("./test_data/starry_night.jpg")
77 |     ny = 300
78 |     nx = img.shape[1] * ny / img.shape[0]
79 |     img = skimage.transform.resize(img, (ny, nx))
80 |     skimage.io.imsave("./test_data/test/output.jpg", img)
81 | 
82 | 
83 | if __name__ == "__main__":
84 |     test()
85 | 


--------------------------------------------------------------------------------
/tfrecord_utils.py:
--------------------------------------------------------------------------------
  1 | '''
  2 | tfrecord utils
  3 | 
  4 | edited version from standford tensorflow class
  5 | '''
  6 | 
  7 | import os
  8 | import sys
  9 | 
 10 | 
 11 | import glob
 12 | from PIL import Image
 13 | import numpy as np
 14 | import tensorflow as tf
 15 | import matplotlib.pyplot as plt
 16 | import cv2 as cv
 17 | 
 18 | IMAGE_SIZE = 224
 19 | 
 20 | def _int64_feature(value):
 21 |   return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
 22 | 
 23 | def _bytes_feature(value):
 24 |   return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
 25 | 
 26 | def get_image_binary(filename):
 27 |     """ You can read in the image using tensorflow too, but it's a drag
 28 |         since you have to create graphs. It's much easier using Pillow and NumPy
 29 |     """
 30 |     # image = Image.open(filename)
 31 |     # image = np.asarray(image, np.uint8)
 32 |     # shape = np.array(image.shape, np.int32)
 33 | 
 34 |     image = cv.imread(filename)
 35 |     image = cv.resize(image, (IMAGE_SIZE, IMAGE_SIZE))
 36 |     # cv.imshow('test',image)
 37 |     # image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
 38 |     # cv.waitKey()
 39 |     image = np.asarray(image, np.uint8)
 40 |     shape = np.array(image.shape, np.int32)
 41 |     return shape.tobytes(), image.tostring() #image.tobytes() # convert image to raw data bytes in the array.
 42 | 
 43 | def write_to_tfrecord_pair_img(writer, binary_image1, binary_image2, tfrecord_file):
 44 |     """ This example is to write a sample to TFRecord file. If you want to write
 45 |     more samples, just use a loop.
 46 |     """
 47 |     # write label, shape, and image content to the TFRecord file
 48 |     example = tf.train.Example(features=tf.train.Features(feature={
 49 |                 'image_input': _bytes_feature(binary_image1),
 50 |                 'image_gt': _bytes_feature(binary_image2)
 51 |                 }))
 52 |     writer.write(example.SerializeToString())
 53 | 
 54 | def write_tfrecord_pair_img(writer, image_file1, image_file2, tfrecord_file):
 55 |     shape1 ,binary_image1 = get_image_binary(image_file1)
 56 |     shape2 ,binary_image2 = get_image_binary(image_file2)
 57 |     write_to_tfrecord_pair_img(writer, binary_image1, binary_image2, tfrecord_file)
 58 | 
 59 | def read_from_tfrecord_pair_img(reader, filenames):
 60 |     tfrecord_file_queue = tf.train.string_input_producer(filenames, name='queue')
 61 |     _, tfrecord_serialized = reader.read(tfrecord_file_queue)
 62 | 
 63 |     # label and image are stored as bytes but could be stored as
 64 |     # int64 or float64 values in a serialized tf.Example protobuf.
 65 |     tfrecord_features = tf.parse_single_example(tfrecord_serialized,
 66 |                         features={
 67 |                             'image_input': tf.FixedLenFeature([], tf.string),
 68 |                             'image_gt': tf.FixedLenFeature([], tf.string),
 69 |                         }, name='features')
 70 | 
 71 |     # image was saved as uint8, so we have to decode as uint8.
 72 |     img_input = tf.decode_raw(tfrecord_features['image_input'], tf.uint8)
 73 |     img_gt    = tf.decode_raw(tfrecord_features['image_gt'],    tf.uint8)
 74 | 
 75 |     # the image tensor is flattened out, so we have to reconstruct the shape
 76 |     img_input = tf.reshape(img_input, [IMAGE_SIZE, IMAGE_SIZE,3])
 77 |     img_gt = tf.reshape(img_gt, [IMAGE_SIZE, IMAGE_SIZE,3])
 78 |     return img_input, img_gt
 79 | 
 80 | def read_tfrecord_pair_img(reader, tfrecord_file,vis):
 81 |     img_input, img_gt = read_from_tfrecord_pair_img(reader, [tfrecord_file])
 82 |     if vis:
 83 |         with tf.Session() as sess:
 84 |             coord = tf.train.Coordinator()
 85 |             threads = tf.train.start_queue_runners(sess=sess, coord=coord)
 86 |             for i in range(4):
 87 |                 shadow, deshadow = sess.run([img_input,img_gt])
 88 |                 pilimg = Image.fromarray(shadow)
 89 |                 pilimg.show()
 90 |                 pilimg = Image.fromarray(deshadow)
 91 |                 pilimg.show()
 92 |             coord.request_stop()
 93 |             coord.join(threads)
 94 |             sess.close()
 95 | 
 96 |     return img_input, img_gt
 97 | 
 98 | def main():
 99 |     paths = glob.glob('./data/*')
100 |     tfrecord_filename = './data.tfrecord'
101 |     writer = tf.python_io.TFRecordWriter(tfrecord_filename)
102 |     print('tf record writing start')
103 | 
104 |     image_size = 128;
105 |     for path in paths:
106 |         path = path + '/*'
107 |         path = glob.glob(path)
108 |         for subpath in path:
109 |             x=[]
110 |             if 'curry' in subpath:
111 |                 print('image name: {}'.format(subpath))
112 |                 # write_tfrecord(writer, subpath,tfrecord_filename)
113 |                 write_tfrecord(writer, subpath,tfrecord_filename)
114 | 
115 |     writer.close()
116 | 
117 |     # testing tf record read1ng
118 |     print('reading tf record')
119 |     reader = tf.TFRecordReader()
120 |     tfrecord_filename = './test.tfrecord'
121 | 
122 |     # closeshape, image = read_tfrecord(reader,tfrecord_filename)
123 |     image, shape = read_vis_tfrecord(reader,tfrecord_filename)
124 |     print('shape:{}:'.format(shape))
125 | 
126 | if __name__ == '__main__':
127 |     main()
128 | 


--------------------------------------------------------------------------------
/vgg16.py:
--------------------------------------------------------------------------------
  1 | import inspect
  2 | import os
  3 | 
  4 | import numpy as np
  5 | import tensorflow as tf
  6 | import time
  7 | 
  8 | VGG_MEAN = [103.939, 116.779, 123.68]
  9 | 
 10 | 
 11 | class Vgg16:
 12 |     def __init__(self, vgg16_npy_path=None):
 13 |         if vgg16_npy_path is None:
 14 |             path = inspect.getfile(Vgg16)
 15 |             path = os.path.abspath(os.path.join(path, os.pardir))
 16 |             # print(path)
 17 |             path = os.path.join(path, "vgg16.npy")
 18 |             vgg16_npy_path = path
 19 | 
 20 |             print('path to the weight of vgg: {}'.format(path))
 21 | 
 22 |         self.data_dict = np.load(vgg16_npy_path, encoding='latin1').item()
 23 |         print(self.data_dict)
 24 |         print("npy file loaded")
 25 | 
 26 |     def build(self, x, batch_size, keep_prob):
 27 |         """
 28 |         load variable from npy to build the VGG
 29 |         :param rgb: rgb image [batch, height, width, 3] values scaled [0, 255]
 30 |         """
 31 | 
 32 |         #with tf.name_scope('norm_vgg_input'):
 33 |         #    # Convert RGB to BGR
 34 |         #    red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=x)
 35 |         #    assert red.get_shape().as_list()[1:] == [224, 224, 1]
 36 |         #    assert green.get_shape().as_list()[1:] == [224, 224, 1]
 37 |         #    assert blue.get_shape().as_list()[1:] == [224, 224, 1]
 38 |         #    bgr = tf.concat(axis=3, values=[
 39 |         #        blue - VGG_MEAN[0],
 40 |         #        green - VGG_MEAN[1],
 41 |         #        red - VGG_MEAN[2],
 42 |         #    ])
 43 |         #    assert bgr.get_shape().as_list()[1:] == [224, 224, 3]
 44 | 
 45 | 
 46 |         sess=tf.Session()
 47 |         print('x')
 48 |         print(sess.run(tf.shape(x)))
 49 |         self.conv1_1 = self.conv_layer(x, keep_prob, "conv1_1")
 50 |         self.conv1_2 = self.conv_layer(self.conv1_1, keep_prob, "conv1_2")
 51 |         self.pool1 = self.max_pool_stride(self.conv1_2, 2,'pool1')
 52 |         print('conv1')
 53 |         print(sess.run(tf.shape(self.pool1)))
 54 | 
 55 |         self.conv2_1 = self.conv_layer(self.pool1, keep_prob, "conv2_1")
 56 |         self.conv2_2 = self.conv_layer(self.conv2_1, keep_prob, "conv2_2")
 57 |         self.pool2 = self.max_pool_stride(self.conv2_2, 2,'pool2')
 58 |         print('conv2')
 59 |         print(sess.run(tf.shape(self.pool2)))
 60 | 
 61 |         self.conv3_1 = self.conv_layer(self.pool2, keep_prob, "conv3_1")
 62 |         self.conv3_2 = self.conv_layer(self.conv3_1, keep_prob, "conv3_2")
 63 |         self.conv3_3 = self.conv_layer(self.conv3_2, keep_prob, "conv3_3")
 64 |         self.pool3 = self.max_pool_stride(self.conv3_3, 2,'pool3')
 65 |         print('conv3')
 66 |         print(sess.run(tf.shape(self.pool3)))
 67 |         self.deconv2_1= self.deconv_layer(self.pool3,[8,8,256,256],[batch_size,112,112,256],4,"deconv2_1")
 68 |         print('deconv2_1')
 69 |         print(sess.run(tf.shape(self.deconv2_1)))
 70 | 
 71 |         self.conv4_1 = self.conv_layer(self.pool3, keep_prob, "conv4_1")
 72 |         self.conv4_2 = self.conv_layer(self.conv4_1, keep_prob, "conv4_2")
 73 |         self.conv4_3 = self.conv_layer(self.conv4_2, keep_prob, "conv4_3")
 74 |         self.pool4 = self.max_pool_stride(self.conv4_3, 1, 'pool4')
 75 |         print('conv4')
 76 |         print(sess.run(tf.shape(self.pool4)))
 77 | 
 78 |         self.conv5_1 = self.conv_layer(self.pool4, keep_prob, "conv5_1")
 79 |         self.conv5_2 = self.conv_layer(self.conv5_1, keep_prob, "conv5_2")
 80 |         self.conv5_3 = self.conv_layer(self.conv5_2, keep_prob, "conv5_3")
 81 |         self.pool5 = self.max_pool_stride(self.conv5_3, 1, 'pool5')
 82 |         print('conv5')
 83 |         print(sess.run(tf.shape(self.pool5)))
 84 | 
 85 | 
 86 |        # fcn layer
 87 |         self.fcn_1 = self.fully_conv_layer(self.pool5, [1,1,512,4096], 1, keep_prob,'fcn1')
 88 |         print('fcn1')
 89 |         print(sess.run(tf.shape(self.fcn_1)))
 90 |         self.fcn_2 = self.fully_conv_layer(self.fcn_1, [1,1,4096,4096], 1, keep_prob, 'fcn2')
 91 |         print('fcn2')
 92 |         print(sess.run(tf.shape(self.fcn_2)))        
 93 |         self.fcn_3 = self.fully_conv_layer(self.fcn_2, [1,1,4096,1000], 1, keep_prob, 'fcn3')
 94 |         print('fcn2')
 95 |         print(sess.run(tf.shape(self.fcn_3)))           
 96 |         self.deconv3_1= self.deconv_layer(self.fcn_3 ,[8,8,256,1000],[batch_size,112,112,256],4,"deconv3_1")
 97 |         print('deconv3_1')
 98 |         print(sess.run(tf.shape(self.deconv3_1)))
 99 |         sess.close()
100 | 
101 |         self.data_dict = None
102 | 
103 |         tf.summary.image('conv1',self.pool1[:,:,:,0:3])
104 |         tf.summary.image('conv2',self.pool2[:,:,:,0:3])
105 |         tf.summary.image('conv3',self.pool3[:,:,:,0:3])
106 |         tf.summary.image('conv4',self.pool4[:,:,:,0:3])
107 |         tf.summary.image('conv5',self.pool5[:,:,:,0:3])
108 |         tf.summary.image('conv5',self.pool5[:,:,:,0:3])
109 |         red = tf.reshape(self.deconv2_1[:,:,:,0], [-1,112,112,1])
110 |         green = tf.reshape(self.deconv2_1[:,:,:,1], [-1,112,112,1])
111 |         blue = tf.reshape(self.deconv2_1[:,:,:,2], [-1,112,112,1])
112 |         tf.summary.image('deconv2_1',red)
113 |         tf.summary.image('deconv2_1',green)
114 |         tf.summary.image('deconv2_1',blue)
115 |         red = tf.reshape(self.deconv3_1[:,:,:,0], [-1,112,112,1])
116 |         green = tf.reshape(self.deconv3_1[:,:,:,1], [-1,112,112,1])
117 |         blue = tf.reshape(self.deconv3_1[:,:,:,2], [-1,112,112,1])
118 |         tf.summary.image('deconv3_1',red)
119 |         tf.summary.image('deconv3_1',green)
120 |         tf.summary.image('deconv3_1',blue)
121 |         return self.deconv2_1, self.deconv3_1
122 | 
123 |     def max_pool_stride(self, bottom, stide, name):
124 |         return tf.nn.max_pool(bottom, ksize=[1, 2, 2, 1], strides=[1, stide, stide, 1], padding='SAME', name=name)
125 | 
126 |     def fully_conv_layer(self, x, filtershape, stride, keep_prob, name):
127 |         with tf.variable_scope(name):
128 |             filters = tf.get_variable(
129 |                 name = 'weight',
130 |                 shape = filtershape,
131 |                 dtype = tf.float32,
132 | 			    initializer = tf.random_normal_initializer(mean=0,stddev=0.001),
133 |                 trainable = True)
134 |             conv = tf.nn.conv2d(x, filters, [1, stride, stride, 1], padding= 'SAME')
135 |             conv_biases = tf.Variable(tf.constant(0.0, shape = [filtershape[3]], dtype = tf.float32),
136 |                                    trainable=True, name ='bias')
137 |             bias = tf.nn.bias_add(conv, conv_biases)
138 |             prelu = self.prelu(bias)
139 |             #output = tf.nn.dropout(prelu, keep_prob=keep_prob)
140 |             return prelu
141 | 
142 |     def conv_layer(self, bottom, keep_prob, name):
143 |         with tf.variable_scope(name):
144 |             filt = self.get_conv_filter(name)
145 |             conv = tf.nn.conv2d(bottom, filt, [1, 1, 1, 1], padding='SAME')
146 |             conv_biases = self.get_bias(name)
147 |             bias = tf.nn.bias_add(conv, conv_biases)
148 |             prelu = self.prelu(bias)
149 |             #output = tf.nn.dropout(prelu, keep_prob=keep_prob)
150 |             return prelu
151 | 
152 |     def get_conv_filter(self, name):
153 |         return tf.Variable(self.data_dict[name][0], name="filter")
154 | 
155 |     def get_bias(self, name):
156 |         return tf.Variable(self.data_dict[name][1], name="biases")
157 | 
158 |     def get_fc_weight(self, name):
159 |         return tf.Variable(self.data_dict[name][0], name="weights")
160 | 
161 |     def prelu(self, x):
162 |         with tf.variable_scope('prelu'):
163 |             alphas = tf.get_variable('alpha', x.get_shape()[-1],
164 | 							initializer=tf.constant_initializer(0.0),
165 | 							dtype=tf.float32)
166 |             pos = tf.nn.relu(x)
167 |             neg = alphas * (x - abs(x)) * 0.5
168 |             return pos + neg
169 | 
170 |     def deconv_layer(self, x, filtershape,output_shape, stride, name):
171 |         with tf.variable_scope(name):
172 |             filters = tf.get_variable(
173 |             name = 'weight',
174 |             shape = filtershape,
175 |             dtype = tf.float32,
176 |             initializer = tf.random_normal_initializer(mean=0,stddev=0.001),
177 |             trainable = True)
178 |             deconv = tf.nn.conv2d_transpose(x, filters, output_shape, [1, stride, stride, 1], padding ='SAME')
179 |             #deconv_biases = tf.Variable(tf.constant(0.0, shape = [filtershape[3]], dtype = tf.float32),
180 |             #                       trainable=True, name ='bias')
181 |             #bias = tf.nn.bias_add(deconv, deconv_biases)
182 |             #prelu = self.prelu(bias)
183 |             #output = tf.nn.dropout(prelu, keep_prob=keep_prob)
184 |             return self.prelu(deconv)
185 | 


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | import tqdm
  2 | import cv2
  3 | import numpy as np
  4 | import tensorflow as tf
  5 | from model import DeshadowNet
  6 | from tfrecord_utils import *
  7 | 
  8 | from tensorflow.python import debug as tf_debug
  9 | 
 10 | IMAGE_SIZE = 224 # 64 128 224
 11 | BATCH_SIZE = 2
 12 | EPOCH = 50000000
 13 | INIT_G_LEARNING_RATE = 1e-5 #1E-5
 14 | INIT_LEARNING_RATE = 1e-4 #1E-4
 15 | MOMENTUM = 0.9
 16 | 
 17 | log_path = './graph/logs' # path to tensorboard graph
 18 | TFRECORD_NAME = 'smalldata.tfrecord'
 19 | 
 20 | # momentum to 0.9 and
 21 | # weight decay to 0.0005 (at this point we do not include)
 22 | 
 23 | '''
 24 | How to use Tensorboard
 25 | Run the command line:
 26 | > tensorboard --logdir=./graph/logs
 27 | '''
 28 | 
 29 | '''
 30 | TODO
 31 | 1. data needs to be normlized 
 32 | 2. make the test set smaller like 20 or 30
 33 | 3. make the test set image size smaller
 34 | 
 35 | '''
 36 | 
 37 | def make_tfrecord():
 38 |     # make test set
 39 |     paths = glob.glob('./data/test/*')
 40 |     # making a list of image
 41 |     print('making pair of image list')
 42 |     image_list = []
 43 |     list_flag = True
 44 |     for path in paths:
 45 |         path = glob.glob(path+'/*')
 46 |         index = 0
 47 |         for sub in path:
 48 |             print('image name:{}'.format(sub))
 49 |             if list_flag:
 50 |                 image_list.append([sub])
 51 |             else:
 52 |                 image_list[index].append(sub)
 53 |                 index += 1
 54 |         list_flag = False
 55 | 
 56 | 
 57 |     # writing tf record
 58 |     print('tf record writing start')
 59 |     tfrecord_filename = TFRECORD_NAME
 60 |     writer = tf.python_io.TFRecordWriter(tfrecord_filename)
 61 | 
 62 |     index = 0
 63 |     for img_pair in image_list:
 64 |         index += 1
 65 |         print('number: {}'.format(index))
 66 |         write_tfrecord_pair_img(writer, img_pair[0],img_pair[1],tfrecord_filename)
 67 | 
 68 |     writer.close()
 69 | 
 70 |     # testing tf record read1ng
 71 |     print('reading tf record')
 72 |     reader = tf.TFRecordReader()
 73 |     # closeshape, image = read_tfrecord(reader,tfrecord_filename)
 74 |     visualize = False
 75 |     read_tfrecord_pair_img(reader,tfrecord_filename,visualize)
 76 | 
 77 | def train(backupFlag):
 78 |     # setting random seed and reset graphs
 79 | 
 80 |     G_LEARNING_RATE = 1e-5 #1e-5
 81 |     LEARNING_RATE = 1e-4 #1e-4
 82 | 
 83 |     tf.set_random_seed(1111)
 84 |     tf.reset_default_graph()
 85 | 
 86 |     #calc step number
 87 |     step_num = int(6 / BATCH_SIZE)
 88 | 
 89 |     # place holders for shadow image input and shadow free image input
 90 |     with tf.variable_scope('Data_Input'):
 91 |         shadow = tf.placeholder(tf.float32, [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, 3], name ='Shadow_image')
 92 |         shadow_free = tf.placeholder(tf.float32,[BATCH_SIZE,IMAGE_SIZE,IMAGE_SIZE,3], name = 'Shadow_free_image')
 93 |         keep_prob = tf.placeholder(tf.float32, name='keep_prob')
 94 |         lr_g = tf.placeholder(tf.float32, name='learn_rate_g')
 95 |         lr = tf.placeholder(tf.float32, name ='learn_rate_as')
 96 |     # init network model
 97 |     model = DeshadowNet(shadow, shadow_free, BATCH_SIZE,keep_prob)
 98 | 
 99 |     # gpu options for some errors
100 |     config = tf.ConfigProto()
101 |     config.gpu_options.allocator_type ='BFC'
102 |     config.gpu_options.per_process_gpu_memory_fraction = 0.80
103 | 
104 |     with tf.Session() as sess:
105 | 
106 |         ## ==================== for tensorflow debugger ========================= ##
107 |         #sess = tf_debug.LocalCLIDebugWrapperSession(sess)
108 |         #sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
109 |         ## ====================================================================== ##
110 | 
111 |         print('initializing hyperparameter ... ')
112 |         #global_step = tf.Variable(0, name='global_step', trainable=False) # step size
113 |         epoch = tf.Variable(0, name='epoch', trainable=False) # epoch
114 | 
115 |         with tf.name_scope('Momentum_Opt'):
116 |             print('init optimization')
117 |             opt_g = tf.train.MomentumOptimizer(lr_g, MOMENTUM)
118 |             train_g_op = opt_g.minimize(model.loss, var_list = model.g_net_variables)
119 |             opt_a = tf.train.MomentumOptimizer(lr, MOMENTUM)
120 |             train_as_op = opt_a.minimize(model.loss, var_list = model.a_net_variables)
121 |             opt_s = tf.train.MomentumOptimizer(lr, MOMENTUM)
122 |             train_as_op = opt_s.minimize(model.loss, var_list = model.s_net_variables)
123 |             train_op = tf.group(train_g_op,train_as_op)
124 | 
125 | 
126 |         # loading dataset ...
127 |         reader = tf.TFRecordReader()
128 |         img_input, img_gt = read_tfrecord_pair_img(reader, TFRECORD_NAME, False)
129 | 
130 |         coord = tf.train.Coordinator()
131 | 
132 |         # shuffle batch ...
133 |         img_input_batch,img_gt_batch = tf.train.shuffle_batch([img_input, img_gt],
134 |                                            BATCH_SIZE,
135 |                                            capacity=10*BATCH_SIZE,
136 |                                            min_after_dequeue = 2*BATCH_SIZE,
137 |                                            num_threads=4,
138 |                                            enqueue_many=False )
139 | 
140 |         print('initializing variables')
141 |         init_op = tf.global_variables_initializer()
142 |         init_local_op = tf.local_variables_initializer()
143 |         sess.run([init_op,init_local_op])
144 |         threads = tf.train.start_queue_runners(coord = coord)
145 | 
146 | 		# parameter
147 |         tf.summary.scalar('epoch',epoch)
148 |         tf.summary.scalar('loss',model.loss)
149 |         tf.summary.scalar('lr_as',lr)
150 |         tf.summary.scalar('lr_g',lr_g)
151 | 
152 | 		# images
153 |         tf.summary.image('F_matte',model.shadow_matte)
154 |         tf.summary.image('GT_matte',model.gt_shadow_matte)
155 |         tf.summary.image('F_free',model.f_shadowfree)
156 |         tf.summary.image('GT_free',model.gt_shadowfree)
157 | 
158 |         summary_op = tf.summary.merge_all()
159 |         print('start training ...')
160 |         writer = tf.summary.FileWriter(log_path, graph=tf.get_default_graph())
161 | 
162 | 
163 |         print('check if there is backup')
164 |         if backupFlag and tf.train.get_checkpoint_state('./backup'):
165 |             print('backup file loading ...')
166 |             saver = tf.train.Saver()
167 |             saver.restore(sess, './backup/latest')
168 |         else:
169 |             print('no backup ...')
170 | 
171 | 
172 |         try:
173 |             while not coord.should_stop():
174 | 
175 |                 sess.run(tf.assign(epoch, tf.add(epoch,1)))
176 |                 print('epoch: {}'.format(sess.run(epoch)))
177 |                 if sess.run(epoch) <= EPOCH:
178 |                     loss_val = 0
179 |                     for i in tqdm.tqdm(range(step_num)):
180 |                         # load image batched
181 |                         img_input_seq, img_gt_seq = sess.run([img_input_batch,img_gt_batch])
182 |                         img_input_seq.reshape([BATCH_SIZE,IMAGE_SIZE,IMAGE_SIZE,3])
183 |                         img_gt_seq.reshape([BATCH_SIZE,IMAGE_SIZE,IMAGE_SIZE,3])
184 | 
185 |                         # type casting
186 |                         img_input_seq = img_input_seq.astype(np.float)
187 |                         img_gt_seq = img_gt_seq.astype(np.float)
188 | 
189 | 
190 |                         _,loss,summary = sess.run([train_op,model.loss,summary_op],
191 |                                             feed_dict={
192 |                                             shadow: img_input_seq,
193 |                                             shadow_free:  img_gt_seq,
194 |                                             keep_prob: 0.5,
195 |                                             lr: LEARNING_RATE,
196 |                                             lr_g: G_LEARNING_RATE
197 |                                             })
198 |                         # print('loss: {}'.format(loss))
199 |                         loss_val += loss
200 |                         # write log
201 |                         writer.add_summary(summary, (sess.run(epoch) -1)*step_num+i)
202 | 
203 | 
204 |                         if i == int(step_num -1):
205 |                             A, S,SM,GT_SM = sess.run([model.A_output,model.S_output,model.shadow_matte,model.gt_shadow_matte],
206 |                                     feed_dict={
207 |                                     shadow: img_input_seq,
208 |                                     shadow_free:  img_gt_seq,
209 |                                     keep_prob: 1.0,
210 |                                     lr: LEARNING_RATE,
211 |                                     lr_g:G_LEARNING_RATE
212 |                                     })
213 |                             A.reshape([BATCH_SIZE,224,224,3])
214 |                             S.reshape([BATCH_SIZE,224,224,3])
215 |                             SM.reshape([BATCH_SIZE,224,224,3])
216 |                             GT_SM.reshape([BATCH_SIZE,224,224,3])
217 |                             np.savetxt('./output/a_out/A{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), A[0,:,:,0], delimiter=',')   # X is an array
218 |                             np.savetxt('./output/s_out/S{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), S[0,:,:,0], delimiter=',')   # X is an array
219 |                             np.savetxt('./output/sm_out/SM{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), SM[0,:,:,0], delimiter=',')   # X is an array
220 |                             np.savetxt('./output/sm_gt_out/GT_SM{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), GT_SM[0,:,:,0], delimiter=',')   # X is an array
221 | 
222 | 
223 | 
224 |                     print('total loss in one epoch: {}'.format(loss_val))
225 | 
226 |                     # check for validation
227 |                     # find shadow matte
228 |                     #f_shadow_free, gt_shadow_free = sess.run([model.f_shadowfree,model.gt_shadowfree],
229 |                     #                                        feed_dict={
230 |                     #                                        shadow: img_input_seq,
231 |                     #                                        shadow_free: img_gt_seq,
232 |                     #                                        keep_prob: 1.0,
233 |                     #                                        lr: LEARNING_RATE,
234 |                     #                                        lr_g:G_LEARNING_RATE
235 |                     #                                        })
236 |                     #np.savetxt('./output/f_shadow/f_shadow{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), f_shadow_free[0,:,:,:], delimiter=',')   # X is an array
237 |                     #np.savetxt('./output/gt_shadow/gt_shadow{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), gt_shadow_free[0,:,:,:], delimiter=',')   # X is an array					
238 |                     # find
239 |                     #out = f_shadow_free[0].astype(np.uint8)
240 |                     #gt = gt_shadow_free[0].astype(np.uint8)
241 |                     #np.savetxt('./output/f_shadow/f_shadow{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), out[:,:,0], delimiter=',')
242 |                     #np.savetxt('./output/gt_shadow/gt_shadow{}.out'.format("{0:06d}".format((sess.run(epoch) -1)*step_num* BATCH_SIZE+i)), gt[:,:,0], delimiter=',')
243 | 
244 |                     #image_show(out)
245 |                     #image_show(gt)
246 | 
247 |                     #cv2.imwrite('./output/f_shadow/{}.jpg'.format("{0:06d}".format(sess.run(epoch))), cv2.cvtColor(out, cv2.COLOR_RGB2BGR))
248 |                     #cv2.imwrite('./output/gt_shadow/{}.jpg'.format("{0:06d}".format(sess.run(epoch))), cv2.cvtColor(gt, cv2.COLOR_RGB2BGR))
249 |                     # saving data
250 |                     saver = tf.train.Saver()
251 |                     saver.save(sess, './backup/latest', write_meta_graph = False)
252 |                     if sess.run(epoch) == EPOCH:
253 |                         saver.save(sess,'./backup/fully_trained',write_meta_graph = False)
254 | 
255 |                     # decay learning rate
256 |                     if (sess.run(epoch) % 200) == 0:
257 |                         LEARNING_RATE = lr_decay(INIT_LEARNING_RATE, 1, sess.run(epoch))
258 |                         G_LEARNING_RATE = lr_decay(INIT_G_LEARNING_RATE, 1, sess.run(epoch))
259 |                         print('decreasing learning rate ...')
260 | 
261 | 
262 |                 else:
263 |                     print('breaking out of while loop ...')
264 |                     break
265 | 
266 |         except tf.errors.OutOfRangeError:
267 |             print('ERROR: out of range\nDone Training -- epoch reached its limit')
268 | 
269 |         finally:
270 |             print('FINAL: stop coordinate and joing thread')
271 |             coord.request_stop()
272 |             coord.join(threads)
273 | 
274 | def image_show(np_image):
275 |     img = Image.fromarray(np_image,'RGB')
276 |     img.show()
277 | 
278 | def lr_decay(lr_input, decay_rate,num_epoch):
279 |     return lr_input / (1 + decay_rate*num_epoch)
280 | 
281 | if __name__ == '__main__':
282 |     make_tfrecord()
283 |     train(False)
284 | 


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | from layer import *
  2 | import numpy as np
  3 | import math
  4 | from PIL import Image
  5 | import vgg16
  6 | import utils_vgg
  7 | 
  8 | from tensorflow.python import debug as tf_debug
  9 | 
 10 | ##import vgg using keras
 11 | ##from keras.applications.vgg16 import VGG16
 12 | ##from keras.preprocessing import image
 13 | ##from keras.applications.vgg16 import preprocess_input
 14 | ##from keras import backend as K
 15 | #from tensorflow.python.keras.models import Model, Sequential
 16 | #from tensorflow.python.keras.layers import Dense, Flatten, Dropout
 17 | #from tensorflow.python.keras.applications import VGG16
 18 | #from tensorflow.python.keras.applications.vgg16 import preprocess_input, decode_predictions
 19 | #from tensorflow.python.keras.preprocessing.image import ImageDataGenerator
 20 | #from tensorflow.python.keras.optimizers import Adam, RMSprop
 21 | 
 22 | ## import vgg using slim
 23 | #import tensorflow.contrib.slim as slim
 24 | #import tensorflow.contrib.slim.nets
 25 | 
 26 | 
 27 | '''
 28 | Tensorflow + Keras Example
 29 |     https://blog.keras.io/keras-as-a-simplified-interface-to-tensorflow-tutorial.html
 30 | 
 31 | Keras Finetune , transfer learning
 32 |     https://github.com/Hvass-Labs/TensorFlow-Tutorials/blob/master/10_Fine-Tuning.ipynb
 33 |     https://keras.io/applications/
 34 | 
 35 | Tensorflow
 36 |     https://github.com/machrisaa/tensorflow-vgg
 37 | '''
 38 | 
 39 | #TODO
 40 | #1. choose one of the following for fine tuning
 41 | #    - use tf slim for loading model
 42 | #    - use keras and tf for loading model
 43 | #2. data generate
 44 | #3. check if the code works using training
 45 | #4. data augmentation
 46 | #    - use keras if in 1 works with keras
 47 | #5. data generation
 48 | #    - use python script to generate data within maya env
 49 | 
 50 | MOMENTUM = 0.9
 51 | VGG_MEAN = [103.939, 116.779, 123.68]
 52 | 
 53 | class DeshadowNet:
 54 | 
 55 |     def __init__(self, x, deshadow_image, batch_size,keep_prob):
 56 |         
 57 |         self.batch_size = batch_size
 58 |         self.norm_x = norm_image(x)
 59 |         #self.norm_deshadow_image = norm_image(deshadow_image)
 60 |         
 61 |         self.A_input, self.S_input = self.G_Net(self.norm_x,keep_prob)
 62 |         #self.A_output = self.A_Net(self.norm_x,self.A_input,keep_prob)
 63 |         #self.S_output = self.S_Net(self.norm_x,self.S_input,keep_prob)
 64 |         self.A_output = self.A_Net(x,self.A_input,keep_prob)
 65 |         self.S_output = self.S_Net(x,self.S_input,keep_prob)
 66 | 
 67 |         self.shadow_matte= self.shadowMatte(self.A_output, self.S_output)
 68 |         self.gt_shadow_matte = self.calcShadowMatte(x, deshadow_image)
 69 | 
 70 | 		# loss function
 71 |         self.loss = self.loss_function(self.shadow_matte, self.gt_shadow_matte)
 72 | 
 73 |         self.f_shadowfree = self.calcShadowFreeImage(x,tf.log(tf.clip_by_value(self.shadow_matte,1e-30,float('Inf'))))
 74 |         self.gt_shadowfree = self.calcShadowFreeImage(x,self.gt_shadow_matte)
 75 | 
 76 |         self.g_net_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='G_Net')
 77 |         self.a_net_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='A_Net')
 78 |         self.s_net_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='S_Net')
 79 | 
 80 |     def G_Net(self,x,keep_prob):
 81 |         vgg = vgg16.Vgg16()
 82 |         print('making g-network')
 83 |         with tf.variable_scope('G_Net'):
 84 |             #x = norm_image(x)
 85 |             A_input, S_input = vgg.build(x, self.batch_size,keep_prob)
 86 |         print('finishing g-network')
 87 |         return A_input, S_input
 88 | 
 89 |     def A_Net(self,x,G_input,keep_prob): # after conv3 in G_Net  256
 90 | 
 91 |         print('making a-network')
 92 |         sess=tf.Session()
 93 |         with tf.variable_scope('A_Net'):
 94 | 
 95 |             # conv2-1
 96 |             conv2_1 = conv_layer(x,[9,9,3,96],1,'conv2-1')
 97 | 
 98 |             # pool5
 99 |             conv2_1_output = max_pool_layer(conv2_1,[1,3,3,1],2,'pool2-1')
100 |             print('conv2-1')
101 |             print(sess.run(tf.shape(conv2_1_output)))
102 | 
103 |             # conv2-2
104 |             conv2_2_output = conv_layer(G_input,[1,1,256,64],1,'conv2-2') 
105 |             print('conv2-2')
106 |             print(sess.run(tf.shape(conv2_2_output)))
107 | 
108 |             # concat conv2-1 and conv2-2
109 |             conv_concat = tf.concat(axis=3, values = [conv2_1_output,conv2_2_output], name = 'concat_a_net')
110 | 
111 |             # conv2-3
112 |             conv2_3 = conv_layer(conv_concat,[5,5,160,64],1,'conv2-3')
113 |             print('conv2-3')
114 |             print(sess.run(tf.shape(conv2_3)))
115 | 
116 |             # conv2-4
117 |             conv2_4 = conv_layer(conv2_3,[5,5,64,64],1,'conv2-4')
118 |             print('conv2-4')
119 |             print(sess.run(tf.shape(conv2_4)))
120 | 
121 |             # conv2-5
122 |             conv2_5 = conv_layer(conv2_4,[5,5,64,64],1,'conv2-5')
123 |             print('conv2-5')
124 |             print(sess.run(tf.shape(conv2_5)))
125 | 
126 |             # conv2-6
127 |             conv2_6 = conv_layer(conv2_5,[5,5,64,64],1,'conv2-6')
128 |             print('conv2-6')
129 |             print(sess.run(tf.shape(conv2_6)))
130 | 
131 |             # deconv2_1
132 |             deconv2_2 = deconv_layer(conv2_6,[4,4,3,64],[self.batch_size,224,224,3],2,'deconv2-2')
133 |             print('deconv2-2')
134 |             print(sess.run(tf.shape(deconv2_2)))
135 |  
136 |             print('finishing a-network')
137 |             
138 |             tf.summary.image('conv2_1',conv2_1_output[:,:,:,0:3])
139 |             tf.summary.image('conv2_2',conv2_2_output[:,:,:,0:3])
140 |             tf.summary.image('conv2_3',conv2_3[:,:,:,0:3])
141 |             tf.summary.image('conv2_4',conv2_4[:,:,:,0:3])
142 |             tf.summary.image('conv2_5',conv2_5[:,:,:,0:3])
143 |             tf.summary.image('conv2_6',conv2_6[:,:,:,0:3])
144 |             red = tf.reshape(deconv2_2[:,:,:,0], [-1,224,224,1])
145 |             green = tf.reshape(deconv2_2[:,:,:,1], [-1,224,224,1])
146 |             blue = tf.reshape(deconv2_2[:,:,:,2], [-1,224,224,1])
147 |             tf.summary.image('deconv3_1',red)
148 |             tf.summary.image('deconv3_1',green)
149 |             tf.summary.image('deconv3_1',blue)
150 |             #tf.summary.image('deconv2_2-1',deconv2_2[:,:,:,0:1])
151 |             #tf.summary.image('deconv2_2-2',deconv2_2[:,:,:,1:2])
152 |             #tf.summary.image('deconv2_2',deconv2_2[:,:,:,:])
153 |             sess.close()
154 |             return deconv2_2
155 | 
156 |     def S_Net(self,x,G_input,keep_prob): # after conv5 in G_Net 512
157 |         
158 |         print('making s-network')
159 |         sess = tf.Session()
160 |         with tf.variable_scope('S_Net'):
161 |             # conv3-1
162 |             conv3_1 = conv_layer(x,[9,9,3,96],1,'conv3-1')
163 |             
164 |             # pool5
165 |             conv3_1_output = max_pool_layer(conv3_1,[1,3,3,1],2,'pool3-1')
166 |             print('conv3-1')
167 |             print(sess.run(tf.shape(conv3_1_output)))
168 |             
169 |             # conv2-2
170 |             conv3_2_output = conv_layer(G_input,[1,1,256,64],1,'conv3-2') # need deconv before, also need to change the size of the channel
171 |             print('conv3-2')
172 |             print(sess.run(tf.shape(conv3_2_output)))
173 | 
174 |             # concat conv2-1 and conv2-2
175 |             conv_concat = tf.concat(axis=3, values = [conv3_1_output,conv3_2_output],name = 'concat_s_net')
176 | 
177 |             # conv3-3
178 |             conv3_3 = conv_layer(conv_concat,[5,5,160,64],1,'conv3-3')
179 |             print('conv3-3')
180 |             print(sess.run(tf.shape(conv3_3)))
181 | 
182 |             # conv3-4
183 |             conv3_4 = conv_layer(conv3_3,[5,5,64,64],1,'conv3-4')
184 |             print('conv3-4')
185 |             print(sess.run(tf.shape(conv3_4)))
186 | 
187 |             # conv3-5
188 |             conv3_5 = conv_layer(conv3_4,[5,5,64,64],1,'conv3-5')
189 |             print('conv3-5')
190 |             print(sess.run(tf.shape(conv3_5)))
191 | 
192 |             # conv3-6
193 |             conv3_6 = conv_layer(conv3_5,[5,5,64,64],1,'conv3-6')
194 |             print('conv3-6')
195 |             print(sess.run(tf.shape(conv3_6)))
196 | 
197 |             # deconv3-2
198 |             deconv3_2 = deconv_layer(conv3_6,[4,4,3,64],[self.batch_size,224,224,3],2,'decov3-2')
199 |             print('decov3-2')
200 |             print(sess.run(tf.shape(deconv3_2)))
201 | 
202 | 
203 |             tf.summary.image('conv3_1',conv3_1_output[:,:,:,0:3])
204 |             tf.summary.image('conv3_2',conv3_2_output[:,:,:,0:3])
205 |             tf.summary.image('conv3_3',conv3_3[:,:,:,0:3])
206 |             tf.summary.image('conv3_4',conv3_4[:,:,:,0:3])
207 |             tf.summary.image('conv3_5',conv3_5[:,:,:,0:3])
208 |             tf.summary.image('conv3_6',conv3_6[:,:,:,0:3])
209 |             red = tf.reshape(deconv3_2[:,:,:,0], [-1,224,224,1])
210 |             green = tf.reshape(deconv3_2[:,:,:,1], [-1,224,224,1])
211 |             blue = tf.reshape(deconv3_2[:,:,:,2], [-1,224,224,1])
212 |             tf.summary.image('deconv3_1',red)
213 |             tf.summary.image('deconv3_1',green)
214 |             tf.summary.image('deconv3_1',blue)
215 |             #tf.summary.image('deconv3_2-1',deconv3_2[:,:,:,0:1])
216 |             #tf.summary.image('deconv3_2-2',deconv3_2[:,:,:,1:2])
217 |             #tf.summary.image('deconv3_2',deconv3_2[:,:,:,:])
218 |             print('finishing s-network')
219 |             sess.close()
220 |             return deconv3_2
221 | 
222 |     def shadowMatte(self, A_input, S_input):
223 |         with tf.variable_scope('F_Matte'):
224 |             x = tf.concat([A_input,S_input],3,name='concat_a_s')
225 |             x = conv_layer(x,[1,1,6,3],1,'1x1conv')
226 |             #x  = tf.clip_by_value(x,0,1) 
227 |             return x
228 | 
229 |     def loss_function(self, x, ground_truth):
230 |         # MSE loss
231 |         with tf.variable_scope('Loss'):
232 |             #diff = tf.log(tf.clip_by_value(x,1e-10,255)) - tf.log(tf.clip_by_value(ground_truth,1e-10,255))
233 |             diff = tf.log(tf.clip_by_value(x,1e-30,float('Inf'))) - ground_truth
234 |             loss = tf.square(diff)
235 |             output = tf.reduce_mean(loss)
236 |         return output
237 | 
238 |     def calcShadowImage(self, deshadow, matte):
239 |         with tf.variable_scope('Shadow_Img'):
240 |             output = tf.multiply(deshadow,matte)
241 |             return output
242 | 
243 |     def calcShadowFreeImage(self, shadow, matte):
244 |         with tf.variable_scope('Shadow_Free_Img'):
245 |             # log (I_s) = log (S_m) + log (I_ns)
246 |             # log(I_ns) = log (I_s) - log (S_m) 
247 |             #unnorm_shadow = unnorm_image(shadow)
248 |             #matte_cliped = tf.clip_by_value(matte, 1e-10,255)
249 |             shadow = tf.clip_by_value(shadow,1e-3,255)
250 |             log_shadow = tf.log(shadow)
251 |             log_shadowfree = log_shadow - matte
252 |             #output = tf.divide(shadow,matte)
253 |             #output = tf.clip_by_value(output, 0,255)
254 |             shadowfree= tf.exp(log_shadowfree)  
255 |             return tf.clip_by_value(shadowfree, 0,255)           
256 |             #return unnorm_image(output)
257 | 
258 |     def calcShadowMatte(self, x, deshadow_image):
259 |         with tf.variable_scope('GT_Matte'):
260 |             # log (I_s) = log (S_m) + log (I_ns)
261 |             # log (S_m) = log (I_s) - log(I_ns)
262 |             # but in this case I_s/I_ns cannot be negative or zero
263 |             # so what we cannot normalize the input images
264 |             # then what happens to the output of network which is normalized?
265 |             # matte should be between 0 to 1
266 |             x = tf.clip_by_value(x,1e-3,255)
267 |             deshadow_image = tf.clip_by_value(deshadow_image,1e-3,255)
268 |             log_shadow_matte = tf.log(x) - tf.log(deshadow_image)
269 |             log_shadow_matte = tf.clip_by_value(log_shadow_matte,-float('Inf'),tf.log(255.0))
270 |             #return tf.exp(log_shadow_matte)
271 |             return log_shadow_matte
272 | def norm_image(x):
273 |     with tf.name_scope('norm_vgg_input'):
274 |         # Convert RGB to BGR
275 |         red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=x)
276 |         assert red.get_shape().as_list()[1:] == [224, 224, 1]
277 |         assert green.get_shape().as_list()[1:] == [224, 224, 1]
278 |         assert blue.get_shape().as_list()[1:] == [224, 224, 1]
279 |         bgr = tf.concat(axis=3, values=[
280 |             blue - VGG_MEAN[0],
281 |             green - VGG_MEAN[1],
282 |             red - VGG_MEAN[2],
283 |         ])
284 |         assert bgr.get_shape().as_list()[1:] == [224, 224, 3]
285 |         return bgr
286 | 
287 | def unnorm_image(x):
288 |     with tf.name_scope('unnorm_vgg_input'):
289 |         # Convert BGR to RGB
290 |         blue, green, red = tf.split(axis=3, num_or_size_splits=3, value=x)
291 |         assert red.get_shape().as_list()[1:] == [224, 224, 1]
292 |         assert green.get_shape().as_list()[1:] == [224, 224, 1]
293 |         assert blue.get_shape().as_list()[1:] == [224, 224, 1]
294 |         rgb = tf.concat(axis=3, values=[
295 |             red + VGG_MEAN[2],
296 |             green + VGG_MEAN[1],
297 |             blue + VGG_MEAN[0],
298 |         ])
299 |         assert rgb.get_shape().as_list()[1:] == [224, 224, 3]
300 |         return rgb
301 | 
302 | def image_show(np_image):
303 |     img = Image.fromarray(np_image,'RGB')
304 |     img.show()


--------------------------------------------------------------------------------