├── FSRN-CNN-train.py ├── FSRN-RNN-train.py ├── Fluid_wave_simulator ├── .DS_Store ├── README.txt ├── Step_1_createwave_0_Ocean.m ├── Step_1_createwave_0_Ripple.m ├── Step_1_createwave_0_Tian.m ├── Step_2_Image_Wave_Index_Mapping.m ├── Step_3_Render_dataset.m ├── cold.m ├── getOceanWave.m ├── getThreeRippleWave.m ├── getTwoRippleWave.m ├── getWarpWater.m ├── npy-matlab-master │ ├── .DS_Store │ └── npy-matlab-master │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── examples │ │ └── exampleMemmap.m │ │ ├── npy-matlab │ │ ├── constructNPYheader.m │ │ ├── datToNPY.m │ │ ├── readNPY.m │ │ ├── readNPYheader.m │ │ └── writeNPY.m │ │ └── tests │ │ ├── data │ │ ├── chelsea_float32.npy │ │ ├── chelsea_float64.npy │ │ ├── chelsea_int16.npy │ │ ├── chelsea_int32.npy │ │ ├── chelsea_int64.npy │ │ ├── chelsea_int8.npy │ │ ├── chelsea_uint16.npy │ │ ├── chelsea_uint32.npy │ │ ├── chelsea_uint64.npy │ │ ├── chelsea_uint8.npy │ │ ├── matlab_chelsea_float32.npy │ │ ├── matlab_chelsea_float64.npy │ │ ├── matlab_chelsea_int16.npy │ │ ├── matlab_chelsea_int32.npy │ │ ├── matlab_chelsea_int64.npy │ │ ├── matlab_chelsea_int8.npy │ │ ├── matlab_chelsea_uint16.npy │ │ ├── matlab_chelsea_uint32.npy │ │ ├── matlab_chelsea_uint64.npy │ │ ├── matlab_chelsea_uint8.npy │ │ ├── matlab_sine_float32.npy │ │ ├── matlab_sine_float64.npy │ │ ├── matlab_sine_int16.npy │ │ ├── matlab_sine_int32.npy │ │ ├── matlab_sine_int64.npy │ │ ├── matlab_sine_int8.npy │ │ ├── matlab_sine_uint16.npy │ │ ├── matlab_sine_uint32.npy │ │ ├── matlab_sine_uint64.npy │ │ ├── matlab_sine_uint8.npy │ │ ├── sine_float32.npy │ │ ├── sine_float64.npy │ │ ├── sine_int16.npy │ │ ├── sine_int32.npy │ │ ├── sine_int64.npy │ │ ├── sine_int8.npy │ │ ├── sine_uint16.npy │ │ ├── sine_uint32.npy │ │ ├── sine_uint64.npy │ │ ├── sine_uint8.npy │ │ └── test.npy │ │ ├── npy.ipynb │ │ ├── test_npy_roundtrip.py │ │ └── test_readNPY.m ├── raytracing_im_generator_ST.m ├── refraction.m ├── simulate.m └── tex1.jpg ├── LICENSE ├── README.md ├── evaluate_metrics.py └── img ├── .DS_Store ├── 3092-teaser.gif ├── real.gif ├── real1.gif ├── rerender.gif ├── rerender1.gif ├── syn.gif └── syn1.gif /FSRN-CNN-train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import matplotlib.pyplot as plt 4 | import PIL 5 | import tensorflow as tf 6 | import numpy as np 7 | import cv2, os 8 | import seaborn as sns 9 | import pandas as pd 10 | import warnings 11 | from scipy.misc import toimage, imsave 12 | import glob 13 | import gc 14 | from sklearn.utils import shuffle 15 | from scipy.linalg import norm 16 | from scipy import sum, average 17 | from scipy.interpolate import RectBivariateSpline 18 | 19 | from tensorflow.python.keras.applications.vgg16 import VGG16 20 | from tensorflow.python.keras.applications.vgg16 import preprocess_input, decode_predictions 21 | from tensorflow.python.keras.preprocessing.image import ImageDataGenerator 22 | import keras, sys, time 23 | from tensorflow.python.keras.models import * 24 | from tensorflow.python.keras.layers import * 25 | from tensorflow.python.keras.optimizers import Adam 26 | # from keras.callbacks import TensorBoard, ModelCheckpoint 27 | from tensorflow.keras.callbacks import TensorBoard 28 | # from keras.callbacks import EarlyStopping 29 | from tensorflow.keras.callbacks import ModelCheckpoint 30 | from tensorflow.keras.callbacks import EarlyStopping 31 | from keras.utils import plot_model 32 | 33 | from our_train_D2N_raytrace_datagen import DataGenerator 34 | #list visible devices 35 | from tensorflow.python.client import device_lib 36 | 37 | TRAIN_NUM = 30600 38 | VAL_NUM = 5400 39 | BATCH_SIZE = 32 40 | 41 | #create network 42 | def FluidNet( nClasses, nClasses1 , input_height=128, input_width=128): 43 | assert input_height%32 == 0 44 | assert input_width%32 == 0 45 | IMAGE_ORDERING = "channels_last" 46 | 47 | img_input = Input(shape=(input_height,input_width, 6), name='combined_input') ## Assume 128,128,6 48 | 49 | ## Block 1 128x128 50 | x = Conv2D(18, (2, 2), activation='relu', padding='same', name='block1_conv1', data_format=IMAGE_ORDERING )(img_input) 51 | x = Conv2D(18, (2, 2), activation='relu', padding='same', name='block1_conv2', data_format=IMAGE_ORDERING )(x) 52 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool', data_format=IMAGE_ORDERING )(x) 53 | f1 = x 54 | 55 | # Block 2 64x64 56 | x = Conv2D(36, (2, 2), activation='relu', padding='same', name='block2_conv1', data_format=IMAGE_ORDERING )(x) 57 | x = Conv2D(36, (2, 2), activation='relu', padding='same', name='block2_conv2', data_format=IMAGE_ORDERING )(x) 58 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool', data_format=IMAGE_ORDERING )(x) 59 | f2 = x 60 | 61 | # Block 3 32x32 62 | x = Conv2D(72, (2, 2), activation='relu', padding='same', name='block3_conv1', data_format=IMAGE_ORDERING )(x) 63 | x = Conv2D(72, (2, 3), activation='relu', padding='same', name='block3_conv2', data_format=IMAGE_ORDERING )(x) 64 | x = Conv2D(72, (2, 2), activation='relu', padding='same', name='block3_conv3', data_format=IMAGE_ORDERING )(x) 65 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool', data_format=IMAGE_ORDERING )(x) 66 | pool3 = x 67 | 68 | # Block 4 16x16 69 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv1', data_format=IMAGE_ORDERING )(x) 70 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv2', data_format=IMAGE_ORDERING )(x) 71 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv3', data_format=IMAGE_ORDERING )(x) 72 | pool4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool', data_format=IMAGE_ORDERING )(x) 73 | 74 | # Block 5 8x8 75 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv1', data_format=IMAGE_ORDERING )(pool4) 76 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv2', data_format=IMAGE_ORDERING )(x) 77 | x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv3', data_format=IMAGE_ORDERING )(x) 78 | pool5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool', data_format=IMAGE_ORDERING )(x) 79 | 80 | # Block Transpose : Depth 81 | #1st deconv layer 4x4 82 | x = (Conv2DTranspose( 72, kernel_size=(4,4) , strides=(2,2) , padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool5" ) (pool5)) 83 | 84 | #concatinate x and pool4 for 2nd Deconv layer 8X8 85 | x = concatenate ([x, pool4],axis = 3) 86 | x = (Conv2DTranspose( 36 , kernel_size=(6,6) , strides=(2,2) ,padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool4")(x)) 87 | 88 | #concatinate x and pool3 for 3rd Deconv layer 28x28 89 | x = concatenate ([x, pool3],axis = 3) 90 | x= (Conv2DTranspose( 18 , kernel_size=(4,4) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool3" )(x)) 91 | 92 | #concatinate x and f2 for 4th Deconv layer 93 | x = concatenate ([x, f2],axis = 3) 94 | x = (Conv2DTranspose( 9 , kernel_size=(4,4) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool2" )(x)) 95 | 96 | #concatinate x and f1 for 5th Deconv layer 97 | 98 | x = concatenate ([x, f1],axis = 3) 99 | x = (Conv2DTranspose( nClasses + nClasses1 , kernel_size=(3,3) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool1" )(x)) 100 | 101 | o = x 102 | o = (Activation('sigmoid', name="depth_out"))(o) 103 | 104 | # Block Transpose : Scale 105 | #1st deconv layer 7x7 106 | x2 = (Conv2DTranspose( 72, kernel_size=(4,4) , strides=(2,2) , padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool5_2" ) (pool5)) 107 | 108 | #concatinate x and pool4 for 2nd Deconv layer 14x14 109 | x2 = concatenate ([x2, pool4],axis = 3) 110 | x2 = (Conv2DTranspose( 36 , kernel_size=(6,6) , strides=(2,2) ,padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool4_2")(x2)) 111 | 112 | #concatinate x and pool3 for 3rd Deconv layer 28x28 113 | x2 = concatenate ([x2, pool3],axis = 3) 114 | x2= (Conv2DTranspose( 18 , kernel_size=(4,4) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool3_2" )(x2)) 115 | 116 | #concatinate x and f2 for 4th Deconv layer 117 | x2 = concatenate ([x2, f2],axis = 3) 118 | x2 = (Conv2DTranspose( 9 , kernel_size=(4,4) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool2_2" )(x2)) 119 | 120 | #concatinate x and f1 for 5th Deconv layer 121 | 122 | x2 = concatenate ([x2, f1],axis = 3) 123 | x2 = (Conv2DTranspose( 7 , kernel_size=(3,3) , strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool1_2" )(x2)) 124 | 125 | o2 = x2 126 | o2 = (Activation('sigmoid', name="scale_out"))(o2) 127 | 128 | singleOut = concatenate([o,o2],axis = 3, name="single_out") 129 | 130 | #model creation 131 | model = Model(img_input, singleOut) 132 | 133 | return model 134 | 135 | # Keras losses 136 | def mean_squared_error(y_true, y_pred): 137 | return K.mean(K.square(y_pred - y_true), axis=-1) 138 | 139 | # Custom loss 140 | def depth_loss (y_true, y_pred): 141 | d = tf.subtract(y_pred,y_true) 142 | n_pixels = 128 * 128 143 | square_n_pixels = n_pixels * n_pixels 144 | square_d = tf.square(d) 145 | sum_square_d = tf.reduce_sum(square_d,1) 146 | sum_d = tf.reduce_sum(d,1) 147 | square_sum_d = tf.square(sum_d) 148 | mid_output = tf.reduce_mean((sum_square_d/n_pixels) - 0.5* (square_sum_d/square_n_pixels)) 149 | 150 | dy_true, dx_true = tf.image.image_gradients(y_true) 151 | dy_pred, dx_pred = tf.image.image_gradients(y_pred) 152 | 153 | paddings_y = tf.constant([[0,0],[1,0],[0,0],[0,0]]) 154 | paddings_x = tf.constant([[0,0],[0,0],[1,0],[0,0]]) 155 | 156 | pad_dy_true = tf.pad(dy_true, paddings_y, "CONSTANT") 157 | pad_dy_pred = tf.pad(dy_pred, paddings_y, "CONSTANT") 158 | pad_dx_true = tf.pad(dx_true, paddings_x, "CONSTANT") 159 | pad_dx_pred = tf.pad(dx_pred, paddings_x, "CONSTANT") 160 | 161 | pad_dy_true = pad_dy_true[:,:-1,:,:] 162 | pad_dy_pred = pad_dy_pred[:,:-1,:,:] 163 | pad_dx_true = pad_dx_true[:,:,:-1,:] 164 | pad_dx_pred = pad_dx_pred[:,:,:-1,:] 165 | 166 | term3 = K.mean(K.abs(dy_pred - dy_true) + K.abs(dx_pred - dx_true) + K.abs(pad_dy_pred - pad_dy_true) + K.abs(pad_dx_pred - pad_dx_true), axis=-1) 167 | 168 | depth_output = mid_output + term3 169 | depth_output = K.mean(depth_output) 170 | return depth_output 171 | 172 | def normal_loss(y_true, y_pred): 173 | d = tf.subtract(y_pred,y_true) 174 | n_pixels = 128 * 128 175 | square_n_pixels = n_pixels * n_pixels 176 | square_d = tf.square(d) 177 | sum_square_d = tf.reduce_sum(square_d,1) 178 | sum_d = tf.reduce_sum(d,1) 179 | square_sum_d = tf.square(sum_d) 180 | normal_output = tf.reduce_mean((sum_square_d/n_pixels) - 0.5* (square_sum_d/square_n_pixels)) 181 | return normal_output 182 | 183 | def depth_to_normal(y_pred_depth,y_true_normal, scale_pred,scale_true): 184 | Scale = 127.5 185 | 186 | depth_min = scale_pred[:,0:1,0:1] 187 | depth_max = scale_pred[:,0:1,1:2] 188 | 189 | normal_min = scale_true[:,0:1,2:3] 190 | normal_max = scale_true[:,0:1,3:4] 191 | 192 | y_pred_depth = depth_min + (depth_max - depth_min) * y_pred_depth 193 | y_true_normal = normal_min + (normal_max - normal_min) * y_true_normal 194 | 195 | zy, zx = tf.image.image_gradients(y_pred_depth) 196 | 197 | zx = zx*Scale 198 | zy = zy*Scale 199 | 200 | normal_ori = tf.concat([zy, -zx, tf.ones_like(y_pred_depth)], 3) 201 | new_normal = tf.sqrt(tf.square(zx) + tf.square(zy) + 1) 202 | normal = normal_ori/new_normal 203 | 204 | normal += 1 205 | normal /= 2 206 | 207 | return normal,y_true_normal 208 | 209 | def vae_loss( y_true, y_pred): 210 | loss1 = mean_squared_error(y_true, y_pred) 211 | loss2 = binary_crossentropy(y_true, y_pred) 212 | return tf.reduce_mean(loss1 + loss2) 213 | 214 | def scale_loss(y_true,y_pred): 215 | pred_depth_min = y_pred[:,0:1,0:1] 216 | pred_depth_max = y_pred[:,0:1,1:2] 217 | pred_normal_min = y_pred[:,0:1,2:3] 218 | pred_normal_max = y_pred[:,0:1,3:4] 219 | 220 | true_depth_min = y_true[:,0:1,0:1] 221 | true_depth_max = y_true[:,0:1,1:2] 222 | true_normal_min = y_true[:,0:1,2:3] 223 | true_normal_max = y_true[:,0:1,3:4] 224 | 225 | loss_depth_min = mean_squared_error(true_depth_min, pred_depth_min) 226 | loss_depth_max = mean_squared_error(true_depth_max, pred_depth_max) 227 | loss_normal_min = mean_squared_error(true_normal_min, pred_normal_min) 228 | loss_normal_max = mean_squared_error(true_normal_max, pred_normal_max) 229 | 230 | return tf.reduce_mean(loss_depth_min + loss_depth_max + loss_normal_min + loss_normal_max) 231 | 232 | def snell_refraction(normal,s1,n1,n2): 233 | this_normal = normal 234 | term_1 = tf.cross(this_normal,tf.cross(-this_normal,s1)) 235 | term_temp = tf.cross(this_normal,s1) 236 | n_sq = (n1/n2)**2 237 | term_2 = tf.sqrt(tf.subtract(1.0,tf.multiply(n_sq,tf.reduce_sum(tf.multiply(term_temp,term_temp),axis = 3)))) 238 | term_3 = tf.stack([term_2, term_2, term_2],axis = 3) 239 | nn = (n1/n2) 240 | s2 = tf.subtract(tf.multiply(nn,term_1) , tf.multiply(this_normal,term_3)) 241 | return s2 242 | 243 | def raytracing_loss(depth,normal,background,scale): 244 | step = 255/2 245 | n1 = 1; 246 | n2 = 1.33; 247 | 248 | depth_min = scale[:,0:1,0:1] 249 | depth_max = scale[:,0:1,1:2] 250 | normal_min = scale[:,0:1,2:3] 251 | normal_max = scale[:,0:1,3:4] 252 | 253 | depth = depth_min + (depth_max - depth_min) * depth 254 | normal = normal_min + (normal_max - normal_min) * normal 255 | 256 | depth = tf.squeeze(depth, axis = -1) 257 | 258 | s1 = tf.Variable(0.0,name="s1") 259 | s1_temp = tf.zeros([K.shape(depth)[0],128,128,1]) 260 | s1 = tf.assign(s1,s1_temp, validate_shape=False) 261 | 262 | s11 = tf.Variable(0.0,name="s11") 263 | s11_temp = -1*tf.ones([K.shape(depth)[0],128,128,1]) 264 | s11 = tf.assign(s11,s11_temp, validate_shape=False) 265 | 266 | assigned_s1 = tf.stack([s1,s1,s11],axis = 3) 267 | assigned_s1 = tf.squeeze(assigned_s1) 268 | 269 | s2 = snell_refraction(normal,assigned_s1,n1,n2) 270 | 271 | x_c_ori, y_c_ori, lamda_ori = tf.split(s2,[1,1,1],axis = 3) 272 | 273 | lamda = -1*tf.divide(depth,tf.squeeze(lamda_ori)) 274 | 275 | x_c = tf.multiply(lamda , tf.squeeze(x_c_ori))*step 276 | y_c = tf.multiply(lamda , tf.squeeze(y_c_ori))*step 277 | 278 | flow = tf.stack([y_c,-x_c],axis = -1) 279 | 280 | out_im_temp = tf.contrib.image.dense_image_warp( 281 | background, 282 | flow, 283 | name='dense_image_warp' 284 | ) 285 | 286 | out_im_tensor = tf.Variable(0.0) 287 | 288 | out_im_tensor = tf.assign(out_im_tensor, out_im_temp, validate_shape=False) 289 | return out_im_tensor 290 | 291 | def combined_loss(y_true,y_pred): 292 | #print(K.int_shape(y_true)[0],K.shape(y_pred)) 293 | 294 | depth_true = y_true[:,:,:,0] 295 | normal_true = y_true[:,:,:,1:4] 296 | img_true = y_true[:,:,:,4:7] 297 | ref_true = y_true[:,:,:,7:10] 298 | scale_true = y_true[:,:,:,10:] 299 | 300 | depth_pred = y_pred[:,:,:,0] 301 | normal_pred = y_pred[:,:,:,1:4] 302 | img_pred = y_pred[:,:,:,4:7] 303 | ref_pred = y_pred[:,:,:,7:10] 304 | scale_pred = y_pred[:,:,:,10:] 305 | 306 | depth_true = tf.expand_dims(depth_true, -1) 307 | depth_pred = tf.expand_dims(depth_pred, -1) 308 | 309 | alpha = 0.2 310 | beta = 0.2 311 | gamma = 0.2 312 | delta = 0.2 313 | theta = 0.2 314 | tau = 0.0 315 | lamda = 0 316 | 317 | #depth loss 318 | loss_depth = alpha*(depth_loss(depth_true,depth_pred)) 319 | 320 | #normal loss 321 | loss_normal = beta*(normal_loss(normal_true,normal_pred)) 322 | 323 | #normal from depth 324 | normal_from_depth, rescaled_true_normal = depth_to_normal(depth_pred,normal_true,scale_pred,scale_true) 325 | loss_depth_to_normal = gamma*(normal_loss(rescaled_true_normal,normal_from_depth)) 326 | 327 | #ray_tracing 328 | ray_traced_tensor= raytracing_loss(depth_pred,normal_pred,ref_true,scale_true) 329 | loss_ray_trace = delta * vae_loss(img_true,ray_traced_tensor) 330 | 331 | #scale_loss 332 | loss_scale = theta * scale_loss(scale_true,scale_pred) 333 | 334 | #reference_loss 335 | loss_reference = tau * vae_loss(ref_true,ref_pred) 336 | 337 | #input image loss 338 | loss_in_img = lamda * vae_loss(img_true,img_pred) 339 | 340 | return (loss_depth + loss_normal + loss_depth_to_normal + loss_ray_trace + loss_scale + loss_reference + loss_in_img) 341 | 342 | # TRAIN 343 | with tf.device("/gpu:0"): 344 | 345 | model = FluidNet(nClasses = 1, 346 | nClasses1 = 3, 347 | input_height = 128, 348 | input_width = 128) 349 | model.summary() 350 | 351 | # Load the preprocessed data 352 | gc.collect() 353 | X_train = np.load(dir_references+"X_train{}.npy".format(TRAIN_NUM)) 354 | X_train = np.array(X_train) 355 | print(X_train.shape) 356 | y_train = np.load(dir_references+"Y_train{}.npy".format(TRAIN_NUM)) 357 | y_train = np.array(y_train) 358 | print(y_train.shape) 359 | 360 | X_test = np.load(dir_references+"X_val{}.npy".format(VAL_NUM)) 361 | X_test = np.array(X_test) 362 | print(X_test.shape) 363 | y_test = np.load(dir_references+"Y_val{}.npy".format(VAL_NUM)) 364 | y_test = np.array(y_test) 365 | print(y_test.shape) 366 | 367 | #create model and train 368 | training_log = TensorBoard(log_folder) 369 | weight_filename = weight_folder + "pretrained_FSRN_CNN.h5" 370 | 371 | stopping = EarlyStopping(monitor='val_loss', patience=2) 372 | 373 | checkpoint = ModelCheckpoint(weight_filename, 374 | monitor = "val_loss", 375 | save_best_only = True, 376 | save_weights_only = True) 377 | #Plot loss 378 | dir_plot = "plot/" 379 | 380 | model = FluidNet(nClasses = 1, 381 | nClasses1 = 3, 382 | input_height = 128, 383 | input_width = 128) 384 | 385 | model.summary() 386 | plot_model(model,to_file=dir_plot+'model.png',show_shapes=True) 387 | 388 | epochs = 35 389 | learning_rate = 0.001 390 | batch_size = BATCH_SIZE 391 | 392 | loss_funcs = { 393 | "single_out": combined_loss, 394 | } 395 | loss_weights = {"single_out": 1.0} 396 | 397 | 398 | print('lr:{},epoch:{},batch_size:{}'.format(learning_rate,epochs,batch_size)) 399 | 400 | adam = Adam(lr=learning_rate) 401 | model.compile(loss= loss_funcs, 402 | optimizer=adam, 403 | loss_weights=loss_weights, 404 | metrics=["accuracy"]) 405 | gc.collect() 406 | 407 | history = model.fit(X_train,y_train, 408 | batch_size=batch_size, 409 | epochs=epochs, 410 | verbose=2, 411 | validation_data=(X_test,y_test), 412 | callbacks = [training_log,checkpoint]) 413 | 414 | fig = plt.figure(figsize=(10,20)) 415 | 416 | ax = fig.add_subplot(1,2,1) 417 | for key in ['loss', 'val_loss']: 418 | ax.plot(history.history[key],label=key) 419 | ax.legend() 420 | 421 | ax = fig.add_subplot(1,2,2) 422 | for key in ['acc', 'val_acc']: 423 | ax.plot(history.history[key],label=key) 424 | ax.legend() 425 | fig.savefig(dir_plot+"Loss_"+str(epochs)+".png") # save the figure to file 426 | plt.close(fig) -------------------------------------------------------------------------------- /FSRN-RNN-train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import matplotlib.pyplot as plt 4 | import PIL 5 | import tensorflow as tf 6 | import numpy as np 7 | import cv2, os 8 | import seaborn as sns 9 | import pandas as pd 10 | import warnings 11 | from scipy.misc import toimage, imsave 12 | import glob 13 | import gc 14 | from sklearn.utils import shuffle 15 | from scipy.linalg import norm 16 | from scipy import sum, average 17 | from scipy.interpolate import RectBivariateSpline 18 | 19 | from tensorflow.python.keras.applications.vgg16 import VGG16 20 | from tensorflow.python.keras.applications.vgg16 import preprocess_input, decode_predictions 21 | from tensorflow.python.keras.preprocessing.image import ImageDataGenerator 22 | import keras, sys, time 23 | from tensorflow.python.keras.models import * 24 | from tensorflow.python.keras.layers import * 25 | from tensorflow.python.keras.optimizers import Adam, SGD 26 | # from keras.callbacks import TensorBoard, ModelCheckpoint 27 | from tensorflow.keras.callbacks import TensorBoard 28 | # from keras.callbacks import EarlyStopping 29 | from tensorflow.keras.callbacks import ModelCheckpoint 30 | from tensorflow.keras.callbacks import EarlyStopping 31 | from keras.utils import plot_model 32 | 33 | from our_train_D2N_raytrace_datagen import DataGenerator 34 | #list visible devices 35 | from tensorflow.python.client import device_lib 36 | 37 | 38 | TRAIN_NUM = 30600 39 | VAL_NUM = 5400 40 | BATCH_SIZE = 32 41 | H = 128 42 | W = 128 43 | 44 | 45 | def FluidNet( nClasses, input_height=128, input_width=128): 46 | assert input_height%32 == 0 47 | assert input_width%32 == 0 48 | IMAGE_ORDERING = "channels_last" 49 | 50 | img_input = Input(shape=(3,input_height,input_width, 4), name='input_0') 51 | 52 | # Temporal Consistency 53 | x = ConvLSTM2D(filters=4, kernel_size=(2, 2) 54 | , data_format=IMAGE_ORDERING 55 | , name='convLSTM_1' 56 | , recurrent_activation='hard_sigmoid' 57 | , activation='sigmoid' 58 | , padding='same', return_sequences=True)(img_input) 59 | x = BatchNormalization()(x) 60 | x = ConvLSTM2D(filters=4, kernel_size=(2, 2) 61 | , data_format=IMAGE_ORDERING 62 | , name='convLSTM_2' 63 | , recurrent_activation='hard_sigmoid' 64 | , activation='sigmoid' 65 | , padding='same', return_sequences=True)(x) 66 | x = BatchNormalization()(x) 67 | x = ConvLSTM2D(filters=4, kernel_size=(2, 2) 68 | , data_format=IMAGE_ORDERING 69 | , name='convLSTM_3' 70 | , recurrent_activation='hard_sigmoid' 71 | , activation='sigmoid' 72 | , padding='same', return_sequences=False)(x) 73 | x = BatchNormalization()(x) 74 | 75 | o = x 76 | o = (Activation('sigmoid', name="refine_out"))(o) 77 | 78 | model = Model(img_input, o) 79 | 80 | return model 81 | 82 | # Custom loss 83 | def depth_loss (y_true, y_pred): 84 | d = tf.subtract(y_pred,y_true) 85 | n_pixels = 128 * 128 86 | square_n_pixels = n_pixels * n_pixels 87 | square_d = tf.square(d) 88 | sum_square_d = tf.reduce_sum(square_d,1) 89 | sum_d = tf.reduce_sum(d,1) 90 | square_sum_d = tf.square(sum_d) 91 | mid_output = tf.reduce_mean((sum_square_d/n_pixels) - 0.5* (square_sum_d/square_n_pixels)) 92 | 93 | dy_true, dx_true = tf.image.image_gradients(y_true) 94 | dy_pred, dx_pred = tf.image.image_gradients(y_pred) 95 | 96 | paddings_y = tf.constant([[0,0],[1,0],[0,0],[0,0]]) 97 | paddings_x = tf.constant([[0,0],[0,0],[1,0],[0,0]]) 98 | 99 | pad_dy_true = tf.pad(dy_true, paddings_y, "CONSTANT") 100 | pad_dy_pred = tf.pad(dy_pred, paddings_y, "CONSTANT") 101 | pad_dx_true = tf.pad(dx_true, paddings_x, "CONSTANT") 102 | pad_dx_pred = tf.pad(dx_pred, paddings_x, "CONSTANT") 103 | 104 | pad_dy_true = pad_dy_true[:,:-1,:,:] 105 | pad_dy_pred = pad_dy_pred[:,:-1,:,:] 106 | pad_dx_true = pad_dx_true[:,:,:-1,:] 107 | pad_dx_pred = pad_dx_pred[:,:,:-1,:] 108 | 109 | term3 = K.mean(K.abs(dy_pred - dy_true) + K.abs(dx_pred - dx_true) + K.abs(pad_dy_pred - pad_dy_true) + K.abs(pad_dx_pred - pad_dx_true), axis=-1) 110 | 111 | depth_output = mid_output + term3 112 | depth_output = K.mean(depth_output) 113 | return depth_output 114 | 115 | def normal_loss(y_true, y_pred): 116 | d = tf.subtract(y_pred,y_true) 117 | n_pixels = 128 * 128 118 | square_n_pixels = n_pixels * n_pixels 119 | square_d = tf.square(d) 120 | sum_square_d = tf.reduce_sum(square_d,1) 121 | sum_d = tf.reduce_sum(d,1) 122 | square_sum_d = tf.square(sum_d) 123 | normal_output = tf.reduce_mean((sum_square_d/n_pixels) - 0.5* (square_sum_d/square_n_pixels)) 124 | return normal_output 125 | 126 | def depth_to_normal(y_pred_depth,y_true_normal, scale_true): 127 | Scale = 127.5 128 | 129 | depth_min = scale_true[:,0:1,0:1] 130 | depth_max = scale_true[:,0:1,1:2] 131 | 132 | normal_min = scale_true[:,0:1,2:3] 133 | normal_max = scale_true[:,0:1,3:4] 134 | 135 | y_pred_depth = depth_min + (depth_max - depth_min) * y_pred_depth 136 | y_true_normal = normal_min + (normal_max - normal_min) * y_true_normal 137 | 138 | zy, zx = tf.image.image_gradients(y_pred_depth) 139 | 140 | zx = zx*Scale 141 | zy = zy*Scale 142 | 143 | normal_ori = tf.concat([zy, -zx, tf.ones_like(y_pred_depth)], 3) 144 | new_normal = tf.sqrt(tf.square(zx) + tf.square(zy) + 1) 145 | normal = normal_ori/new_normal 146 | 147 | normal += 1 148 | normal /= 2 149 | return normal 150 | 151 | def combined_loss(y_true,y_pred): 152 | 153 | depth_true = y_true[:,:,:,0] 154 | normal_true = y_true[:,:,:,1:4] 155 | scale_true = y_true[:,:,:,4:] 156 | 157 | depth_pred = y_pred[:,:,:,0] 158 | normal_pred = y_pred[:,:,:,1:] 159 | #scale_pred = y_pred[:,:,:,4:] 160 | 161 | depth_true = tf.expand_dims(depth_true, -1) 162 | depth_pred = tf.expand_dims(depth_pred, -1) 163 | 164 | alpha = 0.4 165 | beta = 0.4 166 | gamma = 0.2 167 | 168 | #depth loss 169 | loss_depth = alpha*(depth_loss(depth_true,depth_pred)) 170 | 171 | #normal loss 172 | loss_normal = beta*(normal_loss(normal_true,normal_pred))#beta*mean_squared_error(normal_true,normal_pred)#beta*(normal_loss(normal_true,normal_pred)) 173 | 174 | #normal from depth 175 | normal_from_depth = depth_to_normal(depth_pred,normal_true, scale_true) 176 | loss_depth_to_normal = gamma*(normal_loss(normal_true,normal_from_depth)) 177 | 178 | return (loss_depth + loss_normal + loss_depth_to_normal) 179 | 180 | def print_layer_trainable(): 181 | for layer in model_depth.layers: 182 | print("{0}:\t{1}".format(layer.trainable,layer.name)) 183 | 184 | # TRAIN 185 | with tf.device("/gpu:0"): 186 | 187 | model = FluidNet(nClasses = 4, 188 | input_height = 128, 189 | input_width = 128) 190 | 191 | model.summary() 192 | 193 | # Load post-scaled data predicted from FSRN-CNN 194 | gc.collect() 195 | 196 | X_train = np.load(dir_references+"X_FSRN_RNN_train_{}.npy".format(TRAIN_NUM)) 197 | X_train = np.array(X_train) 198 | print(X_train.shape) 199 | 200 | 201 | y_train = np.load(dir_references+"Y_FSRN_RNN_train_{}.npy".format(TRAIN_NUM)) 202 | y_train = np.array(y_train) 203 | print(y_train.shape) 204 | 205 | X_test = np.load(dir_references+"X_FSRN_RNN_val_{}.npy".format(VAL_NUM)) 206 | X_test = np.array(X_test) 207 | print(X_test.shape) 208 | 209 | 210 | y_test = np.load(dir_references+"Y_FSRN_RNN_val_{}.npy".format(VAL_NUM)) 211 | y_test = np.array(y_test) 212 | print(y_test.shape) 213 | 214 | #create model and train 215 | training_log = TensorBoard(log_folder) 216 | weight_filename = weight_folder + "pretrained_FSRN_RNN.h5" 217 | 218 | stopping = EarlyStopping(monitor='val_loss', patience=2) 219 | 220 | checkpoint = ModelCheckpoint(weight_filename, 221 | monitor = "val_loss" 222 | save_best_only = True, 223 | save_weights_only = True) 224 | #Plot loss 225 | dir_plot = "plot/" 226 | 227 | model = FluidNet(nClasses = 4, 228 | input_height = 128, 229 | input_width = 128) 230 | 231 | model.summary() 232 | plot_model(model,to_file=dir_plot+'model_FSRN_RNN_.png',show_shapes=True) 233 | 234 | 235 | epochs = 15 236 | learning_rate = 0.001 237 | batch_size = BATCH_SIZE 238 | loss_funcs = { 239 | "refine_out": combined_loss, 240 | } 241 | loss_weights = {"refine_out": 1.0} 242 | 243 | print('lr:{},epoch:{},batch_size:{}'.format(learning_rate,epochs,batch_size)) 244 | 245 | adam = Adam(lr=learning_rate) 246 | model.compile(loss= loss_funcs, 247 | optimizer=adam, 248 | loss_weights=loss_weights, 249 | metrics=["accuracy"]) 250 | 251 | 252 | gc.collect() 253 | 254 | history = model.fit(X_train,y_train, 255 | batch_size=batch_size, 256 | epochs=epochs, 257 | verbose=2, 258 | validation_data=(X_test,y_test), 259 | callbacks = [training_log,checkpoint]) 260 | 261 | fig = plt.figure(figsize=(10,20)) 262 | 263 | ax = fig.add_subplot(1,2,1) 264 | for key in ['loss', 'val_loss']: 265 | ax.plot(history.history[key],label=key) 266 | ax.legend() 267 | 268 | ax = fig.add_subplot(1,2,2) 269 | for key in ['acc', 'val_acc']: 270 | ax.plot(history.history[key],label=key) 271 | ax.legend() 272 | fig.savefig(dir_plot+"Loss_FSRN_RNN_"+str(epochs)+".png") # save the figure to file 273 | plt.close(fig) -------------------------------------------------------------------------------- /Fluid_wave_simulator/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/.DS_Store -------------------------------------------------------------------------------- /Fluid_wave_simulator/README.txt: -------------------------------------------------------------------------------- 1 | Running steps: 2 | 3 | Create a 'RGB' folder with 'train' and 'val' sub-folders 4 | Add background pattern to 'RGB' folder 5 | 6 | Generate Training set: 7 | - Set variable 'Phase' to 'train' in all the matlab file 8 | Generate Validation set: 9 | - Set variable 'Phase' to 'val' in all the matlab file 10 | 11 | 0: Add npy-matlab-master to the matlab path 12 | 13 | 1: Generate waves for each wave type 14 | - Ocean wave: run Step_1_createwave_0_Ocean 15 | - Ripple wave: run Step_1_createwave_0_Ripple 16 | - Waves from "Seeing through Water:xxx" paper: run Step_1_createwave_0_Tian 17 | 18 | Will save waves in the 'train\WaveSequences\' or 'val\WaveSequences\' folder 19 | 20 | 2: Generate Mapping matrix for the rendering steps: 21 | - Run Step_2_Image_Wave_Index_Mapping 22 | 23 | Will save matrix to the 'Mapping_struct_train.mat' or 'Mapping_struct_val.mat' 24 | 25 | 3. Rendering refracted images for each pattern: 26 | - Run Step_3_Render_dataset 27 | 28 | Will save waves in the 'train\Dataset\' or 'val\Dataset\' folder 29 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/Step_1_createwave_0_Ocean.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | close all 8 | clear 9 | 10 | %% Parameter Setting 11 | seq_Num = '9'; 12 | Phase = 'train'; 13 | % Phase = 'val'; %uncomment this for generating validation set 14 | wave_folder = [ Phase '/WaveSequences/Wave_Ocean/Seq_' seq_Num '/']; 15 | %depth_folder = [wave_folder 'depth/']; 16 | %warp_folder = [wave_folder 'warp/']; 17 | 18 | depth_folder = fullfile(Phase,'WaveSequences', 'Wave_Ocean',strcat('Seq_',seq_Num),'depth'); 19 | warp_folder = fullfile(Phase,'WaveSequences', 'Wave_Ocean',strcat('Seq_',seq_Num),'warp'); 20 | 21 | % Set Image size (W,H) and wave sequence number(Nframe)for this type 22 | W = 128; 23 | H = W; 24 | nFrame = 10; 25 | 26 | %% Generate Waves 27 | if ~exist(depth_folder,'file') 28 | mkdir(depth_folder); 29 | 30 | end 31 | 32 | if ~exist(warp_folder,'file') 33 | mkdir(warp_folder); 34 | end 35 | 36 | %% Attributes 37 | alpha = 5; 38 | c = 0.2; 39 | level_factor = 20; 40 | WindDir = pi/3;%pi/3; % Wind Direction 41 | patchSize = 100;%100; % Resolution 42 | 43 | 44 | rgb = im2double(imread('tex1.jpg')); 45 | pattern_im = imresize(rgb, [W,H]); 46 | rgb = pattern_im; 47 | img = rgb2gray(rgb); 48 | 49 | n1 = 1; 50 | n2 = 1.33; 51 | D0 = 4*alpha; 52 | 53 | %% Wave Generation 54 | 55 | 56 | [warp, levels] = getOceanWave(img, nFrame, alpha, c, WindDir, patchSize); 57 | 58 | levels = levels*level_factor; 59 | 60 | [X,Y] = meshgrid(1:W); 61 | % [X, Y] = meshgrid(linspace(-1,1,256),linspace(-1,1,256)); 62 | h = figure(1); 63 | hh = subplot(2,3,1);handle = surf (X,Y,levels(:,:,1),'FaceColor','interp','edgecolor','none','edgelighting','none'); 64 | colormap(cold); 65 | lighting('gouraud'); 66 | shading('interp'); 67 | 68 | for i = 1: nFrame 69 | disp(num2str(i)); 70 | zh = levels(:,:,i); 71 | zh_new = zh; 72 | zh_nl = zh_new+D0; 73 | [warp_map] = raytracing_im_generator_ST(rgb,zh_nl,n1,n2,X,Y); 74 | 75 | % Save Warp map 76 | 77 | filename_warp=fullfile(warp_folder,strcat('Seq_', seq_Num, '_', num2str(alpha), '_', num2str(i) ,'.npy')); 78 | writeNPY(warp_map,filename_warp); 79 | % Save Depth array 80 | 81 | filename_depth=fullfile(depth_folder, strcat('Seq_', seq_Num, '_', num2str(alpha), '_', num2str(i), '.npy')); 82 | %filename_depth=[depth_folder '/Seq_' seq_Num '_' num2str(alpha) '_' num2str(i) '.npy']; 83 | writeNPY(zh,filename_depth); 84 | 85 | 86 | zh_range = max(zh(:)) - min(zh(:)); 87 | set(handle,'zdata',zh_new); 88 | title(hh,sprintf('# %d: [%.3f , %.3f] --- %.3f', i, min(zh(:)),max(zh(:)), zh_range)); 89 | set(gca,'zlim',[min(levels(:)),max(levels(:))]); 90 | warp_x = warp.Xs(:, :, i)*level_factor; 91 | warp_y = warp.Ys(:, :, i)*level_factor; 92 | frames = simulate(rgb, warp_x,warp_y, false,i); 93 | recovered_in = simulate(frames, -warp_x,-warp_y, false,i); 94 | 95 | subplot(2,3,2),imshow(frames);title('Distorted by Raytracing') 96 | subplot(2,3,3),imshow(frames),title('Distorted by Warp') 97 | subplot(2,3,4),quiver(warp_x,warp_y),axis equal%plot(warp_map,'DecimationFactor',[10 10],'ScaleFactor',10) 98 | subplot(2,3,5),imshow(recovered_in),title('Recoverd By Warp') 99 | subplot(2,3,6),imshow(rgb),title('Undistored GT') 100 | pause(1/25); 101 | end 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/Step_1_createwave_0_Ripple.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | close all 8 | clear 9 | 10 | %% Parameter Setting 11 | seq_Num = '9'; 12 | Phase = 'train'; 13 | % Phase = 'val'; %uncomment this for generating validation set 14 | % wave_folder = [Phase '/WaveSequences/Wave_Ripple/Seq_' seq_Num '/']; 15 | % depth_folder = [wave_folder 'depth/']; 16 | % warp_folder = [wave_folder 'warp/']; 17 | depth_folder = fullfile(Phase,'WaveSequences', 'Wave_Ripple',strcat('Seq_',seq_Num),'depth'); 18 | warp_folder = fullfile(Phase,'WaveSequences', 'Wave_Ripple',strcat('Seq_',seq_Num),'warp'); 19 | % Set Image size (W,H) and wave sequence number (nFrame) for this type 20 | W = 128; 21 | H = W; 22 | nFrame = 100; 23 | 24 | %% Generate Waves 25 | if ~exist(depth_folder,'file') 26 | mkdir(depth_folder); 27 | end 28 | 29 | if ~exist(warp_folder,'file') 30 | mkdir(warp_folder); 31 | end 32 | 33 | xx=-(round(W*0.5)-1):1:round(W*0.5); 34 | yy=xx; 35 | [XX,YY] = meshgrid(xx,yy); % create rectangular mesh 36 | 37 | 38 | 39 | %% Attributes 40 | % Seq_9 41 | alpha = 5; 42 | c = 0.3; 43 | level_factor = 0.25; 44 | R=sqrt((XX+100).^2+(YY+150).^2)/8; %radius %very new 4300-4499 45 | R1=sqrt((XX-100).^2+(YY+150).^2)/8; 46 | k=0.08; % wave vector 47 | R2=sqrt((XX-100).^2+(YY-20).^2)/8; 48 | k2=0.1; 49 | 50 | %% Other Attributes 51 | rgb = im2double(imread('tex1.jpg')); 52 | pattern_im = imresize(rgb, [W,H]); 53 | rgb = pattern_im; 54 | img = rgb2gray(rgb); 55 | 56 | n1 = 1; 57 | n2 = 1.33; 58 | D0 = 4*alpha; 59 | 60 | %% Wave Generation 61 | [warp, levels] = getTwoRippleWave(img, nFrame + 1, alpha, c, R, R1, k); 62 | % [warp, levels] = getThreeRippleWave(img, nFrame + 1, alpha, c, R, R1, k, k2,R2); 63 | 64 | levels = levels*level_factor; 65 | levels = levels(:,:,2:end); 66 | 67 | [X,Y] = meshgrid(1:W); 68 | h = figure(1); 69 | hh = subplot(2,3,1);handle = surf (X,Y,levels(:,:,1),'FaceColor','interp','edgecolor','none','edgelighting','none'); 70 | colormap(cold); 71 | lighting('gouraud'); 72 | shading('interp'); 73 | 74 | % filename_depth=[depth_folder 'Seq_' seq_Num '_' num2str(alpha) '.npy']; 75 | % writeNPY(levels,filename_depth); 76 | 77 | for i = 1: nFrame 78 | disp(num2str(i)); 79 | zh = levels(:,:,i); 80 | zh_new = zh; 81 | zh_nl = zh_new+D0; 82 | 83 | [warp_map] = raytracing_im_generator_ST(rgb,zh_nl,n1,n2,X,Y); 84 | %filename_warp=[warp_folder 'Seq_' seq_Num '_' num2str(alpha) '_' num2str(i) '.npy']; 85 | filename_warp=fullfile(warp_folder,strcat('Seq_', seq_Num, '_', num2str(alpha), '_', num2str(i) ,'.npy')); 86 | 87 | writeNPY(warp_map,filename_warp); 88 | % Save Depth array 89 | %filename_depth=[depth_folder 'Seq_' seq_Num '_' num2str(alpha) '_' num2str(i) '.npy']; 90 | filename_depth=fullfile(depth_folder, strcat('Seq_', seq_Num, '_', num2str(alpha), '_', num2str(i), '.npy')); 91 | writeNPY(zh,filename_depth); 92 | 93 | zh_range = max(zh(:)) - min(zh(:)); 94 | set(handle,'zdata',zh_new); 95 | title(hh,sprintf('# %d: [%.3f , %.3f] --- %.3f', i, min(zh(:)),max(zh(:)), zh_range)); 96 | set(gca,'zlim',[min(levels(:)),max(levels(:))]); 97 | warp_x = warp.Xs(:, :, i)*level_factor; 98 | warp_y = warp.Ys(:, :, i)*level_factor; 99 | frames = simulate(rgb, warp_x,warp_y, false,i); 100 | recovered_in = simulate(frames, -warp_x,-warp_y, false,i); 101 | 102 | subplot(2,3,2),imshow(frames);title('Distorted by Raytracing') 103 | subplot(2,3,3),imshow(frames),title('Distorted by Warp') 104 | subplot(2,3,4),quiver(warp_x,warp_y),axis equal%plot(warp_map,'DecimationFactor',[10 10],'ScaleFactor',10) 105 | subplot(2,3,5),imshow(recovered_in),title('Recoverd By Warp') 106 | subplot(2,3,6),imshow(rgb),title('Undistored GT') 107 | pause(1/25); 108 | end 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/Step_1_createwave_0_Tian.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | close all 8 | clear 9 | 10 | %% Parameter Setting 11 | seq_Num = '9'; 12 | Phase = 'train'; 13 | % Phase = 'val'; %uncomment this for generating validation set 14 | wave_folder = [Phase '/WaveSequences/Wave_Tian/Seq_' seq_Num '/']; 15 | depth_folder = [wave_folder 'depth/']; 16 | warp_folder = [wave_folder 'warp/']; 17 | % Set Image size (W,H) and wave sequence number(Nframe)for this type 18 | W = 128; 19 | H = W; 20 | nFrame = 10; 21 | 22 | %% Generate Waves 23 | if ~exist(depth_folder,'file') 24 | mkdir(depth_folder); 25 | end 26 | 27 | if ~exist(warp_folder,'file') 28 | mkdir(warp_folder); 29 | end 30 | 31 | % img = rgb2gray(im2double(imread('binary.jpg'))); 32 | % img = img(320:463, :); 33 | rgb = im2double(imread('tex1.jpg')); 34 | rgb = imresize(rgb, [W,H]); 35 | img = rgb2gray(rgb); 36 | 37 | %% Attributes 38 | %alpha = 5(0.1563); 39 | randn('state', 1); 40 | c = 0.8; 41 | alpha = 5; 42 | level_factor = 0.15; 43 | 44 | %% 45 | [warp, levels] = getWarpWater(img, nFrame + 1, c, alpha); 46 | % frames = simulate(rgb, warp, true); 47 | 48 | [X,Y] = meshgrid(1:W); 49 | % [X, Y] = meshgrid(linspace(-1,1,256),linspace(-1,1,256)); 50 | h = figure(1); 51 | hh = subplot(2,3,1);handle = surf (X,Y,levels(:,:,1),'FaceColor','interp','edgecolor','none','edgelighting','none'); 52 | colormap(cold); 53 | % camlight('headlight'); 54 | lighting('gouraud'); 55 | shading('interp'); 56 | 57 | n1 = 1; 58 | n2 = 1.33; 59 | levels = levels*level_factor; 60 | levels = levels(:,:,2:end); 61 | 62 | % filename_depth=[depth_folder 'Seq_' seq_Num '_' num2str(alpha) '.npy']; 63 | % writeNPY(levels,filename_depth); 64 | 65 | for i = 1: nFrame 66 | disp(num2str(i)); 67 | zh = levels(:,:,i); 68 | zh_new = zh; 69 | zh_nl = zh_new+alpha*4; 70 | % zh_range = max(zh(:)) - min(zh(:)); 71 | 72 | [warp_map] = raytracing_im_generator_ST(rgb,zh_nl,n1,n2,X,Y); 73 | filename_warp=[warp_folder 'Seq_' seq_Num '_' num2str(alpha) '_' num2str(i) '.npy']; 74 | writeNPY(warp_map,filename_warp); 75 | % Save Depth array 76 | filename_depth=[depth_folder 'Seq_' seq_Num '_' num2str(alpha) '_' num2str(i) '.npy']; 77 | writeNPY(zh,filename_depth); 78 | 79 | zh_range = max(zh(:)) - min(zh(:)); 80 | set(handle,'zdata',zh_new); 81 | title(hh,sprintf('# %d: [%.3f , %.3f] --- %.3f', i, min(zh(:)),max(zh(:)), zh_range)); 82 | set(gca,'zlim',[min(levels(:)),max(levels(:))]); 83 | warp_x = warp.Xs(:, :, i)*level_factor; 84 | warp_y = warp.Ys(:, :, i)*level_factor; 85 | frames = simulate(rgb, warp_x,warp_y, false,i); 86 | recovered_in = simulate(frames, -warp_x,-warp_y, false,i); 87 | subplot(2,3,2),imshow(frames);title('Distorted by Raytracing') 88 | subplot(2,3,3),imshow(frames),title('Distorted by Warp') 89 | subplot(2,3,4),quiver(warp_x,warp_y),axis equal%plot(warp_map,'DecimationFactor',[10 10],'ScaleFactor',10) 90 | subplot(2,3,5),imshow(recovered_in),title('Recoverd By Warp') 91 | subplot(2,3,6),imshow(rgb),title('Undistored GT') 92 | pause(1/25); 93 | end 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/Step_2_Image_Wave_Index_Mapping.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | close all 8 | clear 9 | 10 | %% Parameter Setting 11 | Phase = 'train'; 12 | % Phase = 'val'; 13 | img_folder = ['RGB/' Phase '/']; 14 | 15 | 16 | wave_folder = [Phase '/WaveSequences/']; 17 | % Set number of waves per image(background pattern) 18 | nWaveSeq = 10; 19 | % Set number of images in Training or Val set 20 | nPattern = 4359; %val = 998, train = 4359 21 | 22 | %% Generate The Mapping matrix 23 | img_list = dir([img_folder '*.jpg']); 24 | img_namelist = {img_list.name}; 25 | img_namelist = img_namelist'; 26 | wavetype_list = dir(wave_folder); 27 | % For Mac remove hidden files if any ex: starting with . 28 | wavetype_list(strncmp({wavetype_list.name}, '.', 1)) = []; 29 | wavetype_list = wavetype_list(3:end); 30 | 31 | Mapping_struct= []; 32 | imNum = length(img_namelist); 33 | nWaveType = length(wavetype_list); 34 | 35 | if imNum < nPattern 36 | disp('Do not have enough images. Will use all availab images in the folder.'); 37 | nPattern = imNum; 38 | end 39 | 40 | for i = 1:nPattern 41 | 42 | this_waveType = wavetype_list(randperm(nWaveType,1)); 43 | this_AllSeq = dir([wave_folder this_waveType.name]); 44 | this_AllSeq = this_AllSeq(3:end); 45 | 46 | 47 | this_Seq = this_AllSeq(randperm(length(this_AllSeq),1)); 48 | this_SeqFolder = [this_Seq.folder '/' this_Seq.name '/']; 49 | this_WaveList = dir([this_SeqFolder 'warp/*.npy']); 50 | this_nWave = length(this_WaveList); 51 | this_nvalidIdx = this_nWave - nWaveSeq + 1; 52 | if this_nvalidIdx < 1 53 | error(['nWaveSeq must be less or equal to ' num2str(this_nWave)]); 54 | end 55 | this_startIdx = randperm(this_nvalidIdx,1); 56 | Mapping_struct(i).Image = img_namelist(i); 57 | Mapping_struct(i).WaveFolder = this_SeqFolder; 58 | Mapping_struct(i).WaveType = this_waveType.name; 59 | Mapping_struct(i).SeqNumber = this_Seq.name; 60 | Mapping_struct(i).IndexRange_start = this_startIdx; 61 | end 62 | save( ['Mapping_struct_' Phase], 'Mapping_struct'); -------------------------------------------------------------------------------- /Fluid_wave_simulator/Step_3_Render_dataset.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | close all 8 | clear 9 | %% Parameter Setting 10 | Phase = 'train'; 11 | % Phase = 'val'; 12 | W = 128; 13 | H = 128; 14 | nWaveSeq = 9; 15 | img_folder = ['RGB/' Phase '/']; 16 | 17 | %% Generate Dataset 18 | load(['Mapping_struct_' Phase]); 19 | 20 | Mapping_type = '0'; 21 | Data_result_folder = [Phase '/Dataset/']; 22 | Data_result_folder_depth = [Data_result_folder 'depth/']; 23 | Data_result_folder_warp = [Data_result_folder 'warp/']; 24 | Data_result_folder_RGB = [Data_result_folder 'RGB/']; 25 | render_folder = [Data_result_folder 'Render_Matlab/']; 26 | 27 | if ~exist(render_folder,'file') 28 | mkdir(render_folder); 29 | end 30 | 31 | if ~exist(Data_result_folder_depth,'file') 32 | mkdir(Data_result_folder_depth); 33 | end 34 | 35 | if ~exist(Data_result_folder_warp,'file') 36 | mkdir(Data_result_folder_warp); 37 | end 38 | 39 | if ~exist(Data_result_folder_RGB,'file') 40 | mkdir(Data_result_folder_RGB); 41 | end 42 | 43 | N = length(Mapping_struct); 44 | 45 | for i = 1:N 46 | im_name = Mapping_struct(i).Image{1}; 47 | % if exist([Data_result_folder 'RGB/' im_name],'file') 48 | % continue; 49 | % end 50 | im_name_ori = im_name(1:end-4); 51 | im_rgb = double(imread([img_folder im_name]))./255; 52 | im_rgb = imresize(im_rgb, [W,H]); 53 | ori_waveFolder = Mapping_struct(i).WaveFolder; 54 | depth_folder = [ori_waveFolder 'depth/']; 55 | wave_folder = [ori_waveFolder 'warp/']; 56 | start_loc = Mapping_struct(i).IndexRange_start; 57 | depth_list = dir([depth_folder '*.npy']); 58 | depth_str = depth_list(1).name; 59 | depth_alpha = '_5.npy'; 60 | 61 | A = start_loc:(start_loc+nWaveSeq-1); 62 | seq_n = Mapping_struct(i).SeqNumber; 63 | 64 | % Load Wave and depth from original wavesequence folder 65 | wave_batch = arrayfun(@(x) readNPY([wave_folder seq_n depth_alpha(1:2) '_' num2str(x) '.npy']),A,'Uni',0); 66 | depth_batch_cell = arrayfun(@(x) readNPY([depth_folder seq_n depth_alpha(1:2) '_' num2str(x) '.npy']),A,'Uni',0); 67 | depth_batch_mat = cell2mat(depth_batch_cell); 68 | depth_batch = reshape(depth_batch_mat,[H,W,length(A)]); 69 | % Save Depth to datasest 70 | writeNPY(depth_batch,[Data_result_folder_depth im_name_ori '_' Mapping_type depth_alpha]); 71 | disp(['# Img ' num2str(i) ' : ' im_name]); 72 | 73 | % Render refracted images for each background pattern 74 | for j = A 75 | count = j - start_loc + 1; 76 | warp_xy = wave_batch{count}; 77 | out_im = simulate(im_rgb, warp_xy(:,1:W),warp_xy(:,W+1:end), false,i); 78 | save_name = [im_name_ori '_' Mapping_type '_' num2str(count) depth_alpha(1:2)]; 79 | imwrite(out_im, [render_folder save_name '.png']); 80 | writeNPY(warp_xy,[Data_result_folder 'warp/' save_name '_warp.npy']); 81 | % pause(1/25); 82 | % figure(1), 83 | % subplot(1,2,1),imshow(im_rgb) 84 | % subplot(1,2,2),imshow(out_im) 85 | end 86 | imwrite(im_rgb,[Data_result_folder 'RGB/' im_name]); 87 | end 88 | 89 | 90 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/cold.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function cmap = cold(m) 8 | if nargin < 1 9 | m = size(get(gcf,'colormap'),1); 10 | end 11 | n = fix(3/8*m); 12 | 13 | r = [zeros(2*n,1); (1:m-2*n)'/(m-2*n)]; 14 | g = [zeros(n,1); (1:n)'/n; ones(m-2*n,1)]; 15 | b = [(1:n)'/n; ones(m-n,1)]; 16 | 17 | cmap = [r g b]; 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/getOceanWave.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function [warp, levels] = getOceanWave(img, nFrame, alpha, c, WindDir, patchSize) 8 | 9 | V = 200;%100; % Wind speed m/s 10 | A = 1;%1; % Amplitude 11 | g = 9.81; % Gravitational constant 12 | L = V*V/g; 13 | WindDamp = 0.1;%0.01; % Attenuation 14 | WaveLimit = patchSize/100; 15 | 16 | if size(img, 1) > 1 && size(img, 2) > 1 17 | h = size(img, 1); 18 | w = size(img, 2); 19 | else 20 | w = img(1); 21 | h = img(2); 22 | end 23 | 24 | meshSize = w; % FFT SIZE (128/256/384/512 for speed) 25 | 26 | % GENERATE WAVE SPECTRUM 27 | P = zeros(meshSize,meshSize); 28 | k = zeros(meshSize,meshSize); 29 | for m = 1:meshSize 30 | for n = 1:meshSize 31 | 32 | % CALCULATE WAVE VECTOR 33 | kx = 2*pi*(m-1-meshSize/2)/patchSize; 34 | ky = 2*pi*(n-1-meshSize/2)/patchSize; 35 | 36 | k2 = kx*kx + ky*ky; 37 | kxx = kx/sqrt(k2); 38 | kyy = ky/sqrt(k2); 39 | 40 | k(m,n) = sqrt(kx*kx + ky*ky); 41 | 42 | 43 | % WIND MODULATION 44 | w_dot_k = kxx*cos(WindDir) + kyy*sin(WindDir); 45 | 46 | % SPECTRUM AT GIVEN POINT 47 | P(m,n) = A*exp(- 1.0/((k(m,n)*L)^2))/(k(m,n)^4)*(w_dot_k^2); 48 | P(m,n) = P(m,n)*exp(-(k(m,n)^2) * WaveLimit^2); 49 | 50 | % FILTER WAVES MOVING IN THE WRONG DIRECTION 51 | if(w_dot_k<0) 52 | P(m,n) = P(m,n) * WindDamp; 53 | end 54 | 55 | if(kx == 0 && ky == 0) 56 | P(m,n) = 0; 57 | end 58 | 59 | end 60 | end 61 | 62 | %CALCULATE INITIAL SURFACE IN FREQUENCY DOMAIN 63 | %RANDN - GAUSSIAN | RAND - NORMAL 64 | H0 = 1./sqrt(2)*complex(randn(meshSize),randn(meshSize)).*sqrt(P); 65 | 66 | % GET MIRRORED VALUE OF INITIAL SURFACE 67 | Hm = zeros(meshSize,meshSize); 68 | for m = 1:meshSize 69 | for n=1:meshSize 70 | Hm(m,n) = conj(H0((meshSize-m+1),meshSize-n+1)); 71 | end 72 | end 73 | 74 | %DISPERSION 75 | W = sqrt(g.*k); 76 | 77 | levels = zeros(h, w, nFrame); 78 | warp.Xs = zeros(h, w, nFrame); 79 | warp.Ys = zeros(h, w, nFrame); 80 | tic; 81 | for i = 1:nFrame 82 | time = c*i; 83 | % disp([wave_type ' Time Frame: ' num2str(i)]) 84 | % Update according to the disperion relation 85 | Hkt = H0.*exp(1i.*W*time) + Hm.*exp(-1i.*W*time); 86 | 87 | % Generate HeightField at time t using ifft 88 | Ht = real(ifft2(Hkt)); 89 | for m = 1:meshSize 90 | for n = 1:meshSize 91 | signCorrection = mod(m+n-2,2); 92 | if(signCorrection) 93 | Ht(m,n) = -1.0*Ht(m,n); 94 | end 95 | 96 | end 97 | end 98 | 99 | zh = flip(flip(Ht,3),8); 100 | % [Gx,Gy] = imgradientxy(zh,'central'); 101 | [Gx,Gy] = imgradientxy(zh,'sobel'); 102 | % pred_zh = cumsum([zh(1,:);Gx(2:end-1,:)],2); 103 | % pred_zh = cumsum(Gx,2)/10; 104 | 105 | % figure(2),subplot(1,2,1),imshow(mat2gray(zh)); 106 | % figure(2),subplot(1,2,2),imshow(mat2gray(pred_zh)); 107 | % cum_Gy = cumsum(Gy,2); 108 | % pred_zh = cum_Gx + cum_Gy; 109 | warp.Xs(:, :, i) = -alpha * Gx; 110 | warp.Ys(:, :, i) = -alpha * Gy; 111 | % zh = (zh-min(zh(:)))./(max(zh(:))-min(zh(:))); 112 | levels(:, :, i) = zh; 113 | end 114 | 115 | 116 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/getThreeRippleWave.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function [warp, levels] = getThreeRippleWave(img, nFrame, alpha, c, R, R1, k,k2,R2) 8 | 9 | if size(img, 1) > 1 && size(img, 2) > 1 10 | h = size(img, 1); 11 | w = size(img, 2); 12 | else 13 | w = img(1); 14 | h = img(2); 15 | end 16 | 17 | gKernel = fspecial('gaussian', [81, 81], 10); 18 | lap = fspecial('laplacian'); 19 | soby = fspecial('sobel'); 20 | sobx = soby'; 21 | 22 | % generate random h 23 | % [X, Y] = meshgrid(1:w, 1:h); 24 | % di = X - floor(w/2); 25 | % dj = Y - floor(h/2); 26 | % sigma = 30; 27 | % waterLevel = exp(-(di.^2 + dj.^2) / 2 / sigma^2); 28 | A = rand(1,1); 29 | B = rand(1,1); 30 | C = rand(1,1); 31 | Z=1+sin((2*pi-A)*k*R); 32 | Z1 = 1+sin((2*pi-B)*k*R1); 33 | Z2 = 1+sin((2*pi-C)*k2*R2); 34 | waterLevel = ((Z+Z1+Z2)/3)*(0.05+rand(1,1)*0.1); %0.07 for 1600-1899, 0.04 for 1900-2199, 0.07 for 4300-4499 35 | % waterLevel = ((Z+Z1)/2); %0.07 for 1600-1899, 0.04 for 1900-2199, 0.07 for 4300-4499 36 | % waterLevel = imfilter(randn(h, w), gKernel, 'circular'); 37 | waterLevel = waterLevel / max(waterLevel(:)); 38 | waterLevelPrev = waterLevel; 39 | % only for test. 40 | waterLevel = zeros(h, w); 41 | 42 | warp.Xs = zeros(h, w, nFrame); 43 | warp.Ys = zeros(h, w, nFrame); 44 | levels = zeros(h, w, nFrame); 45 | 46 | for i = 1:nFrame 47 | levels(:, :, i) = waterLevel-mean(waterLevel(:)); 48 | 49 | warp.Xs(:, :, i) = alpha * imfilter(waterLevel, sobx, 'circular'); 50 | warp.Ys(:, :, i) = alpha * imfilter(waterLevel, soby, 'circular'); 51 | 52 | % lapimg = imfilter(waterLevel, lap, 'circular'); 53 | 54 | % waterLevelNext = 2 * waterLevel + c^2 * lapimg - waterLevelPrev; 55 | Z=1+sin((2*pi-A*i)*k*R); 56 | Z1 = 1+sin((2*pi-B*i)*k*R1); 57 | Z2 = 1+sin((2*pi-C*i)*k2*R2); 58 | waterLevel = ((Z+Z1+Z2)/3)*(0.05+rand(1,1)*0.1); 59 | 60 | % waterLevelPrev = waterLevel; 61 | % waterLevel = waterLevelNext; 62 | end 63 | warp.h = h; 64 | warp.w = w; 65 | warp.nFrame = nFrame; -------------------------------------------------------------------------------- /Fluid_wave_simulator/getTwoRippleWave.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function [warp, levels] = getTwoRippleWave(img, nFrame, alpha, c, R, R1, k) 8 | 9 | if size(img, 1) > 1 && size(img, 2) > 1 10 | h = size(img, 1); 11 | w = size(img, 2); 12 | else 13 | w = img(1); 14 | h = img(2); 15 | end 16 | 17 | gKernel = fspecial('gaussian', [81, 81], 10); 18 | lap = fspecial('laplacian'); 19 | soby = fspecial('sobel'); 20 | sobx = soby'; 21 | 22 | % generate random h 23 | % [X, Y] = meshgrid(1:w, 1:h); 24 | % di = X - floor(w/2); 25 | % dj = Y - floor(h/2); 26 | % sigma = 30; 27 | % waterLevel = exp(-(di.^2 + dj.^2) / 2 / sigma^2); 28 | A = rand(1,1); 29 | B = rand(1,1); 30 | Z=1+sin((2*pi-A)*k*R); 31 | Z1 = 1+sin((2*pi-B)*k*R1); 32 | waterLevel = ((Z+Z1)/2)*(0.1+rand(1,1)*0.1); %0.07 for 1600-1899, 0.04 for 1900-2199, 0.07 for 4300-4499 33 | % waterLevel = imfilter(randn(h, w), gKernel, 'circular'); 34 | waterLevel = waterLevel / max(waterLevel(:)); 35 | % waterLevelPrev = waterLevel; 36 | % only for test. 37 | waterLevel = zeros(h, w); 38 | 39 | warp.Xs = zeros(h, w, nFrame); 40 | warp.Ys = zeros(h, w, nFrame); 41 | levels = zeros(h, w, nFrame); 42 | 43 | % count=1; 44 | % for freqrps=0.1:0.006:2*pi 45 | % Z=sin((2*pi-freqrps)*k*R); 46 | 47 | for i = 1:nFrame 48 | levels(:, :, i) = waterLevel-mean(waterLevel(:)); 49 | 50 | warp.Xs(:, :, i) = alpha * imfilter(waterLevel, sobx, 'circular'); 51 | warp.Ys(:, :, i) = alpha * imfilter(waterLevel, soby, 'circular'); 52 | 53 | % lapimg = imfilter(waterLevel, lap, 'circular'); 54 | 55 | % waterLevelNext = 2 * waterLevel + c^2 * lapimg - waterLevelPrev; 56 | % 57 | % waterLevelPrev = waterLevel; 58 | % waterLevel = waterLevelNext; 59 | Z=1+sin((2*pi-i*0.2)*k*R); 60 | Z1 = 1+sin((2*pi-i*0.1)*k*R1); 61 | waterLevel = ((Z+Z1)/2)*(0.1+0.5*5); 62 | 63 | 64 | end 65 | warp.h = h; 66 | warp.w = w; 67 | warp.nFrame = nFrame; -------------------------------------------------------------------------------- /Fluid_wave_simulator/getWarpWater.m: -------------------------------------------------------------------------------- 1 | function [warp, levels] = getWarpWater(img, nFrame, c, alpha) 2 | % Created by Yuandong Tian, Oct. 5, 2009 3 | % Email: yuandong@andrew.cmu.edu 4 | % a simple simulator water simulator with random initial conditions. Note the 5 | % first frame is no distorted. 6 | 7 | if size(img, 1) > 1 && size(img, 2) > 1 8 | h = size(img, 1); 9 | w = size(img, 2); 10 | else 11 | w = img(1); 12 | h = img(2); 13 | end 14 | 15 | gKernel = fspecial('gaussian', [81, 81], 10); 16 | lap = fspecial('laplacian'); 17 | soby = fspecial('sobel'); 18 | sobx = soby'; 19 | 20 | % generate random h 21 | % [X, Y] = meshgrid(1:w, 1:h); 22 | % di = X - floor(w/2); 23 | % dj = Y - floor(h/2); 24 | % sigma = 30; 25 | % waterLevel = exp(-(di.^2 + dj.^2) / 2 / sigma^2); 26 | A = randn(h, w); 27 | waterLevel = imfilter(A, gKernel, 'circular'); 28 | waterLevel = waterLevel / max(waterLevel(:)); 29 | waterLevelPrev = waterLevel; 30 | % only for test. 31 | waterLevel = zeros(h, w); 32 | 33 | warp.Xs = zeros(h, w, nFrame); 34 | warp.Ys = zeros(h, w, nFrame); 35 | levels = zeros(h, w, nFrame); 36 | 37 | for i = 1:nFrame 38 | levels(:, :, i) = waterLevel-mean(waterLevel(:)); 39 | 40 | warp.Xs(:, :, i) = alpha * imfilter(waterLevel, sobx, 'circular'); 41 | warp.Ys(:, :, i) = alpha * imfilter(waterLevel, soby, 'circular'); 42 | 43 | lapimg = imfilter(waterLevel, lap, 'circular'); 44 | 45 | % [warp.Xs(:, :, i), warp.Ys(:, :, i)] = gradient(waterLevel); 46 | % warp.Xs(:, :, i) = warp.Xs(:, :, i) * alpha; 47 | % warp.Ys(:, :, i) = warp.Ys(:, :, i) * alpha; 48 | 49 | waterLevelNext = 2 * waterLevel + c^2 * lapimg - waterLevelPrev; 50 | 51 | waterLevelPrev = waterLevel; 52 | waterLevel = waterLevelNext; 53 | end 54 | warp.h = h; 55 | warp.w = w; 56 | warp.nFrame = nFrame; -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/.DS_Store -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/.gitignore: -------------------------------------------------------------------------------- 1 | *.asv 2 | .ipynb_checkpoints/ 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/.travis.yml: -------------------------------------------------------------------------------- 1 | # Using several other .travis.yml files as inspiration. See for example: 2 | # https://github.com/MOxUnit/MOxUnit 3 | # https://github.com/scottclowe/matlab-continuous-integration/ 4 | # https://github.com/fieldtrip/fieldtrip/blob/master/.travis.yml 5 | 6 | language: python 7 | 8 | cache: 9 | - apt 10 | 11 | before_install: 12 | # to prevent IPv6 being used for APT 13 | - sudo bash -c "echo 'Acquire::ForceIPv4 \"true\";' > /etc/apt/apt.conf.d/99force-ipv4" 14 | - travis_retry sudo apt-get -y -qq update 15 | - travis_retry sudo apt-get install -y -qq software-properties-common python-software-properties 16 | - travis_retry sudo apt-add-repository -y ppa:octave/stable 17 | - travis_retry sudo apt-get -y -qq update 18 | # get Octave 4.0 19 | - travis_retry sudo apt-get -y -qq install octave liboctave-dev 20 | 21 | # Get conda 22 | - wget -q http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh 23 | - chmod +x miniconda.sh 24 | - ./miniconda.sh -b -p /home/travis/miniconda 25 | - export PATH=/home/travis/miniconda/bin:$PATH 26 | - conda update --yes --quiet conda 27 | 28 | install: 29 | - conda create -n testenv --yes python=3.6 30 | - source activate testenv 31 | - conda install --yes --quiet numpy 32 | - conda install --yes -c conda-forge scikit-image 33 | - pip install pytest==3.3.2 pytest-sugar 34 | 35 | script: 36 | - echo "Octave version:" 37 | - octave --no-gui --eval "version()" 38 | - pytest 39 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2015, npy-matlab developers 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/README.md: -------------------------------------------------------------------------------- 1 | [![Travis](https://api.travis-ci.org/kwikteam/npy-matlab.svg?branch=master "Travis")](https://travis-ci.org/kwikteam/npy-matlab) 2 | # npy-matlab 3 | 4 | Code to read and write NumPy's NPY format (`.npy` files) in MATLAB. 5 | 6 | This is experimental code and still work in progress. For example, this code: 7 | - Only reads a subset of all possible NPY files, specifically N-D arrays of 8 | certain data types. 9 | - Only writes little endian, fortran (column-major) ordering 10 | - Only writes with NPY version number 1.0. 11 | - Always outputs a shape according to matlab's convention, e.g. (10, 1) 12 | rather than (10,). 13 | 14 | Feel free to open an issue and/or send a pull request for improving the 15 | state of this project! 16 | 17 | For the complete specification of the NPY format, see the [NumPy documentation](https://www.numpy.org/devdocs/reference/generated/numpy.lib.format.html). 18 | 19 | ## Installation 20 | After downloading npy-matlab as a zip file or via git, just add the 21 | npy-matlab directory to your search path: 22 | 23 | ```matlab 24 | >> addpath('my-idiosyncratic-path/npy-matlab/npy-matlab') 25 | >> savepath 26 | ``` 27 | 28 | ## Usage example 29 | ```matlab 30 | >> a = rand(5,4,3); 31 | >> writeNPY(a, 'a.npy'); 32 | >> b = readNPY('a.npy'); 33 | >> sum(a(:)==b(:)) 34 | ans = 35 | 36 | 60 37 | ``` 38 | 39 | ## Tests 40 | Roundtrip testing is performed using Travis CI and GNU Octave, see 41 | the `.travis.yml` file and `tests/test_npy_roundtrip.py`. 42 | 43 | You can also use two "manual testing scripts": 44 | 45 | - See `tests/npy.ipynb` for Python tests. 46 | - See `tests/test_readNPY.m` for MATLAB reading/writing tests. 47 | 48 | ## Memory mapping npy files 49 | See `examples/exampleMemmap.m` for an example of how to memory map a `.npy` file in MATLAB, which is not trivial when the file uses C-ordering (i.e., row-major order) rather than Fortran-ordering (i.e., column-major ordering). MATLAB's memory mapping only supports Fortran-ordering, but Python's default is C-ordering so `.npy` files created with Python defaults are not straightforward to read in MATLAB. 50 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/examples/exampleMemmap.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | % Example implementation of memory mapping an NPY file using readNPYheader 4 | 5 | filename = 'data/chelsea_float64.npy'; 6 | 7 | [arrayShape, dataType, fortranOrder, littleEndian, totalHeaderLength, npyVersion] = readNPYheader(filename); 8 | 9 | figure; 10 | 11 | if fortranOrder 12 | f = memmapfile(filename, 'Format', {dataType, arrayShape, 'd'}, 'Offset', totalHeaderLength); 13 | image(f.Data.d) 14 | 15 | else 16 | % Note! In this case, the dimensions of the array will be transposed, 17 | % e.g. an AxBxCxD array becomes DxCxBxA. 18 | f = memmapfile(filename, 'Format', {dataType, arrayShape(end:-1:1), 'd'}, 'Offset', totalHeaderLength); 19 | 20 | tmp = f.Data.d; 21 | img = permute(tmp, length(arrayShape):-1:1); % note here you have to reverse the dimensions. 22 | image(img./255) 23 | end -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/npy-matlab/constructNPYheader.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | function header = constructNPYheader(dataType, shape, varargin) 5 | 6 | if ~isempty(varargin) 7 | fortranOrder = varargin{1}; % must be true/false 8 | littleEndian = varargin{2}; % must be true/false 9 | else 10 | fortranOrder = true; 11 | littleEndian = true; 12 | end 13 | 14 | dtypesMatlab = {'uint8','uint16','uint32','uint64','int8','int16','int32','int64','single','double', 'logical'}; 15 | dtypesNPY = {'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8', 'f4', 'f8', 'b1'}; 16 | 17 | magicString = uint8([147 78 85 77 80 89]); %x93NUMPY 18 | 19 | majorVersion = uint8(1); 20 | minorVersion = uint8(0); 21 | 22 | % build the dict specifying data type, array order, endianness, and 23 | % shape 24 | dictString = '{''descr'': '''; 25 | 26 | if littleEndian 27 | dictString = [dictString '<']; 28 | else 29 | dictString = [dictString '>']; 30 | end 31 | 32 | dictString = [dictString dtypesNPY{strcmp(dtypesMatlab,dataType)} ''', ']; 33 | 34 | dictString = [dictString '''fortran_order'': ']; 35 | 36 | if fortranOrder 37 | dictString = [dictString 'True, ']; 38 | else 39 | dictString = [dictString 'False, ']; 40 | end 41 | 42 | dictString = [dictString '''shape'': (']; 43 | 44 | % if length(shape)==1 && shape==1 45 | % 46 | % else 47 | % for s = 1:length(shape) 48 | % if s==length(shape) && shape(s)==1 49 | % 50 | % else 51 | % dictString = [dictString num2str(shape(s))]; 52 | % if length(shape)>1 && s+1==length(shape) && shape(s+1)==1 53 | % dictString = [dictString ',']; 54 | % elseif length(shape)>1 && s %s', tempFilename, inFilename, outFilename)); 38 | 39 | otherwise 40 | fprintf(1, 'I don''t know how to concatenate files for your OS, but you can finish making the NPY youself by concatenating %s with %s.\n', tempFilename, inFilename); 41 | end 42 | 43 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/npy-matlab/readNPY.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | function data = readNPY(filename) 4 | % Function to read NPY files into matlab. 5 | % *** Only reads a subset of all possible NPY files, specifically N-D arrays of certain data types. 6 | % See https://github.com/kwikteam/npy-matlab/blob/master/tests/npy.ipynb for 7 | % more. 8 | % 9 | 10 | [shape, dataType, fortranOrder, littleEndian, totalHeaderLength, ~] = readNPYheader(filename); 11 | 12 | if littleEndian 13 | fid = fopen(filename, 'r', 'l'); 14 | else 15 | fid = fopen(filename, 'r', 'b'); 16 | end 17 | 18 | try 19 | 20 | [~] = fread(fid, totalHeaderLength, 'uint8'); 21 | 22 | % read the data 23 | data = fread(fid, prod(shape), [dataType '=>' dataType]); 24 | 25 | if length(shape)>1 && ~fortranOrder 26 | data = reshape(data, shape(end:-1:1)); 27 | data = permute(data, [length(shape):-1:1]); 28 | elseif length(shape)>1 29 | data = reshape(data, shape); 30 | end 31 | 32 | fclose(fid); 33 | 34 | catch me 35 | fclose(fid); 36 | rethrow(me); 37 | end 38 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/npy-matlab/readNPYheader.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | function [arrayShape, dataType, fortranOrder, littleEndian, totalHeaderLength, npyVersion] = readNPYheader(filename) 4 | % function [arrayShape, dataType, fortranOrder, littleEndian, ... 5 | % totalHeaderLength, npyVersion] = readNPYheader(filename) 6 | % 7 | % parse the header of a .npy file and return all the info contained 8 | % therein. 9 | % 10 | % Based on spec at http://docs.scipy.org/doc/numpy-dev/neps/npy-format.html 11 | 12 | fid = fopen(filename); 13 | 14 | % verify that the file exists 15 | if (fid == -1) 16 | if ~isempty(dir(filename)) 17 | error('Permission denied: %s', filename); 18 | else 19 | error('File not found: %s', filename); 20 | end 21 | end 22 | 23 | try 24 | 25 | dtypesMatlab = {'uint8','uint16','uint32','uint64','int8','int16','int32','int64','single','double', 'logical'}; 26 | dtypesNPY = {'u1', 'u2', 'u4', 'u8', 'i1', 'i2', 'i4', 'i8', 'f4', 'f8', 'b1'}; 27 | 28 | 29 | magicString = fread(fid, [1 6], 'uint8=>uint8'); 30 | 31 | if ~all(magicString == [147,78,85,77,80,89]) 32 | error('readNPY:NotNUMPYFile', 'Error: This file does not appear to be NUMPY format based on the header.'); 33 | end 34 | 35 | majorVersion = fread(fid, [1 1], 'uint8=>uint8'); 36 | minorVersion = fread(fid, [1 1], 'uint8=>uint8'); 37 | 38 | npyVersion = [majorVersion minorVersion]; 39 | 40 | headerLength = fread(fid, [1 1], 'uint16=>uint16'); 41 | 42 | totalHeaderLength = 10+headerLength; 43 | 44 | arrayFormat = fread(fid, [1 headerLength], 'char=>char'); 45 | 46 | % to interpret the array format info, we make some fairly strict 47 | % assumptions about its format... 48 | 49 | r = regexp(arrayFormat, '''descr''\s*:\s*''(.*?)''', 'tokens'); 50 | dtNPY = r{1}{1}; 51 | 52 | littleEndian = ~strcmp(dtNPY(1), '>'); 53 | 54 | dataType = dtypesMatlab{strcmp(dtNPY(2:3), dtypesNPY)}; 55 | 56 | r = regexp(arrayFormat, '''fortran_order''\s*:\s*(\w+)', 'tokens'); 57 | fortranOrder = strcmp(r{1}{1}, 'True'); 58 | 59 | r = regexp(arrayFormat, '''shape''\s*:\s*\((.*?)\)', 'tokens'); 60 | shapeStr = r{1}{1}; 61 | arrayShape = str2num(shapeStr(shapeStr~='L')); 62 | 63 | 64 | fclose(fid); 65 | 66 | catch me 67 | fclose(fid); 68 | rethrow(me); 69 | end 70 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/npy-matlab/writeNPY.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | function writeNPY(var, filename) 4 | % function writeNPY(var, filename) 5 | % 6 | % Only writes little endian, fortran (column-major) ordering; only writes 7 | % with NPY version number 1.0. 8 | % 9 | % Always outputs a shape according to matlab's convention, e.g. (10, 1) 10 | % rather than (10,). 11 | 12 | 13 | shape = size(var); 14 | dataType = class(var); 15 | 16 | header = constructNPYheader(dataType, shape); 17 | 18 | fid = fopen(filename, 'w'); 19 | fwrite(fid, header, 'uint8'); 20 | fwrite(fid, var, dataType); 21 | fclose(fid); 22 | 23 | 24 | end 25 | 26 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_float32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_float32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_float64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_float64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_int8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/chelsea_uint8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_float32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_float32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_float64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_float64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_int8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_chelsea_uint8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_float32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_float32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_float64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_float64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_int8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/matlab_sine_uint8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_float32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_float32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_float64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_float64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_int8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint16.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint16.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint32.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint32.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint64.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint64.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint8.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/sine_uint8.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/test.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/data/test.npy -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/test_npy_roundtrip.py: -------------------------------------------------------------------------------- 1 | """Roundtrip testing of npy-matlab. 2 | 3 | Write data in NPY format to be read and rewritten using npy-matlab. Then 4 | re-read the matlab written data again and check if it is still the same. 5 | 6 | """ 7 | import os 8 | import os.path as op 9 | from subprocess import call 10 | 11 | import numpy as np 12 | import skimage.data 13 | 14 | # For numpy data types see: 15 | # https://docs.scipy.org/doc/numpy-1.15.0/user/basics.types.html 16 | SUPPORTED_DTYPES = ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 17 | 'int32', 'int64', 'float32', 'float64'] 18 | 19 | # Directory of this file 20 | test_dir = op.dirname(op.realpath(__file__)) 21 | 22 | 23 | def _save_testing_data(): 24 | """Write testing data to /tests/data. 25 | 26 | Returns 27 | ------- 28 | all_saved_data : list of tuples 29 | all_saved_data[i][0] is the path to written data 30 | all_saved_data[i][1] is actual data 31 | 32 | """ 33 | # Make a data directory 34 | data_dir = op.join(test_dir, 'data') 35 | if not op.exists(data_dir): 36 | op.mkdir(data_dir) 37 | 38 | # Test 1D data 39 | n = 10000 40 | t = np.linspace(-10., 10., n) 41 | sine = (1 + np.sin(t)) * 64 42 | 43 | # Test 3D data 44 | chelsea = skimage.data.chelsea() 45 | 46 | # Save all test data in NPY format 47 | all_saved_data = list() 48 | for dtype in SUPPORTED_DTYPES: 49 | sine_tr = sine.astype(dtype) 50 | fpath = op.join(data_dir, 'sine_{}.npy'.format(dtype)) 51 | np.save(fpath, sine_tr) 52 | all_saved_data.append((fpath, sine_tr)) 53 | 54 | chelsea_tr = chelsea.astype(dtype) 55 | fpath = op.join(data_dir, 'chelsea_{}.npy'.format(dtype)) 56 | np.save(fpath, chelsea_tr) 57 | all_saved_data.append((fpath, chelsea_tr)) 58 | 59 | return all_saved_data 60 | 61 | 62 | def test_roundtrip(): 63 | """Test roundtrip.""" 64 | # Write testing data from Python 65 | all_saved_data = _save_testing_data() 66 | 67 | # General command structure for calling octave 68 | # note, there is no closing `"` sign. Must be added later. 69 | main_dir = os.sep.join(test_dir.split(os.sep)[:-1]) 70 | func_dir = op.join(main_dir, 'npy-matlab') 71 | cmd_octave = 'octave --no-gui --eval "' 72 | cmd_octave += "addpath('{}');".format(func_dir) 73 | 74 | # Now for each data type and testing data: 75 | # read with octave 76 | # re-write with octave 77 | # re-read with numpy and assert it is the same 78 | for i in range(len(all_saved_data)): 79 | # Get the original data 80 | fpath = all_saved_data[i][0] 81 | original_data = all_saved_data[i][1] 82 | 83 | # Read into octave and write back from octave 84 | new_fpath = op.join(op.split(fpath)[0], 'matlab_' + op.split(fpath)[1]) 85 | cmd_read = "new_data=readNPY('{}');".format(fpath) 86 | cmd_write = "writeNPY(new_data, '{}');".format(new_fpath) 87 | cmd_complete = cmd_octave + cmd_read + cmd_write + '"' 88 | print(cmd_complete) 89 | call(cmd_complete, shell=True) 90 | 91 | # Read back with numpy and assert it's the same 92 | new_data = np.load(new_fpath) 93 | np.testing.assert_array_equal(original_data, new_data.squeeze()) 94 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/npy-matlab-master/npy-matlab-master/tests/test_readNPY.m: -------------------------------------------------------------------------------- 1 | 2 | 3 | %% Test the readNPY function with given data 4 | dtypes = {'uint8','uint16','uint32','uint64','int8','int16','int32','int64','float32','float64'}; 5 | figure; 6 | for d = 1:length(dtypes) 7 | 8 | data = readNPY(['data/sine_' dtypes{d} '.npy']); 9 | 10 | subplot(length(dtypes),1,d) 11 | plot(data) 12 | title(dtypes{d}); 13 | 14 | end 15 | 16 | 17 | figure; 18 | for d = 1:length(dtypes) 19 | 20 | data = readNPY(['data/chelsea_' dtypes{d} '.npy']); 21 | data(1:3,1:3,:) 22 | subplot(length(dtypes),1,d) 23 | imagesc(double(data)./255) 24 | title(dtypes{d}); 25 | 26 | end 27 | 28 | 29 | %% test readNPY and writeNPY 30 | dtypes = {'uint8','uint16','uint32','uint64','int8','int16','int32','int64','float32','float64'}; 31 | 32 | figure; 33 | for d = 1:length(dtypes) 34 | 35 | data = readNPY(['data/sine_' dtypes{d} '.npy']); 36 | writeNPY(data, ['data/matlab_sine_' dtypes{d} '.npy']); 37 | data = readNPY(['data/matlab_sine_' dtypes{d} '.npy']); 38 | 39 | subplot(length(dtypes),1,d) 40 | plot(data) 41 | title(dtypes{d}); 42 | 43 | end 44 | 45 | 46 | figure; 47 | for d = 1:length(dtypes) 48 | 49 | data = readNPY(['data/chelsea_' dtypes{d} '.npy']); 50 | writeNPY(data, ['data/matlab_chelsea_' dtypes{d} '.npy']); 51 | data = readNPY(['data/matlab_chelsea_' dtypes{d} '.npy']); 52 | 53 | data(1:3,1:3,:) 54 | subplot(length(dtypes),1,d) 55 | imagesc(double(data)./255) 56 | title(dtypes{d}); 57 | 58 | end -------------------------------------------------------------------------------- /Fluid_wave_simulator/raytracing_im_generator_ST.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function [warp_map] = raytracing_im_generator_ST(im_rgb,this_depth,n1,n2,x,y) 8 | 9 | step = 1; 10 | [h,w,dim] = size(im_rgb); 11 | % [Nx,Ny,Nz] = surfnorm(x,y,this_depth); 12 | % normal = cat(3, Nx, Ny, Nz); 13 | 14 | [Gx,Gy] = imgradientxy(this_depth); 15 | normal_ori = ones(size(im_rgb)); 16 | normal_ori(:,:,1) = -Gx; 17 | normal_ori(:,:,2) = -Gy; 18 | normal = normal_ori./sqrt(Gx.^2 + Gy.^2 + 1); 19 | % normal = permute(normal,[2 1 3]); 20 | % nx = normal_test(:,:,1); 21 | % ny = normal_test(:,:,2); 22 | % nz = normal_test(:,:,3); 23 | 24 | % figure(3),subplot(1,2,1),quiver3(x(1:10,1:10),y(1:10,1:10),this_depth(1:10,1:10),Nx(1:10,1:10),Ny(1:10,1:10),Nz(1:10,1:10)); 25 | % subplot(1,2,2),quiver3(x(1:10,1:10),y(1:10,1:10),this_depth(1:10,1:10),normal_test(1:10,1:10,1),normal_test(1:10,1:10,2),normal_test(1:10,1:10,3)); 26 | % % quiver3(x,y,this_depth,Nx,Ny,Nz) 27 | % % normal = permute(normal,[2 1 3]); 28 | % figure(2), 29 | % subplot(1,2,1),imshow(mat2gray(this_depth)); 30 | % subplot(1,2,2),imshow(mat2gray(normal_test)); 31 | % normal = imrotate(normal,90); 32 | % out_im = zeros(size(normal)); 33 | % this_depth = 20; 34 | s1 = zeros(size(normal)); 35 | s1(:,:,3) = -1; 36 | s2 = refraction(normal,s1,n1,n2); 37 | a = this_depth./s2(:,:,3); 38 | x_c = round(a.*s2(:,:,1)/step,2); 39 | y_c = round(a.*s2(:,:,2)/step,2); 40 | 41 | warp_map = cat(3,x_c,y_c); 42 | 43 | 44 | 45 | % y_i(y_i>256) = 256; 46 | % x_j(x_j > 256) = 256; 47 | % y_i(y_i<1) = 1; 48 | % x_j(x_j<1) = 1; 49 | 50 | % valid = (x_c >= 1 & x_c <= w & y_c >= 1 & y_c <= h); 51 | 52 | 53 | % if dim == 3 54 | % for d=1:3 55 | % out_im(:,:,d) = interp2(x, y, im_rgb(:,:,d),x_j,y_i,'makima'); 56 | % end 57 | % else 58 | % out_im = interp2(X, Y, im_rgb,x_j,y_i,'makima'); 59 | % end 60 | 61 | % out_im = zeros(h, w, dim); 62 | % currFrame = zeros(h*w, 1); 63 | % for k = 1:dim 64 | % currFrame(valid) = interp2(im_rgb(:, :, k), x_c(valid), y_c(valid)); 65 | % out_im(:, :, k) = reshape(currFrame, h, w); 66 | % end -------------------------------------------------------------------------------- /Fluid_wave_simulator/refraction.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function s2 = refraction(normal,s1,n1,n2) 8 | 9 | this_normal = normal; 10 | s1 = s1./sqrt((s1(:,:,1).^2) + (s1(:,:,2).^2) + (s1(:,:,3).^2)); 11 | term_1 = cross(this_normal,cross(-this_normal,s1,3),3); 12 | term_2 = sqrt(1-(n1/n2)^2*sum(cross(this_normal,s1,3).*cross(this_normal,s1,3),3)); 13 | s2 = (n1/n2).*term_1 - this_normal.*term_2; 14 | s2 = s2./sqrt((s2(:,:,1).^2) + (s2(:,:,2).^2) + (s2(:,:,3).^2)); 15 | 16 | 17 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/simulate.m: -------------------------------------------------------------------------------- 1 | %% 2 | % Dynamic Fluid Surface Reconstruction using Deep Neural Network 3 | % Authors: S Thapa, N Li, J Ye 4 | % CVPR 2020 5 | % contact: sthapa5@lsu.edu 6 | %% 7 | function imgCurr = simulate(img, warp_x,warp_y, isShown,nFrame) 8 | 9 | [h, w, nChannel] = size(img); 10 | % nFrame = size(warp.Xs, 3); 11 | 12 | isShown = exist('isShown', 'var') && isShown; 13 | 14 | [X, Y] = meshgrid(1:w, 1:h); 15 | 16 | i = nFrame; 17 | % for i = 1:nFrame 18 | 19 | x_c = warp_x; 20 | y_c = warp_y; 21 | % 22 | Xnew = reshape(X + x_c, h*w, 1); 23 | Ynew = reshape(Y + y_c, h*w, 1); 24 | 25 | valid = (Xnew >= 1 & Xnew <= w & Ynew >= 1 & Ynew <= h); 26 | 27 | imgCurr = zeros(h, w, nChannel); 28 | currFrame = zeros(h*w, 1); 29 | for k = 1:nChannel 30 | currFrame(valid) = interp2(img(:, :, k), Xnew(valid), Ynew(valid)); 31 | imgCurr(:, :, k) = reshape(currFrame, h, w); 32 | end 33 | 34 | % end; 35 | -------------------------------------------------------------------------------- /Fluid_wave_simulator/tex1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/Fluid_wave_simulator/tex1.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Simron Thapa 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dynamic Fluid Surface Reconstruction Using Deep Neural Network 2 | #### [Web Page](https://ivlab.cse.lsu.edu/FSRN_CVPR20.html) | [Paper](https://ivlab.cse.lsu.edu/pub/fluid_cvpr20.pdf) 3 | [Simron Thapa](http://simronthapa.com/), [Nianyi Li](https://sites.duke.edu/nianyi/), [Jinwei Ye](https://ivlab.cse.lsu.edu/), Imaging and Vision Lab, Louisiana State University. In CVPR 2020 (oral). 4 | 5 |

6 | 7 |

8 | 9 | We present a dynamic fluid surface reconstruction network that recovers time-varying 3D fluid surfaces from a single viewpoint. 10 | 11 | ## Contributions 12 | 1. We design physics-motivated loss functions for network training 13 | 2. We synthesize a large fluid dataset using physics-based modeling and rendering [Check out the folder "Fluid_wave_simulator". It is our synthetic data generation MatLab code.] 14 | 3. Our network is validated on real captured fluid data 15 | 16 | ## Datasets 17 | Complete Training data will be made available soon. 18 | 19 | [Training](https://ivlab.cse.lsu.edu/data/Train.tar), [Validation](https://ivlab.cse.lsu.edu/data/Validation.tar), [Testing](https://ivlab.cse.lsu.edu/data/Test.tar) 20 | 21 | Please remember to cite the paper if you use this dataset. 22 | 23 | ## Training and Testing 24 | The data preprocessing code (before training with FSRN-CNN) and data post-processing code for the predictions (before training with FSRN-RNN) will be made available soon. 25 | ``` 26 | python FSRN-CNN-train.py 27 | python FSRN-RNN-train.py 28 | ``` 29 | 30 | ## Evaluation 31 | The code for evaluating the predictions with ground truth values. We use accuracy and error matrics. 32 | ``` 33 | python evaluate_metrics.py 34 | ``` 35 | ## Results 36 | 1. Synthetic Results 37 | 38 |

39 | 40 | 41 |

42 | 43 | 2. Real Results 44 | 45 |

46 | 47 | 48 |

49 | 50 | 3. Re-rendered Results 51 | 52 |

53 | 54 | 55 |

56 | 57 | ## Citation 58 | 59 | If you find this work useful, please consider citing: 60 | 61 | ``` 62 | @InProceedings{Thapa_2020_CVPR, 63 | author = {Thapa, Simron and Li, Nianyi and Ye, Jinwei}, 64 | title = {Dynamic Fluid Surface Reconstruction Using Deep Neural Network}, 65 | booktitle = {The IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 66 | month = {June}, 67 | year = {2020} 68 | } 69 | ``` 70 | -------------------------------------------------------------------------------- /evaluate_metrics.py: -------------------------------------------------------------------------------- 1 | #EVALUATE 2 | 3 | import numpy as np 4 | 5 | def evaluate_predictions(gt, pred): 6 | 7 | pred_ori = pred 8 | gt_ori = gt 9 | 10 | pred[pred<=0] = 0.00001 11 | gt[gt<=0] = 0.00001 12 | 13 | thresh = np.maximum((gt_ori / pred_ori), (pred_ori / gt_ori)) 14 | 15 | a1 = (thresh < 1.25 ).mean() 16 | a2 = (thresh < 1.25 ** 2).mean() 17 | a3 = (thresh < 1.25 ** 3).mean() 18 | 19 | abs_rel = np.mean(np.abs(gt_ori - pred_ori) / gt_ori) 20 | 21 | rmse = (gt_ori - pred_ori) ** 2 22 | rmse = np.sqrt(rmse.mean()) 23 | 24 | return a1, a2, a3, abs_rel, rmse 25 | 26 | 27 | true_depth = 'act_depth.npy' 28 | pred_depth = 'pred_depth.npy' 29 | 30 | a1, a2, a3, abs_rel, rmse = evaluate_predictions(true_depth,pred_depth) 31 | 32 | print(sum_a1, sum_a2, sum_a3, sum_abs_rel, sum_rmse) 33 | -------------------------------------------------------------------------------- /img/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/.DS_Store -------------------------------------------------------------------------------- /img/3092-teaser.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/3092-teaser.gif -------------------------------------------------------------------------------- /img/real.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/real.gif -------------------------------------------------------------------------------- /img/real1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/real1.gif -------------------------------------------------------------------------------- /img/rerender.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/rerender.gif -------------------------------------------------------------------------------- /img/rerender1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/rerender1.gif -------------------------------------------------------------------------------- /img/syn.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/syn.gif -------------------------------------------------------------------------------- /img/syn1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SimronThapa/FSRN-CVPR2020/fcae987dbd3a92de611a9a0c1c184f31961f48d7/img/syn1.gif --------------------------------------------------------------------------------