├── DL_inference ├── inference │ ├── config.py │ ├── eval2.py │ └── testrealcall.py ├── network7_256 │ ├── __pycache__ │ │ ├── customer_layers_3.cpython-36.pyc │ │ ├── deepvoxel.cpython-36.pyc │ │ └── ops.cpython-36.pyc │ ├── customer_layers_3.py │ ├── deepvoxel.py │ └── ops.py ├── re │ └── testreal-58-0.png ├── utils │ ├── .project │ ├── .pydevproject │ ├── fk.py │ ├── helper.py │ ├── helper.pyc │ ├── lct.py │ └── phasor.py └── utils_pytorch │ ├── .project │ ├── .pydevproject │ ├── tffk.py │ ├── tffkfast.py │ ├── tflct.py │ ├── tflctfast.py │ ├── tfmodule.py │ ├── tfphasor.py │ ├── tfphasor2.py │ └── utils.py ├── README.md ├── cuda-render ├── conversion │ └── preprocess_hdr2video.py └── render │ ├── .gitignore │ ├── pointlight.fragmentshader │ ├── pointlight.vertexshader │ └── src │ ├── copydata.cu │ ├── display_0_initial.cpp │ ├── display_1_cam.cpp │ ├── display_2_fbo.cpp │ ├── display_3_program.cpp │ ├── display_4_loaddata.cpp │ ├── display_5_draw.cpp │ ├── display_6_render.cpp │ ├── main.cpp │ └── renderclass.h ├── data ├── bunny-model │ └── bunny │ │ └── model │ │ ├── model_normalized.mtl │ │ └── model_normalized.obj └── img │ ├── confocal │ ├── video-confocal-gray-full.mp4 │ ├── video-confocal-gray-full_120.png │ ├── video-confocal-gray-full_130.png │ ├── video-confocal-gray-full_140.png │ ├── video-confocal-gray-full_150.png │ ├── video-confocal-gray-full_160.png │ ├── video-confocal-gray-full_170.png │ ├── video-confocal-gray-full_180.png │ ├── video-confocal-gray-full_190.png │ ├── video-confocal-gray-full_200.png │ ├── video-confocal-gray-full_210.png │ ├── video-confocal-gray-full_220.png │ ├── video-confocal-gray-full_230.png │ ├── video-confocal-gray-full_240.png │ ├── video-confocal-gray-full_250.png │ └── video-confocal-gray-full_all.png │ ├── non-confocal │ ├── video-gray-full.mp4 │ ├── video-gray-full_100.png │ ├── video-gray-full_110.png │ ├── video-gray-full_120.png │ ├── video-gray-full_130.png │ ├── video-gray-full_140.png │ ├── video-gray-full_150.png │ ├── video-gray-full_160.png │ ├── video-gray-full_170.png │ ├── video-gray-full_180.png │ ├── video-gray-full_190.png │ ├── video-gray-full_200.png │ ├── video-gray-full_210.png │ ├── video-gray-full_220.png │ ├── video-gray-full_230.png │ ├── video-gray-full_240.png │ ├── video-gray-full_250.png │ └── video-gray-full_all.png │ └── specular │ ├── video-confocal-gray-full.mp4 │ ├── video-confocal-gray-full_100.png │ ├── video-confocal-gray-full_110.png │ ├── video-confocal-gray-full_120.png │ ├── video-confocal-gray-full_130.png │ ├── video-confocal-gray-full_140.png │ ├── video-confocal-gray-full_150.png │ ├── video-confocal-gray-full_160.png │ ├── video-confocal-gray-full_170.png │ ├── video-confocal-gray-full_180.png │ ├── video-confocal-gray-full_190.png │ ├── video-confocal-gray-full_200.png │ ├── video-confocal-gray-full_210.png │ ├── video-confocal-gray-full_220.png │ ├── video-confocal-gray-full_230.png │ ├── video-confocal-gray-full_240.png │ ├── video-confocal-gray-full_250.png │ └── video-confocal-gray-full_all.png └── scenes ├── .DS_Store ├── bike_1.png ├── bike_2.gif ├── bike_2.png ├── classification-table.png ├── compare-synthetic.png ├── detection.png ├── discoball_1.png ├── discoball_2.gif ├── discoball_2.png ├── dragon_1.png ├── dragon_2.gif ├── dragon_2.png ├── resolution_1.png ├── resolution_2.gif ├── resolution_2.png ├── statue_1.png ├── statue_2.gif ├── statue_2.png ├── teaser_1.png ├── teaser_2.gif └── teaser_2.png /DL_inference/inference/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import argparse 3 | 4 | eps = 1e-8 5 | 6 | 7 | ################################################################# 8 | def get_args(): 9 | parser = argparse.ArgumentParser(description='deblur') 10 | 11 | # dataset 12 | parser.add_argument('--datafolder', type=str, default='', 13 | help='dataset folder') 14 | parser.add_argument('--datanum', type=int, default=10000, 15 | help='num of dataset') 16 | parser.add_argument('--thread', type=int, default=8, 17 | help='num of workers') 18 | parser.add_argument('--svfolder', type=str, default='.', 19 | help='save folder') 20 | 21 | parser.add_argument('--h', type=int, default=256, 22 | help='save folder') 23 | parser.add_argument('--w', type=int, default=256, 24 | help='save folder') 25 | 26 | # training 27 | parser.add_argument('--epoch', type=int, default=300, 28 | help='training epoch') 29 | parser.add_argument('--epochsv', type=int, default=1, 30 | help='epoch per model saving') 31 | parser.add_argument('--epochbe', type=int, default=-1, 32 | help='initialize from existing model') 33 | 34 | parser.add_argument('--iter_log', type=int, default=1, 35 | help='iterations per log') 36 | parser.add_argument('--mode', type=str, default='lct', 37 | help='which model') 38 | parser.add_argument('--bs', type=int, default=1, 39 | help='train bs') 40 | parser.add_argument('--dim', type=int, default=1, 41 | help='feature dim') 42 | parser.add_argument('--in_dim', type=int, default=1, 43 | help='input dim') 44 | parser.add_argument('--frame', type=int, default=512, 45 | help='training frame') 46 | parser.add_argument('--lr', type=float, default=2e-4, 47 | help='learning rate') 48 | 49 | #############################################3 50 | parser.add_argument('--tem', type=float, default=1.0, 51 | help='temperaturer') 52 | parser.add_argument('--norm', type=str, default='in', 53 | help='normalization') 54 | parser.add_argument('--normlct', type=int, default=0, 55 | help='norm lct output or not') 56 | parser.add_argument('--addraw', type=int, default=0, 57 | help='add raw data or not') 58 | parser.add_argument('--raytracing', type=int, default=0, 59 | help='do we use raytracining') 60 | 61 | ############################# 62 | parser.add_argument('--netfolder', type=str, default='utils_pytorch', 63 | help='network structure') 64 | parser.add_argument('--netsvfolder', type=str, default='model6_bike', 65 | help='network svfolder') 66 | parser.add_argument('--grid', type=int, default=256, 67 | help='grid') 68 | parser.add_argument('--tres', type=int, default=1, 69 | help='tres') 70 | parser.add_argument('--confocal', type=int, default=2, 71 | help='confocal') 72 | parser.add_argument('--res0', type=int, default=0, 73 | help='do we use res0') 74 | 75 | args = parser.parse_args() 76 | 77 | return args 78 | 79 | -------------------------------------------------------------------------------- /DL_inference/inference/eval2.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # License: BSD 4 | # Author: Ghassen Hamrouni 5 | 6 | from __future__ import print_function 7 | from __future__ import division 8 | 9 | import torch 10 | import torch.nn as nn 11 | import torch.optim as optim 12 | 13 | import numpy as np 14 | import cv2 15 | 16 | from config import get_args 17 | args = get_args() 18 | 19 | import sys 20 | sys.path.append('../%s' % args.netfolder) 21 | from deepvoxel import DeepVoxels as Net 22 | 23 | ############################################ 24 | # Make experiments reproducible 25 | _ = torch.manual_seed(1234569527) 26 | np.random.seed(123456) 27 | 28 | mode = args.mode 29 | in_dim = args.in_dim 30 | out_dim = in_dim * 2 31 | model = Net(img_sidelength=256, 32 | in_channels=in_dim, 33 | out_channels=out_dim, 34 | nf0=args.dim, 35 | grid_dim=args.grid, 36 | mode=mode, 37 | bin_len=0.0096, 38 | # addraw=args.addraw > 0, 39 | # raytracing=args.raytracing > 0 40 | ) 41 | 42 | def rmnan(tensor): 43 | tensor[torch.isnan(tensor)] = 0 44 | tensor[torch.isinf(tensor)] = 0 45 | return tensor 46 | 47 | for p in model.parameters(): 48 | p.register_hook(lambda grad: rmnan(grad)) 49 | 50 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 51 | device_ids = [0] 52 | model = model.to(device) 53 | model.todev(device) 54 | 55 | criterion = nn.BCELoss() 56 | mse = nn.MSELoss() 57 | 58 | 59 | ################################################################# 60 | if __name__ == '__main__': 61 | 62 | prefix = '%s_bike_mode-%s_dim-%d_trainframe-%d_lr-%.5f_addraw-%d_raytracing-%d' % (args.netsvfolder, mode, args.dim, args.frame, args.lr, args.addraw, args.raytracing) 63 | 64 | prefix = '%s_mode-%s_dim-%d_trainframe-%d_lr-%.5f_raytracing-%d' % (args.netsvfolder, mode, args.dim, args.frame, args.lr, args.raytracing) 65 | 66 | prefix = '%s_%s_mode-%s_dim-%d_trainframe-%d_tres-%d_confocal-%d_lr-%.5f' % (args.netsvfolder, args.netfolder, mode, args.dim, args.frame, args.tres, args.confocal, args.lr) 67 | 68 | prefix = '%s_%s_mode-%s_dim-%d_trainnum-%d_trainframe-%d_gtsz-%d-%d_tres-%d_confocal-%d_lr-%.5f' % \ 69 | (args.netsvfolder, args.netfolder, mode, args.dim, args.datanum, args.frame, args.w, args.h, args.tres, args.confocal, args.lr) 70 | 71 | rootfolder = args.datafolder 72 | datafolder = '%s/datasets' % (folder, prefix) 73 | modelsvdir = '%s/%s' % (folder, prefix) 74 | 75 | epochstart = args.epochbe 76 | epochsv = args.epochsv 77 | epochnum = args.epoch 78 | from testrealcall import testreal 79 | for epochbe in range(58, 59): 80 | if epochbe >= -1: 81 | model.load_state_dict(torch.load('%s/%d.pth' % (modelsvdir, epochbe), map_location='cpu')) 82 | print('data loaded %d' % epochbe) 83 | with torch.no_grad(): 84 | testreal(model, epochbe, args.tres, args.in_dim, device, datafolder, '.') 85 | 86 | -------------------------------------------------------------------------------- /DL_inference/inference/testrealcall.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # License: BSD 4 | # Author: Ghassen Hamrouni 5 | 6 | from __future__ import print_function 7 | from __future__ import division 8 | 9 | import torch 10 | import torch.nn as nn 11 | 12 | import numpy as np 13 | import cv2 14 | 15 | 16 | ######################################################## 17 | # Training dataset 18 | 19 | def crop_video(video, framenum, res): 20 | if framenum == 512: 21 | return video, 0, 512 22 | 23 | sumdata = np.zeros((len(video),), dtype=np.float32) 24 | for i, frame in enumerate(video): 25 | sumdata[i] = np.sum(frame) 26 | 27 | idxs = [] 28 | vals = [] 29 | for i in range((len(video) - framenum) // res): 30 | idxs.append([i * res, i * res + framenum]) 31 | vals.append(np.sum(sumdata[i * res:i * res + framenum])) 32 | # find max 33 | valmaxid = np.argmax(vals) 34 | tbe, ten = idxs[valmaxid] 35 | return video[tbe:ten], tbe, ten 36 | 37 | 38 | ############################################## 39 | import scipy.io as sio 40 | ''' 41 | matref = sio.loadmat('/u6/a/wenzheng/remote2/code-nlos-git/nlos-fk-master/bike0.mat') 42 | matref = np.array(matref['measlr'], dtype=np.float32) 43 | matmaxref = np.max(matref) 44 | ''' 45 | 46 | def loadrealdata(datafolder, name, scale, tres=2, isdebug=False): 47 | 48 | matmaxref = 252.0 49 | 50 | tlen = 256 51 | # name = args.realname # 'discoball' 52 | # scale = args.realscale 53 | 54 | if name == 'bike': 55 | tbe2 = 280 56 | # scale = 1.0 57 | if name == 'dragon': 58 | tbe2 = 256 59 | # scale = 1.0 60 | if name == 'statue': 61 | tbe2 = 188 62 | # scale = 1.0 63 | if name == 'resolution': 64 | tbe2 = 180 65 | # scale = 1 66 | if name == 'discoball': 67 | tbe2 = 188 68 | # scale = 1 69 | if name == 'teaser': 70 | tbe2 = 180 71 | tlen = 512 - tres 72 | # scale = 1 73 | 74 | # tbe2 = 0 75 | # tlen = 512 76 | folders = ['/scratch/gobi2/wenzheng/nlos-trans/realdata/%s0.mat' % name] 77 | # folders = ['/u6/a/wenzheng/remote2/code-nlos-git/nlos-fk-master/%s0.mat' % name] 78 | folders = ['%s/%s0.mat' % (datafolder, name)] 79 | 80 | mats = [] 81 | tbes = [] 82 | tens = [] 83 | for df in folders: 84 | mat = sio.loadmat(df) 85 | mat = np.array(mat['measlr'], dtype=np.float32) 86 | mat = np.transpose(mat, [2, 0, 1]); 87 | 88 | matmax = np.max(mat) 89 | print('maxmat%s %.2f, maxmatbike %.2f, scale%.2f' % \ 90 | (name, matmax, matmaxref, matmax / matmaxref)) 91 | 92 | mat = mat / matmax 93 | ''' 94 | if mat99 > 0: 95 | mat = mat / mat99 96 | mat[mat > 1] = 1 97 | ''' 98 | 99 | if isdebug: 100 | tbe = 0 101 | imcrop = mat 102 | for i in range(imcrop.shape[0]): 103 | 104 | impatch = imcrop[i] 105 | impatch2 = imcrop[i] 106 | print('%.2f, %.2f' % ((i + tbe) / 100, np.mean(impatch))) 107 | 108 | # impatch = impatch / np.max(im) 109 | # impatchlog = impatch ** 0.3 110 | # impatch2 = (impatch > 0).astype(np.float32) 111 | impatch = np.concatenate((impatch, np.ones_like(impatch[:, :1]), impatch2 \ 112 | ), axis=1) 113 | # cv2.destroyAllWindows() 114 | # cv2.imshow("im", (impatch > 0).astype(np.float32)) 115 | cv2.imshow("im", impatch) 116 | key = cv2.waitKey(33) 117 | if key == ord('s'): 118 | key = cv2.waitKey(0) 119 | # cv2.imwrite('1-%d.png' % (i + tbe), (impatch * 255).astype(np.uint8)) 120 | cv2.waitKey(0) 121 | 122 | matcrop, tbe, ten = crop_video(mat, tlen, tres) 123 | print('tbe %d, ten %d' % (tbe, ten)) 124 | 125 | # assert tbe2 >= tbe 126 | if tbe2 > tbe: 127 | matcrop = matcrop[tbe2 - tbe:] 128 | tbe = tbe2 129 | 130 | if isdebug: 131 | imcrop = matcrop 132 | for i in range(imcrop.shape[0]): 133 | 134 | impatch = imcrop[i] 135 | impatch2 = imcrop[i] 136 | print('%.2f, %.2f' % ((i + tbe) / 100, np.mean(impatch))) 137 | 138 | # impatch = impatch / np.max(im) 139 | # impatchlog = impatch ** 0.3 140 | # impatch2 = (impatch > 0).astype(np.float32) 141 | impatch = np.concatenate((impatch, np.ones_like(impatch[:, :1]), impatch2 \ 142 | ), axis=1) 143 | # cv2.destroyAllWindows() 144 | # cv2.imshow("im", (impatch > 0).astype(np.float32)) 145 | cv2.imshow("im", impatch) 146 | cv2.waitKey(33) 147 | # cv2.imwrite('1-%d.png' % (i + tbe), (impatch * 255).astype(np.uint8)) 148 | 149 | cv2.waitKey(0) 150 | mats.append(matcrop) 151 | tbes.append(tbe) 152 | tens.append(ten) 153 | 154 | return mats, tbes, tens 155 | 156 | 157 | ######################################################### 158 | def testreal(model, epoch, tres=2, in_dim=1, dev='cuda', datafolder='.', svdir='.'): 159 | model.eval() 160 | 161 | ims = [] 162 | 163 | for ni, name in enumerate(['bike', 'dragon', 'statue', 'resolution', 'discoball', 'teaser']): 164 | 165 | mats, tbes, tens = loadrealdata(datafolder, name, scale=-1, tres=tres, isdebug=False) 166 | mat0 = mats[0] 167 | # matmax = np.max(mat0) 168 | # scales = [-1, 99.9, 99.5, 99.0, 98.0] 169 | # for scale in scales: 170 | if True: 171 | scales = [-1, -1, -1, -1, -1, 99.9] 172 | scale = scales[ni] 173 | mat = mat0 174 | if scale > 0: 175 | mat99 = np.percentile(mat, scale) 176 | print('0.999 %.5f' % mat99) 177 | # scale = matmax / matmaxref 178 | if mat99 > 0: 179 | mat = mat / mat99 180 | mat[mat > 1] = 1 181 | 182 | batch_idx = 0 183 | datanp_txhxw = mat 184 | tbe = tbes[batch_idx] 185 | ten = tens[batch_idx] 186 | 187 | tbe = [tbe // tres] 188 | ten = [ten // tres] 189 | 190 | ######################################################### 191 | data_bxcxdxhxw = torch.from_numpy(datanp_txhxw).unsqueeze(0).unsqueeze(0).repeat(1, in_dim, 1, 1, 1) 192 | 193 | data_bxcxdxhxw = \ 194 | data_bxcxdxhxw.to(dev) 195 | 196 | if torch.min(data_bxcxdxhxw) >= 0: 197 | print('data, max %.5f, min %.5f' % (torch.max(data_bxcxdxhxw), torch.min(data_bxcxdxhxw))) 198 | data_bxcxdxhxw = data_bxcxdxhxw * 2 - 1 199 | 200 | dim = data_bxcxdxhxw.shape[2] // 2 201 | output = model(data_bxcxdxhxw, tbe, ten) 202 | output = torch.clamp(output, -1, 1) 203 | w = output.shape[-1] 204 | 205 | tmp = (output[:, :in_dim] + 1) / 2 206 | tmpmax = tmp.max() 207 | tmp = tmp / (tmpmax.view(-1, 1, 1, 1) + 1e-8) 208 | output[:, :in_dim] = tmp * 2 - 1 209 | 210 | # output[:, -in_dim:] = output[:, -in_dim:].mean(1, keepdim=True).repeat(1, 3, 1, 1) 211 | data = np.concatenate([output[:, :in_dim].detach().cpu().numpy(), 212 | output[:, -in_dim:].detach().cpu().numpy()], axis=3) 213 | ims.append(data) 214 | 215 | ims = np.concatenate(ims, axis=0) 216 | datanum = len(ims) 217 | row = len(scales) 218 | row = 1 219 | a = np.zeros([0, w * 2 * row, in_dim]) 220 | for i in range(datanum // row): 221 | imslice = [ims[d:d + 1] for d in range(i * row, i * row + row)] # ims[i * 4:i * 4 + 4] 222 | imslice = np.concatenate(imslice, axis=3) 223 | a = np.concatenate((a, np.transpose(imslice[0], [1, 2, 0])), axis=0) 224 | 225 | ''' 226 | complct = cv2.imread('/u6/a/wenzheng/remote2/code-nlos-git/nlos-fk-master/%s-lct.png'%name) 227 | compfk = cv2.imread('/u6/a/wenzheng/remote2/code-nlos-git/nlos-fk-master/%s-fk.png'%name) 228 | ''' 229 | ''' 230 | adep = a[:, 256:, :] 231 | a = a[:, :256, :] 232 | a = a / np.max(np.abs(a)) 233 | a = (a * 255).astype(np.uint8) 234 | adep = (adep * 255).astype(np.uint8) 235 | ''' 236 | a = (a + 1) / 2 237 | im = a * 255 238 | cv2.imwrite('%s/testreal-%d-%d.png' % (svdir, epoch, batch_idx), im) 239 | 240 | -------------------------------------------------------------------------------- /DL_inference/network7_256/__pycache__/customer_layers_3.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/DL_inference/network7_256/__pycache__/customer_layers_3.cpython-36.pyc -------------------------------------------------------------------------------- /DL_inference/network7_256/__pycache__/deepvoxel.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/DL_inference/network7_256/__pycache__/deepvoxel.cpython-36.pyc -------------------------------------------------------------------------------- /DL_inference/network7_256/__pycache__/ops.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/DL_inference/network7_256/__pycache__/ops.cpython-36.pyc -------------------------------------------------------------------------------- /DL_inference/network7_256/customer_layers_3.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | import sys 8 | 9 | sys.path.append('../utils_pytorch') 10 | from tfmodule import diffmodule as lct 11 | 12 | from ops import ResConv2D, ResConv3D, Interpsacle2d 13 | import numpy as np 14 | 15 | debugvalue = 1 16 | 17 | 18 | ################################################################################# 19 | class Transient2volumn(nn.Module): 20 | 21 | def __init__(self, nf0, in_channels, \ 22 | norm=nn.InstanceNorm3d): 23 | super(Transient2volumn, self).__init__() 24 | 25 | ############################################### 26 | assert in_channels == 1 27 | 28 | weights = np.zeros((1, 1, 3, 3, 3), dtype=np.float32) 29 | weights[:, :, 1:, 1:, 1:] = 1.0 30 | tfweights = torch.from_numpy(weights / np.sum(weights)) 31 | tfweights.requires_grad = True 32 | self.weights = nn.Parameter(tfweights) 33 | 34 | ############################################## 35 | self.conv1 = nn.Sequential( 36 | # begin, no norm 37 | nn.ReplicationPad3d(1), 38 | nn.Conv3d(in_channels, 39 | nf0 * 1, 40 | kernel_size=[3, 3, 3], 41 | padding=0, 42 | stride=[2, 2, 2], 43 | bias=True), 44 | 45 | ResConv3D(nf0 * 1, inplace=False), 46 | ResConv3D(nf0 * 1, inplace=False) 47 | ) 48 | 49 | def forward(self, x0): 50 | 51 | # x0 is from 0 to 1 52 | x0_conv = F.conv3d(x0, self.weights, \ 53 | bias=None, stride=2, padding=1, dilation=1, groups=1) 54 | 55 | x1 = self.conv1(x0) 56 | 57 | re = torch.cat([x0_conv, x1], dim=1) 58 | 59 | return re 60 | 61 | 62 | ########################################################################### 63 | class VisibleNet(nn.Module): 64 | 65 | def __init__(self, nf0, layernum=0, norm=nn.InstanceNorm3d): 66 | super(VisibleNet, self).__init__() 67 | 68 | self.layernum = layernum 69 | 70 | def forward(self, x): 71 | 72 | x5 = x 73 | 74 | ############################################### 75 | depdim = x5.shape[2] 76 | raw_pred_bxcxhxw, raw_dep_bxcxhxw = x5.max(dim=2) 77 | 78 | # -1 to 1 79 | # the nearer, the bigger 80 | raw_dep_bxcxhxw = depdim - 1 - raw_dep_bxcxhxw.float() 81 | raw_dep_bxcxhxw = raw_dep_bxcxhxw / (depdim - 1) 82 | 83 | xflatdep = torch.cat([raw_pred_bxcxhxw, raw_dep_bxcxhxw], dim=1) 84 | 85 | return xflatdep 86 | 87 | 88 | class Rendering(nn.Module): 89 | 90 | def __init__(self, nf0, out_channels, \ 91 | norm=nn.InstanceNorm2d, isdep=False): 92 | super(Rendering, self).__init__() 93 | 94 | ###################################### 95 | assert out_channels == 1 96 | 97 | weights = np.zeros((1, 2, 1, 1), dtype=np.float32) 98 | if isdep: 99 | weights[:, 1:, :, :] = 1.0 100 | else: 101 | weights[:, :1, :, :] = 1.0 102 | tfweights = torch.from_numpy(weights) 103 | tfweights.requires_grad = True 104 | self.weights = nn.Parameter(tfweights) 105 | 106 | self.resize = Interpsacle2d(factor=2, gain=1, align_corners=False) 107 | 108 | ####################################### 109 | self.conv1 = nn.Sequential( 110 | nn.ReflectionPad2d(1), 111 | nn.Conv2d(nf0 * 1, 112 | nf0 * 1, 113 | kernel_size=3, 114 | padding=0, 115 | stride=1, 116 | bias=True), 117 | ResConv2D(nf0 * 1, inplace=False), 118 | ResConv2D(nf0 * 1, inplace=False), 119 | ) 120 | 121 | self.conv2 = nn.Sequential( 122 | nn.ReflectionPad2d(1), 123 | nn.Conv2d(nf0 * 1 + 1, 124 | nf0 * 2, 125 | kernel_size=3, 126 | padding=0, 127 | stride=1, 128 | bias=True), 129 | ResConv2D(nf0 * 2, inplace=False), 130 | ResConv2D(nf0 * 2, inplace=False), 131 | 132 | nn.ReflectionPad2d(1), 133 | nn.Conv2d(nf0 * 2, 134 | out_channels, 135 | kernel_size=3, 136 | padding=0, 137 | stride=1, 138 | bias=True), 139 | ) 140 | 141 | def forward(self, x0): 142 | 143 | dim = x0.shape[1] // 2 144 | x0_im = x0[:, 0:1, :, :] 145 | x0_dep = x0[:, dim:dim + 1, :, :] 146 | x0_raw_128 = torch.cat([x0_im, x0_dep], dim=1) 147 | x0_raw_256 = self.resize(x0_raw_128) 148 | x0_conv_256 = F.conv2d(x0_raw_256, self.weights, \ 149 | bias=None, stride=1, padding=0, dilation=1, groups=1) 150 | 151 | ################################### 152 | x1 = self.conv1(x0) 153 | x1_up = self.resize(x1) 154 | 155 | x2 = torch.cat([x0_conv_256, x1_up], dim=1) 156 | x2 = self.conv2(x2) 157 | 158 | re = x0_conv_256 + debugvalue * x2 159 | 160 | return re 161 | 162 | 163 | ############################################################ 164 | if __name__ == '__main__': 165 | 166 | tfull = 512 167 | 168 | imsz = 256 169 | tsz = 128 170 | volumnsz = 128 171 | volumntsz = 64 172 | 173 | sres = imsz // volumnsz 174 | tres = tsz // volumntsz 175 | 176 | basedim = 1 177 | bnum = 1 178 | channel = 1 179 | 180 | #################################################### 181 | dev = 'cuda:0' 182 | data = np.zeros((bnum, channel, tsz, imsz, imsz), dtype=np.float32) 183 | 184 | downnet = Transient2volumn(nf0=basedim, in_channels=channel) 185 | downnet = downnet.to(dev) 186 | tfdata = torch.from_numpy(data).to(dev) 187 | tfre = downnet(tfdata) 188 | # tfre = nn.ConstantPad3d((0, 0, 0, 0, 2, 3), 0)(tfre) 189 | print('\n') 190 | print(tfre.shape) 191 | print('\n') 192 | 193 | # next unet 194 | unet3d = lct(spatial=imsz // sres, crop=tfull // tres, bin_len=0.01 * tres, \ 195 | mode='lct') 196 | unet3d = unet3d.to(dev) 197 | unet3d.todev(dev, basedim * 1 + 1) 198 | 199 | tbes = [0] * bnum 200 | tens = [tsz // tres] * bnum 201 | tfre2 = unet3d(tfre, tbes, tens) 202 | print('\n') 203 | print(tfre2.shape) 204 | print('\n') 205 | 206 | layernum = 0 207 | downnet2 = VisibleNet(nf0=basedim * 1 + 1, layernum=layernum) 208 | downnet2 = downnet2.to(dev) 209 | tfre2 = downnet2(tfre2) 210 | print('\n') 211 | print(tfre2.shape) 212 | print('\n') 213 | 214 | rendernet = Rendering(nf0=(basedim * 1 + 1) * (layernum // 2 * 2 + 1 + 1), out_channels=1) 215 | rendernet = rendernet.to(dev) 216 | 217 | rendered_img = rendernet(tfre2) 218 | print('\n') 219 | print(rendered_img.shape) 220 | print('\n') 221 | 222 | -------------------------------------------------------------------------------- /DL_inference/network7_256/deepvoxel.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import numpy as np 6 | from customer_layers_3 import \ 7 | Transient2volumn, \ 8 | VisibleNet, \ 9 | Rendering 10 | 11 | import sys 12 | sys.path.append('../utils_pytorch') 13 | from tfmodule import diffmodule as lct 14 | 15 | 16 | ########################################################################### 17 | def normalize(data_bxcxdxhxw): 18 | b, c, d, h, w = data_bxcxdxhxw.shape 19 | data_bxcxk = data_bxcxdxhxw.reshape(b, c, -1) 20 | 21 | data_min = data_bxcxk.min(2, keepdim=True)[0] 22 | data_zmean = data_bxcxk - data_min 23 | 24 | # most are 0 25 | data_max = data_zmean.max(2, keepdim=True)[0] 26 | data_norm = data_zmean / (data_max + 1e-15) 27 | 28 | return data_norm.view(b, c, d, h, w) 29 | 30 | 31 | ################################################################ 32 | class DeepVoxels(nn.Module): 33 | 34 | def __init__(self, 35 | nf0=16, 36 | in_channels=3, 37 | out_channels=3, 38 | img_sidelength=256, 39 | grid_dim=32, 40 | bin_len=0.01, 41 | wall_size=2.0, 42 | mode='fk', 43 | res0=0): 44 | 45 | super(DeepVoxels, self).__init__() 46 | 47 | ###################################33 48 | # 4 networks 49 | # 1 downsample 50 | # 2 unet 51 | # 3 occlusion 52 | # 4 render 53 | 54 | 55 | imsz = 256 56 | assert imsz == img_sidelength 57 | 58 | volumnsz = 128 59 | assert volumnsz == grid_dim 60 | sres = imsz // volumnsz 61 | 62 | tfull = 512 63 | tsz = 128 64 | volumntsz = 64 65 | tres = tsz // volumntsz 66 | # assert sres == tres 67 | 68 | ######################################################## 69 | basedim = nf0 70 | self.basedim = basedim 71 | # assert not raytracing 72 | 73 | self.downnet = Transient2volumn(nf0=basedim, in_channels=in_channels) 74 | 75 | print('bin_len %.7f' % bin_len) 76 | self.lct = lct(spatial=imsz // sres, crop=tfull // tres, bin_len=bin_len * tres, \ 77 | mode=mode, wall_size=wall_size) 78 | 79 | layernum = 0 80 | self.visnet = VisibleNet(nf0=basedim * 1 + 1, layernum=layernum) 81 | 82 | self.depth = True 83 | assert out_channels == 6 or out_channels == 2 84 | self.rendernet = Rendering(nf0=(basedim * 1 + 1) * (layernum // 2 * 2 + 1 + 1), out_channels=out_channels // 2) 85 | self.depnet = Rendering(nf0=(basedim * 1 + 1) * (layernum // 2 * 2 + 1 + 1), out_channels=out_channels // 2, isdep=True) 86 | 87 | def todev(self, dev): 88 | self.lct.todev(dev, self.basedim * 1 + 1) 89 | 90 | def noise(self, data): 91 | gau = 0.05 + 0.03 * torch.randn_like(data) + data 92 | poi = 0.03 * torch.randn_like(data) * gau + gau 93 | return poi 94 | 95 | def forward(self, input_voxel, tbes, tens): 96 | 97 | if False: 98 | noisedata = self.noise(input_voxel) 99 | else: 100 | noisedata = input_voxel 101 | 102 | ############################### 103 | data_norm = normalize(noisedata) 104 | 105 | tfre = self.downnet(data_norm) 106 | 107 | # lct 108 | tfre2 = self.lct(tfre, tbes, tens) 109 | 110 | # resize 111 | x = tfre2 112 | zdim = x.shape[2] 113 | zdimnew = zdim * 100 // 128 114 | x = x[:, :, :zdimnew] 115 | tfre2 = x 116 | 117 | tfre2 = nn.ReLU()(tfre2) 118 | tfre2 = normalize(tfre2) 119 | 120 | ###################################### 121 | # unet 2 voxel 122 | tfflat = self.visnet(tfre2) 123 | 124 | # render 125 | rendered_img = self.rendernet(tfflat) 126 | 127 | if self.depth: 128 | dep_img = self.depnet(tfflat) 129 | rendered_img = torch.cat([rendered_img, dep_img], dim=1) 130 | 131 | rendered_img = torch.clamp(rendered_img, 0, 1) 132 | rendered_img = rendered_img * 2 - 1 133 | return rendered_img 134 | 135 | 136 | ################################################################# 137 | if __name__ == '__main__': 138 | 139 | basedim = 1 140 | tres = 2 141 | dev = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 142 | 143 | frame = 512 144 | in_channels = 1 145 | data = np.zeros((1, in_channels, frame, 256, 256), dtype=np.float32) 146 | 147 | from scipy.io import loadmat 148 | data = loadmat(file_name='/home/wenzheng/largestore/nlos-phasor/realdata/resolution0.mat') 149 | rect_data_hxwxt = data['measlr'] 150 | rect_data_txhxw = np.transpose(rect_data_hxwxt, axes=[2, 0, 1]) 151 | data = rect_data_txhxw.reshape(1, 1, 512, 256, 256) 152 | tfdata = torch.from_numpy(data).to(dev) 153 | 154 | model = DeepVoxels( 155 | nf0=basedim, 156 | in_channels=in_channels, 157 | out_channels=2, 158 | img_sidelength=256, 159 | grid_dim=128, 160 | mode='lct') 161 | model = model.to(dev) 162 | model.todev(dev) 163 | re = model(tfdata, [0, 0, 0, 0, 0], [frame // tres, 32, 32, 32, 32]) 164 | print('\n') 165 | print(re.shape) 166 | print('\n') 167 | 168 | re = re.detach().cpu().numpy() 169 | re = (re + 1) / 2 170 | im = re[0, 0] 171 | dep = re[0, 1] 172 | im = im / np.max(im) 173 | 174 | import cv2 175 | cv2.imshow('im', im) 176 | cv2.imshow('dep', dep) 177 | cv2.waitKey() 178 | 179 | 180 | 181 | -------------------------------------------------------------------------------- /DL_inference/network7_256/ops.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch.nn.functional as F 4 | import torch.nn as nn 5 | import torch 6 | 7 | import numpy as np 8 | 9 | 10 | ############################################################################# 11 | class Interpsacle2d(nn.Module): 12 | 13 | def __init__(self, factor=2, gain=1, align_corners=False): 14 | """ 15 | the first upsample method in G_synthesis. 16 | :param factor: 17 | :param gain: 18 | """ 19 | super(Interpsacle2d, self).__init__() 20 | self.gain = gain 21 | self.factor = factor 22 | self.align_corners = align_corners 23 | 24 | def forward(self, x): 25 | if self.gain != 1: 26 | x = x * self.gain 27 | 28 | x = nn.functional.interpolate(x, scale_factor=self.factor, mode='bilinear', align_corners=self.align_corners) 29 | 30 | return x 31 | 32 | 33 | ###################################################################### 34 | class ResConv3D(nn.Module): 35 | 36 | def __init__(self, nf0, inplace=False): 37 | super(ResConv3D, self).__init__() 38 | 39 | self.tmp = nn.Sequential( 40 | 41 | nn.ReplicationPad3d(1), 42 | nn.Conv3d(nf0 * 1, 43 | nf0 * 1, 44 | kernel_size=[3, 3, 3], 45 | padding=0, 46 | stride=[1, 1, 1], 47 | bias=True), 48 | 49 | nn.LeakyReLU(negative_slope=0.2, inplace=inplace), 50 | # nn.Dropout3d(0.1, inplace), 51 | 52 | nn.ReplicationPad3d(1), 53 | nn.Conv3d(nf0 * 1, 54 | nf0 * 1, 55 | kernel_size=[3, 3, 3], 56 | padding=0, 57 | stride=[1, 1, 1], 58 | bias=True), 59 | ) 60 | 61 | self.inplace = inplace 62 | 63 | def forward(self, x): 64 | re = F.leaky_relu(self.tmp(x) + x, negative_slope=0.2, inplace=self.inplace) 65 | return re 66 | 67 | 68 | class ResConv2D(nn.Module): 69 | 70 | def __init__(self, nf0, inplace=False): 71 | super(ResConv2D, self).__init__() 72 | 73 | self.tmp = nn.Sequential( 74 | 75 | nn.ReplicationPad2d(1), 76 | nn.Conv2d(nf0 * 1, 77 | nf0 * 1, 78 | kernel_size=[3, 3], 79 | padding=0, 80 | stride=[1, 1], 81 | bias=True), 82 | 83 | nn.LeakyReLU(negative_slope=0.2, inplace=inplace), 84 | # nn.Dropout3d(0.1, inplace), 85 | 86 | nn.ReplicationPad2d(1), 87 | nn.Conv2d(nf0 * 1, 88 | nf0 * 1, 89 | kernel_size=[3, 3], 90 | padding=0, 91 | stride=[1, 1], 92 | bias=True), 93 | ) 94 | 95 | self.inplace = inplace 96 | 97 | def forward(self, x): 98 | re = F.leaky_relu(self.tmp(x) + x, negative_slope=0.2, inplace=self.inplace) 99 | return re 100 | 101 | 102 | ##################################################################### 103 | if __name__ == '__main__': 104 | 105 | import numpy as np 106 | 107 | btf = torch.from_numpy(np.random.rand(10, 16, 256, 256).astype(np.float32)).cuda() 108 | 109 | scaledownlayer = Interpsacle2d(0.5) 110 | ctf = scaledownlayer(btf) 111 | 112 | scaleuplayer = Interpsacle2d(2.0) 113 | dtf = scaleuplayer(ctf) 114 | 115 | print(dtf.shape) 116 | 117 | -------------------------------------------------------------------------------- /DL_inference/re/testreal-58-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/DL_inference/re/testreal-58-0.png -------------------------------------------------------------------------------- /DL_inference/utils/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | utils 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /DL_inference/utils/.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | /${PROJECT_DIR_NAME} 7 | 8 | 9 | -------------------------------------------------------------------------------- /DL_inference/utils/fk.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import cv2 5 | import scipy.interpolate as si 6 | from helper import roll_1 7 | 8 | 9 | def lct(meas_hxwxt, wall_size, crop, bin_len): 10 | 11 | c = 3e8 12 | width = wall_size / 2.0; 13 | bin_resolution = bin_len / c 14 | assert 2 ** int(np.log2(crop)) == crop 15 | 16 | ########################################### 17 | meas_hxwxt = meas_hxwxt[:, :, :crop] # HxWxT 18 | sptial_grid = meas_hxwxt.shape[0] # H, N 19 | temprol_grid = meas_hxwxt.shape[2] # T, M 20 | trange = temprol_grid * c * bin_resolution 21 | 22 | ############################################################ 23 | # 0-1 24 | gridz_M = np.arange(temprol_grid, dtype=np.float32) 25 | gridz_M = gridz_M / (temprol_grid - 1) 26 | gridz_1xMx1x1 = gridz_M.reshape(1, -1, 1, 1) 27 | 28 | ############################################################### 29 | # axis 30 | zdim = np.arange(2 * temprol_grid, dtype=np.float32) 31 | xdim = np.arange(2 * sptial_grid, dtype=np.float32) 32 | 33 | zdim = (zdim - temprol_grid) / temprol_grid 34 | xdim = (xdim - sptial_grid) / sptial_grid 35 | ydim = xdim 36 | 37 | [gridy_2Nx2Nx2M, gridx_2Nx2Nx2M, gridz_2Nx2Nx2M] = np.meshgrid(xdim, ydim, zdim) 38 | gridz_2Mx2Nx2N = np.transpose(gridz_2Nx2Nx2M, [2, 1, 0]) 39 | gridy_2Mx2Nx2N = np.transpose(gridy_2Nx2Nx2M, [2, 1, 0]) 40 | gridx_2Mx2Nx2N = np.transpose(gridx_2Nx2Nx2M, [2, 1, 0]) 41 | 42 | ##########################################################3 43 | M = temprol_grid 44 | N = sptial_grid 45 | 46 | fkrange = ((N * trange) / (M * width * 4)) ** 2 47 | gridznew = fkrange * (gridx_2Mx2Nx2N ** 2 + gridy_2Mx2Nx2N ** 2) + gridz_2Mx2Nx2N ** 2 48 | gridznew = np.sqrt(gridznew) 49 | 50 | ################################################### 51 | # padd data 52 | data_TxHxW = np.transpose(meas_hxwxt, [2, 0, 1]) 53 | data_TxHxW = data_TxHxW * (gridz_1xMx1x1[0] ** 2) 54 | data_TxHxW = np.sqrt(data_TxHxW) 55 | 56 | datapad_2Tx2Hx2W = np.zeros(shape=(2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=np.float32) 57 | datapad_2Tx2Hx2W[:temprol_grid, :sptial_grid, :sptial_grid] = data_TxHxW 58 | 59 | # fft 60 | datafre = np.fft.fftn(datapad_2Tx2Hx2W) 61 | # shift 62 | ''' 63 | datafre = np.roll(datafre, shift=temprol_grid, axis=0) 64 | datafre = np.roll(datafre, shift=sptial_grid, axis=1) 65 | datafre = np.roll(datafre, shift=sptial_grid, axis=2) 66 | ''' 67 | datafre = roll_1(datafre, dim=1, n=temprol_grid) 68 | datafre = roll_1(datafre, dim=2, n=sptial_grid) 69 | datafre = roll_1(datafre, dim=3, n=sptial_grid) 70 | # stolt trick 71 | 72 | # interpolate 73 | tvol = si.interpn(points=(zdim, ydim, xdim), values=datafre, \ 74 | xi=np.stack([gridznew, gridy_2Mx2Nx2N, gridx_2Mx2Nx2N], axis=3), \ 75 | method='linear', bounds_error=False, fill_value=0) 76 | dim = np.where(zdim > 0)[0][0] 77 | print('zzeropos %d' % dim) 78 | tvol[:dim, :, :] = 0 79 | 80 | gridznew = np.maximum(gridznew, 1e-8) 81 | tvol = tvol * np.abs(gridz_2Mx2Nx2N) / gridznew 82 | 83 | ######################################################### 84 | # 0-1 85 | datafre = tvol 86 | ''' 87 | datafre = np.roll(datafre, shift=temprol_grid, axis=0) 88 | datafre = np.roll(datafre, shift=sptial_grid, axis=1) 89 | datafre = np.roll(datafre, shift=sptial_grid, axis=2) 90 | ''' 91 | datafre = roll_1(datafre, dim=1, n=temprol_grid) 92 | datafre = roll_1(datafre, dim=2, n=sptial_grid) 93 | datafre = roll_1(datafre, dim=3, n=sptial_grid) 94 | 95 | volumn_2Mx2Nx2N = np.fft.ifftn(datafre) 96 | volumn_ZxYxX = volumn_2Mx2Nx2N[:temprol_grid, :sptial_grid, :sptial_grid] 97 | volumn_ZxYxX = np.real(volumn_ZxYxX) ** 2 + np.imag(volumn_ZxYxX) ** 2 98 | 99 | front_view_HxW = np.max(volumn_ZxYxX, axis=0) 100 | cv2.imshow("re3", front_view_HxW / np.max(front_view_HxW)) 101 | # cv2.imshow('gt', imgt) 102 | cv2.waitKey() 103 | 104 | for frame in volumn_ZxYxX: 105 | cv2.imshow("re1", frame) 106 | cv2.imshow("re2", frame / np.max(frame)) 107 | cv2.waitKey(0) 108 | 109 | 110 | ######################################################## 111 | if __name__ == '__main__': 112 | 113 | import os 114 | 115 | ''' 116 | fd = '/u6/a/wenzheng/remote2/code-nlos-git/OccludedSceneRep-2/code/pytorch-wz/dataloader_lct'; 117 | ims = [] 118 | tbe = -1 119 | for i in range(512): 120 | name = '%s/1-%d.png' % (fd, i) 121 | if not os.path.isfile(name): 122 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 123 | continue 124 | 125 | if tbe < 0: 126 | tbe = i 127 | 128 | im = cv2.imread(name) 129 | imgt = im[:, :256, :] 130 | im = im[:, -256:, :] 131 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 132 | ims.append(imgray) 133 | 134 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 135 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 136 | ''' 137 | 138 | from scipy.io import loadmat 139 | 140 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/nlos-fk-master/statue.mat') 141 | rect_data_hxwxt = data['data'] 142 | 143 | crop = 512 144 | bin_len = 32e-12 * 3e8 # 0.01 145 | 146 | K = 3 147 | for k in range(K): 148 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 149 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 150 | 151 | ''' 152 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 153 | crop = crop // 2 154 | bin_len = bin_len * 2 155 | ''' 156 | 157 | lct(rect_data_hxwxt, wall_size=2.0, crop=crop, bin_len=bin_len) 158 | 159 | -------------------------------------------------------------------------------- /DL_inference/utils/helper.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import cv2 5 | import scipy.sparse as ssp 6 | 7 | 8 | debug = False 9 | # still not very sure about coordinate! 10 | 11 | 12 | ######################################################### 13 | def filterLaplacian(): 14 | 15 | hszie = 5; 16 | std1 = 1.0 17 | 18 | lim = (hszie - 1) // 2 19 | std2 = std1 ** 2 20 | 21 | dims = np.arange(-lim, lim + 1, dtype=np.float32) 22 | [y, x, z] = np.meshgrid(dims, dims, dims) 23 | w = np.exp(-(x ** 2 + y ** 2 + z ** 2) / (2 * std2)) 24 | w = w / np.sum(w) 25 | 26 | w1 = w * (x ** 2 + y ** 2 + z ** 2 - 3 * std2) 27 | w1 = w1 / (std2 ** 2) 28 | 29 | # w = w1 - np.sum(w1) / (hszie ** 3) 30 | w = w1 - np.mean(w1) 31 | 32 | return w 33 | 34 | 35 | def resamplingOperator(temprol_grid): 36 | 37 | M = temprol_grid 38 | row = M ** 2 39 | col = M 40 | assert 2 ** int(np.log2(M)) == M 41 | 42 | # 1 to M^2 43 | x = np.arange(row, dtype=np.float32) 44 | x = x + 1 45 | 46 | rowidx = np.arange(row) 47 | # 0 to M-1 48 | colidx = np.ceil(np.sqrt(x)) - 1 49 | data = np.ones_like(rowidx, dtype=np.float32) 50 | mtx1 = ssp.csr_matrix((data, (rowidx, colidx)), shape=(row, col), dtype=np.float32) 51 | mtx2 = ssp.spdiags(data=[1.0 / np.sqrt(x)], diags=[0], m=row, n=row) 52 | 53 | mtx = mtx2.dot(mtx1) 54 | K = int(np.log2(M)) 55 | for _ in np.arange(K): 56 | mtx = 0.5 * (mtx[0::2, :] + mtx[1::2]) 57 | # mtxi = 0.5 * (mtxi[:, 0::2] + mtxi[:, 1::2]) 58 | 59 | mtxi = np.transpose(mtx) 60 | 61 | if debug: 62 | print(mtx.toarray()) 63 | 64 | if debug: 65 | print(mtxi.toarray()) 66 | 67 | return mtx.toarray(), mtxi.toarray() 68 | 69 | 70 | def definePsf(sptial_grid, temprol_grid, slope): 71 | 72 | # slop is time_range / wall_size 73 | N = sptial_grid 74 | M = temprol_grid 75 | 76 | # -1 to 1 77 | x_2N = np.arange(2 * sptial_grid, dtype=np.float32) 78 | x_2N = x_2N / (2 * sptial_grid - 1) * 2 - 1 79 | 80 | # here, x and y are symetric 81 | # it doesn't mater y is postive or negative 82 | y_2N = x_2N 83 | 84 | # 0 to 2 85 | z_2M = np.arange(2 * temprol_grid, dtype=np.float32) 86 | z_2M = z_2M / (2 * temprol_grid - 1) * 2 87 | 88 | # grid axis, also in hxwxt 89 | # that's why x is the second axis 90 | # y is the first axis 91 | [gridy_2Nx2Nx2M, gridx_2Nx2Nx2M, gridz_2Nx2Nx2M] = np.meshgrid(x_2N, y_2N, z_2M) 92 | 93 | # dst 94 | a_2Nx2NX2M = (4 * slope) ** 2 * (gridx_2Nx2Nx2M ** 2 + gridy_2Nx2Nx2M ** 2) - gridz_2Nx2Nx2M 95 | b_2Nx2NX2M = np.abs(a_2Nx2NX2M) 96 | 97 | # min data 98 | c_2Nx2NX2M = np.min(b_2Nx2NX2M, axis=2, keepdims=True) 99 | 100 | # should be a ellipse 101 | d_2Nx2NX2M = np.abs(b_2Nx2NX2M - c_2Nx2NX2M) < 1e-8 102 | d_2Nx2NX2M = d_2Nx2NX2M.astype(np.float32) 103 | if debug: 104 | cv2.imshow("0", d_2Nx2NX2M[:, :, 0]) 105 | cv2.imshow("1", d_2Nx2NX2M[:, :, M // 2 - 1]) 106 | cv2.imshow("2", d_2Nx2NX2M[:, :, M - 1]) 107 | cv2.waitKey() 108 | 109 | # norm 110 | e_2Nx2NX2M = d_2Nx2NX2M / np.sqrt(np.sum(d_2Nx2NX2M)) 111 | 112 | # shift 113 | f1_2Nx2NX2M = np.roll(e_2Nx2NX2M, shift=N, axis=0) 114 | f2_2Nx2NX2M = np.roll(f1_2Nx2NX2M, shift=N, axis=1) 115 | if debug: 116 | cv2.imshow("0", f2_2Nx2NX2M[:, :, 0] * 256) 117 | cv2.imshow("1", f2_2Nx2NX2M[:, :, M // 2 - 1] * 256) 118 | cv2.imshow("2", f2_2Nx2NX2M[:, :, M - 1] * 256) 119 | cv2.waitKey() 120 | 121 | psf_2Mx2Nx2N = np.transpose(f2_2Nx2NX2M, [2, 0, 1]) 122 | 123 | return psf_2Mx2Nx2N 124 | 125 | 126 | ######################################################################### 127 | def roll_1(x_bxtxhxwx2, dim, n): 128 | if dim == 1: 129 | a = np.concatenate((x_bxtxhxwx2[-n:], x_bxtxhxwx2[:-n]), axis=dim - 1) 130 | if dim == 2: 131 | a = np.concatenate((x_bxtxhxwx2[:, -n:], x_bxtxhxwx2[:, :-n]), axis=dim - 1) 132 | if dim == 3: 133 | a = np.concatenate((x_bxtxhxwx2[:, :, -n:], x_bxtxhxwx2[:, :, :-n]), axis=dim - 1) 134 | return a 135 | 136 | 137 | ################################################################################## 138 | def gaussianwin(L, alpha): 139 | 140 | N = L - 1; 141 | Nhalf = N / 2.0 142 | n_k = np.arange(N + 1, dtype=np.float32) - Nhalf; 143 | w_k = np.exp(-0.5 * (alpha * n_k / Nhalf) ** 2) 144 | 145 | return w_k 146 | 147 | 148 | def waveconvparam(bin_resolution, virtual_wavelength, cycles): 149 | 150 | c = 3e8; 151 | s_z = bin_resolution * c; 152 | samples = int(round(cycles * virtual_wavelength / (bin_resolution * c))); 153 | num_cycles = samples * s_z / virtual_wavelength; 154 | sigma = 0.3; 155 | 156 | # generate sin/cos signals 157 | grids_k = np.arange(samples, dtype=np.float32) + 1 158 | sin_wave_k = np.sin(2 * np.pi * (num_cycles * grids_k) / samples); 159 | cos_wave_k = np.cos(2 * np.pi * (num_cycles * grids_k) / samples); 160 | 161 | # window = single(gausswin(samples, 1/sigma)); 162 | window = gaussianwin(samples, 1.0 / sigma) 163 | virtual_sin_wave_k = sin_wave_k * window 164 | virtual_cos_wave_k = cos_wave_k * window 165 | 166 | return virtual_cos_wave_k, virtual_sin_wave_k 167 | 168 | 169 | def waveconv(bin_resolution, virtual_wavelength, cycles, data_txhxw): 170 | 171 | c = 3e8; 172 | s_z = bin_resolution * c; 173 | samples = int(round(cycles * virtual_wavelength / (bin_resolution * c))); 174 | num_cycles = samples * s_z / virtual_wavelength; 175 | sigma = 0.3; 176 | 177 | # generate sin/cos signals 178 | grids_k = np.arange(samples, dtype=np.float32) + 1 179 | sin_wave_k = np.sin(2 * np.pi * (num_cycles * grids_k) / samples); 180 | cos_wave_k = np.cos(2 * np.pi * (num_cycles * grids_k) / samples); 181 | 182 | # window = single(gausswin(samples, 1/sigma)); 183 | window = gaussianwin(samples, 1.0 / sigma) 184 | virtual_sin_wave_k = sin_wave_k * window 185 | virtual_cos_wave_k = cos_wave_k * window 186 | 187 | wave_sin = np.zeros_like(data_txhxw) 188 | wave_cos = np.zeros_like(data_txhxw) 189 | 190 | # conv 191 | M, N, _ = data_txhxw.shape 192 | for i in range(N): 193 | for j in range(N): 194 | data_t = data_txhxw[:, i, j] 195 | real = np.convolve(data_t, v=virtual_sin_wave_k, mode='same') 196 | image = np.convolve(data_t, v=virtual_cos_wave_k, mode='same') 197 | wave_sin[:, i, j] = real 198 | wave_cos[:, i, j] = image 199 | 200 | return wave_cos, wave_sin 201 | 202 | -------------------------------------------------------------------------------- /DL_inference/utils/helper.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/DL_inference/utils/helper.pyc -------------------------------------------------------------------------------- /DL_inference/utils/lct.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import cv2 5 | from helper import definePsf, resamplingOperator 6 | 7 | 8 | ########################################################## 9 | def lct(meas_hxwxt, wall_size, crop, bin_len): 10 | 11 | c = 3e8 12 | width = wall_size / 2.0; 13 | bin_resolution = bin_len / c 14 | assert 2 ** int(np.log2(crop)) == crop 15 | 16 | snr = 1e-1 17 | 18 | ########################################### 19 | meas_hxwxt = meas_hxwxt[:, :, :crop] # HxWxT 20 | sptial_grid = meas_hxwxt.shape[0] # H, N 21 | temprol_grid = meas_hxwxt.shape[2] # T, M 22 | trange = temprol_grid * c * bin_resolution 23 | 24 | ######################################################### 25 | # 0-1 26 | gridz_M = np.arange(temprol_grid, dtype=np.float32) 27 | gridz_M = gridz_M / (temprol_grid - 1) 28 | gridz_MxNxN = np.tile(gridz_M.reshape(-1, 1, 1), [1, sptial_grid, sptial_grid]) 29 | 30 | ################################################### 31 | slope = width / trange 32 | psf = definePsf(sptial_grid, temprol_grid, slope) 33 | fpsf = np.fft.fftn(psf) 34 | invpsf = np.conjugate(fpsf) / (1 / snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 35 | # invpsf = np.conjugate(fpsf) 36 | 37 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 38 | 39 | ############################################################# 40 | # diffuse 41 | data_TxHxW = np.transpose(meas_hxwxt, [2, 0, 1]) 42 | data_TxHxW = data_TxHxW * (gridz_MxNxN ** 4) 43 | 44 | datapad_2Tx2Hx2W = np.zeros(shape=(2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=np.float32) 45 | 46 | left = mtx_MxM 47 | right = data_TxHxW.reshape(temprol_grid, -1) 48 | tmp = np.matmul(left, right).reshape(temprol_grid, sptial_grid, sptial_grid) 49 | datapad_2Tx2Hx2W[:temprol_grid, :sptial_grid, :sptial_grid] = tmp 50 | 51 | datafre = np.fft.fftn(datapad_2Tx2Hx2W) 52 | volumn_2Mx2Nx2N = np.fft.ifftn(datafre * invpsf) 53 | volumn_2Mx2Nx2N = np.real(volumn_2Mx2Nx2N) 54 | volumn_ZxYxX = volumn_2Mx2Nx2N[:temprol_grid, :sptial_grid, :sptial_grid] 55 | 56 | left = mtxi_MxM 57 | right = volumn_ZxYxX.reshape(temprol_grid, -1) 58 | tmp = np.matmul(left, right).reshape(temprol_grid, sptial_grid, sptial_grid) 59 | volumn_ZxYxX = tmp 60 | 61 | ################################ 62 | volumn_ZxYxX[volumn_ZxYxX < 0] = 0 63 | 64 | dim = volumn_ZxYxX.shape[0] * 100 // 128 65 | volumn_ZxYxX = volumn_ZxYxX[:dim] 66 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 67 | 68 | front_view_HxW = np.max(volumn_ZxYxX, axis=0) 69 | cv2.imshow("re3", front_view_HxW / np.max(front_view_HxW)) 70 | # cv2.imshow('gt', imgt) 71 | cv2.waitKey() 72 | 73 | for frame in volumn_ZxYxX: 74 | cv2.imshow("re1", frame) 75 | cv2.imshow("re2", frame / np.max(frame)) 76 | cv2.waitKey(0) 77 | 78 | 79 | ######################################################## 80 | if __name__ == '__main__': 81 | 82 | import os 83 | 84 | ''' 85 | fd = '/u6/a/wenzheng/remote2/code-nlos-git/OccludedSceneRep-2/code/pytorch-wz/dataloader_lct'; 86 | ims = [] 87 | tbe = -1 88 | for i in range(512): 89 | name = '%s/1-%d.png' % (fd, i) 90 | if not os.path.isfile(name): 91 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 92 | continue 93 | 94 | if tbe < 0: 95 | tbe = i 96 | 97 | im = cv2.imread(name) 98 | imgt = im[:, :256, :] 99 | im = im[:, -256:, :] 100 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 101 | ims.append(imgray) 102 | 103 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 104 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 105 | ''' 106 | 107 | from scipy.io import loadmat 108 | 109 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/realdata/bike0.mat') 110 | rect_data_hxwxt = data['measlr'] 111 | 112 | crop = 512 113 | bin_len = 32e-12 * 3e8 # 0.01 114 | 115 | K = 0 116 | for k in range(K): 117 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 118 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 119 | 120 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 121 | crop = crop // 2 122 | bin_len = bin_len * 2 123 | 124 | lct(rect_data_hxwxt, wall_size=2.0, crop=crop, bin_len=bin_len) 125 | 126 | -------------------------------------------------------------------------------- /DL_inference/utils/phasor.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import numpy as np 4 | import cv2 5 | from helper import definePsf, resamplingOperator, waveconv 6 | 7 | 8 | def phasor(meas_hxwxt, wall_size, crop, bin_len): 9 | 10 | c = 3e8 11 | width = wall_size / 2.0; 12 | bin_resolution = bin_len / c 13 | assert 2 ** int(np.log2(crop)) == crop 14 | 15 | ########################################### 16 | meas_hxwxt = meas_hxwxt[:, :, :crop] # HxWxT 17 | sptial_grid = meas_hxwxt.shape[0] # H, N 18 | temprol_grid = meas_hxwxt.shape[2] # T, M 19 | trange = temprol_grid * c * bin_resolution 20 | 21 | ################################################### 22 | slope = width / trange 23 | psf = definePsf(sptial_grid, temprol_grid, slope) 24 | fpsf = np.fft.fftn(psf) 25 | # invpsf = np.conjugate(fpsf) / (1 / snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 26 | invpsf = np.conjugate(fpsf) 27 | 28 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 29 | 30 | ############################################################# 31 | # Step 0: define virtual wavelet properties 32 | s_lamda_limit = wall_size / (sptial_grid - 1); # sample spacing on the wall 33 | sampling_coeff = 2; # scale the size of the virtual wavelength (usually 2, optionally 3 for noisy scenes) 34 | virtual_wavelength = sampling_coeff * (s_lamda_limit * 2); # virtual wavelength in units of cm 35 | cycles = 5; # number of wave cycles in the wavelet, typically 4-6 36 | 37 | ########################################################### 38 | data_TxHxW = np.transpose(meas_hxwxt, [2, 0, 1]) 39 | 40 | ############################################################ 41 | # Step 1: convolve measurement volume with virtual wave 42 | phasor_data_cos, phasor_data_sin = waveconv(bin_resolution, virtual_wavelength, cycles, data_TxHxW); 43 | # phasor_data_cos = single(phasor_data_cos); 44 | # phasor_data_sin = single(phasor_data_sin); 45 | 46 | ############################################################# 47 | # Step 2: transform virtual wavefield into LCT domain 48 | M = temprol_grid 49 | N = sptial_grid 50 | phasor_tdata_cos = np.zeros((2 * M, 2 * N, 2 * N), dtype=np.float32); 51 | phasor_tdata_sin = np.zeros((2 * M, 2 * N, 2 * N), dtype=np.float32); 52 | 53 | left = mtx_MxM 54 | right = phasor_data_cos.reshape(temprol_grid, -1) 55 | tmp = np.matmul(left, right).reshape(temprol_grid, sptial_grid, sptial_grid) 56 | phasor_tdata_cos[:temprol_grid, :sptial_grid, :sptial_grid] = tmp 57 | 58 | right = phasor_data_sin.reshape(temprol_grid, -1) 59 | tmp = np.matmul(left, right).reshape(temprol_grid, sptial_grid, sptial_grid) 60 | phasor_tdata_sin[:temprol_grid, :sptial_grid, :sptial_grid] = tmp 61 | 62 | ################################################################### 63 | # Step 3: convolve with backprojection kernel 64 | ''' 65 | tvol_phasorbp_sin = ifftn(fftn(phasor_tdata_sin).*bp_psf); 66 | tvol_phasorbp_sin = tvol_phasorbp_sin(1:end./2,1:end./2,1:end./2); 67 | phasor_tdata_cos = ifftn(fftn(phasor_tdata_cos).*bp_psf); 68 | phasor_tdata_cos = phasor_tdata_cos(1:end./2,1:end./2,1:end./2); 69 | ''' 70 | 71 | datafre = np.fft.fftn(phasor_tdata_sin) 72 | tvol_phasorbp_sin = np.fft.ifftn(datafre * invpsf) 73 | tvol_phasorbp_sin = tvol_phasorbp_sin[:M, :N, :N] 74 | 75 | datafre = np.fft.fftn(phasor_tdata_cos) 76 | tvol_phasorbp_cos = np.fft.ifftn(datafre * invpsf) 77 | tvol_phasorbp_cos = tvol_phasorbp_cos[:M, :N, :N] 78 | 79 | ############################################################### 80 | # Step 4: compute phasor field magnitude and inverse LCT 81 | ''' 82 | tvol = sqrt(tvol_phasorbp_sin.^2 + phasor_tdata_cos.^2); 83 | vol = reshape(mtxi*tvol(:,:),[M N N]); 84 | vol = max(real(vol),0); 85 | ''' 86 | 87 | tvol = np.sqrt(tvol_phasorbp_cos ** 2 + tvol_phasorbp_sin ** 2) 88 | 89 | left = mtxi_MxM 90 | right = tvol.reshape(temprol_grid, -1) 91 | tmp = np.matmul(left, right).reshape(temprol_grid, sptial_grid, sptial_grid) 92 | volumn_ZxYxX = np.real(tmp) 93 | volumn_ZxYxX[volumn_ZxYxX < 0] = 0 94 | 95 | # volumn_ZxYxX[-10:] = 0 96 | 97 | #######################################################33 98 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 99 | 100 | front_view_HxW = np.max(volumn_ZxYxX, axis=0) 101 | cv2.imshow("re3", front_view_HxW / np.max(front_view_HxW)) 102 | # cv2.imshow('gt', imgt) 103 | cv2.waitKey() 104 | 105 | for frame in volumn_ZxYxX: 106 | cv2.imshow("re1", frame) 107 | cv2.imshow("re2", frame / np.max(frame)) 108 | cv2.waitKey(0) 109 | 110 | 111 | ######################################################## 112 | if __name__ == '__main__': 113 | 114 | import os 115 | 116 | ''' 117 | fd = '/u6/a/wenzheng/remote2/code-nlos-git/OccludedSceneRep-2/code/pytorch-wz/dataloader_lct'; 118 | ims = [] 119 | tbe = -1 120 | for i in range(512): 121 | name = '%s/1-%d.png' % (fd, i) 122 | if not os.path.isfile(name): 123 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 124 | continue 125 | 126 | if tbe < 0: 127 | tbe = i 128 | 129 | im = cv2.imread(name) 130 | imgt = im[:, :256, :] 131 | im = im[:, -256:, :] 132 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 133 | ims.append(imgray) 134 | 135 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 136 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 137 | ''' 138 | 139 | from scipy.io import loadmat 140 | 141 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/realdata/resolution0.mat') 142 | rect_data_hxwxt = data['measlr'] 143 | 144 | crop = 512 145 | bin_len = 32e-12 * 3e8 # 0.01 146 | 147 | K = 1 148 | for k in range(K): 149 | ''' 150 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 151 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 152 | ''' 153 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 154 | crop = crop // 2 155 | bin_len = bin_len * 2 156 | 157 | phasor(rect_data_hxwxt, wall_size=2.0, crop=crop, bin_len=bin_len) 158 | 159 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | utils_pytorch 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /${PROJECT_DIR_NAME} 5 | 6 | python 2.7 7 | Default 8 | 9 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/tflct.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | import numpy as np 8 | import sys 9 | sys.path.append('../utils') 10 | from helper import definePsf, resamplingOperator, \ 11 | filterLaplacian 12 | 13 | 14 | class lct(nn.Module): 15 | 16 | def __init__(self, spatial=256, crop=512, \ 17 | bin_len=0.01, wall_size=2.0, \ 18 | method='lct', material='diffuse'): 19 | super(lct, self).__init__() 20 | 21 | self.spatial_grid = spatial 22 | self.crop = crop 23 | assert 2 ** int(np.log2(crop)) == crop 24 | 25 | self.bin_len = bin_len 26 | self.wall_size = wall_size 27 | 28 | ############################################################# 29 | self.method = method 30 | self.material = material 31 | 32 | self.parpareparam() 33 | 34 | ##################################################### 35 | def parpareparam(self): 36 | 37 | self.c = 3e8 38 | self.width = self.wall_size / 2.0; 39 | self.bin_resolution = self.bin_len / self.c 40 | self.trange = self.crop * self.c * self.bin_resolution 41 | 42 | # maybe learnable? 43 | self.snr = 1e-1 44 | 45 | ########################################################3 46 | temprol_grid = self.crop 47 | sptial_grid = self.spatial_grid 48 | 49 | # 0-1 50 | gridz_M = np.arange(temprol_grid, dtype=np.float32) 51 | gridz_M = gridz_M / (temprol_grid - 1) 52 | gridz_1xMx1x1 = gridz_M.reshape(1, -1, 1, 1) 53 | self.gridz_1xMx1x1 = torch.from_numpy(gridz_1xMx1x1.astype(np.float32)) 54 | 55 | ################################################### 56 | slope = self.width / self.trange 57 | psf = definePsf(sptial_grid, temprol_grid, slope) 58 | fpsf = np.fft.fftn(psf) 59 | 60 | if self.method == 'lct': 61 | invpsf = np.conjugate(fpsf) / (1 / self.snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 62 | elif self.method == 'bp': 63 | invpsf = np.conjugate(fpsf) 64 | 65 | self.invpsf_real = torch.from_numpy(np.real(invpsf).astype(np.float32)).unsqueeze(0) 66 | self.invpsf_imag = torch.from_numpy(np.imag(invpsf).astype(np.float32)).unsqueeze(0) 67 | 68 | ###################################################### 69 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 70 | self.mtx_MxM = torch.from_numpy(mtx_MxM.astype(np.float32)) 71 | self.mtxi_MxM = torch.from_numpy(mtxi_MxM.astype(np.float32)) 72 | 73 | ############################################################# 74 | if self.method == 'bp': 75 | lapw_kxkxk = filterLaplacian() 76 | k = lapw_kxkxk.shape[0] 77 | self.pad = nn.ReplicationPad3d(2) 78 | self.lapw = torch.from_numpy(lapw_kxkxk).reshape(1, 1, k, k, k) 79 | 80 | def todev(self, dev, dnum): 81 | self.gridz_1xMx1x1_todev = self.gridz_1xMx1x1.to(dev) 82 | self.datapad_Dx2Tx2Hx2W = torch.zeros((dnum, 2 * self.crop, 2 * self.spatial_grid, 2 * self.spatial_grid), dtype=torch.float32, device=dev) 83 | 84 | self.mtx_MxM_todev = self.mtx_MxM.to(dev) 85 | self.mtxi_MxM_todev = self.mtxi_MxM.to(dev) 86 | 87 | self.invpsf_real_todev = self.invpsf_real.to(dev) 88 | self.invpsf_imag_todev = self.invpsf_imag.to(dev) 89 | 90 | if self.method == 'bp': 91 | self.lapw_todev = self.lapw.to(dev) 92 | 93 | def forward(self, feture_bxdxtxhxw, tbes, tens): 94 | 95 | # 1 padd data with zero 96 | bnum, dnum, tnum, hnum, wnum = feture_bxdxtxhxw.shape 97 | for tbe, ten in zip(tbes, tens): 98 | assert tbe >= 0 99 | assert ten <= self.crop 100 | dev = feture_bxdxtxhxw.device 101 | 102 | featpad_bxdxtxhxw = [] 103 | for i in range(bnum): 104 | featpad_1xdxt1xhxw = torch.zeros((1, dnum, tbes[i], hnum, wnum), dtype=torch.float32, device=dev) 105 | featpad_1xdxt2xhxw = torch.zeros((1, dnum, self.crop - tens[i], hnum, wnum), dtype=torch.float32, device=dev) 106 | featpad_1xdxtxhxw = torch.cat([featpad_1xdxt1xhxw, feture_bxdxtxhxw[i:i + 1], featpad_1xdxt2xhxw], dim=2) 107 | featpad_bxdxtxhxw.append(featpad_1xdxtxhxw) 108 | featpad_bxdxtxhxw = torch.cat(featpad_bxdxtxhxw, dim=0) 109 | 110 | # 2 params 111 | assert hnum == wnum 112 | assert hnum == self.spatial_grid 113 | sptial_grid = hnum 114 | temprol_grid = self.crop 115 | 116 | #################################################### 117 | # 3 run lct 118 | # assert bnum == 1 119 | data_BDxTxHxW = featpad_bxdxtxhxw.view(bnum * dnum, self.crop, hnum, wnum) 120 | 121 | gridz_1xMx1x1 = self.gridz_1xMx1x1_todev 122 | if self.material == 'diffuse': 123 | data_BDxTxHxW = data_BDxTxHxW * (gridz_1xMx1x1 ** 4) 124 | elif self.material == 'specular': 125 | data_BDxTxHxW = data_BDxTxHxW * (gridz_1xMx1x1 ** 2) 126 | 127 | ############################################################### 128 | # datapad_BDx2Tx2Hx2W = torch.zeros((bnum * dnum, 2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=torch.float32, device=dev) 129 | datapad_Dx2Tx2Hx2W = self.datapad_Dx2Tx2Hx2W 130 | # create new variable 131 | datapad_BDx2Tx2Hx2W = datapad_Dx2Tx2Hx2W.repeat(bnum, 1, 1, 1) 132 | 133 | left = self.mtx_MxM_todev 134 | right = data_BDxTxHxW.view(bnum * dnum, temprol_grid, -1) 135 | tmp = torch.matmul(left, right) 136 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 137 | 138 | datapad_BDx2Tx2Hx2W[:, :temprol_grid, :sptial_grid, :sptial_grid] = tmp2 139 | 140 | #################################################################################### 141 | # datapad_BDx2Tx2Hx2Wx2 = torch.stack([datapad_BDx2Tx2Hx2W, torch.zeros_like(datapad_BDx2Tx2Hx2W)], dim=4) 142 | datafre = torch.rfft(datapad_BDx2Tx2Hx2W, 3, onesided=False) 143 | datafre_real = datafre[:, :, :, :, 0] 144 | datafre_imag = datafre[:, :, :, :, 1] 145 | 146 | re_real = datafre_real * self.invpsf_real_todev - datafre_imag * self.invpsf_imag_todev 147 | re_imag = datafre_real * self.invpsf_imag_todev + datafre_imag * self.invpsf_real_todev 148 | refre = torch.stack([re_real, re_imag], dim=4) 149 | re = torch.ifft(refre, 3) 150 | 151 | volumn_BDxTxHxW = re[:, :temprol_grid, :sptial_grid, :sptial_grid, 0] 152 | 153 | ######################################################################### 154 | left = self.mtxi_MxM_todev 155 | right = volumn_BDxTxHxW.reshape(bnum * dnum, temprol_grid, -1) 156 | tmp = torch.matmul(left, right) 157 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 158 | 159 | # volumn_BDxTxHxW = F.relu(tmp2, inplace=False) 160 | volumn_BDxTxHxW = tmp2 161 | 162 | if self.method == 'bp': 163 | volumn_BDx1xTxHxW = volumn_BDxTxHxW.unsqueeze(1) 164 | lapw = self.lapw_todev 165 | volumn_BDx1xTxHxW = self.pad(volumn_BDx1xTxHxW) 166 | volumn_BDx1xTxHxW = F.conv3d(volumn_BDx1xTxHxW, lapw) 167 | volumn_BDxTxHxW = volumn_BDx1xTxHxW.squeeze(1) 168 | # seems border is bad 169 | # if self.crop == 512: 170 | if True: 171 | volumn_BDxTxHxW[:, :1] = 0 172 | # volumn_BDxTxHxW[:, -10:] = 0 173 | # volumn_BDxTxHxW = F.relu(volumn_BDxTxHxW, inplace=False) 174 | 175 | volumn_BxDxTxHxW = volumn_BDxTxHxW.view(bnum, dnum, self.crop, hnum, wnum) 176 | 177 | return volumn_BxDxTxHxW 178 | 179 | 180 | if __name__ == '__main__': 181 | 182 | import os 183 | import cv2 184 | import numpy as np 185 | 186 | ''' 187 | fd = '/u6/a/wenzheng/remote2/code-nlos-git/OccludedSceneRep-2/code/pytorch-wz/dataloader_light22_bbox'; 188 | ims = [] 189 | tbe = -1 190 | for i in range(512): 191 | name = '%s/2-%d.png' % (fd, i) 192 | if not os.path.isfile(name): 193 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 194 | continue 195 | 196 | if tbe < 0: 197 | tbe = i 198 | 199 | im = cv2.imread(name) 200 | imgt = im[:256, :256, :] 201 | im = im[:256, -256:, :] 202 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 203 | ims.append(imgray) 204 | 205 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 206 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 207 | ''' 208 | 209 | from scipy.io import loadmat 210 | 211 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/nlos-fk-master/statue.mat') 212 | rect_data_hxwxt = data['data'] 213 | 214 | sptial_grid = 512 215 | crop = 512 216 | bin_len = 32e-12 * 3e8 # 0.01 217 | 218 | K = 2 219 | temds = False 220 | for k in range(K): 221 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 222 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 223 | sptial_grid = sptial_grid // 2 224 | 225 | if temds: 226 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 227 | crop = crop // 2 228 | bin_len = bin_len * 2 229 | 230 | rect_data_dxhxwxt = np.expand_dims(rect_data_hxwxt, axis=0) 231 | rect_data_bxdxhxwxt = np.expand_dims(rect_data_dxhxwxt, axis=0) 232 | 233 | bnum = 1 234 | dnum = 1 235 | rect_data_bxdxhxwxt = np.tile(rect_data_bxdxhxwxt, [bnum, dnum, 1, 1, 1]) 236 | rect_data_bxdxhxwxt = torch.from_numpy(rect_data_bxdxhxwxt).cuda() 237 | 238 | dev = 'cuda' 239 | 240 | ##################################################################### 241 | lctlayer = lct(spatial=sptial_grid, crop=crop, bin_len=bin_len, 242 | method='bp') 243 | lctlayer.todev(dev, dnum) 244 | 245 | tbe = 0 // (2 ** K) 246 | if temds: 247 | tlen = 512 // (2 ** K) 248 | else: 249 | tlen = 512 250 | 251 | for i in range(10): 252 | print(i) 253 | re = lctlayer(rect_data_bxdxhxwxt[:, :, :, :, tbe:tbe + tlen].permute(0, 1, 4, 2, 3), \ 254 | [tbe, tbe, tbe], [tbe + tlen, tbe + tlen, tbe + tlen]) 255 | 256 | volumn_MxNxN = re.detach().cpu().numpy()[0, -1] 257 | 258 | # get rid of bad points 259 | zdim = volumn_MxNxN.shape[0] * 100 // 128 260 | volumn_MxNxN = volumn_MxNxN[:zdim] 261 | print('volumn min, %f' % volumn_MxNxN.min()) 262 | print('volumn max, %f' % volumn_MxNxN.max()) 263 | # volumn_MxNxN[:5] = 0 264 | # volumn_MxNxN[-5:] = 0 265 | 266 | volumn_MxNxN[volumn_MxNxN < 0] = 0 267 | front_view = np.max(volumn_MxNxN, axis=0) 268 | cv2.imshow("re", front_view / np.max(front_view)) 269 | # cv2.imshow("gt", imgt) 270 | cv2.waitKey() 271 | 272 | volumn_ZxYxX = volumn_MxNxN 273 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 274 | for i, frame in enumerate(volumn_ZxYxX): 275 | print(i) 276 | cv2.imshow("re1", frame) 277 | cv2.imshow("re2", frame / np.max(frame)) 278 | cv2.waitKey(0) 279 | 280 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/tflctfast.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | import numpy as np 8 | import sys 9 | sys.path.append('../utils') 10 | from helper import definePsf, resamplingOperator, \ 11 | filterLaplacian 12 | 13 | 14 | class lct(nn.Module): 15 | 16 | def __init__(self, spatial=256, crop=512, \ 17 | bin_len=0.01, wall_size=2.0, \ 18 | method='lct', material='diffuse'): 19 | super(lct, self).__init__() 20 | 21 | self.spatial_grid = spatial 22 | self.crop = crop 23 | assert 2 ** int(np.log2(crop)) == crop 24 | 25 | self.bin_len = bin_len 26 | self.wall_size = wall_size 27 | 28 | ############################################################# 29 | self.method = method 30 | self.material = material 31 | 32 | self.parpareparam() 33 | 34 | ##################################################### 35 | def parpareparam(self): 36 | 37 | self.c = 3e8 38 | self.width = self.wall_size / 2.0; 39 | self.bin_resolution = self.bin_len / self.c 40 | self.trange = self.crop * self.c * self.bin_resolution 41 | 42 | # maybe learnable? 43 | self.snr = 1e-1 44 | 45 | ########################################################3 46 | temprol_grid = self.crop 47 | sptial_grid = self.spatial_grid 48 | 49 | # 0-1 50 | gridz_M = np.arange(temprol_grid, dtype=np.float32) 51 | gridz_M = gridz_M / (temprol_grid - 1) 52 | gridz_1xMx1x1 = gridz_M.reshape(1, -1, 1, 1) 53 | self.gridz_1xMx1x1 = torch.from_numpy(gridz_1xMx1x1.astype(np.float32)) 54 | 55 | ################################################### 56 | slope = self.width / self.trange 57 | psf = definePsf(sptial_grid, temprol_grid, slope) 58 | fpsf = np.fft.fftn(psf) 59 | 60 | if self.method == 'lct': 61 | invpsf = np.conjugate(fpsf) / (1 / self.snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 62 | elif self.method == 'bp': 63 | invpsf = np.conjugate(fpsf) 64 | 65 | self.invpsf_real = torch.from_numpy(np.real(invpsf).astype(np.float32)).unsqueeze(0) 66 | self.invpsf_imag = torch.from_numpy(np.imag(invpsf).astype(np.float32)).unsqueeze(0) 67 | 68 | ###################################################### 69 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 70 | self.mtx_MxM = torch.from_numpy(mtx_MxM.astype(np.float32)) 71 | self.mtxi_MxM = torch.from_numpy(mtxi_MxM.astype(np.float32)) 72 | 73 | ############################################################# 74 | if self.method == 'bp': 75 | lapw_kxkxk = filterLaplacian() 76 | k = lapw_kxkxk.shape[0] 77 | self.pad = nn.ReplicationPad3d(2) 78 | self.lapw = torch.from_numpy(lapw_kxkxk).reshape(1, 1, k, k, k) 79 | 80 | def todev(self, dev, dnum): 81 | self.gridz_1xMx1x1_todev = self.gridz_1xMx1x1.to(dev) 82 | self.datapad_Dx2Tx2Hx2W = torch.zeros((dnum, 2 * self.crop, 2 * self.spatial_grid, 2 * self.spatial_grid), dtype=torch.float32, device=dev) 83 | self.datazero_Dx2Tx2Hx2W = torch.zeros((dnum, 2 * self.crop, 2 * self.spatial_grid, 2 * self.spatial_grid), dtype=torch.float32, device=dev) 84 | 85 | self.mtx_MxM_todev = self.mtx_MxM.to(dev) 86 | self.mtxi_MxM_todev = self.mtxi_MxM.to(dev) 87 | 88 | self.invpsf_real_todev = self.invpsf_real.to(dev) 89 | self.invpsf_imag_todev = self.invpsf_imag.to(dev) 90 | 91 | if self.method == 'bp': 92 | self.lapw_todev = self.lapw.to(dev) 93 | 94 | def forward(self, feture_bxdxtxhxw): 95 | 96 | # 1 padd data with zero 97 | bnum, dnum, tnum, hnum, wnum = feture_bxdxtxhxw.shape 98 | dev = feture_bxdxtxhxw.device 99 | 100 | ''' 101 | for tbe, ten in zip(tbes, tens): 102 | assert tbe >= 0 103 | assert ten <= self.crop 104 | featpad_bxdxtxhxw = [] 105 | for i in range(bnum): 106 | featpad_1xdxt1xhxw = torch.zeros((1, dnum, tbes[i], hnum, wnum), dtype=torch.float32, device=dev) 107 | featpad_1xdxt2xhxw = torch.zeros((1, dnum, self.crop - tens[i], hnum, wnum), dtype=torch.float32, device=dev) 108 | featpad_1xdxtxhxw = torch.cat([featpad_1xdxt1xhxw, feture_bxdxtxhxw[i:i + 1], featpad_1xdxt2xhxw], dim=2) 109 | featpad_bxdxtxhxw.append(featpad_1xdxtxhxw) 110 | featpad_bxdxtxhxw = torch.cat(featpad_bxdxtxhxw, dim=0) 111 | ''' 112 | # full dim 113 | assert tnum == self.crop 114 | featpad_bxdxtxhxw = feture_bxdxtxhxw 115 | 116 | # 2 params 117 | assert hnum == wnum 118 | assert hnum == self.spatial_grid 119 | sptial_grid = hnum 120 | temprol_grid = self.crop 121 | 122 | #################################################### 123 | # 3 run lct 124 | # assert bnum == 1 125 | data_BDxTxHxW = featpad_bxdxtxhxw.view(bnum * dnum, self.crop, hnum, wnum) 126 | 127 | gridz_1xMx1x1 = self.gridz_1xMx1x1_todev 128 | if self.material == 'diffuse': 129 | data_BDxTxHxW = data_BDxTxHxW * (gridz_1xMx1x1 ** 4) 130 | elif self.material == 'specular': 131 | data_BDxTxHxW = data_BDxTxHxW * (gridz_1xMx1x1 ** 2) 132 | 133 | ############################################################### 134 | # datapad_BDx2Tx2Hx2W = torch.zeros((bnum * dnum, 2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=torch.float32, device=dev) 135 | datapad_Dx2Tx2Hx2W = self.datapad_Dx2Tx2Hx2W 136 | 137 | ''' 138 | # create new variable 139 | datapad_BDx2Tx2Hx2W = datapad_Dx2Tx2Hx2W.repeat(bnum, 1, 1, 1) 140 | ''' 141 | assert bnum == 1 142 | datapad_BDx2Tx2Hx2W = datapad_Dx2Tx2Hx2W 143 | 144 | left = self.mtx_MxM_todev 145 | right = data_BDxTxHxW.view(bnum * dnum, temprol_grid, -1) 146 | tmp = torch.matmul(left, right) 147 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 148 | 149 | datapad_BDx2Tx2Hx2W[:, :temprol_grid, :sptial_grid, :sptial_grid] = tmp2 # .detach() 150 | 151 | #################################################################################### 152 | # datapad_BDx2Tx2Hx2Wx2 = torch.stack([datapad_BDx2Tx2Hx2W, torch.zeros_like(datapad_BDx2Tx2Hx2W)], dim=4) 153 | # datafre = torch.rfft(datapad_BDx2Tx2Hx2W, 3, onesided=False) 154 | datazero_Dx2Tx2Hx2W = self.datazero_Dx2Tx2Hx2W 155 | datazero_BDx2Tx2Hx2W = datazero_Dx2Tx2Hx2W 156 | datapad_BDx2Tx2Hx2Wx2 = torch.stack([datapad_BDx2Tx2Hx2W, datazero_BDx2Tx2Hx2W], dim=4) 157 | datafre = torch.fft(datapad_BDx2Tx2Hx2Wx2, 3) 158 | 159 | datafre_real = datafre[:, :, :, :, 0] 160 | datafre_imag = datafre[:, :, :, :, 1] 161 | 162 | re_real = datafre_real * self.invpsf_real_todev - datafre_imag * self.invpsf_imag_todev 163 | re_imag = datafre_real * self.invpsf_imag_todev + datafre_imag * self.invpsf_real_todev 164 | refre = torch.stack([re_real, re_imag], dim=4) 165 | re = torch.ifft(refre, 3) 166 | 167 | volumn_BDxTxHxW = re[:, :temprol_grid, :sptial_grid, :sptial_grid, 0] 168 | 169 | ######################################################################### 170 | left = self.mtxi_MxM_todev 171 | right = volumn_BDxTxHxW.reshape(bnum * dnum, temprol_grid, -1) 172 | tmp = torch.matmul(left, right) 173 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 174 | 175 | # volumn_BDxTxHxW = F.relu(tmp2, inplace=False) 176 | volumn_BDxTxHxW = tmp2 177 | 178 | volumn_BxDxTxHxW = volumn_BDxTxHxW.view(bnum, dnum, self.crop, hnum, wnum) 179 | 180 | return volumn_BxDxTxHxW 181 | 182 | 183 | if __name__ == '__main__': 184 | 185 | import os 186 | import cv2 187 | import numpy as np 188 | 189 | ''' 190 | fd = '/u6/a/wenzheng/remote2/code-nlos-git/OccludedSceneRep-2/code/pytorch-wz/dataloader_light22_bbox'; 191 | ims = [] 192 | tbe = -1 193 | for i in range(512): 194 | name = '%s/2-%d.png' % (fd, i) 195 | if not os.path.isfile(name): 196 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 197 | continue 198 | 199 | if tbe < 0: 200 | tbe = i 201 | 202 | im = cv2.imread(name) 203 | imgt = im[:256, :256, :] 204 | im = im[:256, -256:, :] 205 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 206 | ims.append(imgray) 207 | 208 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 209 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 210 | ''' 211 | 212 | from scipy.io import loadmat 213 | 214 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/nlos-fk-master/statue.mat') 215 | rect_data_hxwxt = data['data'] 216 | # rect_data_hxwxt = np.zeros((512, 512, 512), dtype=np.float32) 217 | 218 | sptial_grid = 512 219 | crop = 512 220 | bin_len = 32e-12 * 3e8 # 0.01 221 | 222 | K = 2 223 | temds = False 224 | for k in range(K): 225 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 226 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 227 | sptial_grid = sptial_grid // 2 228 | 229 | if temds: 230 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 231 | crop = crop // 2 232 | bin_len = bin_len * 2 233 | 234 | rect_data_dxhxwxt = np.expand_dims(rect_data_hxwxt, axis=0) 235 | rect_data_bxdxhxwxt = np.expand_dims(rect_data_dxhxwxt, axis=0) 236 | 237 | bnum = 1 238 | dnum = 1 239 | rect_data_bxdxhxwxt = np.tile(rect_data_bxdxhxwxt, [bnum, dnum, 1, 1, 1]) 240 | rect_data_bxdxhxwxt = torch.from_numpy(rect_data_bxdxhxwxt).cuda() 241 | 242 | dev = 'cuda' 243 | 244 | ##################################################################### 245 | lctlayer = lct(spatial=sptial_grid, crop=crop, bin_len=bin_len, 246 | method='lct') 247 | lctlayer = lctlayer.to(dev).eval() 248 | lctlayer.todev(dev, dnum) 249 | 250 | tbe = 0 // (2 ** K) 251 | if temds: 252 | tlen = 512 // (2 ** K) 253 | else: 254 | tlen = 512 255 | 256 | import time 257 | for i in range(10): 258 | t1 = time.time() 259 | re = lctlayer(rect_data_bxdxhxwxt[:, :, :, :, tbe:tbe + tlen].permute(0, 1, 4, 2, 3)) # , \ 260 | # [tbe, tbe, tbe], [tbe + tlen, tbe + tlen, tbe + tlen]) 261 | print("cost time: ", time.time() - t1) 262 | 263 | def export_onnx_model(model, input_shape, onnx_path, input_names=None, output_names=None, dynamic_axes=None): 264 | inputs = torch.ones(*input_shape).to(dev) 265 | model(inputs) 266 | torch.onnx.export(model, inputs, onnx_path, input_names=input_names, output_names=output_names) 267 | 268 | input = rect_data_bxdxhxwxt.permute(0, 1, 4, 2, 3) 269 | inputnp = input.detach().cpu().numpy() 270 | np.save(file='ni.npy', arr=inputnp) 271 | shape = input.shape 272 | print(shape) 273 | export_onnx_model(lctlayer, shape, 'test.onnx') 274 | 275 | 276 | 277 | volumn_MxNxN = re.detach().cpu().numpy()[0, -1] 278 | 279 | # get rid of bad points 280 | zdim = volumn_MxNxN.shape[0] * 100 // 128 281 | volumn_MxNxN = volumn_MxNxN[:zdim] 282 | print('volumn min, %f' % volumn_MxNxN.min()) 283 | print('volumn max, %f' % volumn_MxNxN.max()) 284 | # volumn_MxNxN[:5] = 0 285 | # volumn_MxNxN[-5:] = 0 286 | 287 | volumn_MxNxN[volumn_MxNxN < 0] = 0 288 | front_view = np.max(volumn_MxNxN, axis=0) 289 | cv2.imshow("re", front_view / np.max(front_view)) 290 | # cv2.imshow("gt", imgt) 291 | cv2.waitKey() 292 | 293 | volumn_ZxYxX = volumn_MxNxN 294 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 295 | for i, frame in enumerate(volumn_ZxYxX): 296 | print(i) 297 | cv2.imshow("re1", frame) 298 | cv2.imshow("re2", frame / np.max(frame)) 299 | cv2.waitKey(0) 300 | 301 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/tfphasor.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | import numpy as np 8 | import sys 9 | sys.path.append('../utils') 10 | from helper import definePsf, resamplingOperator, \ 11 | waveconvparam, waveconv 12 | 13 | 14 | class phasor(nn.Module): 15 | 16 | def __init__(self, spatial=256, crop=512, \ 17 | bin_len=0.01, wall_size=2.0, \ 18 | sampling_coeff=2.0, \ 19 | cycles=5): 20 | super(phasor, self).__init__() 21 | 22 | self.spatial_grid = spatial 23 | self.crop = crop 24 | assert 2 ** int(np.log2(crop)) == crop 25 | 26 | self.bin_len = bin_len 27 | self.wall_size = wall_size 28 | 29 | self.sampling_coeff = sampling_coeff 30 | self.cycles = cycles 31 | 32 | self.parpareparam() 33 | 34 | ##################################################### 35 | def parpareparam(self): 36 | 37 | self.c = 3e8 38 | self.width = self.wall_size / 2.0; 39 | self.bin_resolution = self.bin_len / self.c 40 | self.trange = self.crop * self.c * self.bin_resolution 41 | 42 | ########################################################3 43 | temprol_grid = self.crop 44 | sptial_grid = self.spatial_grid 45 | 46 | wall_size = self.wall_size 47 | bin_resolution = self.bin_resolution 48 | 49 | sampling_coeff = self.sampling_coeff 50 | cycles = self.cycles 51 | 52 | ###################################################### 53 | # Step 0: define virtual wavelet properties 54 | # s_lamda_limit = wall_size / (sptial_grid - 1); # sample spacing on the wall 55 | # sampling_coeff = 2; # scale the size of the virtual wavelength (usually 2, optionally 3 for noisy scenes) 56 | # virtual_wavelength = sampling_coeff * (s_lamda_limit * 2); # virtual wavelength in units of cm 57 | # cycles = 5; # number of wave cycles in the wavelet, typically 4-6 58 | 59 | s_lamda_limit = wall_size / (sptial_grid - 1); # sample spacing on the wall 60 | virtual_wavelength = sampling_coeff * (s_lamda_limit * 2); # virtual wavelength in units of cm 61 | self.virtual_wavelength = virtual_wavelength 62 | 63 | virtual_cos_wave_k, virtual_sin_wave_k = \ 64 | waveconvparam(bin_resolution, virtual_wavelength, cycles) 65 | 66 | virtual_cos_sin_wave_2xk = np.stack([virtual_cos_wave_k, virtual_sin_wave_k], axis=0) 67 | 68 | # use pytorch conv to replace matlab conv 69 | self.virtual_cos_sin_wave_inv_2x1xk = torch.from_numpy(virtual_cos_sin_wave_2xk[:, ::-1].copy()).unsqueeze(1) 70 | 71 | ################################################### 72 | slope = self.width / self.trange 73 | psf = definePsf(sptial_grid, temprol_grid, slope) 74 | fpsf = np.fft.fftn(psf) 75 | # lct 76 | # invpsf = np.conjugate(fpsf) / (1 / self.snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 77 | # bp 78 | invpsf = np.conjugate(fpsf) 79 | 80 | self.invpsf_real = torch.from_numpy(np.real(invpsf).astype(np.float32)).unsqueeze(0) 81 | self.invpsf_imag = torch.from_numpy(np.imag(invpsf).astype(np.float32)).unsqueeze(0) 82 | 83 | ###################################################### 84 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 85 | self.mtx_MxM = torch.from_numpy(mtx_MxM.astype(np.float32)) 86 | self.mtxi_MxM = torch.from_numpy(mtxi_MxM.astype(np.float32)) 87 | 88 | def todev(self, dev, dnum): 89 | 90 | self.virtual_cos_sin_wave_inv_2x1xk_todev = self.virtual_cos_sin_wave_inv_2x1xk.to(dev) 91 | self.datapad_2Dx2Tx2Hx2W = torch.zeros((2 * dnum, 2 * self.crop, 2 * self.spatial_grid, 2 * self.spatial_grid), dtype=torch.float32, device=dev) 92 | 93 | self.mtx_MxM_todev = self.mtx_MxM.to(dev) 94 | self.mtxi_MxM_todev = self.mtxi_MxM.to(dev) 95 | 96 | self.invpsf_real_todev = self.invpsf_real.to(dev) 97 | self.invpsf_imag_todev = self.invpsf_imag.to(dev) 98 | 99 | def forward(self, feture_bxdxtxhxw, tbes, tens): 100 | 101 | # 1 padd data with zero 102 | bnum, dnum, tnum, hnum, wnum = feture_bxdxtxhxw.shape 103 | for tbe, ten in zip(tbes, tens): 104 | assert tbe >= 0 105 | assert ten <= self.crop 106 | dev = feture_bxdxtxhxw.device 107 | 108 | featpad_bxdxtxhxw = [] 109 | for i in range(bnum): 110 | featpad_1xdxt1xhxw = torch.zeros((1, dnum, tbes[i], hnum, wnum), dtype=torch.float32, device=dev) 111 | featpad_1xdxt2xhxw = torch.zeros((1, dnum, self.crop - tens[i], hnum, wnum), dtype=torch.float32, device=dev) 112 | featpad_1xdxtxhxw = torch.cat([featpad_1xdxt1xhxw, feture_bxdxtxhxw[i:i + 1], featpad_1xdxt2xhxw], dim=2) 113 | featpad_bxdxtxhxw.append(featpad_1xdxtxhxw) 114 | featpad_bxdxtxhxw = torch.cat(featpad_bxdxtxhxw, dim=0) 115 | 116 | # 2 params 117 | assert hnum == wnum 118 | assert hnum == self.spatial_grid 119 | sptial_grid = hnum 120 | temprol_grid = self.crop 121 | tnum = self.crop 122 | 123 | #################################################### 124 | # 3 run lct 125 | # assert bnum == 1 126 | data_BDxTxHxW = featpad_bxdxtxhxw.view(bnum * dnum, tnum, hnum, wnum) 127 | 128 | ############################################################ 129 | # Step 1: convolve measurement volume with virtual wave 130 | 131 | data_BDxHxWxT = data_BDxTxHxW.permute(0, 2, 3, 1) 132 | data_BDHWx1xT = data_BDxHxWxT.reshape(-1, 1, tnum) 133 | knum = self.virtual_cos_sin_wave_inv_2x1xk.shape[2] 134 | phasor_data_cos_sin_BDHWx2x1T = F.conv1d(data_BDHWx1xT, self.virtual_cos_sin_wave_inv_2x1xk_todev, padding=knum // 2) 135 | if knum % 2 == 0: 136 | data_BDHWx2xT = phasor_data_cos_sin_BDHWx2x1T[:, :, 1:] 137 | else: 138 | data_BDHWx2xT = phasor_data_cos_sin_BDHWx2x1T 139 | 140 | data_BDxHxWx2xT = data_BDHWx2xT.reshape(bnum * dnum, hnum, wnum, 2, tnum) 141 | data_2xBDxTxHxW = data_BDxHxWx2xT.permute(3, 0, 4, 1, 2) 142 | data_2BDxTxHxW = data_2xBDxTxHxW.reshape(2 * bnum * dnum, tnum, hnum, wnum) 143 | 144 | ############################################################# 145 | # Step 2: transform virtual wavefield into LCT domain 146 | 147 | # datapad_2BDx2Tx2Hx2W = torch.zeros((2 * bnum * dnum, 2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=torch.float32, device=dev) 148 | datapad_2Dx2Tx2Hx2W = self.datapad_2Dx2Tx2Hx2W 149 | # create new variable 150 | datapad_B2Dx2Tx2Hx2W = datapad_2Dx2Tx2Hx2W.repeat(bnum, 1, 1, 1) 151 | # actually, because it is all zero so it is ok 152 | datapad_2BDx2Tx2Hx2W = datapad_B2Dx2Tx2Hx2W 153 | 154 | left = self.mtx_MxM_todev 155 | right = data_2BDxTxHxW.view(2 * bnum * dnum, temprol_grid, -1) 156 | tmp = torch.matmul(left, right) 157 | tmp2 = tmp.view(2 * bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 158 | 159 | datapad_2BDx2Tx2Hx2W[:, :temprol_grid, :sptial_grid, :sptial_grid] = tmp2 160 | 161 | ###########################################################3 162 | # Step 3: convolve with backprojection kernel 163 | 164 | # datapad_BDx2Tx2Hx2Wx2 = torch.stack([datapad_BDx2Tx2Hx2W, torch.zeros_like(datapad_BDx2Tx2Hx2W)], dim=4) 165 | datafre = torch.rfft(datapad_2BDx2Tx2Hx2W, 3, onesided=False) 166 | datafre_real = datafre[:, :, :, :, 0] 167 | datafre_imag = datafre[:, :, :, :, 1] 168 | 169 | re_real = datafre_real * self.invpsf_real_todev - datafre_imag * self.invpsf_imag_todev 170 | re_imag = datafre_real * self.invpsf_imag_todev + datafre_imag * self.invpsf_real_todev 171 | refre = torch.stack([re_real, re_imag], dim=4) 172 | re = torch.ifft(refre, 3) 173 | volumn_2BDxTxHxWx2 = re[:, :temprol_grid, :sptial_grid, :sptial_grid, :] 174 | 175 | ######################################################################## 176 | # Step 4: compute phasor field magnitude and inverse LCT 177 | 178 | cos_real = volumn_2BDxTxHxWx2[:bnum * dnum, :, :, :, 0] 179 | cos_imag = volumn_2BDxTxHxWx2[:bnum * dnum, :, :, :, 1] 180 | 181 | sin_real = volumn_2BDxTxHxWx2[bnum * dnum:, :, :, :, 0] 182 | sin_imag = volumn_2BDxTxHxWx2[bnum * dnum:, :, :, :, 1] 183 | 184 | sum_real = cos_real ** 2 - cos_imag ** 2 + sin_real ** 2 - sin_imag ** 2 185 | sum_image = 2 * cos_real * cos_imag + 2 * sin_real * sin_imag 186 | 187 | tmp = (torch.sqrt(sum_real ** 2 + sum_image ** 2) + sum_real) / 2 188 | # numerical issue 189 | tmp = F.relu(tmp, inplace=False) 190 | sqrt_sum_real = torch.sqrt(tmp) 191 | 192 | ##################################################################### 193 | left = self.mtxi_MxM_todev 194 | right = sqrt_sum_real.view(bnum * dnum, temprol_grid, -1) 195 | tmp = torch.matmul(left, right) 196 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 197 | 198 | ######################################################################## 199 | # do we force to be > 0? 200 | # volumn_BDxTxHxW = F.relu(tmp2, inplace=False) 201 | volumn_BDxTxHxW = tmp2 202 | 203 | volumn_BxDxTxHxW = volumn_BDxTxHxW.view(bnum, dnum, self.crop, hnum, wnum) 204 | 205 | return volumn_BxDxTxHxW 206 | 207 | 208 | if __name__ == '__main__': 209 | 210 | import os 211 | import cv2 212 | import numpy as np 213 | 214 | syn = False 215 | if syn: 216 | fd = '/home/wenzheng/largestore/nlos-phasor/data/car'; 217 | ims = [] 218 | tbe = -1 219 | tlen = 0 220 | for i in range(512): 221 | name = '%s/2-%d.png' % (fd, i) 222 | if not os.path.isfile(name): 223 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 224 | continue 225 | 226 | if tbe < 0: 227 | tbe = i 228 | tlen += 1 229 | 230 | im = cv2.imread(name) 231 | imgt = im[:256, :256, :] 232 | im = im[:256, -256:, :] 233 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 234 | ims.append(imgray) 235 | 236 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 237 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 238 | 239 | sptial_grid = 256 240 | crop = 512 241 | bin_len = 0.01 242 | else: 243 | 244 | from scipy.io import loadmat 245 | 246 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/nlos-fk-master/statue.mat') 247 | rect_data_hxwxt = data['data'] 248 | 249 | sptial_grid = 512 250 | crop = 512 251 | bin_len = 32e-12 * 3e8 # 0.01 252 | tbe = 0 253 | tlen = crop 254 | 255 | K = 2 256 | temds = True 257 | for k in range(K): 258 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 259 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 260 | sptial_grid = sptial_grid // 2 261 | 262 | if temds: 263 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 264 | crop = crop // 2 265 | bin_len = bin_len * 2 266 | 267 | rect_data_dxhxwxt = np.expand_dims(rect_data_hxwxt, axis=0) 268 | rect_data_bxdxhxwxt = np.expand_dims(rect_data_dxhxwxt, axis=0) 269 | 270 | bnum = 1 271 | dnum = 1 272 | rect_data_bxdxhxwxt = np.tile(rect_data_bxdxhxwxt, [bnum, dnum, 1, 1, 1]) 273 | rect_data_bxdxhxwxt = torch.from_numpy(rect_data_bxdxhxwxt).cuda() 274 | 275 | dev = 'cuda' 276 | 277 | ##################################################################### 278 | lctlayer = phasor(spatial=sptial_grid, crop=crop, bin_len=bin_len, sampling_coeff=2.0, cycles=5) 279 | lctlayer.todev(dev, dnum) 280 | 281 | if temds: 282 | tbe = tbe // (2 ** K) 283 | tlen = tlen // (2 ** K) 284 | 285 | for i in range(10): 286 | print(i) 287 | re = lctlayer(rect_data_bxdxhxwxt[:, :, :, :, tbe:tbe + tlen].permute(0, 1, 4, 2, 3), \ 288 | [tbe, tbe, tbe], [tbe + tlen, tbe + tlen, tbe + tlen]) 289 | 290 | volumn_MxNxN = re.detach().cpu().numpy()[0, -1] 291 | zdim = volumn_MxNxN.shape[0] * 100 // 128 292 | volumn_MxNxN = volumn_MxNxN[:zdim] 293 | print('volumn min, %f' % volumn_MxNxN.min()) 294 | print('volumn max, %f' % volumn_MxNxN.max()) 295 | 296 | volumn_MxNxN[volumn_MxNxN < 0] = 0 297 | front_view = np.max(volumn_MxNxN, axis=0) 298 | cv2.imshow("re", front_view / np.max(front_view)) 299 | # cv2.imshow("gt", imgt) 300 | cv2.waitKey() 301 | 302 | volumn_ZxYxX = volumn_MxNxN 303 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 304 | for i, frame in enumerate(volumn_ZxYxX): 305 | print(i) 306 | cv2.imshow("re1", frame) 307 | cv2.imshow("re2", frame / np.max(frame)) 308 | cv2.waitKey(0) 309 | 310 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/tfphasor2.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | import numpy as np 8 | import sys 9 | sys.path.append('../utils') 10 | from helper import definePsf, resamplingOperator, \ 11 | waveconvparam, waveconv 12 | 13 | 14 | class phasor2(nn.Module): 15 | 16 | def __init__(self, spatial=256, crop=512, \ 17 | bin_len=0.01, wall_size=2.0, \ 18 | sampling_coeff=2.0, \ 19 | cycles=5): 20 | super(phasor2, self).__init__() 21 | 22 | self.spatial_grid = spatial 23 | self.crop = crop 24 | assert 2 ** int(np.log2(crop)) == crop 25 | 26 | self.bin_len = bin_len 27 | self.wall_size = wall_size 28 | 29 | self.sampling_coeff = sampling_coeff 30 | self.cycles = cycles 31 | 32 | self.parpareparam() 33 | 34 | ##################################################### 35 | def parpareparam(self): 36 | 37 | self.c = 3e8 38 | self.width = self.wall_size / 2.0; 39 | self.bin_resolution = self.bin_len / self.c 40 | self.trange = self.crop * self.c * self.bin_resolution 41 | 42 | ########################################################3 43 | temprol_grid = self.crop 44 | sptial_grid = self.spatial_grid 45 | 46 | wall_size = self.wall_size 47 | bin_resolution = self.bin_resolution 48 | 49 | sampling_coeff = self.sampling_coeff 50 | cycles = self.cycles 51 | 52 | ###################################################### 53 | # Step 0: define virtual wavelet properties 54 | # s_lamda_limit = wall_size / (sptial_grid - 1); # sample spacing on the wall 55 | # sampling_coeff = 2; # scale the size of the virtual wavelength (usually 2, optionally 3 for noisy scenes) 56 | # virtual_wavelength = sampling_coeff * (s_lamda_limit * 2); # virtual wavelength in units of cm 57 | # cycles = 5; # number of wave cycles in the wavelet, typically 4-6 58 | 59 | s_lamda_limit = wall_size / (sptial_grid - 1); # sample spacing on the wall 60 | virtual_wavelength = sampling_coeff * (s_lamda_limit * 2); # virtual wavelength in units of cm 61 | self.virtual_wavelength = virtual_wavelength 62 | 63 | virtual_cos_wave_k, virtual_sin_wave_k = \ 64 | waveconvparam(bin_resolution, virtual_wavelength, cycles) 65 | 66 | virtual_cos_sin_wave_2xk = np.stack([virtual_cos_wave_k, virtual_sin_wave_k], axis=0) 67 | 68 | # use pytorch conv to replace matlab conv 69 | self.virtual_cos_sin_wave_inv_2x1xk = torch.from_numpy(virtual_cos_sin_wave_2xk[:, ::-1].copy()).unsqueeze(1) 70 | 71 | ################################################### 72 | slope = self.width / self.trange 73 | psf = definePsf(sptial_grid, temprol_grid, slope) 74 | fpsf = np.fft.fftn(psf) 75 | # lct 76 | # invpsf = np.conjugate(fpsf) / (1 / self.snr + np.real(fpsf) ** 2 + np.imag(fpsf) ** 2) 77 | # bp 78 | invpsf = np.conjugate(fpsf) 79 | 80 | self.invpsf_real = torch.from_numpy(np.real(invpsf).astype(np.float32)).unsqueeze(0) 81 | self.invpsf_imag = torch.from_numpy(np.imag(invpsf).astype(np.float32)).unsqueeze(0) 82 | 83 | ###################################################### 84 | mtx_MxM, mtxi_MxM = resamplingOperator(temprol_grid) 85 | self.mtx_MxM = torch.from_numpy(mtx_MxM.astype(np.float32)) 86 | self.mtxi_MxM = torch.from_numpy(mtxi_MxM.astype(np.float32)) 87 | 88 | def todev(self, dev, dnum): 89 | 90 | self.virtual_cos_sin_wave_inv_2x1xk_todev = self.virtual_cos_sin_wave_inv_2x1xk.to(dev) 91 | # self.datapad_2Dx2Tx2Hx2W = torch.zeros((2 * dnum, 2 * self.crop, 2 * self.spatial_grid, 2 * self.spatial_grid), dtype=torch.float32, device=dev) 92 | 93 | self.mtx_MxM_todev = self.mtx_MxM.to(dev) 94 | self.mtxi_MxM_todev = self.mtxi_MxM.to(dev) 95 | 96 | self.invpsf_real_todev = self.invpsf_real.to(dev) 97 | self.invpsf_imag_todev = self.invpsf_imag.to(dev) 98 | 99 | def forward(self, feture_bxdxtxhxw, tbes, tens): 100 | 101 | # 1 padd data with zero 102 | bnum, dnum, tnum, hnum, wnum = feture_bxdxtxhxw.shape 103 | for tbe, ten in zip(tbes, tens): 104 | assert tbe >= 0 105 | assert ten <= self.crop 106 | dev = feture_bxdxtxhxw.device 107 | 108 | featpad_bxdxtxhxw = [] 109 | for i in range(bnum): 110 | featpad_1xdxt1xhxw = torch.zeros((1, dnum, tbes[i], hnum, wnum), dtype=torch.float32, device=dev) 111 | featpad_1xdxt2xhxw = torch.zeros((1, dnum, self.crop - tens[i], hnum, wnum), dtype=torch.float32, device=dev) 112 | featpad_1xdxtxhxw = torch.cat([featpad_1xdxt1xhxw, feture_bxdxtxhxw[i:i + 1], featpad_1xdxt2xhxw], dim=2) 113 | featpad_bxdxtxhxw.append(featpad_1xdxtxhxw) 114 | featpad_bxdxtxhxw = torch.cat(featpad_bxdxtxhxw, dim=0) 115 | 116 | # 2 params 117 | assert hnum == wnum 118 | assert hnum == self.spatial_grid 119 | sptial_grid = hnum 120 | temprol_grid = self.crop 121 | tnum = self.crop 122 | 123 | #################################################### 124 | # 3 run lct 125 | # assert bnum == 1 126 | data_BDxTxHxW = featpad_bxdxtxhxw.view(bnum * dnum, tnum, hnum, wnum) 127 | 128 | ############################################################ 129 | # Step 1: convolve measurement volume with virtual wave 130 | 131 | data_BDxHxWxT = data_BDxTxHxW.permute(0, 2, 3, 1) 132 | data_BDHWx1xT = data_BDxHxWxT.reshape(-1, 1, tnum) 133 | knum = self.virtual_cos_sin_wave_inv_2x1xk.shape[2] 134 | phasor_data_cos_sin_BDHWx2x1T = F.conv1d(data_BDHWx1xT, self.virtual_cos_sin_wave_inv_2x1xk_todev, padding=knum // 2) 135 | if knum % 2 == 0: 136 | data_BDHWx2xT = phasor_data_cos_sin_BDHWx2x1T[:, :, 1:] 137 | else: 138 | data_BDHWx2xT = phasor_data_cos_sin_BDHWx2x1T 139 | 140 | data_BDxHxWx2xT = data_BDHWx2xT.reshape(bnum * dnum, hnum, wnum, 2, tnum) 141 | data_2xBDxTxHxW = data_BDxHxWx2xT.permute(3, 0, 4, 1, 2) 142 | data_2BDxTxHxW = data_2xBDxTxHxW.reshape(2 * bnum * dnum, tnum, hnum, wnum) 143 | 144 | ############################################################# 145 | # Step 2: transform virtual wavefield into LCT domain 146 | 147 | left = self.mtx_MxM_todev 148 | right = data_2BDxTxHxW.view(2 * bnum * dnum, temprol_grid, -1) 149 | tmp = torch.matmul(left, right) 150 | tmp2 = tmp.view(2 * bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 151 | 152 | 153 | ###############################################33 154 | batch = data_2BDxTxHxW.shape[0] 155 | 156 | res = [] 157 | for i in range(batch): 158 | 159 | dataslice = torch.zeros((1, 2 * temprol_grid, 2 * sptial_grid, 2 * sptial_grid), dtype=torch.float32, device=dev) 160 | dataslice[:, :temprol_grid, :sptial_grid, :sptial_grid] = tmp2[i:i + 1] 161 | 162 | datafre = torch.rfft(dataslice, 3, onesided=False) 163 | datafre_real = datafre[:, :, :, :, 0] 164 | datafre_imag = datafre[:, :, :, :, 1] 165 | 166 | re_real = datafre_real * self.invpsf_real_todev - datafre_imag * self.invpsf_imag_todev 167 | re_imag = datafre_real * self.invpsf_imag_todev + datafre_imag * self.invpsf_real_todev 168 | refre = torch.stack([re_real, re_imag], dim=4) 169 | re = torch.ifft(refre, 3) 170 | volumn_1xTxHxWx2 = re[:, :temprol_grid, :sptial_grid, :sptial_grid, :] 171 | res.append(volumn_1xTxHxWx2) 172 | 173 | volumn_2BDxTxHxWx2 = torch.cat(res, 0) 174 | 175 | ######################################################################## 176 | # Step 4: compute phasor field magnitude and inverse LCT 177 | 178 | cos_real = volumn_2BDxTxHxWx2[:bnum * dnum, :, :, :, 0] 179 | cos_imag = volumn_2BDxTxHxWx2[:bnum * dnum, :, :, :, 1] 180 | 181 | sin_real = volumn_2BDxTxHxWx2[bnum * dnum:, :, :, :, 0] 182 | sin_imag = volumn_2BDxTxHxWx2[bnum * dnum:, :, :, :, 1] 183 | 184 | sum_real = cos_real ** 2 - cos_imag ** 2 + sin_real ** 2 - sin_imag ** 2 185 | sum_image = 2 * cos_real * cos_imag + 2 * sin_real * sin_imag 186 | 187 | tmp = (torch.sqrt(sum_real ** 2 + sum_image ** 2) + sum_real) / 2 188 | # numerical issue 189 | tmp = F.relu(tmp, inplace=False) 190 | sqrt_sum_real = torch.sqrt(tmp) 191 | 192 | ##################################################################### 193 | left = self.mtxi_MxM_todev 194 | right = sqrt_sum_real.view(bnum * dnum, temprol_grid, -1) 195 | tmp = torch.matmul(left, right) 196 | tmp2 = tmp.view(bnum * dnum, temprol_grid, sptial_grid, sptial_grid) 197 | 198 | ######################################################################## 199 | # do we force to be > 0? 200 | # volumn_BDxTxHxW = F.relu(tmp2, inplace=False) 201 | volumn_BDxTxHxW = tmp2 202 | 203 | volumn_BxDxTxHxW = volumn_BDxTxHxW.view(bnum, dnum, self.crop, hnum, wnum) 204 | 205 | return volumn_BxDxTxHxW 206 | 207 | 208 | if __name__ == '__main__': 209 | 210 | import os 211 | import cv2 212 | import numpy as np 213 | 214 | syn = False 215 | if syn: 216 | fd = '/home/wenzheng/largestore/nlos-phasor/data/car'; 217 | ims = [] 218 | tbe = -1 219 | tlen = 0 220 | for i in range(512): 221 | name = '%s/2-%d.png' % (fd, i) 222 | if not os.path.isfile(name): 223 | ims.append(np.zeros((256, 256), dtype=np.uint8)) 224 | continue 225 | 226 | if tbe < 0: 227 | tbe = i 228 | tlen += 1 229 | 230 | im = cv2.imread(name) 231 | imgt = im[:256, :256, :] 232 | im = im[:256, -256:, :] 233 | imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 234 | ims.append(imgray) 235 | 236 | rect_data_txhxw = np.array(ims, dtype=np.float32) / 255.0 237 | rect_data_hxwxt = np.transpose(rect_data_txhxw, [1, 2, 0]) 238 | 239 | sptial_grid = 256 240 | crop = 512 241 | bin_len = 0.01 242 | else: 243 | 244 | from scipy.io import loadmat 245 | 246 | data = loadmat('/home/wenzheng/largestore/nlos-phasor/nlos-fk-master/statue.mat') 247 | rect_data_hxwxt = data['data'] 248 | 249 | sptial_grid = 512 250 | crop = 512 251 | bin_len = 32e-12 * 3e8 # 0.01 252 | tbe = 0 253 | tlen = crop 254 | 255 | K = 2 256 | temds = True 257 | for k in range(K): 258 | rect_data_hxwxt = rect_data_hxwxt[::2, :, :] + rect_data_hxwxt[1::2, :, :] 259 | rect_data_hxwxt = rect_data_hxwxt[:, ::2, :] + rect_data_hxwxt[:, 1::2, :] 260 | sptial_grid = sptial_grid // 2 261 | 262 | if temds: 263 | rect_data_hxwxt = rect_data_hxwxt[:, :, ::2] + rect_data_hxwxt[:, :, 1::2] 264 | crop = crop // 2 265 | bin_len = bin_len * 2 266 | 267 | rect_data_dxhxwxt = np.expand_dims(rect_data_hxwxt, axis=0) 268 | rect_data_bxdxhxwxt = np.expand_dims(rect_data_dxhxwxt, axis=0) 269 | 270 | bnum = 1 271 | dnum = 1 272 | rect_data_bxdxhxwxt = np.tile(rect_data_bxdxhxwxt, [bnum, dnum, 1, 1, 1]) 273 | rect_data_bxdxhxwxt = torch.from_numpy(rect_data_bxdxhxwxt).cuda() 274 | 275 | dev = 'cuda' 276 | 277 | ##################################################################### 278 | lctlayer = phasor(spatial=sptial_grid, crop=crop, bin_len=bin_len, sampling_coeff=2.0, cycles=5) 279 | lctlayer.todev(dev, dnum) 280 | 281 | if temds: 282 | tbe = tbe // (2 ** K) 283 | tlen = tlen // (2 ** K) 284 | 285 | for i in range(10): 286 | print(i) 287 | re = lctlayer(rect_data_bxdxhxwxt[:, :, :, :, tbe:tbe + tlen].permute(0, 1, 4, 2, 3), \ 288 | [tbe, tbe, tbe], [tbe + tlen, tbe + tlen, tbe + tlen]) 289 | 290 | volumn_MxNxN = re.detach().cpu().numpy()[0, -1] 291 | zdim = volumn_MxNxN.shape[0] * 100 // 128 292 | volumn_MxNxN = volumn_MxNxN[:zdim] 293 | print('volumn min, %f' % volumn_MxNxN.min()) 294 | print('volumn max, %f' % volumn_MxNxN.max()) 295 | 296 | volumn_MxNxN[volumn_MxNxN < 0] = 0 297 | front_view = np.max(volumn_MxNxN, axis=0) 298 | cv2.imshow("re", front_view / np.max(front_view)) 299 | # cv2.imshow("gt", imgt) 300 | cv2.waitKey() 301 | 302 | volumn_ZxYxX = volumn_MxNxN 303 | volumn_ZxYxX = volumn_ZxYxX / np.max(volumn_ZxYxX) 304 | for i, frame in enumerate(volumn_ZxYxX): 305 | print(i) 306 | cv2.imshow("re1", frame) 307 | cv2.imshow("re2", frame / np.max(frame)) 308 | cv2.waitKey(0) 309 | 310 | -------------------------------------------------------------------------------- /DL_inference/utils_pytorch/utils.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | import cv2 7 | import numpy as np 8 | 9 | ################################################################################ 10 | ''' 11 | def num_divisible_by_2(number): 12 | i = 0 13 | while not number % 2: 14 | number = number // 2 15 | i += 1 16 | return i 17 | ''' 18 | 19 | 20 | ################################################################################ 21 | def restrctuctre(nf0, norm, inplace=False): 22 | 23 | tmp = nn.Sequential( 24 | norm(nf0 * 1, affine=True), 25 | nn.ReLU(inplace), 26 | # nn.Dropout3d(0.1, inplace), 27 | 28 | nn.ReplicationPad3d(1), 29 | nn.Conv3d(nf0 * 1, 30 | nf0 * 1, 31 | kernel_size=[3, 3, 3], 32 | padding=0, 33 | stride=[1, 1, 1], 34 | bias=False), 35 | 36 | norm(nf0 * 1, affine=True), 37 | nn.ReLU(inplace), 38 | # nn.Dropout3d(0.1, inplace), 39 | 40 | nn.ReplicationPad3d(1), 41 | nn.Conv3d(nf0 * 1, 42 | nf0 * 1, 43 | kernel_size=[3, 3, 3], 44 | padding=0, 45 | stride=[1, 1, 1], 46 | bias=False), 47 | ) 48 | 49 | return tmp 50 | 51 | def restrctuctre3d(nf0, norm, inplace=False): 52 | 53 | tmp = nn.Sequential( 54 | norm(nf0 * 1, affine=True), 55 | nn.ReLU(inplace), 56 | # nn.Dropout3d(0.1, inplace), 57 | 58 | nn.ReplicationPad3d(1), 59 | nn.Conv3d(nf0 * 1, 60 | nf0 * 1, 61 | kernel_size=[3, 3, 3], 62 | padding=0, 63 | stride=[1, 1, 1], 64 | bias=False), 65 | 66 | norm(nf0 * 1, affine=True), 67 | nn.ReLU(inplace), 68 | # nn.Dropout3d(0.1, inplace), 69 | 70 | nn.ReplicationPad3d(1), 71 | nn.Conv3d(nf0 * 1, 72 | nf0 * 1, 73 | kernel_size=[3, 3, 3], 74 | padding=0, 75 | stride=[1, 1, 1], 76 | bias=False), 77 | ) 78 | 79 | return tmp 80 | 81 | 82 | def restrctuctre2d(nf0, norm, kernel=3, pad=1): 83 | 84 | tmp = nn.Sequential( 85 | norm(nf0 * 1, affine=True), 86 | nn.ReLU(False), 87 | # nn.Dropout3d(0.1, False), 88 | 89 | nn.ReflectionPad2d(pad), 90 | nn.Conv2d(nf0 * 1, 91 | nf0 * 1, 92 | kernel_size=[kernel, kernel], 93 | padding=0, 94 | stride=[1, 1], 95 | bias=False), 96 | 97 | norm(nf0 * 1, affine=True), 98 | nn.ReLU(False), 99 | # nn.Dropout3d(0.1, False), 100 | 101 | nn.ReflectionPad2d(pad), 102 | nn.Conv2d(nf0 * 1, 103 | nf0 * 1, 104 | kernel_size=[kernel, kernel], 105 | padding=0, 106 | stride=[1, 1], 107 | bias=False), 108 | ) 109 | 110 | return tmp 111 | 112 | 113 | #################################################### 114 | def init_mats(trans=False): 115 | 116 | mats = torch.Tensor([[1, 0, 0, 0, 1, 0, 0, 0, 1], 117 | [0.760836, 0.0447155, -0.647402, 0.0447155, 0.99164, 118 | 0.121042, 0.647402, -0.121042, 0.752475], 119 | [0.828744, -0.106611, 0.54938, -0.106611, 0.933632, 120 | 0.342001, -0.54938, -0.342001, 0.762376], 121 | [0.983521, -0.0590013, -0.170898, -0.0590013, 122 | 0.788757, -0.611867, 0.170898, 0.611867, 0.772277], 123 | [0.95545, 0.0878597, -0.281775, 0.0878597, 0.826729, 124 | 0.555698, 0.281775, -0.555698, 0.782178], 125 | [0.817867, 0.0685338, 0.571311, 0.0685338, 126 | 0.974212, -0.214976, -0.571311, 0.214976, 0.792079], 127 | [0.829463, -0.0684608, -0.55435, -0.0684608, 128 | 0.972517, -0.22254, 0.55435, 0.22254, 0.80198], 129 | [0.964787, -0.0733777, 0.252591, -0.0733777, 130 | 0.847094, 0.526352, -0.252591, -0.526352, 0.811881], 131 | [0.985013, 0.0494608, 0.165237, 0.0494608, 132 | 0.836769, -0.545317, -0.165237, 0.545317, 0.821782], 133 | [0.87543, 0.0738207, -0.477675, 0.0738207, 0.956254, 134 | 0.283071, 0.477675, -0.283071, 0.831683], 135 | [0.848254, -0.0318132, 0.528634, -0.0318132, 136 | 0.99333, 0.110827, -0.528634, -0.110827, 0.841584], 137 | [0.94951, -0.070351, -0.305746, -0.070351, 138 | 0.901975, -0.426019, 0.305746, 0.426019, 0.851485], 139 | [0.998042, 0.0163562, -0.060365, 0.0163562, 140 | 0.863344, 0.504351, 0.060365, -0.504351, 0.861386], 141 | [0.925979, 0.0636266, 0.372175, 0.0636266, 142 | 0.945308, -0.319913, -0.372175, 0.319913, 0.871287], 143 | [0.881307, -0.00375067, -0.47253, -0.00375067, 144 | 0.999882, -0.0149318, 0.47253, 0.0149318, 0.881188], 145 | [0.94422, -0.0544394, 0.324784, -0.0544394, 0.946869, 146 | 0.316979, -0.324784, -0.316979, 0.891089], 147 | [0.99969, -0.00552741, -0.0242579, -0.00552741, 148 | 0.9013, -0.433161, 0.0242579, 0.433161, 0.90099], 149 | [0.964273, 0.0436712, -0.261286, 0.0436712, 150 | 0.946618, 0.319386, 0.261286, -0.319386, 0.910891], 151 | [0.922411, 0.0112091, 0.386046, 0.0112091, 152 | 0.998381, -0.0557716, -0.386046, 0.0557716, 0.920792], 153 | [0.95267, -0.0322514, -0.302292, -0.0322514, 154 | 0.978024, -0.205985, 0.302292, 0.205985, 0.930693], 155 | [0.996886, -0.0132402, 0.0777394, -0.0132402, 156 | 0.943708, 0.330514, -0.0777394, -0.330514, 0.940594], 157 | [0.988176, 0.0211075, 0.151861, 0.0211075, 158 | 0.962319, -0.271104, -0.151861, 0.271104, 0.950495], 159 | [0.964281, 0.0117793, -0.264621, 0.0117793, 0.996116, 160 | 0.0872645, 0.264621, -0.0872645, 0.960396], 161 | [0.975304, -0.0111198, 0.220587, -0.0111198, 0.994993, 162 | 0.0993228, -0.220587, -0.0993228, 0.970297], 163 | [0.99691, -0.00718622, -0.0782245, -0.00718622, 164 | 0.983288, -0.181914, 0.0782245, 0.181914, 0.980198], 165 | [0.998926, 0.00307839, -0.0462215, 0.00307839, 0.991173, 166 | 0.132543, 0.0462215, -0.132543, 0.990099]]) 167 | n_view = mats.shape[0] 168 | selfmats = torch.zeros(n_view, 3, 4) 169 | selfmats[..., : 3] = mats.reshape(n_view, 3, 3).transpose(1, 2) 170 | 171 | ########################################################### 172 | mtxs = [] 173 | mtxs.append(np.eye(3)) 174 | 175 | samplenum = 25 176 | ratio = 0.5 177 | sam2 = int(samplenum / ratio / ratio) 178 | sambe = sam2 - samplenum 179 | 180 | for i in range(samplenum): 181 | 182 | n = sambe + i + 1.0 183 | N = sam2 + 1.0 184 | 185 | zn = n / N 186 | r = np.sqrt(1 - zn * zn) 187 | 188 | phi = (np.sqrt(5.0) - 1.0) / 2.0 189 | angle = 2.0 * np.pi * n * phi 190 | xn = r * np.cos(angle) 191 | yn = r * np.sin(angle) 192 | 193 | zaxis = np.array([0, 0, 1], dtype=np.float64) 194 | newaxis = np.array([xn, yn, zn], dtype=np.float64) 195 | costheta = zn 196 | theta = np.arccos(costheta) 197 | # print(theta / np.pi * 180) 198 | 199 | rotaxis = np.cross(zaxis, newaxis) 200 | rotaxis = rotaxis / np.sqrt(np.sum(rotaxis ** 2)) 201 | 202 | rotvec = rotaxis * theta 203 | mtx = cv2.Rodrigues(rotvec)[0] 204 | if trans: 205 | mtx = mtx.T 206 | mtxs.append(mtx) 207 | 208 | mtxs = torch.from_numpy(np.array(mtxs, dtype=np.float32)) 209 | mtxs_bx3x4 = torch.cat([mtxs, torch.zeros_like(mtxs[:, :, :1])], dim=2) 210 | 211 | print(torch.max((mtxs_bx3x4 - selfmats).abs())) 212 | 213 | return mtxs_bx3x4 214 | 215 | 216 | if __name__ == '__main__': 217 | 218 | init_mats(False) 219 | 220 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NLOSFeatureEmbeddings Code & Datasets 2 | 3 | This repository contains code for the paper _Learned Feature Embeddings for Non-Line-of-Sight Imaging and Recognition_ by Wenzheng Chen, Fangyin Wei, Kyros Kutulakos, Szymon Rusinkiewicz, and Felix Heide ([project webpage](https://light.cs.princeton.edu/publication/nlos-learnedfeatures/)). 4 | 5 | ## Reconstruction Results on Real Experimental Scenes 6 | 7 | ### Bike 8 | ||| 9 | |---|---| 10 | 11 | - Description: A bike captured at approximately 1 m distance from the wall. 12 | - Resolution: 512 x 512 13 | - Scanned Area: 2 m x 2 m planar wall 14 | - Integration Time: 180 min. 15 | 16 | ### Disco Ball 17 | ||| 18 | |---|---| 19 | 20 | - Description: A specular disco ball captured at approximately 1 m distance from the wall. 21 | - Resolution: 512 x 512 22 | - Scanned Area: 2 m x 2 m planar wall 23 | - Integration Time: 180 min. 24 | 25 | 26 | ### Dragon 27 | ||| 28 | |---|---| 29 | 30 | - Description: A glossy dragon captured at approximately 1 m distance from the wall. 31 | - Resolution: 512 x 512 32 | - Scanned Area: 2 m x 2 m planar wall 33 | - Integration Time: 180 min. 34 | 35 | ### Resolution 36 | ||| 37 | |---|---| 38 | 39 | - Description: A resolution chart captured at approximately 1 m distance from the wall. 40 | - Resolution: 512 x 512 41 | - Scanned Area: 2 m x 2 m planar wall 42 | - Integration Time: 180 min. 43 | 44 | ### Statue 45 | ||| 46 | |---|---| 47 | 48 | - Description: A white stone statue captured at approximately 1 m distance from the wall. 49 | - Resolution: 512 x 512 50 | - Scanned Area: 2 m x 2 m planar wall 51 | - Integration Time: 180 min. 52 | 53 | ### Teaser 54 | ||| 55 | |---|---| 56 | 57 | - Description: The teaser scene used in the paper which includes a number of objects, including a bookshelf, statue, dragon, and disco ball. 58 | - Resolution: 512 x 512 59 | - Scanned Area: 2 m x 2 m planar wall 60 | - Integration Time: 180 min. 61 | 62 | 63 | The realistic experimental scenes above have been captured by [this work](https://github.com/computational-imaging/nlos-fk). 64 | 65 | ## Reconstruction Results on Synthetic Scenes 66 | 67 | 68 | 69 | Qualitative Evaluation for NLOS 2D Imaging. Compared with F-K, LCT, and filtered back-projection (BP), we observe that the proposed method 70 | is able to reconstruct 2D images with clearer boundaries while achieving more 71 | accurate color reconstruction. 72 | 73 | ## NLOS 2.5D Object Detection 74 | 75 | 76 | 77 | Qualitative end-to-end detection results on synthetic (top) and real (bottom) data. 78 | The proposed method correctly predicts the bounding boxes for different classes with various color, shape, and pose. 79 | The model is only trained on synthetic data. Evaluation of such a model on real data 80 | validates its generalization capability. 81 | 82 | ## NLOS Classification 83 | 84 | 85 | 86 | 87 | We compare the classification accuracy of the proposed method, learned to 88 | classify hidden scenes with a monolithic end-to-end network, and sequential NLOS image classification baselines. 89 | The last row in the table reports the confidence scores for the experimental 90 | bike measurement. We note that the proposed model recognizes it as a motorbike 91 | with more than 66% probability. 92 | 93 | ## Description of Files 94 | 95 | The code is organized as in the following directory tree 96 | 97 | ./cuda-render 98 | conversion/ 99 | render/ 100 | ./DL_inference 101 | inference/ 102 | network7_256/ 103 | re/ 104 | utils/ 105 | utils_pytorch/ 106 | ./data 107 | bunny-model/ 108 | img/ 109 | LICENSE 110 | README.md 111 | 112 | ## Usage 113 | 114 | The code base contains two parts. The first part is how to render data and 115 | the second is how to train and test the neural network models. 116 | 117 | ### Rendering 118 | 119 | Please check the cuda-render folder. We recommend opening it in Nsight (tested). 120 | Other IDE should also work. To compile the code, please install cuda (tested for cuda 9.0), 121 | libglm, glew, glfw, and opencv (tested for opencv 3.4). 122 | 123 | ``` 124 | sudo apt-get install libglm-dev 125 | sudo apt-get install libglew-dev 126 | sudo apt-get install libglfw3-dev 127 | sudo apt-get install libopencv-dev 128 | ``` 129 | 130 | To render the 3D model, first create a cuda project in Nsight and put everything in cuda-render/render folder to the created project and compile. To successfully run the code, modify the folder path and data saving path in [main.cpp](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/main.cpp#L32). We provide a bunny model for test. 131 | 132 | ### Rendering Settings 133 | 134 | 1) Change 3D model location and scale. We change the model size in two places. When we load a 3D model, we normalize it by moving it to the origin and load with a specific scale. The code can be modified [here](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/display_4_loaddata.cpp#L337). Next, when we render the model, we may change the model location and rotation [here](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/display_6_render.cpp#L361). 135 | 136 | 2) 3D model normal. For the bunny model, we use point normals. We empirically find that it is better to use face normals for ShapeNet data set. You can change it [here](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/display_4_loaddata.cpp#L464). 137 | 138 | 3) Confocal/Non-confocal renderings. Our rendering algorithm supports both confocal and non-confocal settings. One can change it [here](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/display_6_render.cpp#L613), where conf=0 means non-confocal and conf=1 means confocal. 139 | 140 | 4) Specular rendering. Our rendering algorithm supports both diffuse and specular materials. To render a specular object (metal material), change the switch [here](https://github.com/princeton-computational-imaging/NLOSFeatureEmbeddings/blob/6274ff26c31748c760414664c9f3655d7874de1a/cuda-render/render/src/display_6_render.cpp#L693). 141 | 142 | 5) Video conversion. To convert a rendered hdr file to a video, we provide a script in cuda-render/conversion. Please change the render folder [here](https://github.com/wenzhengchen/Learned-Feature-Embeddings-for-Non-Line-of-Sight-Imaging-and-Recognition/blob/dc12a8c907c7cd6392b7d3a0717ce650b07930fb/cuda-render/conversion/preprocess_hdr2video.py#L284) then run the python script. It will generate a video which is of a much smaller size and easier to load to train the deep learning model. 143 | 144 | 6) SPAD simulation. The rendered hdr file does not have any noise simulation. One can add simple Gaussian noise in dataloader, but we recommend employing a computational method for spad simulation to synthesize noise. We adopt the method from [here](https://graphics.unizar.es/data/spad/). 145 | 146 | 7) Rendered dataset. We provide a motorbike dataset with 3000 motorbike examples [here](https://drive.google.com/file/d/183VAD_wuVtwkyvfaBoguUHZgHu065BNW/view?usp=sharing). 147 | 148 | ### Rendering Examples 149 | 150 | Non-confocal Rendering 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 |
t=1.2m t=1.4m t=1.6m t=1.8m
165 | Confocal Rendering 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 |
t=1.2m t=1.4m t=1.6m t=1.8m
180 | Specular Confocal Rendering 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 |
t=1.2m t=1.4m t=1.6m t=1.8m
195 | 196 | 197 | ### Deep Learning Model 198 | 199 | To run the inference model, please first download the data and pre-trained model [here](https://drive.google.com/drive/folders/17KlddkUmEav-2DeDNYRD013-COqgZc0T?usp=sharing). Next, go to DL_inference/inference folder and run: 200 | 201 | ``` 202 | python eval2.py --datafolder YOUR_DATA_FOLDER --mode fk --netfolder network7_256 --netsvfolder model10_bike --datanum 800 --dim 3 --frame 128 --grid 128 --tres 2 --h 256 --w 256 203 | ``` 204 | 205 | ### Deep Learning Settings 206 | 207 | We provide our reimplementations of different NLOS methods in python and PyTorch. The python implementations are in DL_inference/utils, and the PyTorch implementations are in DL_inference/utils_pytorch. The file name starts with tf. You may directly check tflct.py, tffk.py, and tfphasor.py for NLOS methods LCT (back-projection included), F-K, and Phasor, respectively. 208 | 209 | 210 | ## License 211 | The code and dataset are licensed under the following license: 212 | 213 | > MIT License 214 | > 215 | > Copyright (c) 2020 wenzhengchen 216 | > 217 | > Permission is hereby granted, free of charge, to any person obtaining a copy 218 | of this software and associated documentation files (the "Software"), to deal 219 | in the Software without restriction, including without limitation the rights 220 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 221 | copies of the Software, and to permit persons to whom the Software is 222 | furnished to do so, subject to the following conditions: 223 | > 224 | > The above copyright notice and this permission notice shall be included in all 225 | copies or substantial portions of the Software. 226 | > 227 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 228 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 229 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 230 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 231 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 232 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 233 | SOFTWARE. 234 | 235 | ## Contact 236 | Questions can be addressed to [Wenzheng Chen](mailto:chen1474147@gmail.com) and [Fangyin Wei](mailto:fwei@princeton.edu). 237 | 238 | ## Citation 239 | If you find it is useful, please cite 240 | 241 | ``` 242 | @article{Chen:NLOS:2020, 243 | title = {Learned Feature Embeddings for Non-Line-of-Sight Imaging and Recognition}, 244 | author = {Wenzheng Chen and Fangyin Wei and Kiriakos N. Kutulakos and Szymon Rusinkiewicz and Felix Heide}, 245 | year = {2020}, 246 | issue_date = {December 2020}, 247 | publisher = {Association for Computing Machinery}, 248 | volume = {39}, 249 | number = {6}, 250 | journal = {ACM Transactions on Graphics (Proc. SIGGRAPH Asia)}, 251 | } 252 | ``` 253 | -------------------------------------------------------------------------------- /cuda-render/conversion/preprocess_hdr2video.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from __future__ import print_function 4 | from __future__ import division 5 | 6 | import os 7 | import glob 8 | 9 | import cv2 10 | import numpy as np 11 | from scipy import ndimage 12 | 13 | from torch.utils.data import Dataset, DataLoader 14 | from torch.hub import tqdm 15 | 16 | # make it reproducible 17 | np.random.seed(123456) 18 | 19 | 20 | ####################################################### 21 | class DataProvider(Dataset): 22 | """ 23 | Class for the data provider 24 | """ 25 | 26 | def __init__(self, datafolder, shininess, \ 27 | timeimszs, \ 28 | confocal=1, \ 29 | timebe=0, timeen=6, \ 30 | time_cropbe=0, time_cropen=6, \ 31 | loadnum=-1, \ 32 | mode='train', datadebug=False): 33 | 34 | self.mode = mode 35 | self.datadebug = datadebug 36 | 37 | self.datafolder = datafolder 38 | self.shinness = shininess 39 | 40 | ########################################## 41 | self.timeimszs = timeimszs 42 | self.timebe = timebe 43 | self.timeen = timeen 44 | self.time_cropbe = time_cropbe 45 | self.time_cropen = time_cropen 46 | 47 | ########################################### 48 | self.modeldirs = [] 49 | for fol in datafolder: 50 | for shi in shininess: 51 | modeldirs = glob.glob('%s/%d/*' % (fol, shi)) 52 | for modeldir in modeldirs: 53 | rotdirs = glob.glob('%s/shin*' % (modeldir)) 54 | self.modeldirs.extend(rotdirs) 55 | 56 | self.imnum = len(self.modeldirs) 57 | 58 | ####################################################### 59 | self.transient2video = True 60 | self.videoname = 'video' 61 | self.videoconfocal = confocal 62 | self.gray = True 63 | self.clip = False 64 | self.clipratio = 99.7 65 | if self.videoconfocal == 1: 66 | self.videoname = '%s-confocal' % self.videoname 67 | elif self.videoconfocal == 2: 68 | self.videoname = '%s-confocalspad' % self.videoname 69 | 70 | if self.gray: 71 | timeimszs[-1] = 1 72 | self.videoname = '%s-gray' % self.videoname 73 | else: 74 | self.videoname = '%scolor' % self.videoname 75 | 76 | if self.clip: 77 | self.videoname = '%s-clip%.2f' % (self.videoname, self.clipratio) 78 | else: 79 | self.videoname = '%s-full' % (self.videoname,) 80 | 81 | self.videoformat = 'mp4' 82 | 83 | # direct exit after preprocess 84 | # exit(0) 85 | 86 | ######################################################### 87 | # assert len(self.modeldirs) == 1 # for optimization, it is 1 88 | self.imszs = timeimszs # t h w 3 89 | self.hei = timeimszs[1] 90 | self.wei = timeimszs[2] 91 | self.color = timeimszs[3] 92 | 93 | self.timebe = timebe 94 | self.timeen = timeen 95 | self.time_cropbe = time_cropbe 96 | self.time_cropen = time_cropen 97 | 98 | def __len__(self): 99 | return self.imnum 100 | 101 | def __getitem__(self, idx): 102 | return self.prepare_instance(idx) 103 | 104 | def prepare_instance(self, idx): 105 | 106 | flga = False 107 | # print('progress %.5f' % (idx / self.imnum)) 108 | 109 | # let's do the preprocession 110 | if self.transient2video: 111 | 112 | rotshiftdir = self.modeldirs[idx] 113 | 114 | videofile = '%s/%s.%s' % (rotshiftdir, self.videoname, self.videoformat) 115 | if not os.path.isfile(videofile): 116 | flga = self.preprocess(rotshiftdir, \ 117 | self.timeimszs, self.time_cropbe, \ 118 | self.time_cropen, videofile) 119 | # print('suc signal %d' % int(flga)) 120 | else: 121 | # print('already done') 122 | # redo it 123 | cmd = 'rm %s' % videofile 124 | os.system(cmd) 125 | flga = self.preprocess(rotshiftdir, \ 126 | self.timeimszs, self.time_cropbe, \ 127 | self.time_cropen, videofile) 128 | ''' 129 | flga = True 130 | ''' 131 | 132 | return flga 133 | 134 | def load_hdr(self, rotshiftdir, imszs, time_cropbe, time_cropen): 135 | 136 | if self.videoconfocal > 0: 137 | name = glob.glob('%s/light-%d-*.hdr' % (rotshiftdir, self.videoconfocal)) 138 | else: 139 | name = glob.glob('%s/light-0-*.hdr' % rotshiftdir) 140 | 141 | if len(name) != 1: 142 | print('bad file') 143 | return None 144 | 145 | imname = name[0] 146 | 147 | if self.videoconfocal <= 1: 148 | params = os.path.split(imname)[1][:-4].split('-') 149 | mindist = float(params[-1]) 150 | maxdist = float(params[-2]) 151 | 152 | # care baout 1.5 to 4.5 153 | if mindist < time_cropbe or maxdist >= time_cropen: 154 | print('bad dist') 155 | return None 156 | 157 | try: 158 | # imgtfile = cv2.FileStorage(imname, flags=cv2.FileStorage_READ) 159 | # im = imgtfile.getFirstTopLevelNode().mat() 160 | im = cv2.imread(imname, -1) 161 | im = im / np.max(im) 162 | 163 | if self.gray: 164 | im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 165 | im = im / np.max(im) 166 | 167 | # it should be txhxwx3 168 | im = im.reshape(*imszs) 169 | except: 170 | print('bad capture') 171 | return None 172 | 173 | ################################################################## 174 | # crop meaningful region 175 | timebe = int(100 * time_cropbe) 176 | timeen = int(100 * time_cropen) 177 | imcrop = im[timebe:timeen, :, :, :] 178 | 179 | if self.clip: 180 | # do we need to do the fixed normalization? 181 | data90 = np.percentile(imcrop, self.clipratio) 182 | maxdata = np.max(imcrop) 183 | meandata = np.mean(imcrop) 184 | # data90 = maxdata 185 | imnorm = imcrop / data90 186 | imnorm[imnorm > 1] = 1 187 | else: 188 | imnorm = imcrop 189 | 190 | # smooth to get rid of the artifacts 191 | a = ndimage.filters.uniform_filter 192 | a = ndimage.gaussian_filter 193 | 194 | if False and (self.videoconfocal <= 1): 195 | sig = 0.7 196 | if self.gray: 197 | imsmooth = a(imnorm[:, :, :, 0], sigma=sig, mode='reflect', cval=0.0) 198 | else: 199 | imb = a(imnorm[:, :, :, 0], sigma=sig, mode='reflect', cval=0.0) 200 | img = a(imnorm[:, :, :, 1], sigma=sig, mode='reflect', cval=0.0) 201 | imr = a(imnorm[:, :, :, 2], sigma=sig, mode='reflect', cval=0.0) 202 | imsmooth = np.stack([imb, img, imr], axis=3) 203 | else: 204 | if self.gray: 205 | imsmooth = imnorm[:, :, :, 0] 206 | else: 207 | imsmooth = imnorm 208 | 209 | return imsmooth 210 | 211 | def preprocess(self, modeldir, imszs, time_cropbe, time_cropen, videofile): 212 | 213 | # print(modeldir) 214 | imsmooth = self.load_hdr(modeldir, imszs, time_cropbe, time_cropen) 215 | if imsmooth is None: 216 | return False 217 | 218 | ########################################################################### 219 | frame_width = imsmooth.shape[2] 220 | frame_height = imsmooth.shape[1] 221 | 222 | if self.gray: 223 | imsmooth = np.tile(np.expand_dims(imsmooth, axis=3), [1, 1, 1, 3]) 224 | 225 | # Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file. 226 | fourcc = cv2.VideoWriter_fourcc(*'mp4v') 227 | out = cv2.VideoWriter(videofile, \ 228 | fourcc, \ 229 | 20, \ 230 | (frame_width, frame_height)) 231 | 232 | # write it into videos 233 | imlists = [100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250] 234 | for i in range(imsmooth.shape[0]): 235 | impatch = imsmooth[i] 236 | if i in imlists: 237 | cv2.imwrite(videofile.replace('.mp4', '_%d.png'%i), impatch / impatch.max() *255) 238 | frame = (impatch * 255).astype(np.uint8) 239 | out.write(frame) 240 | impatch = np.mean(imsmooth, 0) 241 | print(impatch.shape) 242 | cv2.imwrite(videofile.replace('.mp4', '_all.png'), impatch / impatch.max() *255) 243 | out.release() 244 | 245 | return True 246 | 247 | 248 | def collate_fn(batch_list): 249 | 250 | return batch_list 251 | 252 | 253 | def get_data_loaders(folders, shininess=[0], imszs=[600, 256, 256, 3], \ 254 | confocal=1, \ 255 | timebe=0, timeen=6, \ 256 | time_cropbe=0, time_cropen=6, \ 257 | mode='train', bs=4, numworkers=32): 258 | 259 | print('Building dataloaders') 260 | 261 | dataset_train = DataProvider(folders, shininess, imszs, \ 262 | confocal, \ 263 | timebe, timeen, \ 264 | time_cropbe, time_cropen, \ 265 | mode=mode, datadebug=False) 266 | 267 | # always true 268 | shuffle = True 269 | if mode == 'train_val' or mode == 'test': 270 | shuffle = False 271 | 272 | train_loader = DataLoader(dataset_train, batch_size=bs, \ 273 | shuffle=shuffle, num_workers=numworkers, collate_fn=collate_fn) 274 | 275 | print('train num {}'.format(len(dataset_train))) 276 | print('train iter'.format(len(train_loader))) 277 | 278 | return train_loader 279 | 280 | 281 | ############################################## 282 | if __name__ == '__main__': 283 | 284 | folders = ['/home/wenzheng/largestore/projects-clean/Learned-Feature-Embeddings-for-Non-Line-of-Sight-Imaging-and-Recognition/data/bunny-renders'] 285 | 286 | imszs = [600, 256, 256, 3] 287 | timebin = 100 288 | 289 | ''' 290 | rootfd = '/u6/a/wenzheng/remote2/datasets/shapenet' 291 | fds = glob.glob('%s/*-multiclassrender' % rootfd) 292 | ''' 293 | 294 | ''' 295 | import tqdm 296 | for i in tqdm.tqdm(np.arange(1000)): 297 | print(i) 298 | ''' 299 | 300 | for conf in [1]: 301 | train_loader = get_data_loaders(folders, \ 302 | imszs=imszs, \ 303 | confocal=conf, \ 304 | mode='test', \ 305 | bs=1, numworkers=0) 306 | 307 | ############################################### 308 | for data in train_loader: 309 | print(data) 310 | 311 | -------------------------------------------------------------------------------- /cuda-render/render/.gitignore: -------------------------------------------------------------------------------- 1 | /Release/ 2 | -------------------------------------------------------------------------------- /cuda-render/render/pointlight.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 420 core 2 | 3 | // Interpolated values from the vertex shaders 4 | in vec3 Normal_cameraspace; 5 | in vec2 UV; 6 | in vec3 LightDirection_cameraspace; 7 | 8 | in float lp; 9 | in float pv; 10 | in float costerm; 11 | in float dist; 12 | in float distfactor; 13 | 14 | // Ouput data 15 | out vec4 color; 16 | 17 | // Values that stay constant for the whole mesh. 18 | uniform sampler2D myTextureSampler; 19 | 20 | uniform float istex; 21 | uniform float isalbedo; 22 | 23 | uniform vec3 MaterialAmbient; 24 | uniform vec3 MaterialDiffuse; 25 | uniform vec3 MaterialSpecular; 26 | 27 | uniform float Shininess; 28 | 29 | void main(){ 30 | 31 | // Material properties 32 | vec3 ObjectColor; 33 | if (istex > 0) 34 | { 35 | ObjectColor = texture( myTextureSampler, UV ).rgb; 36 | } 37 | else 38 | { 39 | ObjectColor = vec3(UV, 1); 40 | } 41 | 42 | /////////////////////////////////////////////////////////////// 43 | // Normal of the computed fragment, in camera space 44 | vec3 n = normalize( Normal_cameraspace ); 45 | 46 | // Direction of the light (from the fragment to the light) 47 | vec3 l = normalize( LightDirection_cameraspace ); 48 | // Cosine of the angle between the normal and the light direction, 49 | // clamped above 0 50 | // - light is at the vertical of the triangle -> 1 51 | // - light is perpendicular to the triangle -> 0 52 | // - light is behind the triangle -> 0 53 | 54 | // do we need this? 55 | if (dot(n, l) < 0) 56 | { 57 | n = -n; 58 | } 59 | 60 | float cosTheta = clamp( dot( n, l ), 0, 1 ); 61 | // not influnced by any light 62 | if (isalbedo > 0) 63 | { 64 | cosTheta = 1.0; 65 | } 66 | 67 | //////////////////////////////////////////////////////////////// 68 | // Light emission properties 69 | // You probably want to put them as uniforms 70 | float LightPower = Shininess; 71 | 72 | vec3 EyeDirection_cameraspace = vec3(0, 0, 1); 73 | // Eye vector (towards the camera) 74 | vec3 E = normalize( EyeDirection_cameraspace ); 75 | // Direction in which the triangle reflects the light 76 | vec3 R = reflect( -l, n ); 77 | // Cosine of the angle between the Eye vector and the Reflect vector, 78 | // clamped to 0 79 | // - Looking into the reflection -> 1 80 | // - Looking elsewhere -> < 1 81 | float cosAlpha = clamp( dot( E, R ), 0, 1 ); 82 | 83 | ////////////////////////////////////////////////////////////////// 84 | float factorlp, factorpv; 85 | // if it is projection mode 86 | // influenced by distance 87 | // distfactor is 0.9f 88 | // else 1.0f 89 | 90 | factorlp = distfactor / ( lp * lp ); 91 | factorpv = distfactor / ( pv * pv ); 92 | 93 | vec3 LightColor = vec3(1, 1, 1); 94 | vec3 LightColorDistance = factorlp * LightColor; 95 | 96 | ///////////////////////////////////////////////////////// 97 | vec3 MaterialAmbientColor; 98 | vec3 MaterialDiffuseColor; 99 | vec3 MaterialSpecularColor; 100 | 101 | MaterialAmbientColor = LightColorDistance * MaterialAmbient; 102 | MaterialDiffuseColor = factorpv * LightColorDistance * MaterialDiffuse * cosTheta; 103 | MaterialSpecularColor = LightColorDistance * MaterialSpecular * pow(cosAlpha, LightPower); 104 | if (isalbedo > 0) 105 | { 106 | MaterialSpecularColor = vec3(0, 0, 0); 107 | } 108 | 109 | vec3 colorrgb = 110 | // Ambient : simulates indirect lighting 111 | (MaterialAmbientColor + 112 | // Diffuse : "color" of the object 113 | MaterialDiffuseColor ) * ObjectColor + 114 | // Specular : reflective highlight, like a mirror 115 | MaterialSpecularColor; 116 | 117 | // ask Felix 118 | // colorrgb = clamp( costerm * color, 0, 1 ); 119 | // colorrgb = costerm * colorrgb; 120 | color = vec4(colorrgb, dist); 121 | } 122 | 123 | -------------------------------------------------------------------------------- /cuda-render/render/pointlight.vertexshader: -------------------------------------------------------------------------------- 1 | #version 420 core 2 | 3 | // Input vertex data, different for all executions of this shader. 4 | layout(location = 0) in vec3 vertexPosition_modelspace; 5 | layout(location = 1) in vec3 vertexNormal_modelspace; 6 | layout(location = 2) in vec2 vertexUV; 7 | 8 | // Output data ; will be interpolated for each fragment. 9 | out vec2 UV; 10 | out vec3 Normal_cameraspace; 11 | out vec3 LightDirection_cameraspace; 12 | 13 | // Values that stay constant for the whole mesh. 14 | uniform mat4 M; 15 | uniform mat4 V; 16 | uniform mat4 P; 17 | 18 | uniform vec3 lightPosition_modelspace; 19 | 20 | uniform float istex; 21 | uniform float isprojection; 22 | uniform float isconfocal; 23 | 24 | out float lp; 25 | out float pv; 26 | out float costerm; 27 | out float dist; 28 | out float distfactor; 29 | 30 | void main(){ 31 | 32 | // UV of the vertex. No special space for this one. 33 | if (istex > 0) 34 | { 35 | UV = vertexUV; 36 | } 37 | else 38 | { 39 | UV = vec2(1, 1); 40 | } 41 | 42 | 43 | //////////////////////////////////////////////////////////// 44 | // point projection 45 | if (isprojection > 0) 46 | { 47 | 48 | // model view 49 | vec4 p0 = M * vec4(vertexPosition_modelspace, 1); 50 | 51 | // camera view 52 | // note that, we didn;t really rotate the point 53 | // acutally, the change of light direction is 54 | // simulated by rotate the camera 55 | vec3 p1 = p0.xyz - vec3(0, 0, 1); 56 | vec4 p2 = V * vec4(p1, 0); 57 | vec3 p3 = p2.xyz + vec3(0, 0, 1); 58 | 59 | // projection view 60 | vec4 p4 = P * vec4(p3, 1); 61 | gl_Position = p4; 62 | 63 | // calculate the projected point on the wall 64 | float k, x, y; 65 | k = (p0.z - 1) / V[2][2]; 66 | x = p0.x - k * V[0][2]; 67 | y = p0.y - k * V[1][2]; 68 | gl_Position.x = x; 69 | gl_Position.y = y; 70 | 71 | // finally projected point on the wall 72 | vec3 pfinal = vec3(x, y, 1); 73 | 74 | // Normal of the the vertex, in camera space 75 | Normal_cameraspace = ( V * M * vec4(vertexNormal_modelspace, 0)).xyz; // Only correct if ModelMatrix does not scale the model ! Use its inverse transpose if not. 76 | 77 | // Vector that goes from the vertex to the camera, in camera space. 78 | vec3 LightDirection_modelspace = lightPosition_modelspace - p0.xyz; 79 | if (isconfocal > 0) 80 | { 81 | LightDirection_modelspace = pfinal - p0.xyz; 82 | } 83 | LightDirection_cameraspace = ( V * vec4(LightDirection_modelspace, 0)).xyz; 84 | 85 | // distance factor 86 | // pv = k; 87 | pv = length(pfinal - p0.xyz); 88 | lp = length(LightDirection_modelspace); 89 | costerm = V[2][2]; 90 | 91 | // distance 92 | // between 0 and 1 93 | dist = 1.0f / (1.0f + lp + pv); 94 | distfactor = 0.9f; 95 | } 96 | 97 | //////////////////////////////////////////////////////////////// 98 | else // normal render 99 | { 100 | 101 | // model view 102 | vec4 p0 = M * vec4(vertexPosition_modelspace, 1); 103 | 104 | // camera view 105 | vec3 p1 = p0.xyz; 106 | vec4 p2 = V * vec4(p1, 0); 107 | vec3 p3 = p2.xyz; 108 | 109 | // projection view 110 | vec4 p4 = P * vec4(p3, 1); 111 | gl_Position = p4; 112 | 113 | // Normal of the the vertex, in camera space 114 | Normal_cameraspace = ( V * M * vec4(vertexNormal_modelspace, 0)).xyz; // Only correct if ModelMatrix does not scale the model ! Use its inverse transpose if not. 115 | 116 | // Vector that goes from the vertex to the camera, in camera space. 117 | vec3 LightDirection_modelspace = lightPosition_modelspace - p0.xyz; 118 | LightDirection_cameraspace = ( V * vec4(LightDirection_modelspace, 0)).xyz; 119 | 120 | // distance factor 121 | // no distance factor 122 | // pv = 1.0f; 123 | // lp = 1.0f; 124 | // costerm = 1.0f; 125 | // dist = 1.0f; 126 | // distfactor = 1.0f; 127 | pv = 1.0f - p3.z; 128 | lp = length(LightDirection_modelspace); 129 | costerm = 1.0f; 130 | dist = 1.0f; 131 | distfactor = 0.9f; 132 | } 133 | } 134 | 135 | -------------------------------------------------------------------------------- /cuda-render/render/src/copydata.cu: -------------------------------------------------------------------------------- 1 | // #include 2 | #include 3 | 4 | // for the older gpus atomicAdd with double arguments does not exist 5 | #if __CUDA_ARCH__ < 600 and defined(__CUDA_ARCH__) 6 | static __inline__ __device__ double atomicAdd(double* address, double val) { 7 | unsigned long long int* address_as_ull = (unsigned long long int*)address; 8 | unsigned long long int old = *address_as_ull, assumed; 9 | do { 10 | assumed = old; 11 | old = atomicCAS(address_as_ull, assumed, 12 | __double_as_longlong(val + __longlong_as_double(assumed))); 13 | // Note: uses integer comparison to avoid hang in case of NaN (since NaN != NaN) } while (assumed != old); 14 | }while (assumed != old); 15 | return __longlong_as_double(old); 16 | } 17 | #endif 18 | 19 | // 4 float values, rgba, where a is the distance factor 20 | texture inTex2; 21 | 22 | __global__ void cudaProcess2(float *data_txhxwx3, int timebin, int imgh, 23 | int imgw, int sz, int maxdepth, int mindepth) { 24 | 25 | ///////////////////////////////////////////////////////////////// 26 | // index 27 | // heiidx * (sz * width) + wididx 28 | 29 | int IMGH = sz * imgh; 30 | int IMGW = sz * imgw; 31 | 32 | int presentthread = blockIdx.x * blockDim.x + threadIdx.x; 33 | 34 | int wididx = presentthread % IMGW; 35 | int heiidx = (presentthread - wididx) / IMGW; 36 | if (heiidx >= IMGH || wididx >= IMGW) { 37 | return; 38 | } 39 | 40 | ////////////////////////////////////////////////////////////// 41 | // 1) what;s the corresponding value? 42 | float4 res = tex2D(inTex2, wididx, heiidx); 43 | float r = res.x; 44 | float g = res.y; 45 | float b = res.z; 46 | float dv = res.w; 47 | 48 | float ress[4]; 49 | ress[0] = r; 50 | ress[1] = g; 51 | ress[2] = b; 52 | ress[3] = dv; 53 | 54 | if (dv > 1.0f / (1.0f + 0.0f + 1e-2f) || dv < 1.0f / (1.0f + 6.0f - 1e-2f)) 55 | return; 56 | 57 | ////////////////////////////////////////////////////////////////////// 58 | // depth: depthvalue = 1 / (1 + depth) 59 | float depth = 1.0f / dv - 1.0f; 60 | 61 | float tidx = depth * 100; 62 | int tidxleft = static_cast(std::floor(tidx)); 63 | int tidxright = tidxleft + 1; 64 | 65 | if (tidxright >= maxdepth) 66 | return; 67 | 68 | //////////////////////////////////////////////////////////// 69 | // 2) which place should it be added 70 | wididx = wididx % imgw; 71 | heiidx = heiidx % imgh; 72 | 73 | // notice that for ogl, height begins from bottom to top 74 | heiidx = imgh - 1 - heiidx; 75 | 76 | /////////////////////////////////////////////////////////////////////// 77 | // interpolation 78 | int idxleft = tidxleft * imgh * imgw * 3 + heiidx * imgw * 3 + wididx * 3; 79 | int idxright = tidxright * imgh * imgw * 3 + heiidx * imgw * 3 + wididx * 3; 80 | // linear 81 | 82 | // linear intepolation 83 | float weightleft = tidxright - tidx; 84 | float weightright = tidx - tidxleft; 85 | for (int kc = 0; kc < 3; kc++) { 86 | // data_txhxwx3[idxleft + kc] += weightleft * res[kc]; 87 | // data_txhxwx3[idxright + kc] += weightright * res[kc]; 88 | atomicAdd(data_txhxwx3 + idxleft + kc, weightleft * ress[kc]); 89 | atomicAdd(data_txhxwx3 + idxright + kc, weightright * ress[kc]); 90 | } 91 | } 92 | 93 | __global__ void cudaProcess3(float *data_hxwx3, int imgh, int imgw, int sz) { 94 | 95 | ///////////////////////////////////////////////////////////////// 96 | // index 97 | // heiidx * width + wididx 98 | 99 | int IMGH = imgh; 100 | int IMGW = imgw; 101 | 102 | int presentthread = blockIdx.x * blockDim.x + threadIdx.x; 103 | 104 | int wididx = presentthread % IMGW; 105 | int heiidx = (presentthread - wididx) / IMGW; 106 | if (heiidx >= IMGH || wididx >= IMGW) { 107 | return; 108 | } 109 | 110 | ///////////////////////////////////////////////////////////////// 111 | int totalidx1 = heiidx * imgw + wididx; 112 | int totalidx3 = totalidx1 * 3; 113 | 114 | //////////////////////////////////////////////////////////////// 115 | // each thread will touch sz * sz data 116 | 117 | // inverse height 118 | heiidx = imgh - 1 - heiidx; 119 | for (int i = 0; i < sz; i++) 120 | for (int j = 0; j < sz; j++) { 121 | int hbig = i * imgh + heiidx; 122 | int wbig = j * imgw + wididx; 123 | 124 | ////////////////////////////////////////////////////////////// 125 | // 1) what;s the corresponding value? 126 | float4 res = tex2D(inTex2, wbig, hbig); 127 | float r = res.x; 128 | float g = res.y; 129 | float b = res.z; 130 | 131 | data_hxwx3[totalidx3 + 0] += r; 132 | data_hxwx3[totalidx3 + 1] += g; 133 | data_hxwx3[totalidx3 + 2] += b; 134 | } 135 | } 136 | 137 | extern "C" void launch_cudaProcess2(cudaArray *g_data_array, 138 | float *g_output_data, int timebin, int imgh, int imgw, int sz, 139 | int maxdepth, int mindepth) { 140 | 141 | ///////////////////////////////////////////////////////////// 142 | // for input 143 | // g_data_array 144 | // it is a (sz*imgh)*(sz*imgw)*4 image 145 | // for output 146 | // g_output_data 147 | // it is a timebin*imgh*imgw*3 image 148 | 149 | ///////////////////////////////////////////////////////////// 150 | // threadnum 151 | const int totalthread = sz * imgh * sz * imgw; 152 | const int threadnum = 1024; 153 | const int blocknum = static_cast(std::ceil( 154 | 1.0f * totalthread / threadnum)); 155 | 156 | const dim3 threads(threadnum, 1, 1); 157 | const dim3 blocks(blocknum, 1, 1); 158 | 159 | ///////////////////////////////////////////////////////////// 160 | // checkCudaErrors(cudaBindTextureToArray(inTex2, g_data_array)); 161 | cudaBindTextureToArray(inTex2, g_data_array); 162 | 163 | struct cudaChannelFormatDesc desc; 164 | // checkCudaErrors(cudaGetChannelDesc(&desc, g_data_array)); 165 | cudaGetChannelDesc(&desc, g_data_array); 166 | 167 | /* 168 | printf("CUDA Array channel descriptor, bits per component:\n"); 169 | printf("X %d Y %d Z %d W %d, kind %d\n", desc.x, desc.y, desc.z, desc.w, 170 | desc.f); 171 | 172 | printf("Possible values for channel format kind: i%d, u%d, f%d:\n", 173 | cudaChannelFormatKindSigned, cudaChannelFormatKindUnsigned, 174 | cudaChannelFormatKindFloat); 175 | 176 | printf("%d\n", inTex2); 177 | */ 178 | 179 | cudaProcess2<<>>(g_output_data, timebin, imgw, imgh, sz, 180 | maxdepth, mindepth); 181 | 182 | } 183 | 184 | extern "C" void launch_cudaProcess3(cudaArray *g_data_array, 185 | float *g_output_data, int imgh, int imgw, int sz) { 186 | 187 | ///////////////////////////////////////////////////////////// 188 | // for input 189 | // g_data_array 190 | // it is a (sz*imgh)*(sz*imgw)*4 image 191 | // for output 192 | // g_output_data 193 | // it is a imgh*imgw*3 image 194 | 195 | ///////////////////////////////////////////////////////////// 196 | // threadnum 197 | const int totalthread = imgh * imgw; 198 | const int threadnum = 1024; 199 | const int blocknum = static_cast(std::ceil( 200 | 1.0f * totalthread / threadnum)); 201 | 202 | const dim3 threads(threadnum, 1, 1); 203 | const dim3 blocks(blocknum, 1, 1); 204 | 205 | ///////////////////////////////////////////////////////////// 206 | // checkCudaErrors(cudaBindTextureToArray(inTex2, g_data_array)); 207 | cudaBindTextureToArray(inTex2, g_data_array); 208 | 209 | struct cudaChannelFormatDesc desc; 210 | // checkCudaErrors(cudaGetChannelDesc(&desc, g_data_array)); 211 | cudaGetChannelDesc(&desc, g_data_array); 212 | 213 | /* 214 | printf("CUDA Array channel descriptor, bits per component:\n"); 215 | printf("X %d Y %d Z %d W %d, kind %d\n", desc.x, desc.y, desc.z, desc.w, 216 | desc.f); 217 | 218 | printf("Possible values for channel format kind: i%d, u%d, f%d:\n", 219 | cudaChannelFormatKindSigned, cudaChannelFormatKindUnsigned, 220 | cudaChannelFormatKindFloat); 221 | 222 | printf("%d\n", inTex2); 223 | */ 224 | 225 | cudaProcess3<<>>(g_output_data, imgw, imgh, sz); 226 | 227 | } 228 | 229 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_0_initial.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | 3 | // Initialize GLFW 4 | bool render::initialGLFW() { 5 | if (!glfwInit()) { 6 | cout << "Failed to initialize GLFW." << endl; 7 | getchar(); 8 | return false; 9 | } 10 | 11 | // we use opengl 4.2 12 | // glfwWindowHint(GLFW_SAMPLES, 4); 13 | glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4); 14 | glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 4); 15 | // glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE); // To make MacOS happy; should not be needed 16 | glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE); 17 | 18 | // osmesa 19 | glfwWindowHint(GLFW_VISIBLE, 0); 20 | 21 | // Open a window and create its OpenGL context 22 | window = glfwCreateWindow(width, height, "Tutorial 08 - Basic Shading", 23 | NULL, 24 | NULL); 25 | if (window == NULL) { 26 | cout << "Failed to open GLFW window. " 27 | "If you have an Intel GPU, they are not 3.3 compatible. " 28 | "Try the 2.1 version of the tutorials." << endl; 29 | glfwTerminate(); 30 | return false; 31 | } 32 | 33 | glfwMakeContextCurrent(window); 34 | std::cout << "initialGLFW error\t" << glGetError() << std::endl; 35 | return true; 36 | } 37 | 38 | // Initialize GLEW 39 | bool render::initialGLEW() { 40 | 41 | glewExperimental = true; // Needed for core profile 42 | if (glewInit() != GLEW_OK) { 43 | cout << "Failed to initialize GLEW." << endl; 44 | glfwTerminate(); 45 | return false; 46 | } 47 | 48 | std::cout << "initialGLEW error\t" << glGetError() << std::endl; 49 | return true; 50 | } 51 | 52 | bool render::initialGL() { 53 | // Dark blue background 54 | glClearColor(0.0f, 0.0f, 0.0f, 1.0f); 55 | 56 | // Enable depth test 57 | glEnable(GL_DEPTH_TEST); 58 | 59 | // Accept fragment if it closer to the camera than the former one 60 | glDepthFunc(GL_LESS); 61 | 62 | // glDisable(GL_DEPTH_TEST); 63 | 64 | glDisable(GL_BLEND); 65 | // glEnable(GL_BLEND); 66 | // glBlendFuncSeparate(GL_ONE, GL_ONE, GL_ZERO, 67 | // GL_ONE); 68 | 69 | // Cull triangles which normal is not towards the camera 70 | glEnable(GL_CULL_FACE); 71 | 72 | std::cout << "initialGL error\t" << glGetError() << std::endl; 73 | return true; 74 | } 75 | 76 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_1_cam.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | 3 | glm::mat4 render::getProjectionMatrix() { 4 | 5 | // Projection matrix : 45��� Field of View, 4:3 ratio, display range : 0.1 unit <-> 100 units 6 | // ProjectionMatrix = glm::perspective(glm::radians(45.0f), 7 | // 1.0f * width / height, 0.1f, 100.0f); 8 | glm::mat4 ProjectionMatrix = glm::ortho(-1.0, 1.0, -1.0, 1.0, -100.0, 9 | 100.0); 10 | 11 | return ProjectionMatrix; 12 | } 13 | 14 | std::vector render::getViewMatrix(int samplenum, double ratio) { 15 | 16 | float pi = 3.141592653589793238463f; 17 | 18 | glm::vec3 zaxis(0.0f, 0.0f, 1.0f); 19 | 20 | int sam2 = static_cast(1.0 * samplenum / ratio / ratio); 21 | int sambe = sam2 - samplenum; 22 | 23 | std::vector re; 24 | re.resize(samplenum); 25 | 26 | for (int i = 0; i < samplenum; i++) { 27 | 28 | float n = sambe + i + 1.0f; 29 | float N = sam2 + 1.0f; 30 | 31 | // large than 0, small than 1 32 | float zn = 1.0f * n / N; 33 | float r = std::sqrt(1.0f - zn * zn); 34 | 35 | float phi = (sqrt(5.0f) - 1.0f) / 2.0f; 36 | float angle = 2.0f * pi * n * phi; 37 | float xn = r * cos(angle); 38 | float yn = r * sin(angle); 39 | 40 | glm::vec3 newaxis(xn, yn, zn); 41 | 42 | float costheta = zn; 43 | float theta = glm::acos(costheta); 44 | 45 | glm::vec3 rotaxis = glm::cross(zaxis, newaxis); 46 | float rotaxislen = std::sqrt( 47 | rotaxis.x * rotaxis.x + rotaxis.y * rotaxis.y 48 | + rotaxis.z * rotaxis.z); 49 | if (rotaxislen > 1e-5) 50 | rotaxis /= rotaxislen; 51 | else { 52 | assert(theta == 0.0f); 53 | std::cout << "rotate axis\t" << newaxis.x << "\t" << newaxis.y 54 | << std::endl; 55 | } 56 | 57 | cv::Mat rotvec = cv::Mat::zeros(3, 1, CV_32FC1); 58 | rotvec.at(0, 0) = theta * rotaxis.x; 59 | rotvec.at(1, 0) = theta * rotaxis.y; 60 | rotvec.at(2, 0) = theta * rotaxis.z; 61 | 62 | cv::Mat rotmtx; 63 | cv::Rodrigues(rotvec, rotmtx); 64 | 65 | re[i] = glm::mat4(1.0); 66 | re[i][0][0] = rotmtx.at(0, 0); 67 | re[i][0][1] = rotmtx.at(1, 0); 68 | re[i][0][2] = rotmtx.at(2, 0); 69 | 70 | re[i][1][0] = rotmtx.at(0, 1); 71 | re[i][1][1] = rotmtx.at(1, 1); 72 | re[i][1][2] = rotmtx.at(2, 1); 73 | 74 | re[i][2][0] = rotmtx.at(0, 2); 75 | re[i][2][1] = rotmtx.at(1, 2); 76 | re[i][2][2] = rotmtx.at(2, 2); 77 | } 78 | 79 | return re; 80 | } 81 | 82 | glm::mat4 render::getModelMatrix(float _x, float _y, float _z, float xshift, 83 | float yshift, float zshift) { 84 | 85 | glm::mat4 Modelx = glm::rotate(glm::mat4(1.0), glm::radians(_x), 86 | glm::vec3(1.0f, 0.0f, 0.0f)); 87 | glm::mat4 Modely = glm::rotate(glm::mat4(1.0), glm::radians(_y), 88 | glm::vec3(0.0f, 1.0f, 0.0f)); 89 | glm::mat4 Modelz = glm::rotate(glm::mat4(1.0), glm::radians(_z), 90 | glm::vec3(0.0f, 0.0f, 1.0f)); 91 | 92 | glm::mat4 Modelshift = glm::mat4(1.0); 93 | Modelshift[3] = glm::vec4(xshift, yshift, zshift, 1.0); 94 | 95 | glm::mat4 ModelMatirx = Modelshift * Modelz * Modely * Modelx; 96 | 97 | return ModelMatirx; 98 | } 99 | 100 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_2_fbo.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | 3 | /////////////////////////////////////////////////////////////////////// 4 | void createfbo(GLuint &fbo, GLuint &tex, int tid, int hei, int wid) { 5 | // GLuint fbo = 0; 6 | glGenFramebuffers(1, &fbo); 7 | glBindFramebuffer(GL_FRAMEBUFFER, fbo); 8 | 9 | // create a texture 10 | // GLuint tex_screen; 11 | glGenTextures(1, &tex); 12 | glActiveTexture(GL_TEXTURE0 + tid); 13 | glBindTexture(GL_TEXTURE_2D, tex); 14 | 15 | /* 16 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, wid, hei, 0, GL_RGB, 17 | GL_UNSIGNED_BYTE, NULL); 18 | 19 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); 20 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); 21 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 22 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, 23 | GL_LINEAR_MIPMAP_LINEAR); 24 | 25 | // ... which requires mipmaps. Generate them automatically. 26 | glGenerateMipmap(GL_TEXTURE_2D); 27 | */ 28 | 29 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, wid, hei, 0, GL_RGBA, 30 | GL_UNSIGNED_BYTE, NULL); 31 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 32 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 33 | 34 | glBindTexture(GL_TEXTURE_2D, 0); 35 | 36 | glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, 37 | tex, 0); 38 | 39 | // depth 40 | GLuint rbo; 41 | glGenRenderbuffers(1, &rbo); 42 | glBindRenderbuffer(GL_RENDERBUFFER, rbo); 43 | glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, wid, hei); 44 | glBindRenderbuffer(GL_RENDERBUFFER, 0); 45 | 46 | glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, 47 | GL_RENDERBUFFER, rbo); 48 | 49 | if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) 50 | cout << "ERROR::FRAMEBUFFER::" 51 | "Framebuffer is not complete!" << endl; 52 | 53 | glBindFramebuffer(GL_FRAMEBUFFER, 0); 54 | cout << "fbo\t" << fbo << "\trbo\t" << rbo << "\ttex_screen\t" << tex 55 | << "\ttex_act\t" << tid << endl; 56 | 57 | std::cout << "createfbo error\t" << glGetError() << std::endl; 58 | } 59 | 60 | void createbigtex(GLuint &tex, int tid, int hei, int wid, int maxsz, 61 | int &maxwidth) { 62 | // GLuint tex_screen; 63 | glGenTextures(1, &tex); 64 | glActiveTexture(GL_TEXTURE0 + tid); 65 | glBindTexture(GL_TEXTURE_2D, tex); 66 | 67 | glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 68 | 69 | /////////////////////////////////////////////////////// 70 | assert(hei == wid); 71 | glGetIntegerv(GL_MAX_TEXTURE_SIZE, &maxwidth); 72 | cout << "maxlen\t" << maxwidth << endl; 73 | if (maxsz * wid > maxwidth) { 74 | cout << "maxsz * width\t" << maxsz * wid << endl; 75 | cout << "too big maxsz" << endl; 76 | exit(1); 77 | } 78 | 79 | maxwidth = wid * maxsz; 80 | cout << "use\t" << maxwidth << endl; 81 | 82 | /////////////////////////////////////////////////////////// 83 | /* 84 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, maxwidth, maxwidth, 0, GL_RGB, 85 | GL_UNSIGNED_BYTE, NULL); 86 | 87 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); 88 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); 89 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 90 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, 91 | GL_LINEAR_MIPMAP_LINEAR); 92 | 93 | // ... which requires mipmaps. Generate them automatically. 94 | glGenerateMipmap(GL_TEXTURE_2D); 95 | */ 96 | 97 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, maxwidth, maxwidth, 0, GL_RGBA, 98 | GL_UNSIGNED_BYTE, NULL); 99 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); 100 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 101 | //glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, 102 | // GL_LINEAR_MIPMAP_LINEAR); 103 | 104 | // ... which requires mipmaps. Generate them automatically. 105 | // glGenerateMipmap(GL_TEXTURE_2D); 106 | 107 | glBindTexture(GL_TEXTURE_2D, 0); 108 | 109 | std::cout << "createbigtex error\t" << glGetError() << std::endl; 110 | } 111 | 112 | void render::initializecuda() { 113 | // create fbo, using first and second texture 114 | // GLuint fbo_obj, tex_screen; 115 | tid = 0; 116 | createfbo(fbo_obj, tex_screen, tid, height, width); 117 | 118 | tid = 1; 119 | createbigtex(tex_saver, tid, height, width, maxsz, maxwidth); 120 | 121 | tid = 2; 122 | 123 | // cuda 124 | initializecuda2(); 125 | } 126 | 127 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_3_program.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | #include 3 | #include 4 | #include 5 | 6 | GLuint render::LoadShaders(const char *vertex_file_path, 7 | const char *fragment_file_path) { 8 | 9 | // Create the shaders 10 | GLuint VertexShaderID = glCreateShader(GL_VERTEX_SHADER); 11 | GLuint FragmentShaderID = glCreateShader(GL_FRAGMENT_SHADER); 12 | 13 | // Read the Vertex Shader code from the file 14 | std::string VertexShaderCode; 15 | std::ifstream VertexShaderStream(vertex_file_path, std::ios::in); 16 | if (VertexShaderStream.is_open()) { 17 | std::stringstream sstr; 18 | sstr << VertexShaderStream.rdbuf(); 19 | VertexShaderCode = sstr.str(); 20 | VertexShaderStream.close(); 21 | } else { 22 | printf( 23 | "Impossible to open %s. Are you in the right directory ? Don't forget to read the FAQ !\n", 24 | vertex_file_path); 25 | getchar(); 26 | return 0; 27 | } 28 | 29 | // Read the Fragment Shader code from the file 30 | std::string FragmentShaderCode; 31 | std::ifstream FragmentShaderStream(fragment_file_path, std::ios::in); 32 | if (FragmentShaderStream.is_open()) { 33 | std::stringstream sstr; 34 | sstr << FragmentShaderStream.rdbuf(); 35 | FragmentShaderCode = sstr.str(); 36 | FragmentShaderStream.close(); 37 | } 38 | 39 | GLint Result = GL_FALSE; 40 | int InfoLogLength; 41 | 42 | // Compile Vertex Shader 43 | printf("Compiling shader : %s\n", vertex_file_path); 44 | char const *VertexSourcePointer = VertexShaderCode.c_str(); 45 | glShaderSource(VertexShaderID, 1, &VertexSourcePointer, NULL); 46 | glCompileShader(VertexShaderID); 47 | 48 | // Check Vertex Shader 49 | glGetShaderiv(VertexShaderID, GL_COMPILE_STATUS, &Result); 50 | glGetShaderiv(VertexShaderID, GL_INFO_LOG_LENGTH, &InfoLogLength); 51 | if (InfoLogLength > 0) { 52 | std::vector VertexShaderErrorMessage(InfoLogLength + 1); 53 | glGetShaderInfoLog(VertexShaderID, InfoLogLength, NULL, 54 | &VertexShaderErrorMessage[0]); 55 | printf("%s\n", &VertexShaderErrorMessage[0]); 56 | } 57 | 58 | // Compile Fragment Shader 59 | printf("Compiling shader : %s\n", fragment_file_path); 60 | char const *FragmentSourcePointer = FragmentShaderCode.c_str(); 61 | glShaderSource(FragmentShaderID, 1, &FragmentSourcePointer, NULL); 62 | glCompileShader(FragmentShaderID); 63 | 64 | // Check Fragment Shader 65 | glGetShaderiv(FragmentShaderID, GL_COMPILE_STATUS, &Result); 66 | glGetShaderiv(FragmentShaderID, GL_INFO_LOG_LENGTH, &InfoLogLength); 67 | if (InfoLogLength > 0) { 68 | std::vector FragmentShaderErrorMessage(InfoLogLength + 1); 69 | glGetShaderInfoLog(FragmentShaderID, InfoLogLength, NULL, 70 | &FragmentShaderErrorMessage[0]); 71 | printf("%s\n", &FragmentShaderErrorMessage[0]); 72 | } 73 | 74 | // Link the program 75 | printf("Linking program\n"); 76 | GLuint ProgramID = glCreateProgram(); 77 | glAttachShader(ProgramID, VertexShaderID); 78 | glAttachShader(ProgramID, FragmentShaderID); 79 | glLinkProgram(ProgramID); 80 | 81 | // Check the program 82 | glGetProgramiv(ProgramID, GL_LINK_STATUS, &Result); 83 | glGetProgramiv(ProgramID, GL_INFO_LOG_LENGTH, &InfoLogLength); 84 | if (InfoLogLength > 0) { 85 | std::vector ProgramErrorMessage(InfoLogLength + 1); 86 | glGetProgramInfoLog(ProgramID, InfoLogLength, NULL, 87 | &ProgramErrorMessage[0]); 88 | printf("%s\n", &ProgramErrorMessage[0]); 89 | } 90 | 91 | glDetachShader(ProgramID, VertexShaderID); 92 | glDetachShader(ProgramID, FragmentShaderID); 93 | 94 | glDeleteShader(VertexShaderID); 95 | glDeleteShader(FragmentShaderID); 96 | 97 | return ProgramID; 98 | } 99 | 100 | ////////////////////////////////////////////////////////////////// 101 | void render::programobj() { 102 | ////////////////////////////////////////////////////////////////////// 103 | // Create and compile our GLSL program from the shaders 104 | programID = LoadShaders("pointlight.vertexshader", 105 | "pointlight.fragmentshader"); 106 | 107 | // first program 108 | glUseProgram(programID); 109 | 110 | ///////////////////////////////////////////////////////////////////// 111 | // M & V will change! 112 | ModelMatrixID = glGetUniformLocation(programID, "M"); 113 | ViewMatrixID = glGetUniformLocation(programID, "V"); 114 | 115 | // P will not change! 116 | GLuint PersepctiveMatrixID = glGetUniformLocation(programID, "P"); 117 | glm::mat4 ProjectionMatrix = getProjectionMatrix(); 118 | glUniformMatrix4fv(PersepctiveMatrixID, 1, 119 | GL_FALSE, &ProjectionMatrix[0][0]); 120 | 121 | lightPosition_modelspace = glGetUniformLocation(programID, 122 | "lightPosition_modelspace"); 123 | 124 | ////////////////////////////////////////////////////// 125 | // it will change 126 | MaterialAmbient = glGetUniformLocation(programID, "MaterialAmbient"); 127 | MaterialDiffuse = glGetUniformLocation(programID, "MaterialDiffuse"); 128 | MaterialSpecular = glGetUniformLocation(programID, "MaterialSpecular"); 129 | 130 | Shininess = glGetUniformLocation(programID, "Shininess"); 131 | 132 | //////////////////////////////////////////////////////////////////////////// 133 | // Get a handle for our "myTextureSampler" uniform 134 | // Set our "myTextureSampler" sampler to use Texture Unit 2 135 | TextureID = glGetUniformLocation(programID, "myTextureSampler"); 136 | istex = glGetUniformLocation(programID, "istex"); 137 | 138 | isprojection = glGetUniformLocation(programID, "isprojection"); 139 | isconfocal = glGetUniformLocation(programID, "isconfocal"); 140 | isalbedo = glGetUniformLocation(programID, "isalbedo"); 141 | 142 | glUseProgram(0); 143 | std::cout << "programobj 1 error\t" << glGetError() << std::endl; 144 | 145 | return; 146 | } 147 | 148 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_4_loaddata.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | ////////////////////////////////////////////////////// 8 | // this is used to get meaningful data 9 | // input "f 1/2/3 1/2/3 1/2/3 " 10 | // output "f" "1/2/3" "1/2/3" "1/2/3" 11 | void SplitString(const std::string &s, std::vector &v, 12 | const std::string &c) { 13 | std::string::size_type pos1, pos2; 14 | pos1 = 0; 15 | pos2 = s.find(c); 16 | while (std::string::npos != pos2) { 17 | if (pos1 < pos2) 18 | v.push_back(s.substr(pos1, pos2 - pos1)); 19 | 20 | pos1 = pos2 + c.size(); 21 | pos2 = s.find(c, pos1); 22 | } 23 | 24 | if (pos1 != s.length()) 25 | v.push_back(s.substr(pos1)); 26 | } 27 | 28 | // this is used to get all data 29 | // input "1//" 30 | // output "1" "" "" 31 | void SplitString2(const std::string &s, std::vector &v, 32 | const std::string &c) { 33 | std::string::size_type pos1, pos2; 34 | pos1 = 0; 35 | pos2 = s.find(c); 36 | vector pos; 37 | while (std::string::npos != pos2) { 38 | pos.push_back(pos2); 39 | pos1 = pos2 + c.size(); 40 | pos2 = s.find(c, pos1); 41 | } 42 | 43 | if (pos.size() == 0) { 44 | v.push_back(s); 45 | return; 46 | } 47 | 48 | int be = -1; 49 | int en = -1; 50 | for (int i = 0; i < pos.size(); i++) { 51 | if (i == 0) { 52 | be = 0; 53 | en = pos[i]; 54 | } else { 55 | be = pos[i - 1] + c.size(); 56 | en = pos[i]; 57 | } 58 | v.push_back(s.substr(be, en - be)); 59 | } 60 | v.push_back(s.substr(pos[pos.size() - 1] + c.size())); 61 | } 62 | 63 | //////////////////////////////////////////////////////////////////// 64 | //mtl 65 | bool readmtl(string folder, string mtlname, vector &mas, 66 | vector &ims, map &m_map //matMap[material_name] = material_ID 67 | ) { 68 | string fullname = folder + "/" + mtlname; 69 | ifstream fp(fullname.c_str()); 70 | if (fp.fail()) { 71 | cout << "error, cannot load mtl file!" << endl; 72 | cout << fullname << endl; 73 | // system("pause"); 74 | return false; 75 | } 76 | 77 | // mas has no material 78 | assert(mas.size() == 0); 79 | assert(ims.size() == 0); 80 | assert(m_map.size() == 0); 81 | 82 | vector all_list; 83 | int midx = -1; 84 | int mconponent = 5; 85 | 86 | string line; 87 | while (getline(fp, line)) { 88 | vector parts; 89 | SplitString(line, parts, " "); 90 | if (parts.size() < 2) 91 | continue; 92 | if (parts[0] == "#") 93 | continue; 94 | 95 | if (parts[0] == "newmtl") { 96 | assert(parts.size() == 2); 97 | // assert(mconponent == 5); 98 | 99 | material tmp; 100 | tmp.mtlname = parts[1]; 101 | all_list.push_back(tmp); 102 | midx += 1; 103 | mconponent = 0; 104 | } 105 | 106 | if (parts[0] == "Ka") { 107 | assert(parts.size() == 4); 108 | for (int i = 0; i < 3; i++) { 109 | double kai = stod(parts[1 + i]); 110 | // assert(kai == 0); 111 | /* 112 | if (kai != 0) { 113 | cout << "non zero ambient" << endl; 114 | return false; 115 | } 116 | */ 117 | all_list[midx].Ka[i] = kai; 118 | } 119 | mconponent++; 120 | } 121 | 122 | if (parts[0] == "Kd") { 123 | assert(parts.size() == 4); 124 | for (int i = 0; i < 3; i++) { 125 | double kdi = stod(parts[1 + i]); 126 | assert(kdi >= 0); 127 | all_list[midx].Kd[i] = kdi; 128 | } 129 | mconponent++; 130 | } 131 | 132 | if (parts[0] == "Ks") { 133 | assert(parts.size() == 4); 134 | for (int i = 0; i < 3; i++) { 135 | double ksi = stod(parts[1 + i]); 136 | assert(ksi >= 0); 137 | all_list[midx].Ks[i] = ksi; 138 | } 139 | mconponent++; 140 | } 141 | 142 | if (parts[0] == "Ns") { 143 | assert(parts.size() == 2); 144 | int ns = stoi(parts[1]); 145 | // assert(ns == 10); 146 | /* 147 | if (ns != 10) { 148 | cout << "ns is not 10" << endl; 149 | return false; 150 | } 151 | */ 152 | all_list[midx].Ns = ns; 153 | mconponent++; 154 | } 155 | 156 | if (parts[0] == "illum") { 157 | assert(parts.size() == 2); 158 | int illum = stoi(parts[1]); 159 | // assert(illum == 2); 160 | all_list[midx].illum = illum; 161 | mconponent++; 162 | } 163 | 164 | if (parts[0] == "map_Kd") { 165 | assert(parts.size() == 2); 166 | string filename = folder + "/" + parts[1]; 167 | all_list[midx].map_Kd = filename; 168 | all_list[midx].istex = true; 169 | } 170 | } 171 | fp.close(); 172 | 173 | // sort mas! 174 | vector gray_list; 175 | vector tex_list; 176 | for (int i = 0; i < all_list.size(); i++) 177 | if (all_list[i].istex) { 178 | // make sure that it is valid texutre 179 | cv::Mat im = cv::imread(all_list[i].map_Kd); 180 | // assert(!im.empty()); 181 | if (im.empty()) { 182 | cout << "empty image" << endl; 183 | return false; 184 | } 185 | 186 | if ((im.type() != CV_8UC1) && (im.type() != CV_8UC3)) { 187 | cout << "wrong type image" << endl; 188 | return false; 189 | } 190 | 191 | tex_list.push_back(all_list[i]); 192 | ims.push_back(im); 193 | 194 | } else 195 | gray_list.push_back(all_list[i]); 196 | 197 | for (int i = 0; i < tex_list.size(); i++) 198 | mas.push_back(tex_list[i]); 199 | for (int i = 0; i < gray_list.size(); i++) 200 | mas.push_back(gray_list[i]); 201 | 202 | for (int i = 0; i < mas.size(); i++) 203 | m_map[mas[i].mtlname] = i; 204 | 205 | return true; 206 | } 207 | 208 | bool readobj(string folder, string obj_name, vector &v_list, 209 | vector &vt_list, vector &fv_list, 210 | vector &fvt_list, vector &fm_list, 211 | vector &m_list, vector &ims) { 212 | 213 | string fullname = folder + "/" + obj_name; 214 | ifstream fp(fullname.c_str()); 215 | if (fp.fail()) { 216 | cout << "error, cannot load obj file!" << endl; 217 | // system("pause"); 218 | return false; 219 | } 220 | 221 | // ��������������������������������������������� 222 | // mtl, use, v, vt, f 223 | float xmin = 10000, ymin = 10000, zmin = 10000, xmax = -10000, 224 | ymax = -10000, zmax = -10000; 225 | 226 | map material_map; // matMap[material_name] = material_ID 227 | int current_material = -1; 228 | 229 | string line; 230 | while (getline(fp, line)) { 231 | vector parts; 232 | SplitString(line, parts, " "); 233 | if (parts.size() < 2) 234 | continue; 235 | if (parts[0] == "#") 236 | continue; 237 | 238 | if (parts[0] == "mtllib") { 239 | assert(parts.size() == 2); 240 | bool suc = readmtl(folder, parts[1], m_list, ims, material_map); 241 | if (!suc) { 242 | cout << "error, cannot load mtl file!" << endl; 243 | // system("pause"); 244 | return false; 245 | } 246 | } 247 | 248 | if (parts[0] == "v") { 249 | assert(parts.size() == 4); 250 | // cout<<"v\t"< xmax) 260 | xmax = tmp[0]; 261 | if (tmp[1] < ymin) 262 | ymin = tmp[1]; 263 | if (tmp[1] > ymax) 264 | ymax = tmp[1]; 265 | if (tmp[2] < zmin) 266 | zmin = tmp[2]; 267 | if (tmp[2] > zmax) 268 | zmax = tmp[2]; 269 | v_list.push_back(tmp); 270 | } 271 | 272 | if (parts[0] == "vt") { 273 | assert(parts.size() == 3 || parts.size() == 4); 274 | if (parts.size() == 4) 275 | assert(stod(parts[3]) == 0.0f); 276 | // cout<<"vt\t"<::iterator iter; 286 | iter = material_map.find(parts[1]); 287 | assert(iter != material_map.end()); 288 | current_material = iter->second; 289 | } 290 | 291 | if (parts[0] == "f") { 292 | // triangles! 293 | assert(parts.size() == 4); 294 | glm::ivec3 fv(0, 0, 0); 295 | glm::ivec3 fvt(0, 0, 0); 296 | bool faceparse = true; 297 | 298 | for (int i = 0; i < 3; i++) { 299 | vector nums; 300 | SplitString2(parts[i + 1], nums, "/"); 301 | 302 | // v/vt/n 303 | fv[i] = stoi(nums[0]); 304 | 305 | if (m_list[current_material].istex) { 306 | // if (debug) 307 | // assert(nums[1] != ""); 308 | // it should be texture cood but there is no coord 309 | // continue 310 | if (nums[1] == "") { 311 | faceparse = false; 312 | break; 313 | } 314 | fvt[i] = stoi(nums[1]); 315 | } 316 | } 317 | 318 | if (faceparse && fv[0] != fv[1] && fv[0] != fv[2] 319 | && fv[1] != fv[2]) { 320 | fm_list.push_back(current_material); 321 | fv_list.push_back(fv); 322 | fvt_list.push_back(fvt); 323 | } 324 | } 325 | } 326 | fp.close(); 327 | 328 | if (xmin < -0.5 || xmax > 0.5 || ymin < -0.5 || ymax > 0.5 || zmin < -0.5 329 | || zmax > 0.5) 330 | return false; 331 | 332 | // resize 333 | cout << "xmin\t" << xmin << "\txmax" << xmax << endl; 334 | cout << "ymin\t" << ymin << "\tymax" << ymax << endl; 335 | cout << "zmin\t" << zmin << "\tzmax" << zmax << endl; 336 | 337 | glm::vec3 shift = glm::vec3((xmin + xmax) / 2, (ymin + ymax) / 2, 338 | (zmin + zmax) / 2); 339 | float scale = std::max(zmax - zmin, std::max(ymax - ymin, xmax - xmin)); 340 | /* 341 | scale = 2 342 | * (static_cast(rand()) / static_cast(RAND_MAX) * 0.2 343 | + 0.5) / scale; 344 | */ 345 | float scale2 = 0.6 346 | + static_cast(rand()) / static_cast(RAND_MAX) * 0.6; 347 | 348 | xmin = 10000; 349 | ymin = 10000; 350 | zmin = 10000; 351 | xmax = -10000; 352 | ymax = -10000; 353 | zmax = -10000; 354 | for (int i = 0; i < v_list.size(); i++) { 355 | glm::vec3 tmp = (v_list[i] - shift) * scale2 / scale; 356 | v_list[i] = tmp; 357 | 358 | if (tmp[0] < xmin) 359 | xmin = tmp[0]; 360 | if (tmp[0] > xmax) 361 | xmax = tmp[0]; 362 | if (tmp[1] < ymin) 363 | ymin = tmp[1]; 364 | if (tmp[1] > ymax) 365 | ymax = tmp[1]; 366 | if (tmp[2] < zmin) 367 | zmin = tmp[2]; 368 | if (tmp[2] > zmax) 369 | zmax = tmp[2]; 370 | } 371 | 372 | // resize 373 | cout << "xmin\t" << xmin << "\txmax" << xmax << endl; 374 | cout << "ymin\t" << ymin << "\tymax" << ymax << endl; 375 | cout << "zmin\t" << zmin << "\tzmax" << zmax << endl; 376 | cout << "scale\t" << scale << endl; 377 | 378 | return true; 379 | } 380 | 381 | // if use texture 382 | // then all the faces must have texture index 383 | // even if the face may use mtl that do not have any texture!!! 384 | mesh render::loadobj(string folder, string obj_name, bool &suc) { 385 | vector v_list; 386 | vector vt_list; 387 | vector fv_list; 388 | vector fvt_list; 389 | vector fm_list; 390 | vector m_list; 391 | vector ims; 392 | 393 | bool objsuc = readobj(folder, obj_name, v_list, vt_list, fv_list, fvt_list, 394 | fm_list, m_list, ims); 395 | if (!objsuc) { 396 | suc = false; 397 | cout << "unsuccessful loading obj!" << endl; 398 | mesh tmp; 399 | return tmp; 400 | } 401 | 402 | // calculate mesh 403 | vector pointnormals; 404 | pointnormals.resize(v_list.size()); 405 | for (int i = 0; i < v_list.size(); i++) 406 | pointnormals[i] = glm::vec3(0, 0, 0); 407 | 408 | vector facenormals; 409 | facenormals.resize(fv_list.size()); 410 | for (int i = 0; i < fv_list.size(); i++) { 411 | glm::vec3 p0 = v_list[fv_list[i][0] - 1]; 412 | glm::vec3 p1 = v_list[fv_list[i][1] - 1]; 413 | glm::vec3 p2 = v_list[fv_list[i][2] - 1]; 414 | 415 | glm::vec3 v01 = p1 - p0; 416 | glm::vec3 v02 = p2 - p0; 417 | facenormals[i] = glm::cross(v01, v02); 418 | 419 | pointnormals[fv_list[i][0] - 1] += facenormals[i]; 420 | pointnormals[fv_list[i][1] - 1] += facenormals[i]; 421 | pointnormals[fv_list[i][2] - 1] += facenormals[i]; 422 | } 423 | 424 | // how many material in total 425 | int texnum = 0; 426 | vector parts; 427 | int fsize = fm_list.size(); 428 | for (int i = 0; i < m_list.size(); i++) { 429 | group tmp; 430 | if (m_list[i].map_Kd != "") { 431 | texnum++; 432 | tmp.istex = true; 433 | tmp.texid = i; 434 | } else { 435 | tmp.istex = false; 436 | tmp.texid = -1; 437 | } 438 | parts.push_back(tmp); 439 | } 440 | 441 | int currentp = -1; 442 | for (int i = 0; i < fm_list.size(); i++) { 443 | currentp = fm_list[i]; 444 | 445 | // Get the indices of its attributes 446 | int vertexIndex0 = fv_list[i][0]; 447 | int vertexIndex1 = fv_list[i][1]; 448 | int vertexIndex2 = fv_list[i][2]; 449 | 450 | glm::vec3 vertex0 = v_list[vertexIndex0 - 1]; 451 | glm::vec3 vertex1 = v_list[vertexIndex1 - 1]; 452 | glm::vec3 vertex2 = v_list[vertexIndex2 - 1]; 453 | 454 | // Put the attributes in buffers 455 | parts[currentp].vertices.push_back(vertex0); 456 | parts[currentp].vertices.push_back(vertex1); 457 | parts[currentp].vertices.push_back(vertex2); 458 | 459 | glm::vec3 diff1 = vertex1 - vertex0; 460 | glm::vec3 diff2 = vertex2 - vertex0; 461 | glm::vec3 cross = glm::cross(diff1, diff2); 462 | glm::vec3 normal = glm::normalize(cross); 463 | 464 | // bunny we use smooth nriomal 465 | /* 466 | parts[currentp].normals.push_back(normal); 467 | parts[currentp].normals.push_back(normal); 468 | parts[currentp].normals.push_back(normal); 469 | */ 470 | 471 | parts[currentp].normals.push_back(pointnormals[vertexIndex0 - 1]); 472 | parts[currentp].normals.push_back(pointnormals[vertexIndex1 - 1]); 473 | parts[currentp].normals.push_back(pointnormals[vertexIndex2 - 1]); 474 | 475 | 476 | if (parts[currentp].istex) { 477 | for (int k = 0; k < 3; k++) { 478 | int uvIndex = fvt_list[i][k]; 479 | assert(uvIndex > 0); 480 | glm::vec2 uv = vt_list[uvIndex - 1]; 481 | parts[currentp].uvs.push_back(uv); 482 | } 483 | } 484 | } 485 | 486 | for (int i = 0; i < parts.size(); i++) 487 | cout << i << "\t" << parts[i].vertices.size() << endl; 488 | 489 | mesh re; 490 | re.vertices = v_list; 491 | re.ims = ims; 492 | re.meshes = parts; 493 | re.mas = m_list; 494 | re.texnum = texnum; 495 | return re; 496 | } 497 | 498 | -------------------------------------------------------------------------------- /cuda-render/render/src/display_5_draw.cpp: -------------------------------------------------------------------------------- 1 | #include "renderclass.h" 2 | 3 | void render::loadmesh(mesh obj) { 4 | objnums = obj.meshes.size(); 5 | 6 | VAOs.resize(objnums); 7 | VBOs.resize(objnums); 8 | normals.resize(objnums); 9 | 10 | glGenVertexArrays(objnums, &VAOs[0]); 11 | glGenBuffers(objnums, &VBOs[0]); 12 | glGenBuffers(objnums, &normals[0]); 13 | 14 | texnums = obj.texnum; 15 | if (texnums > 0) { 16 | uvs.resize(texnums); 17 | textures.resize(texnums); 18 | 19 | glGenBuffers(texnums, &uvs[0]); 20 | glGenTextures(texnums, &textures[0]); 21 | 22 | for (int i = 0; i < texnums; i++) { 23 | cv::Mat img = obj.ims[i]; 24 | int length = img.rows * img.cols * 3; 25 | uchar *buffer = new uchar[length]; 26 | for (int ih = 0; ih < img.rows; ih++) 27 | for (int jw = 0; jw < img.cols; jw++) 28 | for (int kc = 0; kc < 3; kc++) { 29 | int idx = (img.rows - 1 - ih) * img.cols * 3 + jw * 3 30 | + 2 - kc; 31 | if (img.type() == CV_8UC3) 32 | buffer[idx] = img.at(ih, jw)[kc]; 33 | else 34 | buffer[idx] = img.at(ih, jw); 35 | } 36 | 37 | // "Bind" the newly created texture : all future texture functions will modify this texture 38 | glActiveTexture(GL_TEXTURE0 + tid + i); 39 | 40 | glBindTexture(GL_TEXTURE_2D, textures[i]); 41 | glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 42 | 43 | // Give the image to OpenGL 44 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, img.cols, img.rows, 0, 45 | GL_RGB, 46 | GL_UNSIGNED_BYTE, buffer); 47 | 48 | // Poor filtering, or ... 49 | //glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 50 | //glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 51 | 52 | // ... nice trilinear filtering ... 53 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); 54 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); 55 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); 56 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, 57 | GL_LINEAR_MIPMAP_LINEAR); 58 | 59 | // ... which requires mipmaps. Generate them automatically. 60 | glGenerateMipmap(GL_TEXTURE_2D); 61 | 62 | free(buffer); 63 | } 64 | } 65 | 66 | for (int i = 0; i < objnums; i++) { 67 | glBindVertexArray(VAOs[i]); 68 | 69 | group tmp = obj.meshes[i]; 70 | glBindBuffer(GL_ARRAY_BUFFER, VBOs[i]); 71 | glBufferData(GL_ARRAY_BUFFER, tmp.vertices.size() * sizeof(glm::vec3), 72 | &tmp.vertices[0], GL_STATIC_DRAW); 73 | 74 | // 1rst attribute buffer : vertices 75 | glEnableVertexAttribArray(0); 76 | glVertexAttribPointer(0, // attribute 77 | 3, // size 78 | GL_FLOAT, // type 79 | GL_FALSE, // normalized? 80 | 0, // stride 81 | (void*) 0 // array buffer offset 82 | ); 83 | 84 | glBindBuffer(GL_ARRAY_BUFFER, normals[i]); 85 | glBufferData(GL_ARRAY_BUFFER, tmp.normals.size() * sizeof(glm::vec3), 86 | &tmp.normals[0], GL_STATIC_DRAW); 87 | 88 | // 3rd attribute buffer : normals 89 | glEnableVertexAttribArray(1); 90 | glVertexAttribPointer(1, // attribute 91 | 3, // size 92 | GL_FLOAT, // type 93 | GL_FALSE, // normalized? 94 | 0, // stride 95 | (void*) 0 // array buffer offset 96 | ); 97 | 98 | if (tmp.istex) { 99 | glBindBuffer(GL_ARRAY_BUFFER, uvs[i]); 100 | glBufferData(GL_ARRAY_BUFFER, tmp.uvs.size() * sizeof(glm::vec2), 101 | &tmp.uvs[0], 102 | GL_STATIC_DRAW); 103 | 104 | // 2nd attribute buffer : UVs 105 | glEnableVertexAttribArray(2); 106 | glVertexAttribPointer(2, // attribute 107 | 2, // size 108 | GL_FLOAT, // type 109 | GL_FALSE, // normalized? 110 | 0, // stride 111 | (void*) 0 // array buffer offset 112 | ); 113 | } 114 | 115 | } 116 | } 117 | 118 | void render::deletemesh() { 119 | // glDisableVertexAttribArray(0); 120 | // glDisableVertexAttribArray(1); 121 | // glDisableVertexAttribArray(2); 122 | 123 | // Cleanup VBO and shader 124 | glDeleteBuffers(objnums, &VBOs[0]); 125 | glDeleteBuffers(objnums, &normals[0]); 126 | 127 | if (texnums > 0) { 128 | glDeleteBuffers(texnums, &uvs[0]); 129 | glDeleteTextures(texnums, &textures[0]); 130 | } 131 | 132 | glDeleteVertexArrays(objnums, &VAOs[0]); 133 | 134 | VBOs.clear(); 135 | normals.clear(); 136 | uvs.clear(); 137 | textures.clear(); 138 | VAOs.clear(); 139 | } 140 | 141 | -------------------------------------------------------------------------------- /cuda-render/render/src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | using namespace std; 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "renderclass.h" 9 | 10 | void getFiles(string parentFolder, vector &vecFileNames) { 11 | 12 | string cmd = "ls " + parentFolder + " > temp.log"; 13 | system(cmd.c_str()); 14 | 15 | ifstream ifs("temp.log"); 16 | //������������������log��������������������������������������������������������������� 17 | if (ifs.fail()) { 18 | return; 19 | } 20 | 21 | string fileName; 22 | while (getline(ifs, fileName)) { 23 | vecFileNames.push_back(parentFolder + "/" + fileName); 24 | } 25 | 26 | ifs.close(); 27 | return; 28 | } 29 | 30 | int main() { 31 | 32 | string parentFlder = "../../data/bunny-model"; 33 | 34 | string parentSvFolder = "../../data/bunny-renders"; 35 | 36 | // to change to shapenet models 37 | // parentFlder = "/u6/a/wenzheng/remote2/datasets/shapenet/ShapeNetCore.v2/03790512"; 38 | // parentSvFolder = "/u6/a/wenzheng/remote3/dataset-generated/bikenonfocal-shape_0.6_1.2-shift_0.4_0.4_-0.4_0.4"; 39 | 40 | vector folders; 41 | getFiles(parentFlder, folders); 42 | 43 | // use predefined rotation and shift or not 44 | // in this case, we do not rotate the model 45 | bool definerot = true; 46 | float rotx = 0; 47 | float roty = 0; 48 | float rotz = -0; 49 | float shiftx = 0; 50 | float shifty = 0; 51 | float shiftz = -0; 52 | 53 | /* 54 | * To render a model with specific rotation parameters 55 | string mdname = "2d655fc4ecb2df2a747c19778aa6cc0"; 56 | folders.clear(); 57 | folders.push_back(parentFlder + "/" + mdname); 58 | float rotx = 2.9725; 59 | float roty = -88.7643; 60 | float rotz = -7.4680; 61 | float shiftx = 0.1451; 62 | float shifty = -0.1690; 63 | float shiftz = -0.0214; 64 | */ 65 | 66 | char cmd[256]; 67 | sprintf(cmd, "mkdir %s", parentSvFolder.c_str()); 68 | system(cmd); 69 | 70 | for (int i = 0; i < 1; i++) { 71 | string svfolder = parentSvFolder + "/" + to_string(i); 72 | 73 | char cmd[256]; 74 | sprintf(cmd, "mkdir %s", svfolder.c_str()); 75 | system(cmd); 76 | } 77 | 78 | int height = 256; 79 | int width = 256; 80 | 81 | // each model, we sample 100x100, the more the better, but takes longer time. 82 | int samplenum = 600; 83 | // we sample directions over a semisphjere, however, many direction is meaningful, so we can discard some directions 84 | // 0.8 means we sample based on a cone instead of a semisphere. 85 | float sampleratio = 0.8; 86 | 87 | // we will render each sample individually and copy it to a big texture. Finally we merge it. 88 | // the big texture map size is maxsz * height. 89 | int blocksz = 40; 90 | 91 | render *tmp = new render(height, width, blocksz, sampleratio); 92 | tmp->initializecuda(); 93 | tmp->programobj(); 94 | 95 | // how many renders do we want 96 | // in this example, we render only one model. 97 | int rendernum = 1; 98 | int i = -1; 99 | int step = 0; 100 | 101 | // for (int i = 0; i < folders.size(); i++) { 102 | while (true) { 103 | 104 | i = (i + 1) % folders.size(); 105 | 106 | string folder = folders[i] + "/model"; 107 | string name = "model_normalized.obj"; 108 | 109 | // string folder = folders[i]; 110 | // string name = "model.obj"; 111 | 112 | bool suc = true; 113 | mesh tmpobj = tmp->loadobj(folder, name, suc); 114 | if (!suc) { 115 | continue; 116 | } else { 117 | 118 | tmp->loadmesh(tmpobj); 119 | 120 | // how many rotation we have 121 | int rnum = 1; 122 | // how many lights we set, only 1 light for non confocal case. 123 | int lvnum = 1; 124 | int lhnum = 1; 125 | 126 | int shininesslevel = 0; 127 | for (shininesslevel = 0; shininesslevel < 1; shininesslevel++) { 128 | 129 | int pos = folders[i].find_last_of('/'); 130 | string svfolder = parentSvFolder + "/" 131 | + to_string(shininesslevel) + "/" 132 | + folders[i].substr(pos + 1, 133 | folders[i].length() - pos - 1); 134 | cout << folder << "\t" << name << endl; 135 | cout << "svfolder\t" << svfolder << endl; 136 | 137 | char cmd[256]; 138 | sprintf(cmd, "mkdir %s", svfolder.c_str()); 139 | system(cmd); 140 | 141 | // say the smapling number is 80x80 142 | // the blacksz is 40 143 | // so we need 2x2 block 144 | // hnum and vnum should be divided by blocksz!!! 145 | int actualsample = samplenum * sampleratio; 146 | int sz = blocksz; 147 | assert(actualsample % sz == 0); 148 | int hnum = actualsample; 149 | int vnum = actualsample; 150 | 151 | tmp->display(svfolder, tmpobj, shininesslevel, sz, rnum, lhnum, 152 | lvnum, hnum, vnum, definerot, rotx, roty, rotz, shiftx, 153 | shifty, shiftz); 154 | } 155 | 156 | tmp->deletemesh(); 157 | 158 | step++; 159 | if (step >= rendernum) 160 | break; 161 | } 162 | } 163 | 164 | delete tmp; 165 | cout << "done!" << endl; 166 | return 0; 167 | } 168 | 169 | -------------------------------------------------------------------------------- /cuda-render/render/src/renderclass.h: -------------------------------------------------------------------------------- 1 | #ifndef RENDER_CLASS_H_ 2 | #define RENDER_CLASS_H_ 3 | 4 | #include 5 | using namespace std; 6 | 7 | #include 8 | #include 9 | 10 | // Include GLEW 11 | #include 12 | #include 13 | 14 | // Include GLM 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | //////////////////////////////////////////////////////////// 24 | // mesh object 25 | typedef struct { 26 | string mtlname = ""; 27 | float Ka[3] = { 0.0f, 0.0f, 0.0f }; 28 | float Kd[3] = { 1.0f, 1.0f, 1.0f }; 29 | float Ks[3] = { 0.0f, 0.0f, 0.0f }; 30 | int Ns = 10; 31 | int illum = 2; 32 | string map_Kd = ""; 33 | bool istex = false; 34 | } material; 35 | 36 | typedef struct { 37 | std::vector vertices; 38 | std::vector normals; 39 | std::vector uvs; 40 | bool istex = false; 41 | int texid = -1; 42 | } group; 43 | 44 | typedef struct { 45 | vector vertices; 46 | vector meshes; 47 | vector mas; 48 | vector ims; 49 | int texnum = 0; 50 | } mesh; 51 | 52 | // render 53 | class render { 54 | public: 55 | render(int height, int width, int maxsz, double reduceratio = 0.8) { 56 | this->height = height; 57 | this->width = width; 58 | this->maxsz = maxsz; 59 | this->reduceratio = reduceratio; 60 | 61 | initialGLFW(); 62 | initialGLEW(); 63 | initialGL(); 64 | } 65 | 66 | ~render() { 67 | 68 | delete[] result; 69 | delete[] result3; 70 | 71 | glDisableVertexAttribArray(0); 72 | glDisableVertexAttribArray(1); 73 | glDisableVertexAttribArray(2); 74 | 75 | // unregister this buffer object with CUDA 76 | glBindTexture(GL_TEXTURE_2D, 0); 77 | glDeleteTextures(1, &tex_screen); 78 | glDeleteTextures(1, &tex_saver); 79 | 80 | glBindFramebuffer(GL_FRAMEBUFFER, 0); 81 | glDeleteFramebuffers(1, &fbo_obj); 82 | 83 | // Use our shader 84 | glUseProgram(0); 85 | glDeleteProgram(programID); 86 | 87 | // Close OpenGL window and terminate GLFW 88 | glfwTerminate(); 89 | } 90 | 91 | public: 92 | // window 93 | GLFWwindow *window; 94 | int height; 95 | int width; 96 | int maxsz; 97 | double reduceratio; 98 | 99 | // Initialize GLFW & GLEW 100 | bool initialGLFW(); 101 | bool initialGLEW(); 102 | bool initialGL(); 103 | 104 | public: 105 | // fbo & big tex 106 | GLuint fbo_obj; 107 | GLuint tex_screen; 108 | GLuint tex_saver; 109 | int maxwidth; 110 | int tid; 111 | 112 | // cuda 113 | struct cudaGraphicsResource *cuda_tex_saver_resource; 114 | float *cuda_dest_saver_resource; 115 | float *result; 116 | 117 | float *cuda_dest_saver_resource3; 118 | float *result3; 119 | 120 | // void createfbo(GLuint &fbo, GLuint &tex, int tid, int h, int w); 121 | // void createbigtex(); 122 | void initializecuda(); 123 | void initializecuda2(); 124 | 125 | public: 126 | // program 127 | GLuint programID; 128 | 129 | GLuint ModelMatrixID; 130 | GLuint ViewMatrixID; 131 | 132 | GLuint lightPosition_modelspace; 133 | 134 | GLuint MaterialAmbient; 135 | GLuint MaterialDiffuse; 136 | GLuint MaterialSpecular; 137 | GLuint Shininess; 138 | 139 | GLuint TextureID; 140 | GLuint istex; 141 | GLuint isconfocal; 142 | GLuint isprojection; 143 | GLuint isalbedo; 144 | 145 | // shader 146 | GLuint LoadShaders(const char *vertex_file_path, 147 | const char *fragment_file_path); 148 | 149 | void programobj(); 150 | 151 | public: 152 | mesh loadobj(string folder, string name, bool &suc); 153 | 154 | int objnums; 155 | int texnums; 156 | vector VAOs; 157 | vector VBOs; 158 | vector normals; 159 | vector uvs; 160 | vector textures; 161 | 162 | void loadmesh(mesh obj); 163 | void drawmesh(mesh obj); 164 | void deletemesh(); 165 | 166 | public: 167 | // camera 168 | ///////////////////////////////////////////////////// 169 | std::vector getViewMatrix(int N, double ratio); 170 | 171 | glm::mat4 getProjectionMatrix(); 172 | 173 | glm::mat4 getModelMatrix(float _x, float _y, float _z, float xshift, 174 | float yshift, float zshift); 175 | 176 | // render params 177 | void display(string svfolder, mesh obj, int shininesslevel, int sz, 178 | int rnum, int lighthnum, int lightvnum, int hnum, int vnum, 179 | bool inirotshift = false, float rotx = 0, float roty = 0, 180 | float rotz = 0, float shiftx = 0, float shifty = 0, 181 | float shiftz = 0); //, vectorrots, vector shifts); 182 | }; 183 | 184 | //////////////////////////////////////////////////////// 185 | 186 | #endif 187 | 188 | -------------------------------------------------------------------------------- /data/bunny-model/bunny/model/model_normalized.mtl: -------------------------------------------------------------------------------- 1 | newmtl material_0_24 2 | Ka 0 0 0 3 | Kd 1 1 1 4 | Ks 0.4 0.4 0.4 5 | Ke 0 0 0 6 | Ns 10 7 | illum 2 8 | -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full.mp4 -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_120.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_130.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_130.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_140.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_140.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_150.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_160.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_170.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_170.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_180.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_190.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_190.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_200.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_210.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_210.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_220.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_220.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_230.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_230.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_240.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_250.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_250.png -------------------------------------------------------------------------------- /data/img/confocal/video-confocal-gray-full_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/confocal/video-confocal-gray-full_all.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full.mp4 -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_100.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_110.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_110.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_120.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_130.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_130.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_140.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_140.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_150.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_160.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_170.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_170.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_180.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_190.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_190.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_200.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_210.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_210.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_220.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_220.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_230.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_230.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_240.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_250.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_250.png -------------------------------------------------------------------------------- /data/img/non-confocal/video-gray-full_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/non-confocal/video-gray-full_all.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full.mp4 -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_100.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_110.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_110.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_120.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_130.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_130.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_140.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_140.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_150.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_160.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_170.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_170.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_180.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_190.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_190.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_200.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_210.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_210.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_220.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_220.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_230.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_230.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_240.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_250.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_250.png -------------------------------------------------------------------------------- /data/img/specular/video-confocal-gray-full_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/data/img/specular/video-confocal-gray-full_all.png -------------------------------------------------------------------------------- /scenes/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/.DS_Store -------------------------------------------------------------------------------- /scenes/bike_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/bike_1.png -------------------------------------------------------------------------------- /scenes/bike_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/bike_2.gif -------------------------------------------------------------------------------- /scenes/bike_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/bike_2.png -------------------------------------------------------------------------------- /scenes/classification-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/classification-table.png -------------------------------------------------------------------------------- /scenes/compare-synthetic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/compare-synthetic.png -------------------------------------------------------------------------------- /scenes/detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/detection.png -------------------------------------------------------------------------------- /scenes/discoball_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/discoball_1.png -------------------------------------------------------------------------------- /scenes/discoball_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/discoball_2.gif -------------------------------------------------------------------------------- /scenes/discoball_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/discoball_2.png -------------------------------------------------------------------------------- /scenes/dragon_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/dragon_1.png -------------------------------------------------------------------------------- /scenes/dragon_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/dragon_2.gif -------------------------------------------------------------------------------- /scenes/dragon_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/dragon_2.png -------------------------------------------------------------------------------- /scenes/resolution_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/resolution_1.png -------------------------------------------------------------------------------- /scenes/resolution_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/resolution_2.gif -------------------------------------------------------------------------------- /scenes/resolution_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/resolution_2.png -------------------------------------------------------------------------------- /scenes/statue_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/statue_1.png -------------------------------------------------------------------------------- /scenes/statue_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/statue_2.gif -------------------------------------------------------------------------------- /scenes/statue_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/statue_2.png -------------------------------------------------------------------------------- /scenes/teaser_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/teaser_1.png -------------------------------------------------------------------------------- /scenes/teaser_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/teaser_2.gif -------------------------------------------------------------------------------- /scenes/teaser_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/princeton-computational-imaging/NLOSFeatureEmbeddings/f882ca5684e9b6ffb16052a714714f570e606295/scenes/teaser_2.png --------------------------------------------------------------------------------