├── CITATIONS.bib ├── flow_IO.py ├── cameradata.py ├── patches ├── patch_v2.80-release_BW.patch └── patch_v2.80-release_FW.patch ├── README.md └── convert.py /CITATIONS.bib: -------------------------------------------------------------------------------- 1 | @InProceedings{Mehl2023_Spring, 2 | author = {Lukas Mehl and Jenny Schmalfuss and Azin Jahedi and Yaroslava Nalivayko and Andr\'es Bruhn}, 3 | title = {Spring: A High-Resolution High-Detail Dataset and Benchmark for Scene Flow, Optical Flow and Stereo}, 4 | booktitle = {Proc. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 5 | year = {2023} 6 | } -------------------------------------------------------------------------------- /flow_IO.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import h5py 3 | 4 | 5 | def writeFlo5File(flow, filename): 6 | with h5py.File(filename, "w") as f: 7 | f.create_dataset("flow", data=flow, compression="gzip", compression_opts=5) 8 | 9 | 10 | def readFlo5Flow(filename): 11 | with h5py.File(filename, "r") as f: 12 | if "flow" not in f.keys(): 13 | raise IOError(f"File {filename} does not have a 'flow' key. Is this a valid flo5 file?") 14 | return f["flow"][()] 15 | 16 | 17 | def writeDsp5File(disp, filename): 18 | with h5py.File(filename, "w") as f: 19 | f.create_dataset("disparity", data=disp, compression="gzip", compression_opts=5) 20 | 21 | 22 | def readDsp5Disp(filename): 23 | with h5py.File(filename, "r") as f: 24 | if "disparity" not in f.keys(): 25 | raise IOError(f"File {filename} does not have a 'disparity' key. Is this a valid dsp5 file?") 26 | return f["disparity"][()] 27 | 28 | 29 | def writePngMapFile(map_, filename): 30 | Image.fromarray(map_).save(filename) 31 | 32 | -------------------------------------------------------------------------------- /cameradata.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import numpy as np 3 | from mathutils import Matrix 4 | 5 | 6 | def get_intrinsics(focal_length, sizex=1920, sizey=1080, sensor_width=23.76): 7 | # only works for sensor fit "Horizontal" 8 | f = focal_length * sizex / sensor_width 9 | return [f, f, sizex//2, sizey//2] 10 | 11 | 12 | def get_3x4_RT_matrix_from_blender(cam): 13 | R_bcam2cv = Matrix( 14 | ((1, 0, 0), 15 | (0, -1, 0), 16 | (0, 0, -1))) 17 | 18 | location, rotation = cam.matrix_world.decompose()[0:2] 19 | R_world2bcam = rotation.to_matrix().transposed() 20 | 21 | T_world2bcam = -1*R_world2bcam @ location 22 | 23 | R_world2cv = R_bcam2cv@R_world2bcam 24 | T_world2cv = R_bcam2cv@T_world2bcam 25 | 26 | # put into 3x4 matrix 27 | RT = Matrix(( 28 | R_world2cv[0][:] + (T_world2cv[0],), 29 | R_world2cv[1][:] + (T_world2cv[1],), 30 | R_world2cv[2][:] + (T_world2cv[2],) 31 | )) 32 | return RT 33 | 34 | 35 | # TODO: select main camera here 36 | cam = bpy.data.objects["Camera_L"] 37 | 38 | worldmat = get_3x4_RT_matrix_from_blender(cam) 39 | worldmat = np.vstack((worldmat, np.asarray([[0,0,0,1]]))) 40 | np.savetxt(f"extrinsics.txt", worldmat) 41 | 42 | intrinsics = get_intrinsics(cam.data.lens) 43 | np.savetxt(f"intrinsics.txt", intrinsics) 44 | -------------------------------------------------------------------------------- /patches/patch_v2.80-release_BW.patch: -------------------------------------------------------------------------------- 1 | diff --git a/intern/cycles/kernel/geom/geom_primitive.h b/intern/cycles/kernel/geom/geom_primitive.h 2 | index 9a91da79f58..9e66f41885d 100644 3 | --- a/intern/cycles/kernel/geom/geom_primitive.h 4 | +++ b/intern/cycles/kernel/geom/geom_primitive.h 5 | @@ -373,14 +373,21 @@ ccl_device_inline float4 primitive_motion_vector(KernelGlobals *kg, ShaderData * 6 | /* camera motion, for perspective/orthographic motion.pre/post will be a 7 | * world-to-raster matrix, for panorama it's world-to-camera */ 8 | if (kernel_data.cam.type != CAMERA_PANORAMA) { 9 | - ProjectionTransform projection = kernel_data.cam.worldtoraster; 10 | - motion_center = transform_perspective(&projection, center); 11 | + //ProjectionTransform projection = kernel_data.cam.worldtoraster; 12 | + //motion_center = transform_perspective(&projection, center); 13 | 14 | - projection = kernel_data.cam.perspective_pre; 15 | - motion_pre = transform_perspective(&projection, motion_pre); 16 | + //projection = kernel_data.cam.perspective_pre; 17 | + //motion_pre = transform_perspective(&projection, motion_pre); 18 | + tfm = kernel_data.cam.motion_pass_pre; 19 | + motion_pre = transform_point(&tfm, motion_pre); 20 | + 21 | + tfm = kernel_data.cam.worldtocamera; 22 | + motion_center = transform_point(&tfm, center); 23 | 24 | - projection = kernel_data.cam.perspective_post; 25 | - motion_post = transform_perspective(&projection, motion_post); 26 | + //projection = kernel_data.cam.perspective_post; 27 | + //motion_post = transform_perspective(&projection, motion_post); 28 | + tfm = kernel_data.cam.motion_pass_post; 29 | + motion_post = transform_point(&tfm, motion_post); 30 | } 31 | else { 32 | tfm = kernel_data.cam.worldtocamera; 33 | @@ -402,10 +409,10 @@ ccl_device_inline float4 primitive_motion_vector(KernelGlobals *kg, ShaderData * 34 | motion_post.y *= kernel_data.cam.height; 35 | } 36 | 37 | - motion_pre = motion_pre - motion_center; 38 | + motion_pre = motion_center - motion_pre; 39 | motion_post = motion_center - motion_post; 40 | 41 | - return make_float4(motion_pre.x, motion_pre.y, motion_post.x, motion_post.y); 42 | + return make_float4(motion_pre.x, motion_pre.y, motion_pre.z, 0); 43 | } 44 | 45 | CCL_NAMESPACE_END 46 | diff --git a/intern/cycles/render/camera.cpp b/intern/cycles/render/camera.cpp 47 | index 74953afae9d..1bcf89b4b3c 100644 48 | --- a/intern/cycles/render/camera.cpp 49 | +++ b/intern/cycles/render/camera.cpp 50 | @@ -350,10 +350,14 @@ void Camera::update(Scene *scene) 51 | if (have_motion) { 52 | kcam->perspective_pre = cameratoraster * transform_inverse(motion[0]); 53 | kcam->perspective_post = cameratoraster * transform_inverse(motion[motion.size() - 1]); 54 | + kcam->motion_pass_pre = transform_inverse(motion[0]); 55 | + kcam->motion_pass_post = transform_inverse(motion[motion.size() - 1]); 56 | } 57 | else { 58 | kcam->perspective_pre = worldtoraster; 59 | kcam->perspective_post = worldtoraster; 60 | + kcam->motion_pass_pre = kcam->worldtocamera; 61 | + kcam->motion_pass_post = kcam->worldtocamera; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /patches/patch_v2.80-release_FW.patch: -------------------------------------------------------------------------------- 1 | diff --git a/intern/cycles/kernel/geom/geom_primitive.h b/intern/cycles/kernel/geom/geom_primitive.h 2 | index 2c31e5cee03..a478a24bee4 100644 3 | --- a/intern/cycles/kernel/geom/geom_primitive.h 4 | +++ b/intern/cycles/kernel/geom/geom_primitive.h 5 | @@ -302,14 +302,21 @@ ccl_device_inline float4 primitive_motion_vector(KernelGlobals *kg, ShaderData * 6 | /* camera motion, for perspective/orthographic motion.pre/post will be a 7 | * world-to-raster matrix, for panorama it's world-to-camera */ 8 | if (kernel_data.cam.type != CAMERA_PANORAMA) { 9 | - ProjectionTransform projection = kernel_data.cam.worldtoraster; 10 | - motion_center = transform_perspective(&projection, center); 11 | + //ProjectionTransform projection = kernel_data.cam.worldtoraster; 12 | + //motion_center = transform_perspective(&projection, center); 13 | 14 | - projection = kernel_data.cam.perspective_pre; 15 | - motion_pre = transform_perspective(&projection, motion_pre); 16 | + //projection = kernel_data.cam.perspective_pre; 17 | + //motion_pre = transform_perspective(&projection, motion_pre); 18 | + tfm = kernel_data.cam.motion_pass_pre; 19 | + motion_pre = transform_point(&tfm, motion_pre); 20 | + 21 | + tfm = kernel_data.cam.worldtocamera; 22 | + motion_center = transform_point(&tfm, center); 23 | 24 | - projection = kernel_data.cam.perspective_post; 25 | - motion_post = transform_perspective(&projection, motion_post); 26 | + //projection = kernel_data.cam.perspective_post; 27 | + //motion_post = transform_perspective(&projection, motion_post); 28 | + tfm = kernel_data.cam.motion_pass_post; 29 | + motion_post = transform_point(&tfm, motion_post); 30 | } 31 | else { 32 | tfm = kernel_data.cam.worldtocamera; 33 | @@ -331,10 +338,10 @@ ccl_device_inline float4 primitive_motion_vector(KernelGlobals *kg, ShaderData * 34 | motion_post.y *= kernel_data.cam.height; 35 | } 36 | 37 | - motion_pre = motion_pre - motion_center; 38 | + //motion_pre = motion_pre - motion_center; 39 | motion_post = motion_center - motion_post; 40 | 41 | - return make_float4(motion_pre.x, motion_pre.y, motion_post.x, motion_post.y); 42 | + return make_float4(motion_post.x, motion_post.y, motion_post.z, 0); 43 | } 44 | 45 | CCL_NAMESPACE_END 46 | diff --git a/intern/cycles/render/camera.cpp b/intern/cycles/render/camera.cpp 47 | index 327f166f9d8..022adf4b23b 100644 48 | --- a/intern/cycles/render/camera.cpp 49 | +++ b/intern/cycles/render/camera.cpp 50 | @@ -371,10 +371,14 @@ void Camera::update(Scene *scene) 51 | if (have_motion) { 52 | kcam->perspective_pre = cameratoraster * transform_inverse(motion[0]); 53 | kcam->perspective_post = cameratoraster * transform_inverse(motion[motion.size() - 1]); 54 | + kcam->motion_pass_pre = transform_inverse(motion[0]); 55 | + kcam->motion_pass_post = transform_inverse(motion[motion.size() - 1]); 56 | } 57 | else { 58 | kcam->perspective_pre = worldtoraster; 59 | kcam->perspective_post = worldtoraster; 60 | + kcam->motion_pass_pre = kcam->worldtocamera; 61 | + kcam->motion_pass_post = kcam->worldtocamera; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 3D motion vectors and scene flow from Blender 2 | This repository contains the code required to get forward or backward 3D motion vectors / scene flow from the Blender cycles renderer. 3 | 4 | 5 | ## How does it work? 6 | While it is possible to extract forward and backward optical flow (2D motion vectors) from Blender using the [Vector pass](https://docs.blender.org/manual/en/latest/render/layers/passes.html), there is no direct way of exporting point-wise 3D motion vectors or scene flow. 7 | For the project [Spring: A High-Resolution High-Detail Dataset and Benchmark for Scene Flow, Optical Flow and Stereo](https://spring-benchmark.org/), we developed a patch for Blender to export forward or backward **3D** motion vectors by reusing the vector pass. 8 | 9 | 10 | ## Setup 11 | - Clone the Blender source code, e.g. via https://github.com/blender/blender. We tested the branch `blender-v2.80-release`. 12 | - Apply the provided FW patch via `git apply ` 13 | - Compile Blender; see https://wiki.blender.org/wiki/Building_Blender for details. 14 | 15 | With the resulting blender executable, the export of forward 3D motion vectors (relative to the camera coordinate system) is possible. 16 | The motion vectors are available in the Blender Vector pass (instead of optical flow). 17 | In order to export backward 3D motion vectors, please use the patch `patch_v2.80-release_BW.patch`. 18 | 19 | ## Blender Scene Setup 20 | - Make sure to select the cycles rendering engine and enable the vector pass in the "View Layer" menu. Note that the vector pass is only available if motion blur is turned off. 21 | - Make sure that in the "Output" Menu under "Post Processing", "Compositing" is turned on, as well as "Use Nodes" in the "Compositing" view. 22 | - In the "Composting" view add the following: 23 | - Add a "Separate RGBA" node and connect the "Render Layers"/Vector output to its input. 24 | - Add a "Combine RGBA" node and connect all 4 outputs of the "Separate RGBA" node to its corresponding inputs. 25 | - Add a "File Output" node and connect its input with the output of the "Combine RGBA" node. 26 | - In its node properties set the file format to "OpenEXR" and the color setting to RGBA Float (Full). 27 | 28 | With this strategy, EXR files with 3D motion vectors in their R, G and B channels, respectively, are generated during rendering. 29 | In order to also generate depth maps, report the above steps analogously for the Z-pass / Depth pass. 30 | 31 | Additionally, extrinsic and intrinsic camera parameters are required, which can be extracted by running cameradata.py within the Blender scene. 32 | 33 | ## Conversion to scene flow 34 | Given depth, 3D motion vectors as well as intrinsic and extrinsic camera parameters, scene flow and evaluation maps can be computed: 35 | - Install the required python libraries numpy, OpenEXR and h5py. (Tested with Python 3.9) 36 | - Use convert.py to generate scene flow data. 37 | 38 | 39 | ## Citation 40 | If you make use of this code, please cite our paper: 41 | ```bibtex 42 | @InProceedings{Mehl2023_Spring, 43 | author = {Lukas Mehl and Jenny Schmalfuss and Azin Jahedi and Yaroslava Nalivayko and Andr\'es Bruhn}, 44 | title = {Spring: A High-Resolution High-Detail Dataset and Benchmark for Scene Flow, Optical Flow and Stereo}, 45 | booktitle = {Proc. IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 46 | year = {2023} 47 | } 48 | ``` 49 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | import OpenEXR 2 | import numpy as np 3 | import flow_IO 4 | 5 | 6 | THRESH1 = 0.01 7 | THRESH2 = 0.5 8 | 9 | 10 | def disp1ToFlow(disp1, is_left=True): 11 | multiplier = -1.0 if is_left else 1.0 12 | return multiplier * np.dstack((disp1, np.zeros_like(disp1))) 13 | 14 | 15 | def disp2ToFlow(flow, disp2, is_left=True): 16 | multiplier = -1.0 if is_left else 1.0 17 | flow[...,0] += multiplier * disp2 18 | return flow 19 | 20 | 21 | def matchmap(flow_fw, flow_bw, factor=1.0): 22 | flow_fw = flow_fw.astype(np.float64) 23 | flow_bw = flow_bw.astype(np.float64) 24 | flow_fw *= factor 25 | flow_bw *= factor 26 | flow_bw_flat = flow_bw.reshape(-1,2) 27 | 28 | h, w, _ = flow_fw.shape 29 | indices = np.indices((h, w)) 30 | indices = np.dstack((indices[1], indices[0])) 31 | indices = indices.astype(np.float64) 32 | 33 | indices += flow_fw 34 | 35 | h, w = indices.shape[:2] 36 | nanmap = np.isnan(indices[...,0]) | np.isnan(indices[...,1]) 37 | indices_ = np.round(indices).astype(int) 38 | 39 | outwards_map = (indices_[...,0] < 0) | (indices_[...,1] < 0) | (indices_[...,0] >= w) | (indices_[...,1] >= h) 40 | outwards_map[nanmap] = False 41 | indices_[indices_[...,0] < 0, 0] = 0 42 | indices_[indices_[...,1] < 0, 1] = 0 43 | indices_[indices_[...,0] >= w, 0] = w - 1 44 | indices_[indices_[...,1] >= h, 1] = h - 1 45 | 46 | indices_ = indices_[...,1] * w + indices_[...,0] 47 | 48 | warped = flow_bw_flat[indices_] 49 | 50 | l2_sq = ((flow_fw + warped)**2).sum(axis=-1) 51 | sq_sum = (flow_fw**2).sum(axis=-1) + (warped**2).sum(axis=-1) 52 | 53 | if factor == 2.0: 54 | l2_sq = np.dstack((l2_sq[::2,::2], l2_sq[::2,1::2], l2_sq[1::2,::2], l2_sq[1::2,1::2])) 55 | l2_sq = np.nansum(l2_sq, axis=-1) / np.maximum(4 - np.isnan(l2_sq).sum(axis=-1), 1) 56 | 57 | sq_sum = np.dstack((sq_sum[::2,::2], sq_sum[::2,1::2], sq_sum[1::2,::2], sq_sum[1::2,1::2])) 58 | sq_sum = np.nansum(sq_sum, axis=-1) / np.maximum(4 - np.isnan(sq_sum).sum(axis=-1), 1) 59 | 60 | outwards_map = np.dstack((outwards_map[::2,::2], outwards_map[::2,1::2], outwards_map[1::2,::2], outwards_map[1::2,1::2])).astype(np.float32).mean(axis=-1) 61 | outwards_map = outwards_map >= 0.5 62 | 63 | result = l2_sq > THRESH1 * sq_sum + THRESH2 64 | 65 | result = result | outwards_map 66 | return result 67 | 68 | 69 | def readEXR(filepath): 70 | exrfile = OpenEXR.InputFile(filepath) 71 | 72 | # Compute the size 73 | dw = exrfile.header()['dataWindow'] 74 | w = dw.max.x - dw.min.x + 1 75 | h = dw.max.y - dw.min.y + 1 76 | 77 | full_img = np.dstack([np.frombuffer(exrfile.channel(c), dtype=np.float32).reshape((h,w)) for c in ["R", "G", "B"]]) 78 | return full_img 79 | 80 | 81 | def get_invalid(depth): 82 | return np.isnan(depth) | (depth>10000) 83 | 84 | 85 | def get_sky(depth, clip_end): 86 | return (depth > clip_end * 0.98) & (~get_invalid(depth)) 87 | 88 | 89 | def matmul3D(mat, tensor): 90 | """compute matrix multiplication mat @ vec for every vec in tensor""" 91 | return np.einsum('ijk,lk->ijl', tensor, mat) 92 | 93 | 94 | def project(Xs, intrinsics): 95 | """ Pinhole camera projection """ 96 | X, Y, Z = Xs[:,:,0], Xs[:,:,1], Xs[:,:,2] 97 | fx, fy, cx, cy = intrinsics 98 | 99 | x = fx * (X / Z) + cx 100 | y = fy * (Y / Z) + cy 101 | d = 1.0 / Z 102 | 103 | coords = np.stack([x, y, d], axis=-1) 104 | return coords 105 | 106 | 107 | def inv_project(depths, intrinsics): 108 | """ Pinhole camera inverse-projection """ 109 | 110 | ht, wd = depths.shape 111 | 112 | fx, fy, cx, cy = intrinsics 113 | 114 | y, x = np.meshgrid(np.arange(ht), np.arange(wd)) 115 | 116 | X = depths * ((x.T - cx) / fx) 117 | Y = depths * ((y.T - cy) / fy) 118 | Z = depths 119 | 120 | return np.stack([X, Y, Z], axis=-1) 121 | 122 | 123 | def invert_transformation_matrix(mat): 124 | rot_part = mat[:3,:3] 125 | trans_part = mat[:3,3] 126 | trans_part_ = - rot_part.T @ trans_part 127 | result_mat = mat.copy() 128 | result_mat[:3,:3] = rot_part.T 129 | result_mat[:3,3] = trans_part_ 130 | return result_mat 131 | 132 | 133 | def depth_conversion(A_int, width, height): 134 | A_int_inv = np.linalg.inv(A_int) 135 | point1 = np.indices((width, height)) 136 | point1 = np.transpose(point1, (1,2,0)) 137 | 138 | point1_hom = np.dstack((point1, np.ones((width, height, 1)))) 139 | point_3D = np.einsum('ijk,lk->ijl', point1_hom, A_int_inv) 140 | return np.linalg.norm(point_3D, axis=-1).T 141 | 142 | 143 | def get_sceneflow(vec3dpath, extrinsics1, extrinsics2, intrinsics1, intrinsics2, depth, disparity, skymap, baseline_width, factor=1.0): 144 | 145 | vec3d = readEXR(vec3dpath) 146 | 147 | intrinsics1 *= factor 148 | intrinsics2 *= factor 149 | 150 | rel_matrix = extrinsics2 @ invert_transformation_matrix(extrinsics1) 151 | 152 | vec3d *= -1 153 | vec3d[:,:,1] *= -1 154 | 155 | # compute optical flow + disparity change 156 | point1_3D = inv_project(depth, intrinsics1) 157 | point2_3D = point1_3D + vec3d 158 | pos1 = project(point1_3D, intrinsics1) 159 | pos2 = project(point2_3D, intrinsics2) 160 | 161 | pos1[...,2] *= baseline_width * intrinsics1[0] 162 | pos2[...,2] *= baseline_width * intrinsics2[0] 163 | 164 | flow2d, dispchange = np.split(pos2-pos1, [2], axis=-1) 165 | dispchange /= factor 166 | disparity2 = disparity + dispchange[:,:,0] 167 | 168 | # rigidity map 169 | point1_3D_hom = np.dstack((point1_3D, np.ones((point1_3D.shape[0],point1_3D.shape[1])))) 170 | point2_3D_hom = matmul3D(rel_matrix, point1_3D_hom) 171 | point2_3D_RIGID = point2_3D_hom[:,:,:3] / point2_3D_hom[:,:,3, None] 172 | 173 | difference = np.linalg.norm(point2_3D - point2_3D_RIGID, axis=-1) 174 | rigidmap = difference>1e-3 175 | 176 | # compute optical flow for sky pixels that are at infinity 177 | point1_3D_sky = inv_project(np.ones_like(depth), intrinsics1) 178 | point1_3D_sky = np.dstack((point1_3D_sky, np.zeros_like(depth))) 179 | point2_3D_sky = matmul3D(rel_matrix, point1_3D_sky) 180 | point2_3D_sky = point2_3D_sky[:,:,:3] 181 | pos1_sky = project(point1_3D_sky, intrinsics1) 182 | pos2_sky = project(point2_3D_sky, intrinsics2) 183 | flow2d_sky, _ = np.split(pos2_sky-pos1_sky, [2], axis=-1) 184 | 185 | # replace optical flow at sky pixels: 186 | flow2d[skymap] = flow2d_sky[skymap] 187 | 188 | # disparity2 is zero at sky pixels: 189 | disparity2[skymap] = 0.0 190 | 191 | flow2d /= factor 192 | 193 | detailmap_disp2 = get_detailmap(disparity2) 194 | detailmap_flow = get_detailmap_flow(flow2d) 195 | 196 | return disparity2, flow2d, rigidmap, detailmap_disp2, detailmap_flow 197 | 198 | 199 | def get_detailmap(img): 200 | img = np.dstack((img[::2,::2], img[1::2,::2], img[::2,1::2], img[1::2,1::2])) 201 | med = np.median(img, axis=-1) 202 | maxmeddev = (img-med[...,None]).max(axis=-1) 203 | 204 | return maxmeddev>1 205 | 206 | 207 | def get_detailmap_flow(flow): 208 | return get_detailmap(flow[...,0]) | get_detailmap(flow[...,1]) 209 | 210 | 211 | def get_depth(path, intrinsics, baseline_width, factor=1, clip_end=10000): 212 | 213 | depth = readEXR(path)[:,:,0] 214 | 215 | invalid_map1 = get_invalid(depth) 216 | depth[invalid_map1] = np.nan 217 | 218 | skymap = get_sky(depth, clip_end) 219 | 220 | depth[skymap] = np.nan 221 | 222 | A_int = np.asarray([[intrinsics[0], 0, intrinsics[2]], [0, intrinsics[1], intrinsics[3]], [0,0,1]]) 223 | correction = depth_conversion(A_int, 1920*factor, 1080*factor) 224 | depth /= correction 225 | 226 | disparity = baseline_width * intrinsics[0] / depth 227 | disparity /= factor 228 | 229 | # the disparity is zero at sky pixels: 230 | disparity[skymap] = 0.0 231 | 232 | detailmap_disp1 = get_detailmap(disparity) 233 | 234 | return depth, disparity, skymap, detailmap_disp1 235 | 236 | 237 | 238 | if __name__ == "__main__": 239 | # reference frame intrinsics 240 | intrinsics1 = np.loadtxt("intrinsics1.txt") 241 | # target frame intrinsics 242 | intrinsics2 = np.loadtxt("intrinsics2.txt") 243 | 244 | # reference frame extrinsics 245 | extrinsics1 = np.loadtxt("extrinsics1.txt") 246 | # target frame extrinsics 247 | extrinsics2 = np.loadtxt("extrinsics2.txt") 248 | 249 | depth_path = "depth.exr" 250 | vec3d_path = "vec3d.exr" 251 | 252 | baseline_width = 0.065 253 | 254 | depth, disparity, skymap, detailmap_disp1 = get_depth(depth_path, intrinsics1, baseline_width) 255 | 256 | disparity2, flow2d, rigidmap, detailmap_disp2, detailmap_flow = get_sceneflow(vec3d_path, extrinsics1, extrinsics2, intrinsics1, intrinsics2, depth, disparity, skymap, baseline_width) 257 | 258 | flow_IO.writeDsp5File(disparity, "disparity.dsp5") 259 | flow_IO.writeDsp5File(disparity2, "disparity2.dsp5") 260 | flow_IO.writeFlo5File(flow2d, "flow.flo5") 261 | 262 | flow_IO.writePngMapFile(skymap, "skymap.png") 263 | flow_IO.writePngMapFile(detailmap_disp1, "detailmap_disp1.png") 264 | flow_IO.writePngMapFile(detailmap_disp2, "detailmap_disp2.png") 265 | flow_IO.writePngMapFile(detailmap_flow, "detailmap_flow.png") 266 | flow_IO.writePngMapFile(rigidmap, "rigidmap.png") 267 | 268 | # matchmaps are computed with matchmap(fw, bw) 269 | # where fw, bw are either optical flows or disparities after conversion with disp1ToFlow or disp2ToFlow 270 | --------------------------------------------------------------------------------