├── .gitignore ├── LICENSE ├── README.md ├── index.html ├── lightbulb.png └── main.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kevin Kwok 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lit-splat 2 | 3 | A 3D Gaussian Splat viewer with dynamic lighting. Based on the [splat](https://github.com/antimatter15/splat) viewer by [antimatter15](https://github.com/antimatter15). 4 | 5 | 6 | 7 | https://github.com/andrewkchan/lit-splat/assets/8591901/f61c4633-4a1a-4bbb-80d5-e7f73044635c 8 | 9 | 10 | 11 | ## Controls 12 | 13 | movement (arrow keys) 14 | - left/right arrow keys to strafe side to side 15 | - up/down arrow keys to move forward/back 16 | - space to jump 17 | 18 | camera angle (wasd) 19 | - a/d to turn camera left/right 20 | - w/s to tilt camera up/down 21 | - q/e to roll camera counterclockwise/clockwise 22 | - i/k and j/l to orbit 23 | 24 | trackpad 25 | - scroll up/down/left/right to orbit 26 | - pinch to move forward/back 27 | - ctrl key + scroll to move forward/back 28 | - shift + scroll to move up/down or strafe 29 | 30 | mouse 31 | - click and drag to orbit 32 | - right click (or ctrl/cmd key) and drag up/down to move 33 | - click and drag a light to move it in X and Y axes 34 | - right click and drag a light to move it in Z axis 35 | 36 | touch (mobile) 37 | - one finger to orbit 38 | - two finger pinch to move forward/back 39 | - two finger rotate to rotate camera clockwise/counterclockwise 40 | - two finger pan to move side-to-side and up-down 41 | 42 | gamepad 43 | - if you have a game controller connected it should work 44 | 45 | other 46 | - press M to switch between lighting mode and no lighting 47 | - press N to switch between explicit normals and pseudo-normals 48 | - press 0-9 to switch to one of the pre-loaded camera views 49 | - press '-' or '+'key to cycle loaded cameras 50 | - press p to resume default animation 51 | - drag and drop .ply file to convert to .lsplat 52 | - drag and drop cameras.json to load cameras 53 | 54 | ## Examples 55 | 56 | - https://andrewkchan.dev/lit-splat/?url=garden.lsplat 57 | - https://andrewkchan.dev/lit-splat/?url=lego.lsplat 58 | - https://andrewkchan.dev/lit-splat/?url=ship.lsplat 59 | - https://andrewkchan.dev/lit-splat/?url=mic.lsplat 60 | - https://andrewkchan.dev/lit-splat/?url=materials.lsplat 61 | - https://andrewkchan.dev/lit-splat/?url=chair.lsplat -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Relightable Gaussian Splats 5 | 6 | 10 | 11 | 15 | 200 | 201 | 202 | 205 |
206 |

Relightable Gaussian Splats

207 |

208 | 209 | By Andrew Chan, based on the viewer by Kevin Kwok. 210 | Code on 211 | Github. 213 | 214 |

215 | 216 |
217 | Use mouse or arrow keys to navigate. M to toggle lighting. 218 | 219 |
movement (arrow keys) 220 | - left/right arrow keys to strafe side to side 221 | - up/down arrow keys to move forward/back 222 | - space to jump 223 | 224 | camera angle (wasd) 225 | - a/d to turn camera left/right 226 | - w/s to tilt camera up/down 227 | - q/e to roll camera counterclockwise/clockwise 228 | - i/k and j/l to orbit 229 | 230 | trackpad 231 | - scroll up/down/left/right to orbit 232 | - pinch to move forward/back 233 | - ctrl key + scroll to move forward/back 234 | - shift + scroll to move up/down or strafe 235 | 236 | mouse 237 | - click and drag to orbit 238 | - right click (or ctrl/cmd key) and drag up/down to move 239 | - click and drag a light to move it in X and Y axes 240 | - right click and drag a light to move it in Z axis 241 | 242 | touch (mobile) 243 | - one finger to orbit 244 | - two finger pinch to move forward/back 245 | - two finger rotate to rotate camera clockwise/counterclockwise 246 | - two finger pan to move side-to-side and up-down 247 | 248 | gamepad 249 | - if you have a game controller connected it should work 250 | 251 | other 252 | - press M to switch between lighting mode and no lighting 253 | - press N to switch between explicit normals and pseudo-normals 254 | - press 0-9 to switch to one of the pre-loaded camera views 255 | - press '-' or '+'key to cycle loaded cameras 256 | - press p to resume default animation 257 | - drag and drop .ply file to convert to .lsplat 258 | - drag and drop cameras.json to load cameras 259 |
260 | 261 |
262 | 263 |
264 | 265 |
266 | 267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 | 283 | 284 |
285 | 💡 Add Light 286 |
287 | 288 |
289 | 290 |
291 |
292 | 293 |
294 | 295 | 296 | 297 | -------------------------------------------------------------------------------- /lightbulb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewkchan/lit-splat/ca24ecef3719dfc12643c630ac495d0cef6a54dc/lightbulb.png -------------------------------------------------------------------------------- /main.js: -------------------------------------------------------------------------------- 1 | const PACKED_SPLAT_LENGTH = ( 2 | 3*4 + // XYZ - Position (Float32) 3 | 3*4 + // XYZ - Scale (Float32) 4 | 4 + // RGBA - colors (uint8) 5 | 4 + // IJKL - quaternion/rot (uint8) 6 | 3*4 + // XYZ - Normal (Float32) 7 | 3 + // RGB - PBR base colors (uint8) 8 | 1 + // ... padding out to 4-byte alignment 9 | 2*4 // RM - PBR materials (Float32) 10 | // ... padding 11 | ); 12 | const PACKED_RENDERABLE_SPLAT_LENGTH = ( 13 | 3*4 + // XYZ - Position (Float32) 14 | 4 + // ... padding out to BYTES_PER_TEXEL 15 | 6*2 + // 6 parameters of covariance matrix (Float16) 16 | 4 + // RGBA - colors (uint8) 17 | 3*4 + // XYZ - Normal (Float32) 18 | 3 + // RGB - PBR base colors (uint8) 19 | 1 + // ... padding out to 4-byte alignment 20 | 2*4 // RM - PBR materials (Float32) 21 | // ... padding 22 | ); 23 | const BYTES_PER_TEXEL = 16; // RGBA32UI = 32 bits per channel * 4 channels = 4*4 bytes 24 | const TEXELS_PER_PACKED_SPLAT = Math.ceil(PACKED_RENDERABLE_SPLAT_LENGTH / BYTES_PER_TEXEL); 25 | const PADDED_RENDERABLE_SPLAT_LENGTH = TEXELS_PER_PACKED_SPLAT * BYTES_PER_TEXEL; 26 | const PADDED_SPLAT_LENGTH = 4 * Math.ceil(PACKED_SPLAT_LENGTH / 4); 27 | 28 | const PLY_MAGIC_HEADER = new Uint8Array([112, 108, 121, 10]); // "ply\n" 29 | const LSPLAT_MAGIC_HEADER = new Uint8Array([108, 115, 112, 108, 97, 116, 10]); // "lsplat\n" 30 | 31 | let cameras = [ 32 | { 33 | id: 0, 34 | img_name: "00001", 35 | width: 1959, 36 | height: 1090, 37 | position: [ 38 | -3.0089893469241797, -0.11086489695181866, -3.7527640949141428, 39 | ], 40 | rotation: [ 41 | [0.876134201218856, 0.06925962026449776, 0.47706599800804744], 42 | [-0.04747421839895102, 0.9972110940209488, -0.057586739349882114], 43 | [-0.4797239414934443, 0.027805376500959853, 0.8769787916452908], 44 | ], 45 | fy: 1164.6601287484507, 46 | fx: 1159.5880733038064, 47 | }, 48 | { 49 | id: 1, 50 | img_name: "00009", 51 | width: 1959, 52 | height: 1090, 53 | position: [ 54 | -2.5199776022057296, -0.09704735754873686, -3.6247725540304545, 55 | ], 56 | rotation: [ 57 | [0.9982731285632193, -0.011928707708098955, -0.05751927260507243], 58 | [0.0065061360949636325, 0.9955928229282383, -0.09355533724430458], 59 | [0.058381769258182864, 0.09301955098900708, 0.9939511719154457], 60 | ], 61 | fy: 1164.6601287484507, 62 | fx: 1159.5880733038064, 63 | }, 64 | { 65 | id: 2, 66 | img_name: "00017", 67 | width: 1959, 68 | height: 1090, 69 | position: [ 70 | -0.7737533667465242, -0.3364271945329695, -2.9358969417573753, 71 | ], 72 | rotation: [ 73 | [0.9998813418672372, 0.013742375651625236, -0.0069605529394208224], 74 | [-0.014268370388586709, 0.996512943252834, -0.08220929105659476], 75 | [0.00580653013657589, 0.08229885200307129, 0.9965907801935302], 76 | ], 77 | fy: 1164.6601287484507, 78 | fx: 1159.5880733038064, 79 | }, 80 | { 81 | id: 3, 82 | img_name: "00025", 83 | width: 1959, 84 | height: 1090, 85 | position: [ 86 | 1.2198221749590001, -0.2196687861401182, -2.3183162007028453, 87 | ], 88 | rotation: [ 89 | [0.9208648867765482, 0.0012010625395201253, 0.389880004297208], 90 | [-0.06298204172269357, 0.987319521752825, 0.14571693239364383], 91 | [-0.3847611242348369, -0.1587410451475895, 0.9092635249821667], 92 | ], 93 | fy: 1164.6601287484507, 94 | fx: 1159.5880733038064, 95 | }, 96 | { 97 | id: 4, 98 | img_name: "00033", 99 | width: 1959, 100 | height: 1090, 101 | position: [ 102 | 1.742387858893817, -0.13848225198886954, -2.0566370113193146, 103 | ], 104 | rotation: [ 105 | [0.24669889292141334, -0.08370189346592856, -0.9654706879349405], 106 | [0.11343747891376445, 0.9919082664242816, -0.05700815184573074], 107 | [0.9624300466054861, -0.09545671285663988, 0.2541976029815521], 108 | ], 109 | fy: 1164.6601287484507, 110 | fx: 1159.5880733038064, 111 | }, 112 | { 113 | id: 5, 114 | img_name: "00041", 115 | width: 1959, 116 | height: 1090, 117 | position: [ 118 | 3.6567309419223935, -0.16470990600750707, -1.3458085590422042, 119 | ], 120 | rotation: [ 121 | [0.2341293058324528, -0.02968330457755884, -0.9717522161434825], 122 | [0.10270823606832301, 0.99469554638321, -0.005638106875665722], 123 | [0.9667649592295676, -0.09848690996657204, 0.2359360976431732], 124 | ], 125 | fy: 1164.6601287484507, 126 | fx: 1159.5880733038064, 127 | }, 128 | { 129 | id: 6, 130 | img_name: "00049", 131 | width: 1959, 132 | height: 1090, 133 | position: [ 134 | 3.9013554243203497, -0.2597500978038105, -0.8106154188297828, 135 | ], 136 | rotation: [ 137 | [0.6717235545638952, -0.015718162115524837, -0.7406351366386528], 138 | [0.055627354673906296, 0.9980224478387622, 0.029270992841185218], 139 | [0.7387104058127439, -0.060861588786650656, 0.6712695459756353], 140 | ], 141 | fy: 1164.6601287484507, 142 | fx: 1159.5880733038064, 143 | }, 144 | { 145 | id: 7, 146 | img_name: "00057", 147 | width: 1959, 148 | height: 1090, 149 | position: [4.742994605467533, -0.05591660945412069, 0.9500365976084458], 150 | rotation: [ 151 | [-0.17042655709210375, 0.01207080756938, -0.9852964448542146], 152 | [0.1165090336695526, 0.9931575292530063, -0.00798543433078162], 153 | [0.9784581921120181, -0.1161568667478904, -0.1706667764862097], 154 | ], 155 | fy: 1164.6601287484507, 156 | fx: 1159.5880733038064, 157 | }, 158 | { 159 | id: 8, 160 | img_name: "00065", 161 | width: 1959, 162 | height: 1090, 163 | position: [4.34676307626522, 0.08168160516967145, 1.0876221470355405], 164 | rotation: [ 165 | [-0.003575447631888379, -0.044792503246552894, -0.9989899137764799], 166 | [0.10770152645126597, 0.9931680875192705, -0.04491693593046672], 167 | [0.9941768441149182, -0.10775333677534978, 0.0012732004866391048], 168 | ], 169 | fy: 1164.6601287484507, 170 | fx: 1159.5880733038064, 171 | }, 172 | { 173 | id: 9, 174 | img_name: "00073", 175 | width: 1959, 176 | height: 1090, 177 | position: [3.264984351114202, 0.078974937336732, 1.0117200284114904], 178 | rotation: [ 179 | [-0.026919994628162257, -0.1565891128261527, -0.9872968974090509], 180 | [0.08444552208239385, 0.983768234577625, -0.1583319754069128], 181 | [0.9960643893290491, -0.0876350978794554, -0.013259786205163005], 182 | ], 183 | fy: 1164.6601287484507, 184 | fx: 1159.5880733038064, 185 | }, 186 | ]; 187 | 188 | let camera = cameras[0]; 189 | 190 | function getProjectionMatrix(fx, fy, width, height) { 191 | // Returns a matrix in column-major order. 192 | // TODO: Why does this look so different from the OpenGL projection matrix? 193 | const znear = 0.2; 194 | const zfar = 200; 195 | return [ 196 | [(2 * fx) / width, 0, 0, 0], 197 | [0, -(2 * fy) / height, 0, 0], 198 | [0, 0, zfar / (zfar - znear), 1], 199 | [0, 0, -(zfar * znear) / (zfar - znear), 0], 200 | ].flat(); 201 | } 202 | 203 | function getViewMatrix(camera) { 204 | // Returns a 4x4 matrix in column-major order. 205 | const R = camera.rotation.flat(); 206 | const t = camera.position; 207 | const camToWorld = [ 208 | [R[0], R[1], R[2], 0], 209 | [R[3], R[4], R[5], 0], 210 | [R[6], R[7], R[8], 0], 211 | [ 212 | -t[0] * R[0] - t[1] * R[3] - t[2] * R[6], 213 | -t[0] * R[1] - t[1] * R[4] - t[2] * R[7], 214 | -t[0] * R[2] - t[1] * R[5] - t[2] * R[8], 215 | 1, 216 | ], 217 | ].flat(); 218 | return camToWorld; 219 | } 220 | function add3(a, b) { 221 | return [a[0] + b[0], a[1] + b[1], a[2] + b[2]]; 222 | } 223 | function sub3(a, b) { 224 | return [a[0] - b[0], a[1] - b[1], a[2] - b[2]]; 225 | } 226 | function normalize3(v) { 227 | const len = Math.max(Math.hypot(v[0], v[1], v[2]), 1e-7); 228 | return [v[0] / len, v[1] / len, v[2] / len]; 229 | } 230 | function cross3(a, b) { 231 | return [ 232 | a[1] * b[2] - a[2] * b[1], 233 | a[2] * b[0] - a[0] * b[2], 234 | a[0] * b[1] - a[1] * b[0], 235 | ]; 236 | } 237 | function lookAt(eye, center, up) { 238 | // Returns a 4x4 matrix in column-major order. 239 | const forward = normalize3(sub3(center, eye)); 240 | const side = normalize3(cross3(forward, up)); 241 | up = normalize3(cross3(side, forward)); 242 | return multiply4( 243 | [ 244 | side[0], up[0], -forward[0], 0, 245 | side[1], up[1], -forward[1], 0, 246 | side[2], up[2], -forward[2], 0, 247 | 0, 0, 0, 1, 248 | ], 249 | translate4(identity4(), -eye[0], -eye[1], -eye[2]) 250 | ); 251 | } 252 | 253 | function multiply4(a, b) { 254 | return [ 255 | b[0] * a[0] + b[1] * a[4] + b[2] * a[8] + b[3] * a[12], 256 | b[0] * a[1] + b[1] * a[5] + b[2] * a[9] + b[3] * a[13], 257 | b[0] * a[2] + b[1] * a[6] + b[2] * a[10] + b[3] * a[14], 258 | b[0] * a[3] + b[1] * a[7] + b[2] * a[11] + b[3] * a[15], 259 | b[4] * a[0] + b[5] * a[4] + b[6] * a[8] + b[7] * a[12], 260 | b[4] * a[1] + b[5] * a[5] + b[6] * a[9] + b[7] * a[13], 261 | b[4] * a[2] + b[5] * a[6] + b[6] * a[10] + b[7] * a[14], 262 | b[4] * a[3] + b[5] * a[7] + b[6] * a[11] + b[7] * a[15], 263 | b[8] * a[0] + b[9] * a[4] + b[10] * a[8] + b[11] * a[12], 264 | b[8] * a[1] + b[9] * a[5] + b[10] * a[9] + b[11] * a[13], 265 | b[8] * a[2] + b[9] * a[6] + b[10] * a[10] + b[11] * a[14], 266 | b[8] * a[3] + b[9] * a[7] + b[10] * a[11] + b[11] * a[15], 267 | b[12] * a[0] + b[13] * a[4] + b[14] * a[8] + b[15] * a[12], 268 | b[12] * a[1] + b[13] * a[5] + b[14] * a[9] + b[15] * a[13], 269 | b[12] * a[2] + b[13] * a[6] + b[14] * a[10] + b[15] * a[14], 270 | b[12] * a[3] + b[13] * a[7] + b[14] * a[11] + b[15] * a[15], 271 | ]; 272 | } 273 | 274 | function transform4(T, v) { 275 | return [ 276 | T[0] * v[0] + T[4] * v[1] + T[8] * v[2] + T[12] * v[3], 277 | T[1] * v[0] + T[5] * v[1] + T[9] * v[2] + T[13] * v[3], 278 | T[2] * v[0] + T[6] * v[1] + T[10] * v[2] + T[14] * v[3], 279 | T[3] * v[0] + T[7] * v[1] + T[11] * v[2] + T[15] * v[3], 280 | ]; 281 | } 282 | 283 | function identity4() { 284 | return [ 285 | 1, 0, 0, 0, 286 | 0, 1, 0, 0, 287 | 0, 0, 1, 0, 288 | 0, 0, 0, 1, 289 | ]; 290 | } 291 | 292 | function mat3From4(a) { 293 | return [ 294 | a[0], a[1], a[2], 295 | a[4], a[5], a[6], 296 | a[8], a[9], a[10], 297 | ]; 298 | } 299 | 300 | function invert4(a) { 301 | let b00 = a[0] * a[5] - a[1] * a[4]; 302 | let b01 = a[0] * a[6] - a[2] * a[4]; 303 | let b02 = a[0] * a[7] - a[3] * a[4]; 304 | let b03 = a[1] * a[6] - a[2] * a[5]; 305 | let b04 = a[1] * a[7] - a[3] * a[5]; 306 | let b05 = a[2] * a[7] - a[3] * a[6]; 307 | let b06 = a[8] * a[13] - a[9] * a[12]; 308 | let b07 = a[8] * a[14] - a[10] * a[12]; 309 | let b08 = a[8] * a[15] - a[11] * a[12]; 310 | let b09 = a[9] * a[14] - a[10] * a[13]; 311 | let b10 = a[9] * a[15] - a[11] * a[13]; 312 | let b11 = a[10] * a[15] - a[11] * a[14]; 313 | let det = 314 | b00 * b11 - b01 * b10 + b02 * b09 + b03 * b08 - b04 * b07 + b05 * b06; 315 | if (!det) return null; 316 | return [ 317 | (a[5] * b11 - a[6] * b10 + a[7] * b09) / det, 318 | (a[2] * b10 - a[1] * b11 - a[3] * b09) / det, 319 | (a[13] * b05 - a[14] * b04 + a[15] * b03) / det, 320 | (a[10] * b04 - a[9] * b05 - a[11] * b03) / det, 321 | (a[6] * b08 - a[4] * b11 - a[7] * b07) / det, 322 | (a[0] * b11 - a[2] * b08 + a[3] * b07) / det, 323 | (a[14] * b02 - a[12] * b05 - a[15] * b01) / det, 324 | (a[8] * b05 - a[10] * b02 + a[11] * b01) / det, 325 | (a[4] * b10 - a[5] * b08 + a[7] * b06) / det, 326 | (a[1] * b08 - a[0] * b10 - a[3] * b06) / det, 327 | (a[12] * b04 - a[13] * b02 + a[15] * b00) / det, 328 | (a[9] * b02 - a[8] * b04 - a[11] * b00) / det, 329 | (a[5] * b07 - a[4] * b09 - a[6] * b06) / det, 330 | (a[0] * b09 - a[1] * b07 + a[2] * b06) / det, 331 | (a[13] * b01 - a[12] * b03 - a[14] * b00) / det, 332 | (a[8] * b03 - a[9] * b01 + a[10] * b00) / det, 333 | ]; 334 | } 335 | 336 | function rotate4(a, rad, x, y, z) { 337 | let len = Math.hypot(x, y, z); 338 | x /= len; 339 | y /= len; 340 | z /= len; 341 | let s = Math.sin(rad); 342 | let c = Math.cos(rad); 343 | let t = 1 - c; 344 | let b00 = x * x * t + c; 345 | let b01 = y * x * t + z * s; 346 | let b02 = z * x * t - y * s; 347 | let b10 = x * y * t - z * s; 348 | let b11 = y * y * t + c; 349 | let b12 = z * y * t + x * s; 350 | let b20 = x * z * t + y * s; 351 | let b21 = y * z * t - x * s; 352 | let b22 = z * z * t + c; 353 | return [ 354 | a[0] * b00 + a[4] * b01 + a[8] * b02, 355 | a[1] * b00 + a[5] * b01 + a[9] * b02, 356 | a[2] * b00 + a[6] * b01 + a[10] * b02, 357 | a[3] * b00 + a[7] * b01 + a[11] * b02, 358 | a[0] * b10 + a[4] * b11 + a[8] * b12, 359 | a[1] * b10 + a[5] * b11 + a[9] * b12, 360 | a[2] * b10 + a[6] * b11 + a[10] * b12, 361 | a[3] * b10 + a[7] * b11 + a[11] * b12, 362 | a[0] * b20 + a[4] * b21 + a[8] * b22, 363 | a[1] * b20 + a[5] * b21 + a[9] * b22, 364 | a[2] * b20 + a[6] * b21 + a[10] * b22, 365 | a[3] * b20 + a[7] * b21 + a[11] * b22, 366 | ...a.slice(12, 16), 367 | ]; 368 | } 369 | 370 | function translate4(a, x, y, z) { 371 | return [ 372 | ...a.slice(0, 12), 373 | a[0] * x + a[4] * y + a[8] * z + a[12], 374 | a[1] * x + a[5] * y + a[9] * z + a[13], 375 | a[2] * x + a[6] * y + a[10] * z + a[14], 376 | a[3] * x + a[7] * y + a[11] * z + a[15], 377 | ]; 378 | } 379 | 380 | function createWorker(self) { 381 | // These constants all need to be redefined because of how the worker is created 382 | const PACKED_SPLAT_LENGTH = ( 383 | 3*4 + // XYZ - Position (Float32) 384 | 3*4 + // XYZ - Scale (Float32) 385 | 4 + // RGBA - colors (uint8) 386 | 4 + // IJKL - quaternion/rot (uint8) 387 | 3*4 + // XYZ - Normal (Float32) 388 | 3 + // RGB - PBR base colors (uint8) 389 | 1 + // ... padding out to 4-byte alignment 390 | 2*4 // RM - PBR materials (Float32) 391 | // ... padding 392 | ); 393 | const PACKED_RENDERABLE_SPLAT_LENGTH = ( 394 | 3*4 + // XYZ - Position (Float32) 395 | 4 + // ... padding out to BYTES_PER_TEXEL 396 | 6*2 + // 6 parameters of covariance matrix (Float16) 397 | 4 + // RGBA - colors (uint8) 398 | 3*4 + // XYZ - Normal (Float32) 399 | 3 + // RGB - PBR base colors (uint8) 400 | 1 + // ... padding out to 4-byte alignment 401 | 2*4 // RM - PBR materials (Float32) 402 | // ... padding 403 | ); 404 | const BYTES_PER_TEXEL = 16; // RGBA32UI = 32 bits per channel * 4 channels = 4*4 bytes 405 | const TEXELS_PER_PACKED_SPLAT = Math.ceil(PACKED_RENDERABLE_SPLAT_LENGTH / BYTES_PER_TEXEL); 406 | const PADDED_RENDERABLE_SPLAT_LENGTH = TEXELS_PER_PACKED_SPLAT * BYTES_PER_TEXEL; 407 | const PADDED_SPLAT_LENGTH = 4 * Math.ceil(PACKED_SPLAT_LENGTH / 4); 408 | const FLOAT32_PER_PADDED_RENDERABLE_SPLAT = PADDED_RENDERABLE_SPLAT_LENGTH / 4; 409 | const UINT32_PER_PADDED_RENDERABLE_SPLAT = FLOAT32_PER_PADDED_RENDERABLE_SPLAT; 410 | const FLOAT32_PER_PADDED_SPLAT = PADDED_SPLAT_LENGTH / 4; 411 | 412 | let buffer; 413 | let gaussianCount = 0; 414 | let lastProj = []; 415 | let depthIndex = new Uint32Array(); 416 | let lastGaussianCount = 0; 417 | 418 | var _floatView = new Float32Array(1); 419 | var _int32View = new Int32Array(_floatView.buffer); 420 | 421 | function floatToHalf(float) { 422 | _floatView[0] = float; 423 | var f = _int32View[0]; 424 | 425 | var sign = (f >> 31) & 0x0001; 426 | var exp = (f >> 23) & 0x00ff; 427 | var frac = f & 0x007fffff; 428 | 429 | var newExp; 430 | if (exp == 0) { 431 | newExp = 0; 432 | } else if (exp < 113) { 433 | newExp = 0; 434 | frac |= 0x00800000; 435 | frac = frac >> (113 - exp); 436 | if (frac & 0x01000000) { 437 | newExp = 1; 438 | frac = 0; 439 | } 440 | } else if (exp < 142) { 441 | newExp = exp - 112; 442 | } else { 443 | newExp = 31; 444 | frac = 0; 445 | } 446 | 447 | return (sign << 15) | (newExp << 10) | (frac >> 13); 448 | } 449 | 450 | function packHalf2x16(x, y) { 451 | return (floatToHalf(x) | (floatToHalf(y) << 16)) >>> 0; 452 | } 453 | 454 | function generateTexture() { 455 | if (!buffer) return; 456 | const f_buffer = new Float32Array(buffer); 457 | const u_buffer = new Uint8Array(buffer); 458 | 459 | var texwidth = 1024 * TEXELS_PER_PACKED_SPLAT; // Set to your desired width 460 | var texheight = Math.ceil((TEXELS_PER_PACKED_SPLAT * gaussianCount) / texwidth); // Set to your desired height 461 | var texdata = new Uint32Array(texwidth * texheight * 4); // 4 Uint32 components per pixel in RGBAUI 462 | var texdata_c = new Uint8Array(texdata.buffer); 463 | var texdata_f = new Float32Array(texdata.buffer); 464 | var hasNormals = false; 465 | 466 | // Here we convert from a .splat file buffer into a texture 467 | // With a little bit more foresight perhaps this texture file 468 | // should have been the native format as it'd be very easy to 469 | // load it into webgl. 470 | for (let i = 0; i < gaussianCount; i++) { 471 | // position x, y, z 472 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 0] = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 0]; 473 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 1] = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 1]; 474 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 2] = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 2]; 475 | 476 | // r, g, b, a 477 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 7) + 0] = u_buffer[PADDED_SPLAT_LENGTH * i + 24 + 0]; 478 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 7) + 1] = u_buffer[PADDED_SPLAT_LENGTH * i + 24 + 1]; 479 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 7) + 2] = u_buffer[PADDED_SPLAT_LENGTH * i + 24 + 2]; 480 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 7) + 3] = u_buffer[PADDED_SPLAT_LENGTH * i + 24 + 3]; 481 | 482 | // quaternions 483 | let scale = [ 484 | f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 3 + 0], 485 | f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 3 + 1], 486 | f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 3 + 2], 487 | ]; 488 | let rot = [ 489 | (u_buffer[PADDED_SPLAT_LENGTH * i + 28 + 0] - 128) / 128, 490 | (u_buffer[PADDED_SPLAT_LENGTH * i + 28 + 1] - 128) / 128, 491 | (u_buffer[PADDED_SPLAT_LENGTH * i + 28 + 2] - 128) / 128, 492 | (u_buffer[PADDED_SPLAT_LENGTH * i + 28 + 3] - 128) / 128, 493 | ]; 494 | 495 | // Compute the matrix product of S and R (M = S * R) 496 | const M = [ 497 | 1.0 - 2.0 * (rot[2] * rot[2] + rot[3] * rot[3]), 498 | 2.0 * (rot[1] * rot[2] + rot[0] * rot[3]), 499 | 2.0 * (rot[1] * rot[3] - rot[0] * rot[2]), 500 | 501 | 2.0 * (rot[1] * rot[2] - rot[0] * rot[3]), 502 | 1.0 - 2.0 * (rot[1] * rot[1] + rot[3] * rot[3]), 503 | 2.0 * (rot[2] * rot[3] + rot[0] * rot[1]), 504 | 505 | 2.0 * (rot[1] * rot[3] + rot[0] * rot[2]), 506 | 2.0 * (rot[2] * rot[3] - rot[0] * rot[1]), 507 | 1.0 - 2.0 * (rot[1] * rot[1] + rot[2] * rot[2]), 508 | ].map((k, i) => k * scale[Math.floor(i / 3)]); 509 | 510 | const sigma = [ 511 | M[0] * M[0] + M[3] * M[3] + M[6] * M[6], 512 | M[0] * M[1] + M[3] * M[4] + M[6] * M[7], 513 | M[0] * M[2] + M[3] * M[5] + M[6] * M[8], 514 | M[1] * M[1] + M[4] * M[4] + M[7] * M[7], 515 | M[1] * M[2] + M[4] * M[5] + M[7] * M[8], 516 | M[2] * M[2] + M[5] * M[5] + M[8] * M[8], 517 | ]; 518 | 519 | texdata[UINT32_PER_PADDED_RENDERABLE_SPLAT * i + 4] = packHalf2x16(4 * sigma[0], 4 * sigma[1]); 520 | texdata[UINT32_PER_PADDED_RENDERABLE_SPLAT * i + 5] = packHalf2x16(4 * sigma[2], 4 * sigma[3]); 521 | texdata[UINT32_PER_PADDED_RENDERABLE_SPLAT * i + 6] = packHalf2x16(4 * sigma[4], 4 * sigma[5]); 522 | 523 | // normal x, y, z 524 | var n_x = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 8 + 0]; 525 | var n_y = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 8 + 1]; 526 | var n_z = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 8 + 2]; 527 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 8 + 0] = n_x; 528 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 8 + 1] = n_y; 529 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 8 + 2] = n_z; 530 | if (n_x != 0 || n_y != 0 || n_z != 0) { 531 | hasNormals = true; 532 | } 533 | 534 | // PBR base colors r, g, b 535 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 11) + 0] = u_buffer[PADDED_SPLAT_LENGTH * i + 4*11 + 0]; 536 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 11) + 1] = u_buffer[PADDED_SPLAT_LENGTH * i + 4*11 + 1]; 537 | texdata_c[4 * (FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 11) + 2] = u_buffer[PADDED_SPLAT_LENGTH * i + 4*11 + 2]; 538 | 539 | // PBR roughness, metallic 540 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 12 + 0] = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 12 + 0]; 541 | texdata_f[FLOAT32_PER_PADDED_RENDERABLE_SPLAT * i + 12 + 1] = f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 12 + 1]; 542 | } 543 | 544 | self.postMessage({ texdata, texwidth, texheight, hasNormals }, [texdata.buffer]); 545 | } 546 | 547 | function runSort(viewProj, label) { 548 | if (!buffer) return; 549 | const f_buffer = new Float32Array(buffer); 550 | if (lastGaussianCount == gaussianCount) { 551 | let dot = 552 | lastProj[2] * viewProj[2] + 553 | lastProj[6] * viewProj[6] + 554 | lastProj[10] * viewProj[10]; 555 | if (Math.abs(dot - 1) < 0.01) { 556 | return; 557 | } 558 | } else { 559 | generateTexture(); 560 | lastGaussianCount = gaussianCount; 561 | } 562 | 563 | // console.time("sort"); 564 | let maxDepth = -Infinity; 565 | let minDepth = Infinity; 566 | let sizeList = new Int32Array(gaussianCount); 567 | for (let i = 0; i < gaussianCount; i++) { 568 | let depth = 569 | ((viewProj[2] * f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 0] + 570 | viewProj[6] * f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 1] + 571 | viewProj[10] * f_buffer[FLOAT32_PER_PADDED_SPLAT * i + 2]) * 572 | 4096) | 573 | 0; 574 | sizeList[i] = depth; 575 | if (depth > maxDepth) maxDepth = depth; 576 | if (depth < minDepth) minDepth = depth; 577 | } 578 | 579 | // This is a 16 bit single-pass counting sort 580 | let depthInv = (256 * 256) / (maxDepth - minDepth); 581 | let counts0 = new Uint32Array(256 * 256); 582 | for (let i = 0; i < gaussianCount; i++) { 583 | sizeList[i] = ((sizeList[i] - minDepth) * depthInv) | 0; 584 | counts0[sizeList[i]]++; 585 | } 586 | let starts0 = new Uint32Array(256 * 256); 587 | for (let i = 1; i < 256 * 256; i++) 588 | starts0[i] = starts0[i - 1] + counts0[i - 1]; 589 | depthIndex = new Uint32Array(gaussianCount); 590 | for (let i = 0; i < gaussianCount; i++) 591 | depthIndex[starts0[sizeList[i]]++] = i; 592 | 593 | // console.timeEnd("sort"); 594 | 595 | lastProj = viewProj; 596 | self.postMessage({ depthIndex, viewProj, gaussianCount, label }, [ 597 | depthIndex.buffer, 598 | ]); 599 | } 600 | 601 | function processPlyBuffer(inputBuffer) { 602 | const ubuf = new Uint8Array(inputBuffer); 603 | // 10KB ought to be enough for a header... 604 | const header = new TextDecoder().decode(ubuf.slice(0, 1024 * 10)); 605 | const header_end = "end_header\n"; 606 | const header_end_index = header.indexOf(header_end); 607 | if (header_end_index < 0) 608 | throw new Error("Unable to read .ply file header"); 609 | const gaussianCount = parseInt(/element vertex (\d+)\n/.exec(header)[1]); 610 | console.log("Gaussian Count", gaussianCount); 611 | let row_offset = 0, 612 | offsets = {}, 613 | types = {}; 614 | const TYPE_MAP = { 615 | double: "getFloat64", 616 | int: "getInt32", 617 | uint: "getUint32", 618 | float: "getFloat32", 619 | short: "getInt16", 620 | ushort: "getUint16", 621 | uchar: "getUint8", 622 | }; 623 | for (let prop of header 624 | .slice(0, header_end_index) 625 | .split("\n") 626 | .filter((k) => k.startsWith("property "))) { 627 | const [p, type, name] = prop.split(" "); 628 | const arrayType = TYPE_MAP[type] || "getInt8"; 629 | types[name] = arrayType; 630 | offsets[name] = row_offset; 631 | row_offset += parseInt(arrayType.replace(/[^\d]/g, "")) / 8; 632 | } 633 | 634 | let dataView = new DataView( 635 | inputBuffer, 636 | header_end_index + header_end.length, 637 | ); 638 | let row = 0; 639 | const attrs = new Proxy( 640 | {}, 641 | { 642 | get(target, prop) { 643 | if (!types[prop]) throw new Error(prop + " not found"); 644 | return dataView[types[prop]]( 645 | row * row_offset + offsets[prop], 646 | true, 647 | ); 648 | }, 649 | }, 650 | ); 651 | 652 | console.time("calculate importance"); 653 | let sizeList = new Float32Array(gaussianCount); 654 | let sizeIndex = new Uint32Array(gaussianCount); 655 | for (row = 0; row < gaussianCount; row++) { 656 | sizeIndex[row] = row; 657 | if (!types["scale_0"]) continue; 658 | const size = 659 | Math.exp(attrs.scale_0) * 660 | Math.exp(attrs.scale_1) * 661 | Math.exp(attrs.scale_2); 662 | const opacity = 1 / (1 + Math.exp(-attrs.opacity)); 663 | sizeList[row] = size * opacity; 664 | } 665 | console.timeEnd("calculate importance"); 666 | 667 | console.time("sort"); 668 | sizeIndex.sort((b, a) => sizeList[a] - sizeList[b]); 669 | console.timeEnd("sort"); 670 | 671 | const buffer = new ArrayBuffer(PADDED_SPLAT_LENGTH * gaussianCount); 672 | 673 | console.time("build buffer"); 674 | for (let j = 0; j < gaussianCount; j++) { 675 | row = sizeIndex[j]; 676 | 677 | const position = new Float32Array(buffer, j * PADDED_SPLAT_LENGTH, 3); 678 | const scales = new Float32Array(buffer, j * PADDED_SPLAT_LENGTH + 4 * 3, 3); 679 | const rgba = new Uint8ClampedArray( 680 | buffer, 681 | j*PADDED_SPLAT_LENGTH + 3*4 + 3*4, 682 | 4, 683 | ); 684 | const rot = new Uint8ClampedArray( 685 | buffer, 686 | j*PADDED_SPLAT_LENGTH + 3*4 + 3*4 + 4, 687 | 4, 688 | ); 689 | const normal = new Float32Array( 690 | buffer, 691 | j*PADDED_SPLAT_LENGTH + 3*4 + 3*4 + 4 + 4, 692 | 3, 693 | ); 694 | const pbrRGB = new Uint8ClampedArray( 695 | buffer, 696 | j*PADDED_SPLAT_LENGTH + 3*4 + 3*4 + 4 + 4 + 3*4, 697 | 3, 698 | ); 699 | const pbrRM = new Float32Array( 700 | buffer, 701 | j*PADDED_SPLAT_LENGTH + 3*4 + 3*4 + 4 + 4 + 3*4 + 3 + 1, 702 | 2, 703 | ); 704 | 705 | if (types["scale_0"]) { 706 | const qlen = Math.sqrt( 707 | attrs.rot_0 ** 2 + 708 | attrs.rot_1 ** 2 + 709 | attrs.rot_2 ** 2 + 710 | attrs.rot_3 ** 2, 711 | ); 712 | 713 | rot[0] = (attrs.rot_0 / qlen) * 128 + 128; 714 | rot[1] = (attrs.rot_1 / qlen) * 128 + 128; 715 | rot[2] = (attrs.rot_2 / qlen) * 128 + 128; 716 | rot[3] = (attrs.rot_3 / qlen) * 128 + 128; 717 | 718 | scales[0] = Math.exp(attrs.scale_0); 719 | scales[1] = Math.exp(attrs.scale_1); 720 | scales[2] = Math.exp(attrs.scale_2); 721 | } else { 722 | scales[0] = 0.01; 723 | scales[1] = 0.01; 724 | scales[2] = 0.01; 725 | 726 | rot[0] = 255; 727 | rot[1] = 0; 728 | rot[2] = 0; 729 | rot[3] = 0; 730 | } 731 | 732 | position[0] = attrs.x; 733 | position[1] = attrs.y; 734 | position[2] = attrs.z; 735 | 736 | if (types["f_dc_0"]) { 737 | const SH_C0 = 0.28209479177387814; 738 | rgba[0] = (0.5 + SH_C0 * attrs.f_dc_0) * 255; 739 | rgba[1] = (0.5 + SH_C0 * attrs.f_dc_1) * 255; 740 | rgba[2] = (0.5 + SH_C0 * attrs.f_dc_2) * 255; 741 | } else { 742 | rgba[0] = attrs.red; 743 | rgba[1] = attrs.green; 744 | rgba[2] = attrs.blue; 745 | } 746 | if (types["opacity"]) { 747 | rgba[3] = (1 / (1 + Math.exp(-attrs.opacity))) * 255; 748 | } else { 749 | rgba[3] = 255; 750 | } 751 | if (types["nx"] && types["ny"] && types["nz"]) { 752 | normal[0] = attrs.nx; 753 | normal[1] = attrs.ny; 754 | normal[2] = attrs.nz; 755 | } 756 | if (types["base_color_0"] && types["base_color_1"] && types["base_color_2"]) { 757 | pbrRGB[0] = attrs.base_color_0 * 255; 758 | pbrRGB[1] = attrs.base_color_1 * 255; 759 | pbrRGB[2] = attrs.base_color_2 * 255; 760 | } 761 | if (types["roughness"] && types["metallic"]) { 762 | pbrRM[0] = attrs.roughness; 763 | pbrRM[1] = attrs.metallic; 764 | } 765 | } 766 | console.timeEnd("build buffer"); 767 | return buffer; 768 | } 769 | 770 | const labelsToSorters = {}; 771 | const getOrCreateThrottledSorter = (label) => { 772 | if (!labelsToSorters[label]) { 773 | var currViewProj = null; 774 | var sortRunning = false; 775 | const self = { 776 | resortCurrent: () => { 777 | // Call this when something invalidates the current positions or gaussian count, 778 | // e.g. progressively loading another chunk of gaussians 779 | if (currViewProj) { 780 | self.throttledSort(currViewProj); 781 | } 782 | }, 783 | throttledSort: (viewProj) => { 784 | currViewProj = viewProj; 785 | if (!sortRunning) { 786 | sortRunning = true; 787 | let lastView = viewProj; 788 | runSort(lastView, label); 789 | setTimeout(() => { 790 | sortRunning = false; 791 | if (lastView !== currViewProj) { 792 | self.throttledSort(currViewProj); 793 | } 794 | }, 0); 795 | } 796 | }, 797 | }; 798 | labelsToSorters[label] = self; 799 | } 800 | return labelsToSorters[label]; 801 | }; 802 | 803 | self.onmessage = (e) => { 804 | if (e.data.ply) { 805 | gaussianCount = 0; 806 | buffer = processPlyBuffer(e.data.ply); 807 | gaussianCount = Math.floor(buffer.byteLength / PADDED_SPLAT_LENGTH); 808 | postMessage({ buffer: buffer }); 809 | } else if (e.data.buffer) { 810 | buffer = e.data.buffer; 811 | gaussianCount = e.data.gaussianCount; 812 | Object.keys(labelsToSorters).forEach((k) => { 813 | labelsToSorters[k].resortCurrent(); 814 | }); 815 | } else if (e.data.gaussianCount) { 816 | gaussianCount = e.data.gaussianCount; 817 | } else if (e.data.view) { 818 | if (!e.data.label) { 819 | console.error("Expected label for sort"); 820 | } else { 821 | getOrCreateThrottledSorter(e.data.label).throttledSort(e.data.view); 822 | } 823 | } 824 | }; 825 | } 826 | 827 | const gaussianVertexSource = ` 828 | #version 300 es 829 | precision highp float; 830 | precision highp int; 831 | 832 | uniform highp usampler2D u_texture; 833 | uniform mat4 projection, view; 834 | uniform vec2 focal; 835 | uniform vec2 viewport; 836 | uniform int mode; 837 | 838 | in vec2 position; 839 | in int index; 840 | 841 | out vec4 vColor; 842 | out vec3 vPBRColor; 843 | out vec2 vPosition; 844 | out vec3 vNormal; 845 | out float vRoughness; 846 | out float vMetallic; 847 | 848 | void main () { 849 | uvec4 bytes_00_15 = texelFetch(u_texture, ivec2((uint(index) & 0x3ffu) * uint(${TEXELS_PER_PACKED_SPLAT}), uint(index) >> 10), 0); 850 | vec4 cam = view * vec4(uintBitsToFloat(bytes_00_15.xyz), 1); 851 | vec4 pos2d = projection * cam; 852 | 853 | float clip = 1.2 * pos2d.w; 854 | if (pos2d.z < -clip || pos2d.x < -clip || pos2d.x > clip || pos2d.y < -clip || pos2d.y > clip) { 855 | gl_Position = vec4(0.0, 0.0, 2.0, 1.0); 856 | return; 857 | } 858 | 859 | uvec4 bytes_16_31 = texelFetch(u_texture, ivec2(((uint(index) & 0x3ffu) * uint(${TEXELS_PER_PACKED_SPLAT})) | 1u, uint(index) >> 10), 0); 860 | vec2 u1 = unpackHalf2x16(bytes_16_31.x), 861 | u2 = unpackHalf2x16(bytes_16_31.y), 862 | u3 = unpackHalf2x16(bytes_16_31.z); 863 | mat3 Vrk = mat3(u1.x, u1.y, u2.x, 864 | u1.y, u2.y, u3.x, 865 | u2.x, u3.x, u3.y); 866 | 867 | mat3 J = mat3( 868 | focal.x / cam.z, 0., -(focal.x * cam.x) / (cam.z * cam.z), 869 | 0., -focal.y / cam.z, (focal.y * cam.y) / (cam.z * cam.z), 870 | 0., 0., 0. 871 | ); 872 | 873 | mat3 T = transpose(mat3(view)) * J; 874 | mat3 cov2d = transpose(T) * Vrk * T; 875 | 876 | float mid = (cov2d[0][0] + cov2d[1][1]) / 2.0; 877 | float radius = length( 878 | vec2( 879 | (cov2d[0][0] - cov2d[1][1]) / 2.0, 880 | cov2d[0][1] 881 | ) 882 | ); 883 | float lambda1 = mid + radius, lambda2 = mid - radius; 884 | 885 | if(lambda2 < 0.0) return; 886 | vec2 diagonalVector = normalize(vec2(cov2d[0][1], lambda1 - cov2d[0][0])); 887 | vec2 majorAxis = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector; 888 | vec2 minorAxis = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x); 889 | 890 | uvec4 bytes_32_47 = texelFetch(u_texture, ivec2(((uint(index) & 0x3ffu) * uint(${TEXELS_PER_PACKED_SPLAT})) | 2u, uint(index) >> 10), 0); 891 | uvec4 bytes_48_63 = texelFetch(u_texture, ivec2(((uint(index) & 0x3ffu) * uint(${TEXELS_PER_PACKED_SPLAT})) | 3u, uint(index) >> 10), 0); 892 | // TODO handle splat data without normals 893 | vNormal = normalize(vec3(uintBitsToFloat(bytes_32_47.xyz))); 894 | uint opacity255 = (bytes_16_31.w >> 24) & 0xffu; 895 | 896 | if (mode == 0) { 897 | // Color mode 898 | vColor = 899 | clamp(pos2d.z/pos2d.w+1.0, 0.0, 1.0) * 900 | vec4( 901 | (bytes_16_31.w) & 0xffu, 902 | (bytes_16_31.w >> 8) & 0xffu, 903 | (bytes_16_31.w >> 16) & 0xffu, 904 | opacity255 905 | ) / 255.0; 906 | vPBRColor = 907 | clamp(pos2d.z/pos2d.w+1.0, 0.0, 1.0) * 908 | vec3( 909 | (bytes_32_47.w) & 0xffu, 910 | (bytes_32_47.w >> 8) & 0xffu, 911 | (bytes_32_47.w >> 16) & 0xffu 912 | ) / 255.0; 913 | vRoughness = uintBitsToFloat(bytes_48_63.x); 914 | vMetallic = uintBitsToFloat(bytes_48_63.y); 915 | } else { 916 | // Depth mode 917 | // TODO(achan): We should compute the depth for each individual fragment based 918 | // on its position within the rasterized Gaussian quad, rather than pretend all 919 | // fragments of the quad have the depth of the center. 920 | // 921 | // This seems important for getting the correct world-space point of a fragment 922 | // for accurate lighting. 923 | float depth = pos2d.w; 924 | vColor = vec4( 925 | depth, 926 | 0., 927 | length(cam.xyz), 928 | float(opacity255) / 255.0 929 | ); 930 | } 931 | vPosition = position; 932 | 933 | vec2 vCenter = vec2(pos2d) / pos2d.w; 934 | gl_Position = vec4( 935 | vCenter 936 | + position.x * majorAxis / viewport 937 | + position.y * minorAxis / viewport, 0.0, 1.0); 938 | 939 | } 940 | `.trim(); 941 | 942 | const overlayVertexSource = ` 943 | #version 300 es 944 | precision highp float; 945 | 946 | uniform mat4 projection, view; 947 | uniform vec3 worldCameraPosition; 948 | uniform vec3 worldCameraUp; 949 | uniform vec2 size; 950 | 951 | in vec2 uv; 952 | in vec3 worldCenter; 953 | 954 | out vec2 vUv; 955 | 956 | void main () { 957 | vec3 worldP = worldCenter; 958 | // Overlay quad should always face the camera 959 | vec3 dirToCamera = normalize(worldCameraPosition - worldP); 960 | vec3 up = worldCameraUp; 961 | vec3 right = normalize(cross(up, dirToCamera)); 962 | 963 | vec4 world_p = vec4(worldP, 1.) + vec4(size.x * right * (uv.x-0.5) + size.y * up * (uv.y-0.5), 0); 964 | vec4 eye_p = view * world_p; 965 | vec4 clip_p = projection * eye_p; 966 | 967 | vUv = uv; 968 | gl_Position = clip_p; 969 | } 970 | `.trim(); 971 | 972 | const cubeMapDebugFragmentSource = ` 973 | #version 300 es 974 | precision highp float; 975 | 976 | uniform samplerCube overlayTexture; 977 | 978 | in vec2 vUv; 979 | 980 | out vec4 fragColor; 981 | 982 | void main () { 983 | fragColor.rgb = vec3(0.0, 0.0, 0.2); 984 | 985 | vec3 samplePos = vec3(0.0f); 986 | 987 | // Crude statement to visualize different cube map faces based on UV coordinates 988 | int x = int(floor(vUv.x / 0.25f)); 989 | int y = int(floor(vUv.y / (1.0 / 3.0))); 990 | if (y == 1) { 991 | vec2 uv = vec2(vUv.x * 4.0f, (vUv.y - 1.0/3.0) * 3.0); 992 | uv = 2.0 * vec2(uv.x - float(x) * 1.0, uv.y) - 1.0; 993 | switch (x) { 994 | case 0: // NEGATIVE_X 995 | samplePos = vec3(-1.0f, uv.y, uv.x); 996 | break; 997 | case 1: // POSITIVE_Z 998 | samplePos = vec3(uv.x, uv.y, 1.0f); 999 | break; 1000 | case 2: // POSITIVE_X 1001 | samplePos = vec3(1.0, uv.y, -uv.x); 1002 | break; 1003 | case 3: // NEGATIVE_Z 1004 | samplePos = vec3(-uv.x, uv.y, -1.0f); 1005 | break; 1006 | } 1007 | } else { 1008 | if (x == 1) { 1009 | vec2 uv = vec2((vUv.x - 0.25) * 4.0, (vUv.y - float(y) / 3.0) * 3.0); 1010 | uv = 2.0 * uv - 1.0; 1011 | switch (y) { 1012 | case 0: // NEGATIVE_Y 1013 | samplePos = vec3(uv.x, -1.0f, uv.y); 1014 | break; 1015 | case 2: // POSITIVE_Y 1016 | samplePos = vec3(uv.x, 1.0f, -uv.y); 1017 | break; 1018 | } 1019 | } 1020 | } 1021 | 1022 | if ((samplePos.x != 0.0f) && (samplePos.y != 0.0f)) { 1023 | fragColor = vec4(texture(overlayTexture, samplePos).r, 0., 0., 1.); 1024 | } 1025 | } 1026 | 1027 | `.trim(); 1028 | 1029 | const overlayFragmentSource = ` 1030 | #version 300 es 1031 | precision highp float; 1032 | 1033 | uniform sampler2D overlayTexture; 1034 | 1035 | in vec2 vUv; 1036 | 1037 | out vec4 fragColor; 1038 | 1039 | void main () { 1040 | fragColor = texture(overlayTexture, vUv); 1041 | } 1042 | 1043 | `.trim(); 1044 | 1045 | const colorFragmentSource = ` 1046 | #version 300 es 1047 | precision highp float; 1048 | precision highp int; 1049 | 1050 | uniform int mode; 1051 | uniform float alphaThreshold; 1052 | 1053 | in vec4 vColor; 1054 | in vec2 vPosition; 1055 | 1056 | out vec4 fragColor; 1057 | 1058 | void main () { 1059 | if (vColor.a < alphaThreshold) discard; 1060 | float A = -dot(vPosition, vPosition); 1061 | if (A < -4.0) discard; 1062 | float B = exp(A) * vColor.a; 1063 | fragColor = vec4(B * vColor.rgb, B); 1064 | } 1065 | 1066 | `.trim(); 1067 | 1068 | const MAX_LIGHTS = 8; 1069 | const lightingFragmentSource = ` 1070 | #version 300 es 1071 | precision highp float; 1072 | precision highp int; 1073 | 1074 | #define M_PI 3.1415926535897932384626433832795 1075 | 1076 | uniform vec2 screenSize; 1077 | uniform int usePseudoNormals; 1078 | uniform int usePBR; 1079 | uniform sampler2D depthTexture; 1080 | uniform mat4 invProjection, invView; 1081 | uniform vec3 lightPositions[${MAX_LIGHTS}]; 1082 | uniform mat4 lightViewProjMatrices[${MAX_LIGHTS}]; 1083 | uniform samplerCube shadowMaps[${MAX_LIGHTS}]; 1084 | uniform int numLights; 1085 | 1086 | in vec4 vColor; 1087 | in vec2 vPosition; 1088 | in vec3 vNormal; 1089 | in vec3 vPBRColor; 1090 | in float vMetallic; 1091 | in float vRoughness; 1092 | 1093 | out vec4 fragColor; 1094 | 1095 | float computeShadow(samplerCube shadowMap, vec3 lightToPoint) { 1096 | float depth = length(lightToPoint); 1097 | 1098 | float shadow = 1.0; 1099 | float bias = 0.05; 1100 | float shadowDepth = texture(shadowMap, vec3(lightToPoint.x, -lightToPoint.y, -lightToPoint.z)).b; 1101 | if (depth - bias <= shadowDepth) { 1102 | shadow = 0.0; 1103 | } 1104 | return shadow; 1105 | } 1106 | 1107 | vec3 computePBR(float radiance, vec3 normal, vec3 pointToLight, vec3 pointToCamera, vec3 albedo, float roughness, float metallic) { 1108 | vec3 halfVector = normalize(pointToLight + pointToCamera); 1109 | vec3 fd = (1. - metallic) * albedo / M_PI; 1110 | 1111 | // D 1112 | float r2 = max(roughness * roughness, 0.0000001); 1113 | float amp = 1.0 / (r2 * M_PI); 1114 | float sharp = 2.0 / r2; 1115 | float D = amp * exp(sharp * (dot(halfVector, normal) - 1.0)); 1116 | 1117 | // F 1118 | vec3 F_0 = 0.04 * (1.0 - metallic) + albedo * metallic; 1119 | vec3 F = F_0 + (1.0f - F_0) * pow(1.0 - dot(halfVector, pointToCamera), 5.0); 1120 | 1121 | r2 = pow(1.0 + roughness, 2.0) / 8.0; 1122 | float V = 1123 | (0.5 / max(dot(pointToLight, normal) * (1. - r2) + r2, 0.0000001)) * 1124 | (0.5 / max(dot(pointToCamera, normal) * (1. - r2) + r2, 0.0000001)); 1125 | vec3 fs = D * F * V; 1126 | float transport = radiance * (2.0 * M_PI * dot(normal, pointToLight)); 1127 | 1128 | return (fd + fs) * transport; 1129 | } 1130 | 1131 | void main () { 1132 | float A = -dot(vPosition, vPosition); 1133 | if (A < -4.0) discard; 1134 | float B = exp(A) * vColor.a; 1135 | 1136 | float du = 1.0 / float(textureSize(depthTexture, 0).x); 1137 | float dv = 1.0 / float(textureSize(depthTexture, 0).y); 1138 | vec2 uv0 = gl_FragCoord.xy / screenSize; 1139 | vec2 uv1 = uv0 + vec2(du, 0.); 1140 | vec2 uv2 = uv0 + vec2(0., dv); 1141 | // Reconstruct clip-space points. 1142 | float clip_w0 = texture(depthTexture, uv0).r; 1143 | float clip_w1 = texture(depthTexture, uv1).r; 1144 | float clip_w2 = texture(depthTexture, uv2).r; 1145 | vec2 ndc0 = 2.0 * uv0 - 1.0; 1146 | vec2 ndc1 = 2.0 * uv1 - 1.0; 1147 | vec2 ndc2 = 2.0 * uv2 - 1.0; 1148 | 1149 | float zfar = 200.; 1150 | float znear = 0.2; 1151 | float zw = zfar/(zfar - znear); 1152 | float zb = -zfar*znear/(zfar - znear); 1153 | 1154 | vec4 clip_p0 = clip_w0 * vec4(ndc0, zw, 1.) + vec4(0., 0., zb, 0.); 1155 | vec4 clip_p1 = clip_w1 * vec4(ndc1, zw, 1.) + vec4(0., 0., zb, 0.); 1156 | vec4 clip_p2 = clip_w2 * vec4(ndc2, zw, 1.) + vec4(0., 0., zb, 0.); 1157 | 1158 | vec4 world_p0 = invView * invProjection * clip_p0; 1159 | vec4 world_p1 = invView * invProjection * clip_p1; 1160 | vec4 world_p2 = invView * invProjection * clip_p2; 1161 | 1162 | vec4 world_camera = invView * vec4(0., 0., 0., 1.); 1163 | vec3 pointToCamera = world_camera.xyz - world_p0.xyz; 1164 | 1165 | vec3 normal = vec3(0., 0., 0.); 1166 | if (usePseudoNormals == 1) { 1167 | normal = normalize(cross(world_p1.xyz - world_p0.xyz, world_p2.xyz - world_p0.xyz)); 1168 | } else { 1169 | normal = vNormal; 1170 | } 1171 | vec3 albedo = vColor.rgb; 1172 | if (usePBR == 1) { 1173 | albedo = vPBRColor; 1174 | } 1175 | vec3 result = vec3(0.0, 0.0, 0.0); 1176 | for (int i = 0; i < numLights; i++) { 1177 | vec3 pointToLight = lightPositions[i] - world_p0.xyz; 1178 | float shadow = 0.; 1179 | switch(i) { 1180 | ${ 1181 | // Due to GLSL stupidity we can only index into sampler arrays with constants and so need to 1182 | // codegen this switch statement to allow using the loop counter as array index. 1183 | new Array(MAX_LIGHTS).fill(0).map((_, i) => 1184 | `case ${i}: shadow = computeShadow(shadowMaps[${i}], -pointToLight); break;` 1185 | ).join("\n") 1186 | }} 1187 | float lightDistance = length(pointToLight); 1188 | float R = 4.0; 1189 | float attenuation = 1. / (pow(lightDistance / R, 2.) + 1.); 1190 | float radiance = (1. - shadow) * attenuation; 1191 | if (usePBR == 1) { 1192 | result += computePBR( 1193 | radiance, normal, normalize(pointToLight), 1194 | normalize(pointToCamera), albedo, 1195 | vRoughness, vMetallic 1196 | ); 1197 | } else { 1198 | result += radiance * albedo * max(dot(normal, normalize(pointToLight)), 0.0); 1199 | } 1200 | } 1201 | if (usePBR == 1) { 1202 | // gamma correct 1203 | // TODO: this should use the learned gamma parameter from R3DG if available. 1204 | result = result / (result + vec3(1.0)); 1205 | result = pow(result, vec3(1.0/2.2)); 1206 | } else { 1207 | float ambient = 0.2; 1208 | result += ambient * albedo; 1209 | } 1210 | 1211 | fragColor = vec4(B * result, B); 1212 | } 1213 | 1214 | `.trim(); 1215 | 1216 | const filterVertexSource = ` 1217 | #version 300 es 1218 | precision highp float; 1219 | 1220 | in vec2 uv; 1221 | 1222 | out vec2 vUv; 1223 | 1224 | void main () { 1225 | vUv = uv; 1226 | gl_Position = vec4(2.*uv.x - 1., 2.*uv.y - 1., 0., 1.); 1227 | } 1228 | `.trim(); 1229 | 1230 | const filterFragmentSource = ` 1231 | #version 300 es 1232 | precision highp float; 1233 | 1234 | uniform sampler2D tex; 1235 | uniform int kernelSize; 1236 | uniform float sigma_range; 1237 | uniform float sigma_domain; 1238 | 1239 | in vec2 vUv; 1240 | 1241 | out vec4 fragColor; 1242 | 1243 | vec4 sampleBilateralFiltered(sampler2D tex, vec2 uv) { 1244 | vec2 duv = 1.0 / vec2(textureSize(tex, 0)); 1245 | float sigma_range_sq = sigma_range * sigma_range; 1246 | float sigma_domain_sq = sigma_domain * sigma_domain; 1247 | vec4 s0 = texture(tex, uv); 1248 | 1249 | vec4 result = vec4(0.); 1250 | vec4 totalWeight = vec4(0.); 1251 | for (int i = -kernelSize; i <= kernelSize; i++) { 1252 | for (int j = -kernelSize; j <= kernelSize; j++) { 1253 | vec2 delta = vec2(i, j) * duv; 1254 | float uvSqDist = dot(delta, delta); 1255 | vec2 uvi = uv + delta; 1256 | vec4 si = texture(tex, uvi); 1257 | vec4 rangeSqDist = (si - s0) * (si - s0); 1258 | vec4 wi = exp(- vec4(uvSqDist / (2. * sigma_domain_sq)) - rangeSqDist / (2. * sigma_range_sq)); 1259 | result += si * wi; 1260 | totalWeight += wi; 1261 | } 1262 | } 1263 | result = result / totalWeight; 1264 | return result; 1265 | } 1266 | 1267 | void main () { 1268 | fragColor = sampleBilateralFiltered(tex, vUv); 1269 | } 1270 | 1271 | `.trim(); 1272 | 1273 | let defaultViewMatrix = [ 1274 | 0.47, 0.04, 0.88, 0, -0.11, 0.99, 0.02, 0, -0.88, -0.11, 0.47, 0, 0.07, 1275 | 0.03, 6.55, 1, 1276 | ]; 1277 | let viewMatrix = defaultViewMatrix; 1278 | const MODES = { 1279 | "COLOR": 0, 1280 | "DEPTH": 1, 1281 | "LIGHTING": 2, 1282 | }; 1283 | 1284 | async function main() { 1285 | let carousel = true; 1286 | const params = new URLSearchParams(location.search); 1287 | try { 1288 | viewMatrix = JSON.parse(decodeURIComponent(location.hash.slice(1))); 1289 | carousel = false; 1290 | } catch (err) {} 1291 | const url = new URL( 1292 | params.get("url") || "garden.lsplat", 1293 | "https://huggingface.co/datasets/andrewkchan/lit-splat-data/resolve/main/" 1294 | ); 1295 | const req = await fetch(url, { 1296 | mode: "cors", // no-cors, *cors, same-origin 1297 | credentials: "omit", // include, *same-origin, omit 1298 | }); 1299 | console.log(req); 1300 | if (req.status != 200) 1301 | throw new Error(req.status + " Unable to load " + req.url); 1302 | 1303 | const reader = req.body.getReader(); 1304 | let splatData = new Uint8Array(req.headers.get("content-length") - LSPLAT_MAGIC_HEADER.length); 1305 | 1306 | const downsample = 1307 | splatData.length / PADDED_SPLAT_LENGTH > 500000 ? 1 : 1 / devicePixelRatio; 1308 | console.log(splatData.length / PADDED_SPLAT_LENGTH, downsample); 1309 | 1310 | const worker = new Worker( 1311 | URL.createObjectURL( 1312 | new Blob(["(", createWorker.toString(), ")(self)"], { 1313 | type: "application/javascript", 1314 | }), 1315 | ), 1316 | ); 1317 | 1318 | const canvas = document.getElementById("canvas"); 1319 | const fps = document.getElementById("fps"); 1320 | const camid = document.getElementById("camid"); 1321 | const addLightButton = document.getElementById("add-light"); 1322 | 1323 | let projectionMatrix; 1324 | 1325 | const gl = canvas.getContext("webgl2", { 1326 | antialias: false, 1327 | }); 1328 | gl.getExtension('EXT_color_buffer_float'); 1329 | 1330 | let LAST_TEX_ID = 0; 1331 | function createTextureObject(filter, textureType) { 1332 | const texId = LAST_TEX_ID++; 1333 | gl.activeTexture(gl.TEXTURE0 + texId); 1334 | let texture = gl.createTexture(); 1335 | gl.bindTexture(textureType, texture); 1336 | gl.texParameteri(textureType, gl.TEXTURE_MIN_FILTER, filter); 1337 | gl.texParameteri(textureType, gl.TEXTURE_MAG_FILTER, filter); 1338 | gl.texParameteri(textureType, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); 1339 | gl.texParameteri(textureType, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); 1340 | if (textureType == gl.TEXTURE_CUBE_MAP) { 1341 | gl.texParameteri(textureType, gl.TEXTURE_WRAP_R, gl.CLAMP_TO_EDGE); 1342 | } 1343 | return { 1344 | texture, 1345 | texId, 1346 | textureType, 1347 | }; 1348 | } 1349 | function createFBO(w, h, internalFormat, format, type, filter, textureType) { 1350 | let fbo = gl.createFramebuffer(); 1351 | var { texture, texId } = createTextureObject(filter, textureType); 1352 | gl.activeTexture(gl.TEXTURE0 + texId); 1353 | gl.bindTexture(textureType, texture); 1354 | if (textureType == gl.TEXTURE_2D) { 1355 | gl.texImage2D( 1356 | gl.TEXTURE_2D, 1357 | 0, 1358 | internalFormat, 1359 | w, 1360 | h, 1361 | 0, 1362 | format, 1363 | type, 1364 | null, 1365 | ); 1366 | gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); 1367 | gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0); 1368 | gl.viewport(0, 0, w, h); 1369 | gl.clear(gl.COLOR_BUFFER_BIT); 1370 | } else { 1371 | gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); 1372 | for (let i = 0; i < 6; i++) { 1373 | gl.texImage2D( 1374 | gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 1375 | 0, 1376 | internalFormat, 1377 | w, 1378 | h, 1379 | 0, 1380 | format, 1381 | type, 1382 | null, 1383 | ) 1384 | gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, texture, 0); 1385 | } 1386 | } 1387 | return { 1388 | texture, 1389 | textureType, 1390 | fbo, 1391 | texId, 1392 | }; 1393 | } 1394 | 1395 | var gaussianDataTexture = createTextureObject(gl.NEAREST, gl.TEXTURE_2D); 1396 | 1397 | const depthWidth = 640; 1398 | const depthHeight = 480; 1399 | let depthFBO = createFBO(depthWidth, depthHeight, gl.RGBA16F, gl.RGBA, gl.HALF_FLOAT, gl.LINEAR, gl.TEXTURE_2D); 1400 | let filteredDepthFBO = createFBO(depthWidth, depthHeight, gl.RGBA16F, gl.RGBA, gl.HALF_FLOAT, gl.LINEAR, gl.TEXTURE_2D); 1401 | 1402 | const shadowMapSize = 400; 1403 | let shadowMapFBOs = []; 1404 | let lights = []; 1405 | const DUMMY_SHADOW_MAP_FBO = createFBO(shadowMapSize, shadowMapSize, gl.RGBA16F, gl.RGBA, gl.HALF_FLOAT, gl.LINEAR, gl.TEXTURE_CUBE_MAP); 1406 | 1407 | var lightPositions = new Float32Array(MAX_LIGHTS * 3); // Fixed length for easy upload to GPU 1408 | var ndcSpaceLightBoundingBoxes = []; // At the end of each frame, is guaranteed to have `numLights` elements 1409 | var numLights = 0; 1410 | var selectedLight = -1; 1411 | function addLight() { 1412 | if (numLights >= MAX_LIGHTS) return; 1413 | const i = numLights++; 1414 | shadowMapFBOs.push(createFBO(shadowMapSize, shadowMapSize, gl.RGBA16F, gl.RGBA, gl.HALF_FLOAT, gl.LINEAR, gl.TEXTURE_CUBE_MAP)); 1415 | const light = { 1416 | position: null, 1417 | faces: {}, 1418 | needsShadowMapUpdate: false, 1419 | } 1420 | for (let i = 0; i < 6; i++) { 1421 | light.faces[i] = { 1422 | viewMatrix: null, 1423 | projMatrix: null, 1424 | viewProj: null, 1425 | indexBuffer: gl.createBuffer(), 1426 | }; 1427 | } 1428 | lights.push(light); 1429 | updateLightPosition(i, [0, 0, 0]); 1430 | addLightButton.innerHTML = `💡 Add Light (${numLights}/${MAX_LIGHTS})`; 1431 | } 1432 | function updateLightPosition(i, position) { 1433 | const light = lights[i]; 1434 | light.position = position; 1435 | light.needsShadowMapUpdate = true; 1436 | for (let f = 0; f < 6; f++) { 1437 | switch (f) { 1438 | case 0: { 1439 | // positive x 1440 | light.faces[f].viewMatrix = lookAt(position, add3(position, [-1, 0, 0]), [0, -1, 0]); 1441 | break; 1442 | } 1443 | case 1: { 1444 | // negative x 1445 | light.faces[f].viewMatrix = lookAt(position, add3(position, [1, 0, 0]), [0, -1, 0]); 1446 | break; 1447 | } 1448 | case 2: { 1449 | // positive y 1450 | light.faces[f].viewMatrix = lookAt(position, add3(position, [0, 1, 0]), [0, 0, 1]); 1451 | break; 1452 | } 1453 | case 3: { 1454 | // negative y 1455 | light.faces[f].viewMatrix = lookAt(position, add3(position, [0, -1, 0]), [0, 0, -1]); 1456 | break; 1457 | } 1458 | case 4: { 1459 | // positive z 1460 | light.faces[f].viewMatrix = lookAt(position, add3(position, [0, 0, 1]), [0, -1, 0]); 1461 | break; 1462 | } 1463 | case 5: { 1464 | // negative z 1465 | light.faces[f].viewMatrix = lookAt(position, add3(position, [0, 0, -1]), [0, -1, 0]); 1466 | break; 1467 | } 1468 | } 1469 | light.faces[f].projMatrix = getProjectionMatrix(shadowMapSize / 2, shadowMapSize / 2, shadowMapSize, shadowMapSize); 1470 | light.faces[f].viewProj = multiply4(light.faces[f].projMatrix, light.faces[f].viewMatrix); 1471 | worker.postMessage({ view: light.faces[f].viewProj, label: `light-${i}-${f}` }); 1472 | } 1473 | lightPositions[i * 3] = position[0]; 1474 | lightPositions[i * 3 + 1] = position[1]; 1475 | lightPositions[i * 3 + 2] = position[2]; 1476 | } 1477 | addLightButton.addEventListener("click", addLight); 1478 | 1479 | const lightOverlayTexture = createTextureObject(gl.LINEAR, gl.TEXTURE_2D); 1480 | const image = new Image(); 1481 | image.src = "./lightbulb.png"; 1482 | image.onload = function() { 1483 | gl.activeTexture(gl.TEXTURE0 + lightOverlayTexture.texId); 1484 | gl.bindTexture(gl.TEXTURE_2D, lightOverlayTexture.texture); 1485 | gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); 1486 | gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); 1487 | gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); 1488 | }; 1489 | 1490 | const indexBuffer = gl.createBuffer(); 1491 | 1492 | const GAUSSIAN_QUAD_VERTICES = new Float32Array([ 1493 | -2, -2, 1494 | 2, -2, 1495 | 2, 2, 1496 | -2, 2 1497 | ]); 1498 | const gaussianQuadVertexBuffer = gl.createBuffer(); 1499 | gl.bindBuffer(gl.ARRAY_BUFFER, gaussianQuadVertexBuffer); 1500 | gl.bufferData(gl.ARRAY_BUFFER, GAUSSIAN_QUAD_VERTICES, gl.STATIC_DRAW); 1501 | 1502 | const QUAD_UVS = new Float32Array([ 1503 | 0, 1, 1504 | 1, 1, 1505 | 1, 0, 1506 | 0, 0 1507 | ]); 1508 | const quadUVBuffer = gl.createBuffer(); 1509 | gl.bindBuffer(gl.ARRAY_BUFFER, quadUVBuffer); 1510 | gl.bufferData(gl.ARRAY_BUFFER, QUAD_UVS, gl.STATIC_DRAW); 1511 | 1512 | const gaussianVertexShader = gl.createShader(gl.VERTEX_SHADER); 1513 | gl.shaderSource(gaussianVertexShader, gaussianVertexSource); 1514 | gl.compileShader(gaussianVertexShader); 1515 | if (!gl.getShaderParameter(gaussianVertexShader, gl.COMPILE_STATUS)) { 1516 | console.error(gl.getShaderInfoLog(gaussianVertexShader)); 1517 | } 1518 | 1519 | const colorFragmentShader = gl.createShader(gl.FRAGMENT_SHADER); 1520 | gl.shaderSource(colorFragmentShader, colorFragmentSource); 1521 | gl.compileShader(colorFragmentShader); 1522 | if (!gl.getShaderParameter(colorFragmentShader, gl.COMPILE_STATUS)) { 1523 | console.error(gl.getShaderInfoLog(colorFragmentShader)); 1524 | } 1525 | 1526 | const lightingFragmentShader = gl.createShader(gl.FRAGMENT_SHADER); 1527 | gl.shaderSource(lightingFragmentShader, lightingFragmentSource); 1528 | gl.compileShader(lightingFragmentShader); 1529 | if (!gl.getShaderParameter(lightingFragmentShader, gl.COMPILE_STATUS)) { 1530 | console.error(gl.getShaderInfoLog(lightingFragmentShader)); 1531 | } 1532 | 1533 | const overlayVertexShader = gl.createShader(gl.VERTEX_SHADER); 1534 | gl.shaderSource(overlayVertexShader, overlayVertexSource); 1535 | gl.compileShader(overlayVertexShader); 1536 | if (!gl.getShaderParameter(overlayVertexShader, gl.COMPILE_STATUS)) { 1537 | console.error(gl.getShaderInfoLog(overlayVertexShader)); 1538 | } 1539 | 1540 | const overlayFragmentShader = gl.createShader(gl.FRAGMENT_SHADER); 1541 | gl.shaderSource(overlayFragmentShader, overlayFragmentSource); 1542 | gl.compileShader(overlayFragmentShader); 1543 | if (!gl.getShaderParameter(overlayFragmentShader, gl.COMPILE_STATUS)) { 1544 | console.error(gl.getShaderInfoLog(overlayFragmentShader)); 1545 | } 1546 | 1547 | const filterVertexShader = gl.createShader(gl.VERTEX_SHADER); 1548 | gl.shaderSource(filterVertexShader, filterVertexSource); 1549 | gl.compileShader(filterVertexShader); 1550 | if (!gl.getShaderParameter(filterVertexShader, gl.COMPILE_STATUS)) { 1551 | console.error(gl.getShaderInfoLog(filterVertexShader)); 1552 | } 1553 | 1554 | const filterFragmentShader = gl.createShader(gl.FRAGMENT_SHADER); 1555 | gl.shaderSource(filterFragmentShader, filterFragmentSource); 1556 | gl.compileShader(filterFragmentShader); 1557 | if (!gl.getShaderParameter(filterFragmentShader, gl.COMPILE_STATUS)) { 1558 | console.error(gl.getShaderInfoLog(filterFragmentShader)); 1559 | } 1560 | 1561 | const colorProgram = gl.createProgram(); 1562 | gl.attachShader(colorProgram, gaussianVertexShader); 1563 | gl.attachShader(colorProgram, colorFragmentShader); 1564 | gl.linkProgram(colorProgram); 1565 | gl.useProgram(colorProgram); 1566 | if (!gl.getProgramParameter(colorProgram, gl.LINK_STATUS)) { 1567 | console.error(gl.getProgramInfoLog(colorProgram)); 1568 | } 1569 | const colorProgramUniforms = { 1570 | u_viewport: gl.getUniformLocation(colorProgram, "viewport"), 1571 | u_projection: gl.getUniformLocation(colorProgram, "projection"), 1572 | u_view: gl.getUniformLocation(colorProgram, "view"), 1573 | u_focal: gl.getUniformLocation(colorProgram, "focal"), 1574 | u_textureLocation: gl.getUniformLocation(colorProgram, "u_texture"), 1575 | u_mode: gl.getUniformLocation(colorProgram, "mode"), 1576 | u_alphaThreshold: gl.getUniformLocation(colorProgram, "alphaThreshold"), 1577 | }; 1578 | const colorProgramAttributes = { 1579 | a_position: gl.getAttribLocation(colorProgram, "position"), 1580 | a_index: gl.getAttribLocation(colorProgram, "index"), 1581 | }; 1582 | 1583 | const lightingProgram = gl.createProgram(); 1584 | gl.attachShader(lightingProgram, gaussianVertexShader); 1585 | gl.attachShader(lightingProgram, lightingFragmentShader); 1586 | gl.linkProgram(lightingProgram); 1587 | gl.useProgram(lightingProgram); 1588 | if (!gl.getProgramParameter(lightingProgram, gl.LINK_STATUS)) { 1589 | console.error(gl.getProgramInfoLog(lightingProgram)); 1590 | } 1591 | const lightingProgramUniforms = { 1592 | u_viewport: gl.getUniformLocation(lightingProgram, "viewport"), 1593 | u_projection: gl.getUniformLocation(lightingProgram, "projection"), 1594 | u_view: gl.getUniformLocation(lightingProgram, "view"), 1595 | u_focal: gl.getUniformLocation(lightingProgram, "focal"), 1596 | u_textureLocation: gl.getUniformLocation(lightingProgram, "u_texture"), 1597 | u_depthTextureLocation: gl.getUniformLocation(lightingProgram, "depthTexture"), 1598 | u_mode: gl.getUniformLocation(lightingProgram, "mode"), 1599 | u_screenSize: gl.getUniformLocation(lightingProgram, "screenSize"), 1600 | u_invProjection: gl.getUniformLocation(lightingProgram, "invProjection"), 1601 | u_invView: gl.getUniformLocation(lightingProgram, "invView"), 1602 | u_lightPositions: gl.getUniformLocation(lightingProgram, "lightPositions"), 1603 | u_lightViewProjMatrices: gl.getUniformLocation(lightingProgram, "lightViewProjMatrices"), 1604 | u_shadowMaps: gl.getUniformLocation(lightingProgram, "shadowMaps"), 1605 | u_numLights: gl.getUniformLocation(lightingProgram, "numLights"), 1606 | u_sigma_range: gl.getUniformLocation(lightingProgram, "sigma_range"), 1607 | u_sigma_domain: gl.getUniformLocation(lightingProgram, "sigma_domain"), 1608 | u_kernelSize: gl.getUniformLocation(lightingProgram, "kernelSize"), 1609 | u_usePseudoNormals: gl.getUniformLocation(lightingProgram, "usePseudoNormals"), 1610 | u_usePBR: gl.getUniformLocation(lightingProgram, "usePBR"), 1611 | }; 1612 | let alphaThreshold = 0.0; 1613 | let usePseudoNormals = 0; 1614 | let usePBR = 0; 1615 | const lightingProgramAttributes = { 1616 | a_position: gl.getAttribLocation(lightingProgram, "position"), 1617 | a_index: gl.getAttribLocation(lightingProgram, "index"), 1618 | }; 1619 | 1620 | const filterProgram = gl.createProgram(); 1621 | gl.attachShader(filterProgram, filterVertexShader); 1622 | gl.attachShader(filterProgram, filterFragmentShader); 1623 | gl.linkProgram(filterProgram); 1624 | gl.useProgram(filterProgram); 1625 | if (!gl.getProgramParameter(filterProgram, gl.LINK_STATUS)) { 1626 | console.error(gl.getProgramInfoLog(filterProgram)); 1627 | } 1628 | const filterProgramUniforms = { 1629 | u_tex: gl.getUniformLocation(filterProgram, "tex"), 1630 | u_sigma_range: gl.getUniformLocation(filterProgram, "sigma_range"), 1631 | u_sigma_domain: gl.getUniformLocation(filterProgram, "sigma_domain"), 1632 | u_kernelSize: gl.getUniformLocation(filterProgram, "kernelSize"), 1633 | }; 1634 | const sigma_range_step = 0.01; 1635 | const sigma_domain_step = 1.0 / 640; 1636 | let sigma_range = 0.1; 1637 | let sigma_domain = 0.1; 1638 | let kernelSize = 1; 1639 | const filterProgramAttributes = { 1640 | a_uv: gl.getAttribLocation(filterProgram, "uv"), 1641 | }; 1642 | 1643 | const overlayProgram = gl.createProgram(); 1644 | gl.attachShader(overlayProgram, overlayVertexShader); 1645 | gl.attachShader(overlayProgram, overlayFragmentShader); 1646 | gl.linkProgram(overlayProgram); 1647 | gl.useProgram(overlayProgram); 1648 | if (!gl.getProgramParameter(overlayProgram, gl.LINK_STATUS)) { 1649 | console.error(gl.getProgramInfoLog(overlayProgram)); 1650 | } 1651 | const overlayProgramUniforms = { 1652 | u_texture: gl.getUniformLocation(overlayProgram, "overlayTexture"), 1653 | u_projection: gl.getUniformLocation(overlayProgram, "projection"), 1654 | u_view: gl.getUniformLocation(overlayProgram, "view"), 1655 | u_worldCameraPosition: gl.getUniformLocation(overlayProgram, "worldCameraPosition"), 1656 | u_worldCameraUp: gl.getUniformLocation(overlayProgram, "worldCameraUp"), 1657 | u_size: gl.getUniformLocation(overlayProgram, "size"), 1658 | }; 1659 | const overlayProgramAttributes = { 1660 | a_uv: gl.getAttribLocation(overlayProgram, "uv"), 1661 | a_worldCenter: gl.getAttribLocation(overlayProgram, "worldCenter"), 1662 | }; 1663 | // Setup attributes for overlay center positions and allocate space for an array of light overlays 1664 | const lightOverlayCenterBuffer = gl.createBuffer(); 1665 | gl.bindBuffer(gl.ARRAY_BUFFER, lightOverlayCenterBuffer); 1666 | // Just allocate, don't upload yet 1667 | gl.bufferData(gl.ARRAY_BUFFER, lightPositions.byteLength, gl.DYNAMIC_DRAW); // DYNAMIC_DRAW because we will change this often 1668 | 1669 | gl.disable(gl.DEPTH_TEST); // Disable depth testing 1670 | 1671 | // Enable blending 1672 | gl.enable(gl.BLEND); 1673 | 1674 | var currentMode = params.get('mode') == 'color' ? MODES.COLOR : MODES.LIGHTING; 1675 | 1676 | const resize = () => { 1677 | projectionMatrix = getProjectionMatrix( 1678 | camera.fx, 1679 | camera.fy, 1680 | innerWidth, 1681 | innerHeight, 1682 | ); 1683 | 1684 | gl.canvas.width = Math.round(innerWidth / downsample); 1685 | gl.canvas.height = Math.round(innerHeight / downsample); 1686 | gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); 1687 | }; 1688 | 1689 | window.addEventListener("resize", resize); 1690 | resize(); 1691 | 1692 | worker.onmessage = (e) => { 1693 | if (e.data.buffer) { 1694 | splatData = new Uint8Array(e.data.buffer); 1695 | const exportedSplatData = new Uint8Array(LSPLAT_MAGIC_HEADER.length + splatData.length); 1696 | exportedSplatData.set(LSPLAT_MAGIC_HEADER, 0); 1697 | exportedSplatData.set(splatData, LSPLAT_MAGIC_HEADER.length); 1698 | const blob = new Blob([exportedSplatData.buffer], { 1699 | type: "application/octet-stream", 1700 | }); 1701 | const link = document.createElement("a"); 1702 | link.download = "model.lsplat"; 1703 | link.href = URL.createObjectURL(blob); 1704 | document.body.appendChild(link); 1705 | link.click(); 1706 | worker.postMessage({ 1707 | buffer: splatData.buffer, 1708 | gaussianCount: Math.floor(splatData.length / PADDED_SPLAT_LENGTH), 1709 | }); 1710 | } else if (e.data.texdata) { 1711 | const { texdata, texwidth, texheight, hasNormals } = e.data; 1712 | gl.activeTexture(gl.TEXTURE0 + gaussianDataTexture.texId); 1713 | gl.bindTexture(gl.TEXTURE_2D, gaussianDataTexture.texture); 1714 | gl.texImage2D( 1715 | gl.TEXTURE_2D, 1716 | 0, 1717 | gl.RGBA32UI, 1718 | texwidth, 1719 | texheight, 1720 | 0, 1721 | gl.RGBA_INTEGER, 1722 | gl.UNSIGNED_INT, 1723 | texdata, 1724 | ); 1725 | if (!hasNormals && !usePseudoNormals) { 1726 | usePseudoNormals = true; 1727 | } 1728 | } else if (e.data.depthIndex) { 1729 | const { depthIndex, label } = e.data; 1730 | if (!label) { 1731 | console.error("Expected label for sort result"); 1732 | } else if (label == "main") { 1733 | gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer); 1734 | gl.bufferData(gl.ARRAY_BUFFER, depthIndex, gl.DYNAMIC_DRAW); 1735 | } else if (label.startsWith("light-")) { 1736 | // looks like `light-{i}-{face}` 1737 | const i = parseInt(label.slice(`light-`.length, `light-`.length + 1)); 1738 | const face = parseInt(label.slice(`light-0-`.length, `light-0-`.length + 1)); 1739 | const light = lights[i]; 1740 | gl.bindBuffer(gl.ARRAY_BUFFER, light.faces[face].indexBuffer); 1741 | gl.bufferData(gl.ARRAY_BUFFER, depthIndex, gl.DYNAMIC_DRAW); 1742 | light.needsShadowMapUpdate = true; 1743 | } 1744 | gaussianCount = e.data.gaussianCount; 1745 | } 1746 | }; 1747 | 1748 | let activeKeys = []; 1749 | let currentCameraIndex = 0; 1750 | 1751 | window.addEventListener("keydown", (e) => { 1752 | // if (document.activeElement != document.body) return; 1753 | carousel = false; 1754 | if (!activeKeys.includes(e.code)) activeKeys.push(e.code); 1755 | if (/\d/.test(e.key)) { 1756 | currentCameraIndex = parseInt(e.key) 1757 | camera = cameras[currentCameraIndex]; 1758 | viewMatrix = getViewMatrix(camera); 1759 | } 1760 | if (['-', '_'].includes(e.key)){ 1761 | currentCameraIndex = (currentCameraIndex + cameras.length - 1) % cameras.length; 1762 | viewMatrix = getViewMatrix(cameras[currentCameraIndex]); 1763 | } 1764 | if (['+', '='].includes(e.key)){ 1765 | currentCameraIndex = (currentCameraIndex + 1) % cameras.length; 1766 | viewMatrix = getViewMatrix(cameras[currentCameraIndex]); 1767 | } 1768 | if (['m', 'M'].includes(e.key)) { 1769 | if (currentMode == MODES.LIGHTING) { 1770 | currentMode = MODES.COLOR; 1771 | } else { 1772 | currentMode = MODES.LIGHTING; 1773 | } 1774 | } 1775 | if (['['].includes(e.key)) { 1776 | sigma_range = Math.max(0, sigma_range - sigma_range_step); 1777 | console.log("bilateral filter sigma_range:", sigma_range); 1778 | } 1779 | if ([']'].includes(e.key)) { 1780 | sigma_range += sigma_range_step; 1781 | console.log("bilateral filter sigma_range:", sigma_range); 1782 | } 1783 | if ([',','<'].includes(e.key)) { 1784 | alphaThreshold = Math.max(0, alphaThreshold - 0.01); 1785 | console.log("alphaThreshold:", alphaThreshold); 1786 | } 1787 | if (['.','>'].includes(e.key)) { 1788 | alphaThreshold = Math.min(1, alphaThreshold + 0.01); 1789 | console.log("alphaThreshold:", alphaThreshold); 1790 | } 1791 | if ([';'].includes(e.key)) { 1792 | kernelSize = Math.max(0, kernelSize - 1); 1793 | console.log("kernelSize:", kernelSize); 1794 | } 1795 | if (["'"].includes(e.key)) { 1796 | kernelSize += 1; 1797 | console.log("kernelSize:", kernelSize); 1798 | } 1799 | if (["n", "N"].includes(e.key)) { 1800 | usePseudoNormals = 1 - usePseudoNormals; 1801 | console.log("usePseudoNormals:", usePseudoNormals); 1802 | } 1803 | if (["|", "\\"].includes(e.key)) { 1804 | usePBR = 1 - usePBR; 1805 | console.log("usePBR:", usePBR); 1806 | } 1807 | camid.innerText = "cam " + currentCameraIndex; 1808 | if (e.code == "KeyV") { 1809 | location.hash = 1810 | "#" + 1811 | JSON.stringify( 1812 | viewMatrix.map((k) => Math.round(k * 100) / 100), 1813 | ); 1814 | camid.innerText ="" 1815 | } else if (e.code === "KeyP") { 1816 | carousel = true; 1817 | camid.innerText ="" 1818 | } 1819 | }); 1820 | window.addEventListener("keyup", (e) => { 1821 | activeKeys = activeKeys.filter((k) => k !== e.code); 1822 | }); 1823 | window.addEventListener("blur", () => { 1824 | activeKeys = []; 1825 | }); 1826 | 1827 | window.addEventListener( 1828 | "wheel", 1829 | (e) => { 1830 | carousel = false; 1831 | e.preventDefault(); 1832 | const lineHeight = 10; 1833 | const scale = 1834 | e.deltaMode == 1 1835 | ? lineHeight 1836 | : e.deltaMode == 2 1837 | ? innerHeight 1838 | : 1; 1839 | let inv = invert4(viewMatrix); 1840 | if (e.shiftKey) { 1841 | inv = translate4( 1842 | inv, 1843 | (e.deltaX * scale) / innerWidth, 1844 | (e.deltaY * scale) / innerHeight, 1845 | 0, 1846 | ); 1847 | } else if (e.ctrlKey || e.metaKey) { 1848 | // inv = rotate4(inv, (e.deltaX * scale) / innerWidth, 0, 0, 1); 1849 | // inv = translate4(inv, 0, (e.deltaY * scale) / innerHeight, 0); 1850 | // let preY = inv[13]; 1851 | inv = translate4( 1852 | inv, 1853 | 0, 1854 | 0, 1855 | (-10 * (e.deltaY * scale)) / innerHeight, 1856 | ); 1857 | // inv[13] = preY; 1858 | } else { 1859 | let d = 4; 1860 | inv = translate4(inv, 0, 0, d); 1861 | inv = rotate4(inv, -(e.deltaX * scale) / innerWidth, 0, 1, 0); 1862 | inv = rotate4(inv, (e.deltaY * scale) / innerHeight, 1, 0, 0); 1863 | inv = translate4(inv, 0, 0, -d); 1864 | } 1865 | 1866 | viewMatrix = invert4(inv); 1867 | }, 1868 | { passive: false }, 1869 | ); 1870 | 1871 | let startX, startY, down; 1872 | canvas.addEventListener("mousedown", (e) => { 1873 | carousel = false; 1874 | startX = e.clientX; 1875 | startY = e.clientY; 1876 | const ndcX = 2.0 * (e.clientX / innerWidth) - 1.0; 1877 | const ndcY = 2.0 * (1.0 - e.clientY / innerHeight) - 1.0; 1878 | selectedLight = getSelectedLight(ndcX, ndcY); 1879 | down = e.ctrlKey || e.metaKey ? 2 : 1; 1880 | }); 1881 | canvas.addEventListener("contextmenu", (e) => { 1882 | carousel = false; 1883 | e.preventDefault(); 1884 | startX = e.clientX; 1885 | startY = e.clientY; 1886 | down = 2; 1887 | }); 1888 | 1889 | function getSelectedLight(ndcX, ndcY) { 1890 | // TODO break ties with distance from camera 1891 | for (let i = 0; i < ndcSpaceLightBoundingBoxes.length; i++) { 1892 | const [minX, maxY, maxX, minY] = ndcSpaceLightBoundingBoxes[i]; 1893 | if (ndcX >= minX && ndcX <= maxX && ndcY >= minY && ndcY <= maxY) { 1894 | return i; 1895 | } 1896 | } 1897 | return -1; 1898 | } 1899 | 1900 | canvas.addEventListener("mousemove", (e) => { 1901 | e.preventDefault(); 1902 | let useHoverCursor = false; 1903 | if (down == 0) { 1904 | const ndcX = 2.0 * (e.clientX / innerWidth) - 1.0; 1905 | const ndcY = 2.0 * (1.0 - e.clientY / innerHeight) - 1.0; 1906 | useHoverCursor = getSelectedLight(ndcX, ndcY) >= 0; 1907 | } else if (down == 1) { 1908 | if (selectedLight >= 0) { 1909 | let lightPos = lightPositions.slice(selectedLight * 3, selectedLight * 3 + 3); 1910 | // Reposition light to the projection of the mouse cursor on the XY-plane at the light's current depth in camera space 1911 | const viewProj = multiply4(projectionMatrix, viewMatrix); 1912 | let invViewProj = invert4(viewProj); 1913 | const lightPosClip = transform4(viewProj, [...lightPos, 1]); 1914 | const ndcCursor = [2.0 * (e.clientX / innerWidth) - 1.0, 2.0 * (1.0 - e.clientY / innerHeight) - 1.0]; 1915 | const clipCursor = [lightPosClip[3] * ndcCursor[0], lightPosClip[3] * ndcCursor[1], lightPosClip[2], lightPosClip[3]]; 1916 | const worldCursor = transform4(invViewProj, clipCursor); 1917 | updateLightPosition(selectedLight, worldCursor); 1918 | } else { 1919 | let inv = invert4(viewMatrix); 1920 | let dx = (5 * (e.clientX - startX)) / innerWidth; 1921 | let dy = (5 * (e.clientY - startY)) / innerHeight; 1922 | let d = 4; 1923 | 1924 | inv = translate4(inv, 0, 0, d); 1925 | inv = rotate4(inv, dx, 0, 1, 0); 1926 | inv = rotate4(inv, -dy, 1, 0, 0); 1927 | inv = translate4(inv, 0, 0, -d); 1928 | viewMatrix = invert4(inv); 1929 | 1930 | startX = e.clientX; 1931 | startY = e.clientY; 1932 | } 1933 | } else if (down == 2) { 1934 | let inv = invert4(viewMatrix); 1935 | if (selectedLight >= 0) { 1936 | // Translate light on Z-axis in camera-space 1937 | let lightPos = lightPositions.slice(selectedLight * 3, selectedLight * 3 + 3); 1938 | let delta = [ 1939 | 0, 1940 | 0, 1941 | (5 * (e.clientY - startY)) / innerHeight, 1942 | 1, 1943 | ]; 1944 | delta = transform4([...inv.slice(0, 12), 0, 0, 0, 1], delta); 1945 | lightPos[0] += delta[0]; 1946 | lightPos[1] += delta[1]; 1947 | lightPos[2] += delta[2]; 1948 | updateLightPosition(selectedLight, lightPos); 1949 | } else { 1950 | // Translate camera on XZ-plane in camera-space 1951 | inv = translate4( 1952 | inv, 1953 | (-10 * (e.clientX - startX)) / innerWidth, 1954 | 0, 1955 | (10 * (e.clientY - startY)) / innerHeight, 1956 | ); 1957 | viewMatrix = invert4(inv); 1958 | } 1959 | startX = e.clientX; 1960 | startY = e.clientY; 1961 | } 1962 | if (useHoverCursor) { 1963 | document.documentElement.style.cursor = 'grab'; 1964 | } else { 1965 | document.documentElement.style.cursor = 'default'; 1966 | } 1967 | }); 1968 | canvas.addEventListener("mouseup", (e) => { 1969 | e.preventDefault(); 1970 | down = 0; 1971 | selectedLight = -1; 1972 | startX = 0; 1973 | startY = 0; 1974 | }); 1975 | 1976 | let altX = 0, 1977 | altY = 0; 1978 | canvas.addEventListener( 1979 | "touchstart", 1980 | (e) => { 1981 | e.preventDefault(); 1982 | if (e.touches.length === 1) { 1983 | carousel = false; 1984 | startX = e.touches[0].clientX; 1985 | startY = e.touches[0].clientY; 1986 | down = 1; 1987 | } else if (e.touches.length === 2) { 1988 | // console.log('beep') 1989 | carousel = false; 1990 | startX = e.touches[0].clientX; 1991 | altX = e.touches[1].clientX; 1992 | startY = e.touches[0].clientY; 1993 | altY = e.touches[1].clientY; 1994 | down = 1; 1995 | } 1996 | }, 1997 | { passive: false }, 1998 | ); 1999 | canvas.addEventListener( 2000 | "touchmove", 2001 | (e) => { 2002 | e.preventDefault(); 2003 | if (e.touches.length === 1 && down) { 2004 | let inv = invert4(viewMatrix); 2005 | let dx = (4 * (e.touches[0].clientX - startX)) / innerWidth; 2006 | let dy = (4 * (e.touches[0].clientY - startY)) / innerHeight; 2007 | 2008 | let d = 4; 2009 | inv = translate4(inv, 0, 0, d); 2010 | inv = rotate4(inv, dx, 0, 1, 0); 2011 | inv = rotate4(inv, -dy, 1, 0, 0); 2012 | inv = translate4(inv, 0, 0, -d); 2013 | 2014 | viewMatrix = invert4(inv); 2015 | 2016 | startX = e.touches[0].clientX; 2017 | startY = e.touches[0].clientY; 2018 | } else if (e.touches.length === 2) { 2019 | const dtheta = 2020 | Math.atan2(startY - altY, startX - altX) - 2021 | Math.atan2( 2022 | e.touches[0].clientY - e.touches[1].clientY, 2023 | e.touches[0].clientX - e.touches[1].clientX, 2024 | ); 2025 | const dscale = 2026 | Math.hypot(startX - altX, startY - altY) / 2027 | Math.hypot( 2028 | e.touches[0].clientX - e.touches[1].clientX, 2029 | e.touches[0].clientY - e.touches[1].clientY, 2030 | ); 2031 | const dx = 2032 | (e.touches[0].clientX + 2033 | e.touches[1].clientX - 2034 | (startX + altX)) / 2035 | 2; 2036 | const dy = 2037 | (e.touches[0].clientY + 2038 | e.touches[1].clientY - 2039 | (startY + altY)) / 2040 | 2; 2041 | let inv = invert4(viewMatrix); 2042 | inv = rotate4(inv, dtheta, 0, 0, 1); 2043 | 2044 | inv = translate4(inv, -dx / innerWidth, -dy / innerHeight, 0); 2045 | 2046 | inv = translate4(inv, 0, 0, 3 * (1 - dscale)); 2047 | 2048 | viewMatrix = invert4(inv); 2049 | 2050 | startX = e.touches[0].clientX; 2051 | altX = e.touches[1].clientX; 2052 | startY = e.touches[0].clientY; 2053 | altY = e.touches[1].clientY; 2054 | } 2055 | }, 2056 | { passive: false }, 2057 | ); 2058 | canvas.addEventListener( 2059 | "touchend", 2060 | (e) => { 2061 | e.preventDefault(); 2062 | down = 0; 2063 | selectedLight = -1; 2064 | startX = 0; 2065 | startY = 0; 2066 | }, 2067 | { passive: false }, 2068 | ); 2069 | 2070 | let jumpDelta = 0; 2071 | let gaussianCount = 0; 2072 | 2073 | let lastFrame = 0; 2074 | let avgFps = 0; 2075 | let start = 0; 2076 | 2077 | window.addEventListener("gamepadconnected", (e) => { 2078 | const gp = navigator.getGamepads()[e.gamepad.index]; 2079 | console.log( 2080 | `Gamepad connected at index ${gp.index}: ${gp.id}. It has ${gp.buttons.length} buttons and ${gp.axes.length} axes.`, 2081 | ); 2082 | }); 2083 | window.addEventListener("gamepaddisconnected", (e) => { 2084 | console.log("Gamepad disconnected"); 2085 | }); 2086 | 2087 | let leftGamepadTrigger, rightGamepadTrigger; 2088 | 2089 | const frame = (now) => { 2090 | let inv = invert4(viewMatrix); 2091 | let shiftKey = activeKeys.includes("Shift") || activeKeys.includes("ShiftLeft") || activeKeys.includes("ShiftRight") 2092 | 2093 | if (activeKeys.includes("ArrowUp")) { 2094 | if (shiftKey) { 2095 | inv = translate4(inv, 0, -0.03, 0); 2096 | } else { 2097 | inv = translate4(inv, 0, 0, 0.1); 2098 | } 2099 | } 2100 | if (activeKeys.includes("ArrowDown")) { 2101 | if (shiftKey) { 2102 | inv = translate4(inv, 0, 0.03, 0); 2103 | } else { 2104 | inv = translate4(inv, 0, 0, -0.1); 2105 | } 2106 | } 2107 | if (activeKeys.includes("ArrowLeft")) 2108 | inv = translate4(inv, -0.03, 0, 0); 2109 | // 2110 | if (activeKeys.includes("ArrowRight")) 2111 | inv = translate4(inv, 0.03, 0, 0); 2112 | // inv = rotate4(inv, 0.01, 0, 1, 0); 2113 | if (activeKeys.includes("KeyA")) inv = rotate4(inv, -0.01, 0, 1, 0); 2114 | if (activeKeys.includes("KeyD")) inv = rotate4(inv, 0.01, 0, 1, 0); 2115 | if (activeKeys.includes("KeyQ")) inv = rotate4(inv, 0.01, 0, 0, 1); 2116 | if (activeKeys.includes("KeyE")) inv = rotate4(inv, -0.01, 0, 0, 1); 2117 | if (activeKeys.includes("KeyW")) inv = rotate4(inv, 0.005, 1, 0, 0); 2118 | if (activeKeys.includes("KeyS")) inv = rotate4(inv, -0.005, 1, 0, 0); 2119 | 2120 | const gamepads = navigator.getGamepads ? navigator.getGamepads() : []; 2121 | let isJumping = activeKeys.includes("Space"); 2122 | for (let gamepad of gamepads) { 2123 | if (!gamepad) continue; 2124 | 2125 | const axisThreshold = 0.1; // Threshold to detect when the axis is intentionally moved 2126 | const moveSpeed = 0.06; 2127 | const rotateSpeed = 0.02; 2128 | 2129 | // Assuming the left stick controls translation (axes 0 and 1) 2130 | if (Math.abs(gamepad.axes[0]) > axisThreshold) { 2131 | inv = translate4(inv, moveSpeed * gamepad.axes[0], 0, 0); 2132 | carousel = false; 2133 | } 2134 | if (Math.abs(gamepad.axes[1]) > axisThreshold) { 2135 | inv = translate4(inv, 0, 0, -moveSpeed * gamepad.axes[1]); 2136 | carousel = false; 2137 | } 2138 | if(gamepad.buttons[12].pressed || gamepad.buttons[13].pressed){ 2139 | inv = translate4(inv, 0, -moveSpeed*(gamepad.buttons[12].pressed - gamepad.buttons[13].pressed), 0); 2140 | carousel = false; 2141 | } 2142 | 2143 | if(gamepad.buttons[14].pressed || gamepad.buttons[15].pressed){ 2144 | inv = translate4(inv, -moveSpeed*(gamepad.buttons[14].pressed - gamepad.buttons[15].pressed), 0, 0); 2145 | carousel = false; 2146 | } 2147 | 2148 | // Assuming the right stick controls rotation (axes 2 and 3) 2149 | if (Math.abs(gamepad.axes[2]) > axisThreshold) { 2150 | inv = rotate4(inv, rotateSpeed * gamepad.axes[2], 0, 1, 0); 2151 | carousel = false; 2152 | } 2153 | if (Math.abs(gamepad.axes[3]) > axisThreshold) { 2154 | inv = rotate4(inv, -rotateSpeed * gamepad.axes[3], 1, 0, 0); 2155 | carousel = false; 2156 | } 2157 | 2158 | let tiltAxis = gamepad.buttons[6].value - gamepad.buttons[7].value; 2159 | if (Math.abs(tiltAxis) > axisThreshold) { 2160 | inv = rotate4(inv, rotateSpeed * tiltAxis, 0, 0, 1); 2161 | carousel = false; 2162 | } 2163 | if (gamepad.buttons[4].pressed && !leftGamepadTrigger) { 2164 | camera = cameras[(cameras.indexOf(camera)+1)%cameras.length] 2165 | inv = invert4(getViewMatrix(camera)); 2166 | carousel = false; 2167 | } 2168 | if (gamepad.buttons[5].pressed && !rightGamepadTrigger) { 2169 | camera = cameras[(cameras.indexOf(camera)+cameras.length-1)%cameras.length] 2170 | inv = invert4(getViewMatrix(camera)); 2171 | carousel = false; 2172 | } 2173 | leftGamepadTrigger = gamepad.buttons[4].pressed; 2174 | rightGamepadTrigger = gamepad.buttons[5].pressed; 2175 | if (gamepad.buttons[0].pressed) { 2176 | isJumping = true; 2177 | carousel = false; 2178 | } 2179 | if(gamepad.buttons[3].pressed){ 2180 | carousel = true; 2181 | } 2182 | } 2183 | 2184 | if ( 2185 | ["KeyJ", "KeyK", "KeyL", "KeyI"].some((k) => activeKeys.includes(k)) 2186 | ) { 2187 | let d = 4; 2188 | inv = translate4(inv, 0, 0, d); 2189 | inv = rotate4( 2190 | inv, 2191 | activeKeys.includes("KeyJ") 2192 | ? -0.05 2193 | : activeKeys.includes("KeyL") 2194 | ? 0.05 2195 | : 0, 2196 | 0, 2197 | 1, 2198 | 0, 2199 | ); 2200 | inv = rotate4( 2201 | inv, 2202 | activeKeys.includes("KeyI") 2203 | ? 0.05 2204 | : activeKeys.includes("KeyK") 2205 | ? -0.05 2206 | : 0, 2207 | 1, 2208 | 0, 2209 | 0, 2210 | ); 2211 | inv = translate4(inv, 0, 0, -d); 2212 | } 2213 | 2214 | viewMatrix = invert4(inv); 2215 | 2216 | if (carousel) { 2217 | let inv = invert4(defaultViewMatrix); 2218 | 2219 | const t = Math.sin((Date.now() - start) / 5000); 2220 | inv = translate4(inv, 2.5 * t, 0, 6 * (1 - Math.cos(t))); 2221 | inv = rotate4(inv, -0.6 * t, 0, 1, 0); 2222 | 2223 | viewMatrix = invert4(inv); 2224 | } 2225 | 2226 | if (isJumping) { 2227 | jumpDelta = Math.min(1, jumpDelta + 0.05); 2228 | } else { 2229 | jumpDelta = Math.max(0, jumpDelta - 0.05); 2230 | } 2231 | 2232 | let inv2 = invert4(viewMatrix); 2233 | inv2 = translate4(inv2, 0, -jumpDelta, 0); 2234 | inv2 = rotate4(inv2, -0.1 * jumpDelta, 1, 0, 0); 2235 | let actualViewMatrix = invert4(inv2); 2236 | 2237 | const viewProj = multiply4(projectionMatrix, actualViewMatrix); 2238 | worker.postMessage({ view: viewProj, label: "main" }); 2239 | 2240 | // Hit-test light overlays 2241 | ndcSpaceLightBoundingBoxes = []; 2242 | const BBOX_WIDTH = 0.05; 2243 | const BBOX_HEIGHT = 0.1; 2244 | for (let i = 0; i < numLights; i++) { 2245 | const lightPosition = [lightPositions[i * 3], lightPositions[i * 3 + 1], lightPositions[i * 3 + 2], 1.]; 2246 | const clipLightPosition = transform4(viewProj, lightPosition); 2247 | const ndcLightPositionXY = [ 2248 | clipLightPosition[0] / clipLightPosition[3], clipLightPosition[1] / clipLightPosition[3] 2249 | ]; 2250 | ndcSpaceLightBoundingBoxes.push([ 2251 | // top-left 2252 | ndcLightPositionXY[0] - BBOX_WIDTH, ndcLightPositionXY[1] + BBOX_HEIGHT, 2253 | // bottom-right 2254 | ndcLightPositionXY[0] + BBOX_WIDTH, ndcLightPositionXY[1] - BBOX_HEIGHT, 2255 | ]); 2256 | } 2257 | 2258 | const currentFps = 1000 / (now - lastFrame) || 0; 2259 | avgFps = avgFps * 0.9 + currentFps * 0.1; 2260 | 2261 | gl.useProgram(colorProgram); 2262 | gl.uniform1i(colorProgramUniforms.u_textureLocation, gaussianDataTexture.texId); 2263 | gl.uniform2fv(colorProgramUniforms.u_focal, new Float32Array([camera.fx, camera.fy])); 2264 | gl.uniform2fv(colorProgramUniforms.u_viewport, new Float32Array([innerWidth, innerHeight])); 2265 | gl.uniformMatrix4fv(colorProgramUniforms.u_projection, false, projectionMatrix); 2266 | // See eqn. 3 of the 3D gaussian splats paper for alpha blending. 2267 | // Note in the fragment shader we also pre-multiply the color by the source alpha. 2268 | gl.blendFunc( 2269 | gl.ONE_MINUS_DST_ALPHA, 2270 | gl.ONE, 2271 | ); 2272 | gl.blendEquationSeparate(gl.FUNC_ADD, gl.FUNC_ADD); 2273 | if (gaussianCount > 0) { 2274 | document.getElementById("spinner").style.display = "none"; 2275 | 2276 | // 1. write to depth framebuffer which will be used for surface normal reconstruction 2277 | gl.uniformMatrix4fv(colorProgramUniforms.u_view, false, actualViewMatrix); 2278 | gl.uniform1i(colorProgramUniforms.u_mode, MODES.DEPTH); 2279 | gl.uniform1f(colorProgramUniforms.u_alphaThreshold, alphaThreshold); 2280 | 2281 | gl.enableVertexAttribArray(colorProgramAttributes.a_position); 2282 | gl.bindBuffer(gl.ARRAY_BUFFER, gaussianQuadVertexBuffer); 2283 | gl.vertexAttribPointer(colorProgramAttributes.a_position, 2, gl.FLOAT, false, 0, 0); 2284 | gl.enableVertexAttribArray(colorProgramAttributes.a_index); 2285 | gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer); 2286 | gl.vertexAttribIPointer(colorProgramAttributes.a_index, 1, gl.INT, false, 0, 0); 2287 | gl.vertexAttribDivisor(colorProgramAttributes.a_index, 1); 2288 | 2289 | gl.bindFramebuffer(gl.FRAMEBUFFER, depthFBO.fbo); 2290 | gl.viewport(0, 0, depthWidth, depthHeight); 2291 | gl.clear(gl.COLOR_BUFFER_BIT); 2292 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, gaussianCount); 2293 | 2294 | // Filter depth bilaterally 2295 | gl.useProgram(filterProgram); 2296 | 2297 | gl.uniform1i(filterProgramUniforms.u_tex, depthFBO.texId); 2298 | gl.uniform1i(filterProgramUniforms.u_kernelSize, kernelSize); 2299 | gl.uniform1f(filterProgramUniforms.u_sigma_domain, sigma_domain); 2300 | gl.uniform1f(filterProgramUniforms.u_sigma_range, sigma_range); 2301 | 2302 | gl.enableVertexAttribArray(filterProgramAttributes.a_uv); 2303 | gl.bindBuffer(gl.ARRAY_BUFFER, quadUVBuffer); 2304 | gl.vertexAttribPointer(filterProgramAttributes.a_uv, 2, gl.FLOAT, false, 0, 0); 2305 | 2306 | gl.bindFramebuffer(gl.FRAMEBUFFER, filteredDepthFBO.fbo); 2307 | gl.viewport(0, 0, depthWidth, depthHeight); 2308 | gl.clear(gl.COLOR_BUFFER_BIT); 2309 | gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); 2310 | 2311 | if (currentMode == MODES.LIGHTING) { 2312 | // 2. draw to shadow maps 2313 | for (let i = 0; i < numLights; i++) { 2314 | const light = lights[i]; 2315 | if (!light.needsShadowMapUpdate) { 2316 | continue; 2317 | } 2318 | const shadowMapFBO = shadowMapFBOs[i]; 2319 | for (let f = 0; f < 6; f++) { 2320 | gl.useProgram(colorProgram); 2321 | 2322 | gl.uniformMatrix4fv(colorProgramUniforms.u_view, false, light.faces[f].viewMatrix); 2323 | gl.uniform2fv(colorProgramUniforms.u_focal, new Float32Array([shadowMapSize / 2, shadowMapSize / 2])); 2324 | gl.uniform2fv(colorProgramUniforms.u_viewport, new Float32Array([shadowMapSize, shadowMapSize])); 2325 | gl.uniformMatrix4fv(colorProgramUniforms.u_projection, false, light.faces[f].projMatrix); 2326 | gl.uniform1i(colorProgramUniforms.u_mode, MODES.DEPTH); 2327 | 2328 | gl.enableVertexAttribArray(colorProgramAttributes.a_position); 2329 | gl.bindBuffer(gl.ARRAY_BUFFER, gaussianQuadVertexBuffer); 2330 | gl.vertexAttribPointer(colorProgramAttributes.a_position, 2, gl.FLOAT, false, 0, 0); 2331 | gl.enableVertexAttribArray(colorProgramAttributes.a_index); 2332 | gl.bindBuffer(gl.ARRAY_BUFFER, light.faces[f].indexBuffer); 2333 | gl.vertexAttribIPointer(colorProgramAttributes.a_index, 1, gl.INT, false, 0, 0); 2334 | gl.vertexAttribDivisor(colorProgramAttributes.a_index, 1); 2335 | 2336 | gl.bindFramebuffer(gl.FRAMEBUFFER, shadowMapFBO.fbo); 2337 | gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_CUBE_MAP_POSITIVE_X + f, shadowMapFBO.texture, 0); 2338 | gl.viewport(0, 0, shadowMapSize, shadowMapSize); 2339 | gl.clear(gl.COLOR_BUFFER_BIT); 2340 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, gaussianCount); 2341 | } 2342 | light.needsShadowMapUpdate = false; 2343 | } 2344 | 2345 | // 3. draw scene with lighting 2346 | gl.useProgram(lightingProgram); 2347 | gl.uniformMatrix4fv(lightingProgramUniforms.u_view, false, actualViewMatrix); 2348 | gl.uniform1i(lightingProgramUniforms.u_textureLocation, gaussianDataTexture.texId); 2349 | gl.uniform2fv(lightingProgramUniforms.u_focal, new Float32Array([camera.fx, camera.fy])); 2350 | gl.uniform2fv(lightingProgramUniforms.u_viewport, new Float32Array([innerWidth, innerHeight])); 2351 | gl.uniformMatrix4fv(lightingProgramUniforms.u_projection, false, projectionMatrix); 2352 | gl.uniform2fv(lightingProgramUniforms.u_screenSize, new Float32Array([gl.canvas.width, gl.canvas.height])); 2353 | gl.uniform1i(lightingProgramUniforms.u_depthTextureLocation, filteredDepthFBO.texId); 2354 | gl.uniformMatrix4fv(lightingProgramUniforms.u_invProjection, false, invert4(projectionMatrix)); 2355 | gl.uniformMatrix4fv(lightingProgramUniforms.u_invView, false, invert4(actualViewMatrix)); 2356 | gl.uniform3fv(lightingProgramUniforms.u_lightPositions, lightPositions); 2357 | gl.uniformMatrix4fv(lightingProgramUniforms.u_lightViewProjMatrices, false, new Float32Array(lights.map(l => l.viewProj).flat())); 2358 | gl.uniform1iv(lightingProgramUniforms.u_shadowMaps, new Int32Array(new Array(MAX_LIGHTS).fill(0).map((_, i) => { 2359 | if (i < shadowMapFBOs.length) { 2360 | return shadowMapFBOs[i].texId; 2361 | } else { 2362 | // Pad with a dummy sampler unit that won't get used 2363 | return DUMMY_SHADOW_MAP_FBO.texId; 2364 | } 2365 | }))); 2366 | gl.uniform1i(lightingProgramUniforms.u_numLights, numLights); 2367 | gl.uniform1f(lightingProgramUniforms.u_sigma_range, sigma_range); 2368 | gl.uniform1f(lightingProgramUniforms.u_sigma_domain, sigma_domain); 2369 | gl.uniform1i(lightingProgramUniforms.u_kernelSize, kernelSize); 2370 | gl.uniform1i(lightingProgramUniforms.u_usePseudoNormals, usePseudoNormals); 2371 | gl.uniform1i(lightingProgramUniforms.u_usePBR, usePBR); 2372 | 2373 | gl.enableVertexAttribArray(lightingProgramAttributes.a_position); 2374 | gl.bindBuffer(gl.ARRAY_BUFFER, gaussianQuadVertexBuffer); 2375 | gl.vertexAttribPointer(lightingProgramAttributes.a_position, 2, gl.FLOAT, false, 0, 0); 2376 | gl.enableVertexAttribArray(lightingProgramAttributes.a_index); 2377 | gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer); 2378 | gl.vertexAttribIPointer(lightingProgramAttributes.a_index, 1, gl.INT, false, 0, 0); 2379 | gl.vertexAttribDivisor(lightingProgramAttributes.a_index, 1); 2380 | 2381 | gl.bindFramebuffer(gl.FRAMEBUFFER, null); 2382 | gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); 2383 | gl.clear(gl.COLOR_BUFFER_BIT); 2384 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, gaussianCount); 2385 | 2386 | // 4. draw overlays 2387 | gl.useProgram(overlayProgram); 2388 | 2389 | gl.uniform1i(overlayProgramUniforms.u_texture, lightOverlayTexture.texId); 2390 | gl.uniformMatrix4fv(overlayProgramUniforms.u_projection, false, projectionMatrix); 2391 | gl.uniformMatrix4fv(overlayProgramUniforms.u_view, false, actualViewMatrix); 2392 | gl.uniform3fv(overlayProgramUniforms.u_worldCameraPosition, new Float32Array([inv2[12], inv2[13], inv2[14]])); 2393 | gl.uniform3fv(overlayProgramUniforms.u_worldCameraUp, new Float32Array([inv2[4], inv2[5], inv2[6]])); 2394 | gl.uniform2fv(overlayProgramUniforms.u_size, new Float32Array([0.2, 0.2])); 2395 | // Use normal over blending 2396 | gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); 2397 | gl.blendEquation(gl.FUNC_ADD); 2398 | 2399 | gl.enableVertexAttribArray(overlayProgramAttributes.a_uv); 2400 | gl.bindBuffer(gl.ARRAY_BUFFER, quadUVBuffer); 2401 | gl.vertexAttribPointer(overlayProgramAttributes.a_uv, 2, gl.FLOAT, false, 0, 0); 2402 | gl.enableVertexAttribArray(overlayProgramAttributes.a_worldCenter); 2403 | gl.bindBuffer(gl.ARRAY_BUFFER, lightOverlayCenterBuffer); 2404 | gl.bufferSubData(gl.ARRAY_BUFFER, 0, lightPositions); 2405 | const bytesPerOverlayCenter = 4 * 3; 2406 | gl.vertexAttribPointer(overlayProgramAttributes.a_worldCenter, 3, gl.FLOAT, false, bytesPerOverlayCenter, 0); 2407 | gl.vertexAttribDivisor(overlayProgramAttributes.a_worldCenter, 1); 2408 | 2409 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, numLights); 2410 | } else { 2411 | // 2. If not in lighting mode, just draw scene with color shader 2412 | gl.useProgram(colorProgram); 2413 | 2414 | gl.uniform1i(colorProgramUniforms.u_mode, currentMode); 2415 | gl.uniform1f(colorProgramUniforms.u_alphaThreshold, alphaThreshold); 2416 | 2417 | gl.enableVertexAttribArray(colorProgramAttributes.a_position); 2418 | gl.bindBuffer(gl.ARRAY_BUFFER, gaussianQuadVertexBuffer); 2419 | gl.vertexAttribPointer(colorProgramAttributes.a_position, 2, gl.FLOAT, false, 0, 0); 2420 | gl.enableVertexAttribArray(colorProgramAttributes.a_index); 2421 | gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer); 2422 | gl.vertexAttribIPointer(colorProgramAttributes.a_index, 1, gl.INT, false, 0, 0); 2423 | gl.vertexAttribDivisor(colorProgramAttributes.a_index, 1); 2424 | 2425 | gl.bindFramebuffer(gl.FRAMEBUFFER, null); 2426 | gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); 2427 | gl.clear(gl.COLOR_BUFFER_BIT); 2428 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, gaussianCount); 2429 | 2430 | // ... and draw the overlays too 2431 | gl.useProgram(overlayProgram); 2432 | 2433 | gl.uniform1i(overlayProgramUniforms.u_texture, lightOverlayTexture.texId); 2434 | gl.uniformMatrix4fv(overlayProgramUniforms.u_projection, false, projectionMatrix); 2435 | gl.uniformMatrix4fv(overlayProgramUniforms.u_view, false, actualViewMatrix); 2436 | gl.uniform3fv(overlayProgramUniforms.u_worldCameraPosition, new Float32Array([inv2[12], inv2[13], inv2[14]])); 2437 | gl.uniform3fv(overlayProgramUniforms.u_worldCameraUp, new Float32Array([inv2[4], inv2[5], inv2[6]])); 2438 | gl.uniform2fv(overlayProgramUniforms.u_size, new Float32Array([0.2, 0.2])); 2439 | // Use normal over blending 2440 | gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); 2441 | gl.blendEquation(gl.FUNC_ADD); 2442 | 2443 | gl.enableVertexAttribArray(overlayProgramAttributes.a_uv); 2444 | gl.bindBuffer(gl.ARRAY_BUFFER, quadUVBuffer); 2445 | gl.vertexAttribPointer(overlayProgramAttributes.a_uv, 2, gl.FLOAT, false, 0, 0); 2446 | gl.enableVertexAttribArray(overlayProgramAttributes.a_worldCenter); 2447 | gl.bindBuffer(gl.ARRAY_BUFFER, lightOverlayCenterBuffer); 2448 | gl.bufferSubData(gl.ARRAY_BUFFER, 0, lightPositions); 2449 | const bytesPerOverlayCenter = 4 * 3; 2450 | gl.vertexAttribPointer(overlayProgramAttributes.a_worldCenter, 3, gl.FLOAT, false, bytesPerOverlayCenter, 0); 2451 | gl.vertexAttribDivisor(overlayProgramAttributes.a_worldCenter, 1); 2452 | 2453 | gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, numLights); 2454 | } 2455 | } else { 2456 | gl.bindFramebuffer(gl.FRAMEBUFFER, null); 2457 | gl.clear(gl.COLOR_BUFFER_BIT); 2458 | document.getElementById("spinner").style.display = ""; 2459 | start = Date.now() + 2000; 2460 | } 2461 | const progress = (100 * gaussianCount) / (splatData.length / PADDED_SPLAT_LENGTH); 2462 | if (progress < 100) { 2463 | document.getElementById("progress").style.width = progress + "%"; 2464 | } else { 2465 | document.getElementById("progress").style.display = "none"; 2466 | } 2467 | fps.innerText = Math.round(avgFps) + " fps"; 2468 | if (isNaN(currentCameraIndex)){ 2469 | camid.innerText = ""; 2470 | } 2471 | lastFrame = now; 2472 | requestAnimationFrame(frame); 2473 | }; 2474 | 2475 | frame(); 2476 | 2477 | const selectFile = (file) => { 2478 | const fr = new FileReader(); 2479 | if (/\.json$/i.test(file.name)) { 2480 | fr.onload = () => { 2481 | cameras = JSON.parse(fr.result); 2482 | viewMatrix = getViewMatrix(cameras[0]); 2483 | projectionMatrix = getProjectionMatrix( 2484 | camera.fx / downsample, 2485 | camera.fy / downsample, 2486 | canvas.width, 2487 | canvas.height, 2488 | ); 2489 | 2490 | console.log("Loaded Cameras"); 2491 | }; 2492 | fr.readAsText(file); 2493 | } else { 2494 | stopLoading = true; 2495 | fr.onload = () => { 2496 | splatData = new Uint8Array(fr.result); 2497 | console.log("Loaded", Math.floor(splatData.length / PADDED_SPLAT_LENGTH)); 2498 | 2499 | if ( 2500 | splatData.length >= PLY_MAGIC_HEADER.length && 2501 | PLY_MAGIC_HEADER.every((v, i) => splatData[i] === v) 2502 | ) { 2503 | // .ply file 2504 | worker.postMessage({ ply: splatData.buffer }); 2505 | } else if ( 2506 | splatData.length >= LSPLAT_MAGIC_HEADER.length && 2507 | LSPLAT_MAGIC_HEADER.every((v, i) => splatData[i] === v) 2508 | ) { 2509 | splatData = splatData.slice(LSPLAT_MAGIC_HEADER.length); 2510 | // .lsplat file 2511 | worker.postMessage({ 2512 | buffer: splatData.buffer, 2513 | gaussianCount: Math.floor(splatData.length / PADDED_SPLAT_LENGTH), 2514 | }); 2515 | } else { 2516 | throw new Error("Unsupported file format"); 2517 | } 2518 | }; 2519 | fr.readAsArrayBuffer(file); 2520 | } 2521 | }; 2522 | 2523 | window.addEventListener("hashchange", (e) => { 2524 | try { 2525 | viewMatrix = JSON.parse(decodeURIComponent(location.hash.slice(1))); 2526 | carousel = false; 2527 | } catch (err) {} 2528 | }); 2529 | 2530 | const preventDefault = (e) => { 2531 | e.preventDefault(); 2532 | e.stopPropagation(); 2533 | }; 2534 | document.addEventListener("dragenter", preventDefault); 2535 | document.addEventListener("dragover", preventDefault); 2536 | document.addEventListener("dragleave", preventDefault); 2537 | document.addEventListener("drop", (e) => { 2538 | e.preventDefault(); 2539 | e.stopPropagation(); 2540 | selectFile(e.dataTransfer.files[0]); 2541 | }); 2542 | 2543 | let bytesRead = 0; 2544 | let lastGaussianCount = -1; 2545 | let stopLoading = false; 2546 | 2547 | while (true) { 2548 | const { done, value } = await reader.read(); 2549 | if (done || stopLoading) break; 2550 | 2551 | if (bytesRead + value.length > LSPLAT_MAGIC_HEADER.length) { 2552 | splatData.set(value.subarray(Math.max(0, LSPLAT_MAGIC_HEADER.length - bytesRead)), Math.max(0, bytesRead - LSPLAT_MAGIC_HEADER.length)); 2553 | } 2554 | bytesRead += value.length; 2555 | 2556 | if (gaussianCount > lastGaussianCount) { 2557 | worker.postMessage({ 2558 | buffer: splatData.buffer, 2559 | gaussianCount: Math.floor(bytesRead / PADDED_SPLAT_LENGTH), 2560 | }); 2561 | lastGaussianCount = gaussianCount; 2562 | } 2563 | } 2564 | if (!stopLoading) { 2565 | worker.postMessage({ 2566 | buffer: splatData.buffer, 2567 | gaussianCount: Math.floor(bytesRead / PADDED_SPLAT_LENGTH), 2568 | }); 2569 | } 2570 | addLight(); // add a light in after everything finishes loading 2571 | } 2572 | 2573 | main().catch((err) => { 2574 | document.getElementById("spinner").style.display = "none"; 2575 | document.getElementById("message").innerText = err.toString(); 2576 | throw err; 2577 | }); 2578 | --------------------------------------------------------------------------------