├── README.md
├── neuralangelo_colab.ipynb
├── test_colab.ipynb
└── visualize_colmap.ipynb
/README.md:
--------------------------------------------------------------------------------
1 | 🐣 Please follow me for new updates https://twitter.com/camenduru
2 | 🔥 Please join our discord server https://discord.gg/k5BwmmvJJU
3 | 🥳 Please join my patreon community https://patreon.com/camenduru
4 |
5 | # 🚦WIP🚦
6 |
7 | ## 🦒 Colab
8 |
9 | | Colab | Info
10 | | --- | --- |
11 | [](https://colab.research.google.com/github/camenduru/neuralangelo-colab/blob/main/neuralangelo_colab.ipynb) | neuralangelo_colab
12 | [](https://colab.research.google.com/drive/13u8DX9BNzQwiyPPCB7_4DbSxiQ5-_nGF) | neuralangelo author colab
13 |
14 | ## Tutorial
15 |
16 | ### Input
17 | https://github.com/camenduru/neuralangelo-colab/assets/54370274/196960fb-4828-483a-b2d1-9f31ee911e8a
18 |
19 | ### Output
20 | https://github.com/camenduru/neuralangelo-colab/assets/54370274/b483bc07-3330-45fc-94af-41d1854c1821
21 |
22 | ## Blender addon to inspect and preprocess COLMAP data for Neuralangelo (Optional)
23 |
24 | https://github.com/mli0603/BlenderNeuralangelo
25 |
26 | https://github.com/camenduru/neuralangelo-colab/assets/54370274/9e4b267d-fdbf-410a-b269-0d9515d70d70
27 |
28 | ## Output 20000 Steps (NEW CODE)
29 | 
30 |
31 | ## Output 15000 Steps (OLD CODE)
32 | 
33 |
34 | 
35 |
36 | ## Output 25000 Steps (OLD CODE)
37 | 
38 |
39 | 
40 |
41 | ## Training Video (25000 Steps) (OLD CODE)
42 | https://www.youtube.com/watch?v=yZokUyHaOb0
43 |
44 | ## Main Repo
45 | https://github.com/NVlabs/neuralangelo
46 |
47 | ## Page
48 | https://research.nvidia.com/labs/dir/neuralangelo/
49 |
50 | ## Paper
51 | https://arxiv.org/abs/2306.03092
52 |
53 | ## Output Data
54 | https://huggingface.co/camenduru/neuralangelo/tree/main/lego
55 |
--------------------------------------------------------------------------------
/neuralangelo_colab.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "view-in-github"
7 | },
8 | "source": [
9 | "[](https://colab.research.google.com/github/camenduru/neuralangelo-colab/blob/main/neuralangelo_colab.ipynb)"
10 | ]
11 | },
12 | {
13 | "cell_type": "code",
14 | "execution_count": null,
15 | "metadata": {
16 | "id": "VjYy0F2gZIPR"
17 | },
18 | "outputs": [],
19 | "source": [
20 | "%cd /content\n",
21 | "\n",
22 | "!apt-get install \\\n",
23 | " cmake \\\n",
24 | " libgoogle-glog-dev \\\n",
25 | " libgflags-dev \\\n",
26 | " libatlas-base-dev \\\n",
27 | " libeigen3-dev \\\n",
28 | " libsuitesparse-dev \\\n",
29 | " libboost-program-options-dev \\\n",
30 | " libboost-filesystem-dev \\\n",
31 | " libboost-graph-dev \\\n",
32 | " libboost-system-dev \\\n",
33 | " libboost-test-dev \\\n",
34 | " libfreeimage-dev \\\n",
35 | " libmetis-dev \\\n",
36 | " libglew-dev \\\n",
37 | " qtbase5-dev \\\n",
38 | " libqt5opengl5-dev \\\n",
39 | " libcgal-dev\n",
40 | "\n",
41 | "!wget https://huggingface.co/camenduru/neuralangelo/resolve/main/colmap.zip\n",
42 | "!unzip /content/colmap.zip -d colmap\n",
43 | "\n",
44 | "!cp -r /content/colmap/lib/. /usr/local/lib\n",
45 | "!chmod 755 /content/colmap/bin/colmap\n",
46 | "!cp -r /content/colmap/bin/. /usr/local/bin\n",
47 | "\n",
48 | "!pip install -q commentjson addict pynvml wandb trimesh PyMCubes\n",
49 | "!pip install -q https://github.com/camenduru/wheels/releases/download/colab/tinycudann-1.7-cp310-cp310-linux_x86_64.whl\n",
50 | "\n",
51 | "%cd /content\n",
52 | "!git clone -b v1.0 --recursive https://github.com/camenduru/neuralangelo"
53 | ]
54 | },
55 | {
56 | "cell_type": "code",
57 | "execution_count": null,
58 | "metadata": {},
59 | "outputs": [],
60 | "source": [
61 | "%cd /content/neuralangelo\n",
62 | "!wget https://huggingface.co/camenduru/neuralangelo/resolve/main/lego.mp4 -O /content/lego.mp4\n",
63 | "SEQUENCE=\"lego\"\n",
64 | "PATH_TO_VIDEO=\"/content/lego.mp4\"\n",
65 | "DOWNSAMPLE_RATE=2\n",
66 | "SCENE_TYPE=\"object\"\n",
67 | "!bash /content/neuralangelo/projects/neuralangelo/scripts/preprocess.sh {SEQUENCE} {PATH_TO_VIDEO} {DOWNSAMPLE_RATE} {SCENE_TYPE}"
68 | ]
69 | },
70 | {
71 | "cell_type": "code",
72 | "execution_count": null,
73 | "metadata": {},
74 | "outputs": [],
75 | "source": [
76 | "# from https://github.com/NVlabs/neuralangelo\n",
77 | "# @title Visulize COLMAP (Optional)\n",
78 | "!pip install -q k3d\n",
79 | "\n",
80 | "colmap_path='/content/neuralangelo/datasets/lego_ds2'\n",
81 | "%cd /content/neuralangelo\n",
82 | "# Import Python libraries.\n",
83 | "import numpy as np\n",
84 | "import torch\n",
85 | "import k3d\n",
86 | "import json\n",
87 | "import plotly.graph_objs as go\n",
88 | "from collections import OrderedDict\n",
89 | "# Import imaginaire modules.\n",
90 | "from projects.nerf.utils import camera, visualize\n",
91 | "from third_party.colmap.scripts.python.read_write_model import read_model\n",
92 | "# Read the COLMAP data.\n",
93 | "cameras, images, points_3D = read_model(path=f\"{colmap_path}/sparse\", ext=\".bin\")\n",
94 | "# Convert camera poses.\n",
95 | "images = OrderedDict(sorted(images.items()))\n",
96 | "qvecs = torch.from_numpy(np.stack([image.qvec for image in images.values()]))\n",
97 | "tvecs = torch.from_numpy(np.stack([image.tvec for image in images.values()]))\n",
98 | "Rs = camera.quaternion.q_to_R(qvecs)\n",
99 | "poses = torch.cat([Rs, tvecs[..., None]], dim=-1) # [N,3,4]\n",
100 | "print(f\"# images: {len(poses)}\")\n",
101 | "# Get the sparse 3D points and the colors.\n",
102 | "xyzs = torch.from_numpy(np.stack([point.xyz for point in points_3D.values()]))\n",
103 | "rgbs = np.stack([point.rgb for point in points_3D.values()])\n",
104 | "rgbs_int32 = (rgbs[:, 0] * 2**16 + rgbs[:, 1] * 2**8 + rgbs[:, 2]).astype(np.uint32)\n",
105 | "print(f\"# points: {len(xyzs)}\")\n",
106 | "\n",
107 | "# Visualize the bounding sphere.\n",
108 | "json_fname = f\"{colmap_path}/transforms.json\"\n",
109 | "with open(json_fname) as file:\n",
110 | " meta = json.load(file)\n",
111 | "center = meta[\"sphere_center\"]\n",
112 | "radius = meta[\"sphere_radius\"]\n",
113 | "# ------------------------------------------------------------------------------------\n",
114 | "# These variables can be adjusted to make the bounding sphere fit the region of interest.\n",
115 | "# The adjusted values can then be set in the config as data.readjust.center and data.readjust.scale\n",
116 | "readjust_x = 0. # @param {type:\"number\"}\n",
117 | "readjust_y = 0. # @param {type:\"number\"}\n",
118 | "readjust_z = 0. # @param {type:\"number\"}\n",
119 | "readjust_scale = 1. # @param {type:\"number\"}\n",
120 | "readjust_center = np.array([readjust_x, readjust_y, readjust_z])\n",
121 | "# ------------------------------------------------------------------------------------\n",
122 | "center += readjust_center\n",
123 | "radius *= readjust_scale\n",
124 | "# Make some points to hallucinate a bounding sphere.\n",
125 | "sphere_points = np.random.randn(100000, 3)\n",
126 | "sphere_points = sphere_points / np.linalg.norm(sphere_points, axis=-1, keepdims=True)\n",
127 | "sphere_points = sphere_points * radius + center\n",
128 | "\n",
129 | "vis_depth = 0.2\n",
130 | "# Visualize with Plotly.\n",
131 | "x, y, z = *xyzs.T,\n",
132 | "colors = rgbs / 255.0\n",
133 | "sphere_x, sphere_y, sphere_z = *sphere_points.T,\n",
134 | "sphere_colors = [\"#4488ff\"] * len(sphere_points)\n",
135 | "traces_poses = visualize.plotly_visualize_pose(poses, vis_depth=vis_depth, xyz_length=0.02, center_size=0.01, xyz_width=0.005, mesh_opacity=0.05)\n",
136 | "trace_points = go.Scatter3d(x=x, y=y, z=z, mode=\"markers\", marker=dict(size=1, color=colors, opacity=1), hoverinfo=\"skip\")\n",
137 | "trace_sphere = go.Scatter3d(x=sphere_x, y=sphere_y, z=sphere_z, mode=\"markers\", marker=dict(size=0.5, color=sphere_colors, opacity=0.7), hoverinfo=\"skip\")\n",
138 | "traces_all = traces_poses + [trace_points, trace_sphere]\n",
139 | "layout = go.Layout(scene=dict(xaxis=dict(showspikes=False, backgroundcolor=\"rgba(0,0,0,0)\", gridcolor=\"rgba(0,0,0,0.1)\"),\n",
140 | " yaxis=dict(showspikes=False, backgroundcolor=\"rgba(0,0,0,0)\", gridcolor=\"rgba(0,0,0,0.1)\"),\n",
141 | " zaxis=dict(showspikes=False, backgroundcolor=\"rgba(0,0,0,0)\", gridcolor=\"rgba(0,0,0,0.1)\"),\n",
142 | " xaxis_title=\"X\", yaxis_title=\"Y\", zaxis_title=\"Z\", dragmode=\"orbit\",\n",
143 | " aspectratio=dict(x=1, y=1, z=1), aspectmode=\"data\"), height=800)\n",
144 | "fig = go.Figure(data=traces_all, layout=layout)\n",
145 | "fig.show()"
146 | ]
147 | },
148 | {
149 | "cell_type": "code",
150 | "execution_count": null,
151 | "metadata": {},
152 | "outputs": [],
153 | "source": [
154 | "!wandb login"
155 | ]
156 | },
157 | {
158 | "cell_type": "code",
159 | "execution_count": null,
160 | "metadata": {},
161 | "outputs": [],
162 | "source": [
163 | "%cd /content/neuralangelo\n",
164 | "EXPERIMENT=\"lego\"\n",
165 | "GROUP=\"lego_group\"\n",
166 | "NAME=\"lego\"\n",
167 | "CONFIG=f\"/content/neuralangelo/projects/neuralangelo/configs/custom/{EXPERIMENT}.yaml\"\n",
168 | "GPUS=1\n",
169 | "CHECKPOINT_PATH=\"/content/checkpoint\"\n",
170 | "!torchrun --nproc_per_node={GPUS} train.py \\\n",
171 | " --wandb \\\n",
172 | " --wandb_name=neuralangelo \\\n",
173 | " --logdir=logs/{GROUP}/{NAME} \\\n",
174 | " --config={CONFIG} \\\n",
175 | " --show_pbar \\\n",
176 | " --data.readjust.scale=0.5 \\\n",
177 | " --max_iter=25000 \\\n",
178 | " --validation_iter=99999999 \\\n",
179 | " --model.object.sdf.encoding.coarse2fine.step=200 \\\n",
180 | " --model.object.sdf.encoding.hashgrid.dict_size=19 \\\n",
181 | " --optim.sched.warm_up_end=200 \\\n",
182 | " --optim.sched.two_steps=[12000,16000]"
183 | ]
184 | },
185 | {
186 | "cell_type": "code",
187 | "execution_count": null,
188 | "metadata": {},
189 | "outputs": [],
190 | "source": [
191 | "%cd /content/neuralangelo\n",
192 | "CONFIG=f\"/content/neuralangelo/projects/neuralangelo/configs/custom/lego.yaml\"\n",
193 | "GPUS=1\n",
194 | "CHECKPOINT=\"/content/neuralangelo/logs/lego_group/lego/epoch_00150_iteration_000015000_checkpoint.pt\"\n",
195 | "OUTPUT_MESH=\"/content/lego.ply\"\n",
196 | "RESOLUTION=2048\n",
197 | "BLOCK_RES=128\n",
198 | "!torchrun --nproc_per_node={GPUS} projects/neuralangelo/scripts/extract_mesh.py \\\n",
199 | " --config={CONFIG} \\\n",
200 | " --checkpoint={CHECKPOINT} \\\n",
201 | " --output_file={OUTPUT_MESH} \\\n",
202 | " --resolution={RESOLUTION} \\\n",
203 | " --block_res={BLOCK_RES} \\\n",
204 | " --textured"
205 | ]
206 | }
207 | ],
208 | "metadata": {
209 | "accelerator": "GPU",
210 | "colab": {
211 | "gpuType": "T4",
212 | "provenance": []
213 | },
214 | "kernelspec": {
215 | "display_name": "Python 3",
216 | "name": "python3"
217 | },
218 | "language_info": {
219 | "name": "python"
220 | }
221 | },
222 | "nbformat": 4,
223 | "nbformat_minor": 0
224 | }
225 |
--------------------------------------------------------------------------------
/test_colab.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "view-in-github"
7 | },
8 | "source": [
9 | "[](https://colab.research.google.com/github/camenduru/neuralangelo-colab/blob/main/test_colab.ipynb)"
10 | ]
11 | },
12 | {
13 | "cell_type": "code",
14 | "execution_count": null,
15 | "metadata": {
16 | "id": "VjYy0F2gZIPR"
17 | },
18 | "outputs": [],
19 | "source": [
20 | "%cd /content\n",
21 | "# !apt -y install -qq aria2\n",
22 | "\n",
23 | "!apt-get install \\\n",
24 | " cmake \\\n",
25 | " libgoogle-glog-dev \\\n",
26 | " libgflags-dev \\\n",
27 | " libatlas-base-dev \\\n",
28 | " libeigen3-dev \\\n",
29 | " libsuitesparse-dev \\\n",
30 | " libboost-program-options-dev \\\n",
31 | " libboost-filesystem-dev \\\n",
32 | " libboost-graph-dev \\\n",
33 | " libboost-system-dev \\\n",
34 | " libboost-test-dev \\\n",
35 | " libfreeimage-dev \\\n",
36 | " libmetis-dev \\\n",
37 | " libglew-dev \\\n",
38 | " qtbase5-dev \\\n",
39 | " libqt5opengl5-dev \\\n",
40 | " libcgal-dev\n",
41 | "\n",
42 | "!wget https://huggingface.co/camenduru/neuralangelo/resolve/main/colmap.zip\n",
43 | "!unzip /content/colmap.zip -d colmap\n",
44 | "\n",
45 | "!cp -r /content/colmap/lib/. /usr/local/lib\n",
46 | "!chmod 755 /content/colmap/bin/colmap\n",
47 | "!cp -r /content/colmap/bin/. /usr/local/bin\n",
48 | "\n",
49 | "!pip install -q commentjson addict pynvml wandb trimesh PyMCubes\n",
50 | "!pip install -q https://github.com/camenduru/wheels/releases/download/colab/tinycudann-1.7-cp310-cp310-linux_x86_64.whl\n",
51 | "\n",
52 | "# %cd /content\n",
53 | "# !git clone --recursive https://github.com/nvlabs/tiny-cuda-nn\n",
54 | "# %cd /content/tiny-cuda-nn\n",
55 | "# !cmake . -B build\n",
56 | "# !cmake --build build --config RelWithDebInfo -j\n",
57 | "\n",
58 | "%cd /content\n",
59 | "!git clone -b dev --recursive https://github.com/camenduru/neuralangelo\n",
60 | "\n",
61 | "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/neuralangelo/resolve/main/toy_example.MOV -d /content/test -o toy_example.MOV\n",
62 | "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/neuralangelo/resolve/main/neus.zip -d /content/datasets/dtu -o neus.zip\n",
63 | "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/neuralangelo/resolve/main/trainingdata.zip -d /content/test -o trainingdata.zip\n",
64 | "# !aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/neuralangelo/raw/main/downloader.py -d /content -o downloader.py\n",
65 | "# %cd /content/datasets/dtu\n",
66 | "# !unzip -q neus.zip -d /content/datasets/dtu\n",
67 | "# %cd /content/neuralangelo\n",
68 | "# !python /content/neuralangelo/projects/neuralangelo/scripts/convert_dtu_to_json.py --dtu_path /content/datasets/dtu\n",
69 | "# !bash projects/neuralangelo/scripts/preprocess_dtu.sh /content/datasets/dtu\n",
70 | "# !python downloader.py --modality video --group both\n",
71 | "# !python downloader.py --modality image --group intermediate\n",
72 | "# !python downloader.py -s"
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": null,
78 | "metadata": {},
79 | "outputs": [],
80 | "source": [
81 | "# !wget https://huggingface.co/camenduru/neuralangelo/resolve/main/fox.zip -O /content/fox.zip\n",
82 | "# !unzip /content/fox.zip -d /content\n",
83 | "\n",
84 | "# !mkdir /content/fox/sparse /content/fox/dense\n",
85 | "\n",
86 | "# !colmap feature_extractor \\\n",
87 | "# --database_path /content/fox/database.db \\\n",
88 | "# --image_path /content/fox \\\n",
89 | "# --ImageReader.camera_model=RADIAL \\\n",
90 | "# --SiftExtraction.use_gpu=true \\\n",
91 | "# --SiftExtraction.num_threads=32 \\\n",
92 | "# --ImageReader.single_camera=true # assuming single camera\n",
93 | "\n",
94 | "# !colmap sequential_matcher \\\n",
95 | "# --database_path /content/fox/database.db \\\n",
96 | "# --SiftMatching.use_gpu=true\n",
97 | "\n",
98 | "# !colmap mapper \\\n",
99 | "# --database_path /content/fox/database.db \\\n",
100 | "# --image_path /content/fox \\\n",
101 | "# --output_path /content/fox/sparse\n",
102 | "\n",
103 | "# !colmap image_undistorter \\\n",
104 | "# --image_path /content/fox \\\n",
105 | "# --input_path /content/fox/sparse/0 \\\n",
106 | "# --output_path /content/fox/dense \\\n",
107 | "# --output_type COLMAP \\\n",
108 | "# --max_image_size 2000\n",
109 | "\n",
110 | "%cd /content/neuralangelo\n",
111 | "!wget https://huggingface.co/camenduru/neuralangelo/resolve/main/lego.mp4 -O /content/lego.mp4\n",
112 | "EXPERIMENT=\"lego\"\n",
113 | "PATH_TO_VIDEO=\"/content/lego.mp4\"\n",
114 | "SKIP_FRAME_RATE=2\n",
115 | "SCENE_TYPE=\"object\"\n",
116 | "!bash /content/neuralangelo/projects/neuralangelo/scripts/preprocess.sh {EXPERIMENT} {PATH_TO_VIDEO} {SKIP_FRAME_RATE} {SCENE_TYPE}"
117 | ]
118 | },
119 | {
120 | "cell_type": "code",
121 | "execution_count": null,
122 | "metadata": {},
123 | "outputs": [],
124 | "source": [
125 | "!wandb login"
126 | ]
127 | },
128 | {
129 | "cell_type": "code",
130 | "execution_count": null,
131 | "metadata": {},
132 | "outputs": [],
133 | "source": [
134 | "%cd /content/neuralangelo\n",
135 | "EXPERIMENT=\"lego\"\n",
136 | "GROUP=\"lego_group\"\n",
137 | "NAME=\"lego\"\n",
138 | "CONFIG=f\"/content/neuralangelo/projects/neuralangelo/configs/custom/{EXPERIMENT}.yaml\"\n",
139 | "GPUS=1\n",
140 | "CHECKPOINT_PATH=\"/content/checkpoint\"\n",
141 | "!torchrun --nproc_per_node={GPUS} train.py \\\n",
142 | " --wandb \\\n",
143 | " --wandb_name=neuralangelo \\\n",
144 | " --logdir=logs/{GROUP}/{NAME} \\\n",
145 | " --config={CONFIG} \\\n",
146 | " --show_pbar"
147 | ]
148 | },
149 | {
150 | "cell_type": "code",
151 | "execution_count": null,
152 | "metadata": {},
153 | "outputs": [],
154 | "source": [
155 | "%cd /content/neuralangelo\n",
156 | "CONFIG=f\"/content/neuralangelo/projects/neuralangelo/configs/custom/lego.yaml\"\n",
157 | "GPUS=1\n",
158 | "CHECKPOINT=\"/content/neuralangelo/logs/lego_group/lego/epoch_00150_iteration_000015000_checkpoint.pt\"\n",
159 | "OUTPUT_MESH=\"/content/lego.ply\"\n",
160 | "RESOLUTION=2048\n",
161 | "BLOCK_RES=128\n",
162 | "!torchrun --nproc_per_node={GPUS} projects/neuralangelo/scripts/extract_mesh.py \\\n",
163 | " --config={CONFIG} \\\n",
164 | " --checkpoint={CHECKPOINT} \\\n",
165 | " --output_file={OUTPUT_MESH} \\\n",
166 | " --resolution={RESOLUTION} \\\n",
167 | " --block_res={BLOCK_RES} \\\n",
168 | " --textured"
169 | ]
170 | },
171 | {
172 | "cell_type": "code",
173 | "execution_count": null,
174 | "metadata": {},
175 | "outputs": [],
176 | "source": [
177 | "# !rm -rf /content/neuralangelo/projects/neuralangelo/configs/custom/lego.yaml\n",
178 | "# !rm -rf /content/neuralangelo/datasets/lego_skip2\n",
179 | "# !rm -rf /content/neuralangelo/logs/lego_group"
180 | ]
181 | }
182 | ],
183 | "metadata": {
184 | "accelerator": "GPU",
185 | "colab": {
186 | "gpuType": "T4",
187 | "provenance": []
188 | },
189 | "kernelspec": {
190 | "display_name": "Python 3",
191 | "name": "python3"
192 | },
193 | "language_info": {
194 | "name": "python"
195 | }
196 | },
197 | "nbformat": 4,
198 | "nbformat_minor": 0
199 | }
200 |
--------------------------------------------------------------------------------
/visualize_colmap.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "e1897673",
6 | "metadata": {
7 | "id": "view-in-github"
8 | },
9 | "source": [
10 | "[](https://colab.research.google.com/github/camenduru/neuralangelo-colab/blob/main/colmap_colab.ipynb)"
11 | ]
12 | },
13 | {
14 | "cell_type": "code",
15 | "execution_count": null,
16 | "id": "8b8d7b17-af50-42cd-b531-ef61c49c9e61",
17 | "metadata": {},
18 | "outputs": [],
19 | "source": [
20 | "# Set the work directory to the imaginaire root.\n",
21 | "import os, sys, time\n",
22 | "import pathlib\n",
23 | "root_dir = pathlib.Path().absolute().parents[2]\n",
24 | "os.chdir(root_dir)\n",
25 | "print(f\"Root Directory Path: {root_dir}\")"
26 | ]
27 | },
28 | {
29 | "cell_type": "code",
30 | "execution_count": null,
31 | "id": "2b5b9e2f-841c-4815-92e0-0c76ed46da62",
32 | "metadata": {},
33 | "outputs": [],
34 | "source": [
35 | "# Import Python libraries.\n",
36 | "import numpy as np\n",
37 | "import torch\n",
38 | "import k3d\n",
39 | "import json\n",
40 | "from collections import OrderedDict\n",
41 | "# Import imaginaire modules.\n",
42 | "from projects.nerf.utils import camera, visualize\n",
43 | "from third_party.colmap.scripts.python.read_write_model import read_model"
44 | ]
45 | },
46 | {
47 | "cell_type": "code",
48 | "execution_count": null,
49 | "id": "76033016-2d92-4a5d-9e50-3978553e8df4",
50 | "metadata": {},
51 | "outputs": [],
52 | "source": [
53 | "# Read the COLMAP data.\n",
54 | "colmap_path = \"datasets/iphone/climbingnet_skip12\"\n",
55 | "cameras, images, points_3D = read_model(path=f\"{colmap_path}/dense/sparse\", ext=\".bin\")\n",
56 | "# Convert camera poses.\n",
57 | "images = OrderedDict(sorted(images.items()))\n",
58 | "qvecs = torch.from_numpy(np.stack([image.qvec for image in images.values()]))\n",
59 | "tvecs = torch.from_numpy(np.stack([image.tvec for image in images.values()]))\n",
60 | "Rs = camera.quaternion.q_to_R(qvecs)\n",
61 | "poses = torch.cat([Rs, tvecs[..., None]], dim=-1) # [N,3,4]\n",
62 | "print(f\"# images: {len(poses)}\")\n",
63 | "# Get the sparse 3D points and the colors.\n",
64 | "xyzs = torch.from_numpy(np.stack([point.xyz for point in points_3D.values()]))\n",
65 | "rgbs = np.stack([point.rgb for point in points_3D.values()])\n",
66 | "rgbs = (rgbs[:, 0] * 2**16 + rgbs[:, 1] * 2**8 + rgbs[:, 2]).astype(np.uint32)\n",
67 | "print(f\"# points: {len(xyzs)}\")"
68 | ]
69 | },
70 | {
71 | "cell_type": "code",
72 | "execution_count": null,
73 | "id": "b6cf60ec-fe6a-43ba-9aaf-e3c7afd88208",
74 | "metadata": {},
75 | "outputs": [],
76 | "source": [
77 | "# Visualize the bounding sphere.\n",
78 | "json_fname = f\"{colmap_path}/dense/transforms.json\"\n",
79 | "with open(json_fname) as file:\n",
80 | " meta = json.load(file)\n",
81 | "center = meta[\"sphere_center\"]\n",
82 | "radius = meta[\"sphere_radius\"]\n",
83 | "# ------------------------------------------------------------------------------------\n",
84 | "# These variables can be adjusted to make the bounding sphere fit the region of interest.\n",
85 | "# The adjusted values can then be set in the config as data.readjust.center and data.readjust.scale\n",
86 | "readjust_center = np.array([0., 0., 0.])\n",
87 | "readjust_scale = 0.25\n",
88 | "# ------------------------------------------------------------------------------------\n",
89 | "center += readjust_center\n",
90 | "radius *= readjust_scale\n",
91 | "# Make some points to hallucinate a bounding sphere.\n",
92 | "sphere_points = np.random.randn(100000, 3)\n",
93 | "sphere_points = sphere_points / np.linalg.norm(sphere_points, axis=-1, keepdims=True)\n",
94 | "sphere_points = sphere_points * radius + center"
95 | ]
96 | },
97 | {
98 | "cell_type": "code",
99 | "execution_count": null,
100 | "id": "fdde170b-4546-4617-9162-a9fcb936347d",
101 | "metadata": {},
102 | "outputs": [],
103 | "source": [
104 | "# Visualize with K3D.\n",
105 | "vis_scale = 0.5\n",
106 | "plot = visualize.k3d_visualize_pose(poses,\n",
107 | " vis_depth=(0.5 * vis_scale),\n",
108 | " xyz_length=(0.1 * vis_scale),\n",
109 | " center_size=(0.05 * vis_scale),\n",
110 | " xyz_width=(0.02 * vis_scale))\n",
111 | "plot += k3d.points(xyzs, colors=rgbs, point_size=(0.05 * vis_scale), shader=\"flat\")\n",
112 | "plot += k3d.points(sphere_points, color=0x4488ff, point_size=0.02, shader=\"flat\")\n",
113 | "plot.display()\n",
114 | "plot.camera_fov = 30.0"
115 | ]
116 | },
117 | {
118 | "cell_type": "code",
119 | "execution_count": null,
120 | "id": "7ff98977",
121 | "metadata": {},
122 | "outputs": [],
123 | "source": [
124 | "from google.colab import output\n",
125 | "output.enable_custom_widget_manager()\n",
126 | "\n",
127 | "# from google.colab import output\n",
128 | "# output.disable_custom_widget_manager()\n",
129 | "\n",
130 | "import numpy as np\n",
131 | "import torch\n",
132 | "import k3d\n",
133 | "import json\n",
134 | "from collections import OrderedDict\n",
135 | "from projects.nerf.utils import camera, visualize\n",
136 | "from third_party.colmap.scripts.python.read_write_model import read_model\n",
137 | "\n",
138 | "colmap_path = \"/content/neuralangelo/datasets/lego_skip2\"\n",
139 | "cameras, images, points_3D = read_model(path=f\"{colmap_path}/dense/sparse\", ext=\".bin\")\n",
140 | "images = OrderedDict(sorted(images.items()))\n",
141 | "qvecs = torch.from_numpy(np.stack([image.qvec for image in images.values()]))\n",
142 | "tvecs = torch.from_numpy(np.stack([image.tvec for image in images.values()]))\n",
143 | "Rs = camera.quaternion.q_to_R(qvecs)\n",
144 | "poses = torch.cat([Rs, tvecs[..., None]], dim=-1) # [N,3,4]\n",
145 | "print(f\"# images: {len(poses)}\")\n",
146 | "xyzs = torch.from_numpy(np.stack([point.xyz for point in points_3D.values()]))\n",
147 | "rgbs = np.stack([point.rgb for point in points_3D.values()])\n",
148 | "rgbs = (rgbs[:, 0] * 2**16 + rgbs[:, 1] * 2**8 + rgbs[:, 2]).astype(np.uint32)\n",
149 | "print(f\"# points: {len(xyzs)}\")\n",
150 | "\n",
151 | "json_fname = f\"{colmap_path}/dense/transforms.json\"\n",
152 | "with open(json_fname) as file:\n",
153 | " meta = json.load(file)\n",
154 | "center = meta[\"sphere_center\"]\n",
155 | "radius = meta[\"sphere_radius\"]\n",
156 | "readjust_center = np.array([0., 0., 0.])\n",
157 | "readjust_scale = 0.25\n",
158 | "center += readjust_center\n",
159 | "radius *= readjust_scale\n",
160 | "sphere_points = np.random.randn(100000, 3)\n",
161 | "sphere_points = sphere_points / np.linalg.norm(sphere_points, axis=-1, keepdims=True)\n",
162 | "sphere_points = sphere_points * radius + center\n",
163 | "\n",
164 | "vis_scale = 0.5\n",
165 | "plot = visualize.k3d_visualize_pose(poses,\n",
166 | " vis_depth=(0.5 * vis_scale),\n",
167 | " xyz_length=(0.1 * vis_scale),\n",
168 | " center_size=(0.05 * vis_scale),\n",
169 | " xyz_width=(0.02 * vis_scale))\n",
170 | "plot += k3d.points(xyzs, colors=rgbs, point_size=(0.05 * vis_scale), shader=\"flat\")\n",
171 | "plot += k3d.points(sphere_points, color=0x4488ff, point_size=0.02, shader=\"flat\")\n",
172 | "plot.display()\n",
173 | "plot.camera_fov = 30.0"
174 | ]
175 | }
176 | ],
177 | "metadata": {
178 | "kernelspec": {
179 | "display_name": "Python 3 (ipykernel)",
180 | "language": "python",
181 | "name": "python3"
182 | },
183 | "language_info": {
184 | "codemirror_mode": {
185 | "name": "ipython",
186 | "version": 3
187 | },
188 | "file_extension": ".py",
189 | "mimetype": "text/x-python",
190 | "name": "python",
191 | "nbconvert_exporter": "python",
192 | "pygments_lexer": "ipython3",
193 | "version": "3.9.13"
194 | }
195 | },
196 | "nbformat": 4,
197 | "nbformat_minor": 5
198 | }
199 |
--------------------------------------------------------------------------------