├── README.md ├── checkpoints └── put_model_here.txt ├── configs ├── finetune.txt ├── finetune_ec.txt ├── finetune_ec_weight.txt ├── finetune_gso.txt └── finetune_gso_ec.txt ├── data ├── One2345_training_pose.json ├── cake │ ├── view_0.png │ ├── view_0_0_10.png │ ├── view_0_0_10_gt.png │ ├── view_0_0_10_gt_depth_mm.png │ ├── view_0_1_10.png │ ├── view_0_1_10_gt.png │ ├── view_0_1_10_gt_depth_mm.png │ ├── view_0_2_10.png │ ├── view_0_2_10_gt.png │ ├── view_0_2_10_gt_depth_mm.png │ ├── view_0_3_10.png │ ├── view_0_3_10_gt.png │ ├── view_0_3_10_gt_depth_mm.png │ ├── view_0_depth.png │ ├── view_0_depth_mm.png │ ├── view_1.png │ ├── view_1_0_10.png │ ├── view_1_0_10_gt.png │ ├── view_1_0_10_gt_depth_mm.png │ ├── view_1_1_10.png │ ├── view_1_1_10_gt.png │ ├── view_1_1_10_gt_depth_mm.png │ ├── view_1_2_10.png │ ├── view_1_2_10_gt.png │ ├── view_1_2_10_gt_depth_mm.png │ ├── view_1_3_10.png │ ├── view_1_3_10_gt.png │ ├── view_1_3_10_gt_depth_mm.png │ ├── view_1_depth.png │ ├── view_1_depth_mm.png │ ├── view_2.png │ ├── view_2_0_10.png │ ├── view_2_0_10_gt.png │ ├── view_2_0_10_gt_depth_mm.png │ ├── view_2_1_10.png │ ├── view_2_1_10_gt.png │ ├── view_2_1_10_gt_depth_mm.png │ ├── view_2_2_10.png │ ├── view_2_2_10_gt.png │ ├── view_2_2_10_gt_depth_mm.png │ ├── view_2_3_10.png │ ├── view_2_3_10_gt.png │ ├── view_2_3_10_gt_depth_mm.png │ ├── view_2_depth.png │ ├── view_2_depth_mm.png │ ├── view_3.png │ ├── view_3_0_10.png │ ├── view_3_0_10_gt.png │ ├── view_3_0_10_gt_depth_mm.png │ ├── view_3_1_10.png │ ├── view_3_1_10_gt.png │ ├── view_3_1_10_gt_depth_mm.png │ ├── view_3_2_10.png │ ├── view_3_2_10_gt.png │ ├── view_3_2_10_gt_depth_mm.png │ ├── view_3_3_10.png │ ├── view_3_3_10_gt.png │ ├── view_3_3_10_gt_depth_mm.png │ ├── view_3_depth.png │ ├── view_3_depth_mm.png │ ├── view_4.png │ ├── view_4_0_10.png │ ├── view_4_0_10_gt.png │ ├── view_4_0_10_gt_depth_mm.png │ ├── view_4_1_10.png │ ├── view_4_1_10_gt.png │ ├── view_4_1_10_gt_depth_mm.png │ ├── view_4_2_10.png │ ├── view_4_2_10_gt.png │ ├── view_4_2_10_gt_depth_mm.png │ ├── view_4_3_10.png │ ├── view_4_3_10_gt.png │ ├── view_4_3_10_gt_depth_mm.png │ ├── view_4_depth.png │ ├── view_4_depth_mm.png │ ├── view_5.png │ ├── view_5_0_10.png │ ├── view_5_0_10_gt.png │ ├── view_5_0_10_gt_depth_mm.png │ ├── view_5_1_10.png │ ├── view_5_1_10_gt.png │ ├── view_5_1_10_gt_depth_mm.png │ ├── view_5_2_10.png │ ├── view_5_2_10_gt.png │ ├── view_5_2_10_gt_depth_mm.png │ ├── view_5_3_10.png │ ├── view_5_3_10_gt.png │ ├── view_5_3_10_gt_depth_mm.png │ ├── view_5_depth.png │ ├── view_5_depth_mm.png │ ├── view_6.png │ ├── view_6_0_10.png │ ├── view_6_0_10_gt.png │ ├── view_6_0_10_gt_depth_mm.png │ ├── view_6_1_10.png │ ├── view_6_1_10_gt.png │ ├── view_6_1_10_gt_depth_mm.png │ ├── view_6_2_10.png │ ├── view_6_2_10_gt.png │ ├── view_6_2_10_gt_depth_mm.png │ ├── view_6_3_10.png │ ├── view_6_3_10_gt.png │ ├── view_6_3_10_gt_depth_mm.png │ ├── view_6_depth.png │ ├── view_6_depth_mm.png │ ├── view_7.png │ ├── view_7_0_10.png │ ├── view_7_0_10_gt.png │ ├── view_7_0_10_gt_depth_mm.png │ ├── view_7_1_10.png │ ├── view_7_1_10_gt.png │ ├── view_7_1_10_gt_depth_mm.png │ ├── view_7_2_10.png │ ├── view_7_2_10_gt.png │ ├── view_7_2_10_gt_depth_mm.png │ ├── view_7_3_10.png │ ├── view_7_3_10_gt.png │ ├── view_7_3_10_gt_depth_mm.png │ ├── view_7_depth.png │ └── view_7_depth_mm.png ├── camera │ ├── view_0.png │ ├── view_0_0_10.png │ ├── view_0_0_10_gt.png │ ├── view_0_0_10_gt_depth_mm.png │ ├── view_0_1_10.png │ ├── view_0_1_10_gt.png │ ├── view_0_1_10_gt_depth_mm.png │ ├── view_0_2_10.png │ ├── view_0_2_10_gt.png │ ├── view_0_2_10_gt_depth_mm.png │ ├── view_0_3_10.png │ ├── view_0_3_10_gt.png │ ├── view_0_3_10_gt_depth_mm.png │ ├── view_0_depth.png │ ├── view_0_depth_mm.png │ ├── view_1.png │ ├── view_1_0_10.png │ ├── view_1_0_10_gt.png │ ├── view_1_0_10_gt_depth_mm.png │ ├── view_1_1_10.png │ ├── view_1_1_10_gt.png │ ├── view_1_1_10_gt_depth_mm.png │ ├── view_1_2_10.png │ ├── view_1_2_10_gt.png │ ├── view_1_2_10_gt_depth_mm.png │ ├── view_1_3_10.png │ ├── view_1_3_10_gt.png │ ├── view_1_3_10_gt_depth_mm.png │ ├── view_1_depth.png │ ├── view_1_depth_mm.png │ ├── view_2.png │ ├── view_2_0_10.png │ ├── view_2_0_10_gt.png │ ├── view_2_0_10_gt_depth_mm.png │ ├── view_2_1_10.png │ ├── view_2_1_10_gt.png │ ├── view_2_1_10_gt_depth_mm.png │ ├── view_2_2_10.png │ ├── view_2_2_10_gt.png │ ├── view_2_2_10_gt_depth_mm.png │ ├── view_2_3_10.png │ ├── view_2_3_10_gt.png │ ├── view_2_3_10_gt_depth_mm.png │ ├── view_2_depth.png │ ├── view_2_depth_mm.png │ ├── view_3.png │ ├── view_3_0_10.png │ ├── view_3_0_10_gt.png │ ├── view_3_0_10_gt_depth_mm.png │ ├── view_3_1_10.png │ ├── view_3_1_10_gt.png │ ├── view_3_1_10_gt_depth_mm.png │ ├── view_3_2_10.png │ ├── view_3_2_10_gt.png │ ├── view_3_2_10_gt_depth_mm.png │ ├── view_3_3_10.png │ ├── view_3_3_10_gt.png │ ├── view_3_3_10_gt_depth_mm.png │ ├── view_3_depth.png │ ├── view_3_depth_mm.png │ ├── view_4.png │ ├── view_4_0_10.png │ ├── view_4_0_10_gt.png │ ├── view_4_0_10_gt_depth_mm.png │ ├── view_4_1_10.png │ ├── view_4_1_10_gt.png │ ├── view_4_1_10_gt_depth_mm.png │ ├── view_4_2_10.png │ ├── view_4_2_10_gt.png │ ├── view_4_2_10_gt_depth_mm.png │ ├── view_4_3_10.png │ ├── view_4_3_10_gt.png │ ├── view_4_3_10_gt_depth_mm.png │ ├── view_4_depth.png │ ├── view_4_depth_mm.png │ ├── view_5.png │ ├── view_5_0_10.png │ ├── view_5_0_10_gt.png │ ├── view_5_0_10_gt_depth_mm.png │ ├── view_5_1_10.png │ ├── view_5_1_10_gt.png │ ├── view_5_1_10_gt_depth_mm.png │ ├── view_5_2_10.png │ ├── view_5_2_10_gt.png │ ├── view_5_2_10_gt_depth_mm.png │ ├── view_5_3_10.png │ ├── view_5_3_10_gt.png │ ├── view_5_3_10_gt_depth_mm.png │ ├── view_5_depth.png │ ├── view_5_depth_mm.png │ ├── view_6.png │ ├── view_6_0_10.png │ ├── view_6_0_10_gt.png │ ├── view_6_0_10_gt_depth_mm.png │ ├── view_6_1_10.png │ ├── view_6_1_10_gt.png │ ├── view_6_1_10_gt_depth_mm.png │ ├── view_6_2_10.png │ ├── view_6_2_10_gt.png │ ├── view_6_2_10_gt_depth_mm.png │ ├── view_6_3_10.png │ ├── view_6_3_10_gt.png │ ├── view_6_3_10_gt_depth_mm.png │ ├── view_6_depth.png │ ├── view_6_depth_mm.png │ ├── view_7.png │ ├── view_7_0_10.png │ ├── view_7_0_10_gt.png │ ├── view_7_0_10_gt_depth_mm.png │ ├── view_7_1_10.png │ ├── view_7_1_10_gt.png │ ├── view_7_1_10_gt_depth_mm.png │ ├── view_7_2_10.png │ ├── view_7_2_10_gt.png │ ├── view_7_2_10_gt_depth_mm.png │ ├── view_7_3_10.png │ ├── view_7_3_10_gt.png │ ├── view_7_3_10_gt_depth_mm.png │ ├── view_7_depth.png │ └── view_7_depth_mm.png └── gso │ ├── Android_Figure_Orange │ ├── r_000.png │ ├── r_001.png │ ├── r_002.png │ ├── r_003.png │ ├── r_004.png │ ├── r_005.png │ ├── r_006.png │ ├── r_007.png │ ├── r_008.png │ ├── r_009.png │ ├── r_010.png │ ├── r_011.png │ ├── r_012.png │ ├── r_013.png │ ├── r_014.png │ ├── r_015.png │ ├── r_016.png │ ├── r_017.png │ ├── r_018.png │ ├── r_019.png │ ├── r_020.png │ ├── r_021.png │ ├── r_022.png │ ├── r_023.png │ ├── r_024.png │ ├── r_025.png │ ├── r_026.png │ ├── r_027.png │ ├── r_028.png │ ├── r_029.png │ ├── r_030.png │ ├── r_031.png │ └── transforms.json │ └── DINNING_ROOM_FURNITURE_SET_1 │ ├── r_000.png │ ├── r_001.png │ ├── r_002.png │ ├── r_003.png │ ├── r_004.png │ ├── r_005.png │ ├── r_006.png │ ├── r_007.png │ ├── r_008.png │ ├── r_009.png │ ├── r_010.png │ ├── r_011.png │ ├── r_012.png │ ├── r_013.png │ ├── r_014.png │ ├── r_015.png │ ├── r_016.png │ ├── r_017.png │ ├── r_018.png │ ├── r_019.png │ ├── r_020.png │ ├── r_021.png │ ├── r_022.png │ ├── r_023.png │ ├── r_024.png │ ├── r_025.png │ ├── r_026.png │ ├── r_027.png │ ├── r_028.png │ ├── r_029.png │ ├── r_030.png │ ├── r_031.png │ └── transforms.json ├── dataset ├── gso_dataset.py └── obj_dataset.py ├── entropy_coding ├── base.py ├── latent_codecs.py └── weight_entropy.py ├── finetune.py ├── finetune_ec.py ├── finetune_ec_weight.py ├── models ├── criterion.py ├── model.py ├── render_image.py ├── render_ray.py └── sample_ray.py ├── network ├── attention.py ├── codec.py ├── encoders │ ├── __init__.py │ ├── dino_wrapper.py │ ├── dinov2 │ │ ├── __init__.py │ │ ├── hub │ │ │ ├── __init__.py │ │ │ ├── backbones.py │ │ │ ├── classifiers.py │ │ │ ├── depth │ │ │ │ ├── __init__.py │ │ │ │ ├── decode_heads.py │ │ │ │ ├── encoder_decoder.py │ │ │ │ └── ops.py │ │ │ ├── depthers.py │ │ │ └── utils.py │ │ ├── layers │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── block.py │ │ │ ├── dino_head.py │ │ │ ├── drop_path.py │ │ │ ├── layer_scale.py │ │ │ ├── mlp.py │ │ │ ├── patch_embed.py │ │ │ └── swiglu_ffn.py │ │ └── models │ │ │ ├── __init__.py │ │ │ └── vision_transformer.py │ └── dinov2_wrapper.py ├── spa_block.py ├── tri_block.py ├── tri_new.py └── tri_res.py ├── opt.py ├── run.sh └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # CodecNeRF: Toward Fast Encoding and Decoding, Compact, and High-quality Novel-view Synthesis 2 | 3 | 4 | ### AAAI 2025 5 | [[`arXiv`](https://arxiv.org/abs/2404.04913)] 6 | [[`Project Page`](https://gynjn.github.io/CodecNeRF/)] 7 | 8 | This repository contains code for "CodecNeRF: Toward Fast Encoding and Decoding, Compact, and High-quality Novel-view Synthesis". 9 | 10 | Clone the repository: 11 | ``` 12 | git clone https://github.com/Gynjn/CodecNeRF.git 13 | ``` 14 | 15 | ## Setting up Environment 16 | 17 | ### Install required packages (CUDA 11.8 example) 18 | ``` 19 | conda create -n codecnerf python=3.10 -y 20 | conda activate codecnerf 21 | pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118 22 | pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch 23 | pip install opencv-python imageio imageio-ffmpeg timm configargparse einops dahuffman vector-quantize-pytorch compressai scipy torchmetrics matplotlib loralib pandas gdown transformers jaxtyping tensorboard tensorboardX lpips 24 | pip install -U xformers --index-url https://download.pytorch.org/whl/cu118 25 | ``` 26 | 27 | # Run example 28 | Download the pretrained model weights from https://huggingface.co/anonymous-submit/submission/tree/main place them to checkpoints folder. 29 | 30 | Run finetune (ec means W/ entropy coding). 31 | ``` 32 | python finetune.py --config configs/finetune.txt 33 | python finetune_ec.py --config configs/finetune_ec.txt 34 | ``` 35 | If you want to perform entropy coding on tensor and MLP both, then 36 | ``` 37 | python finetune_ec_weight.py --config configs/finetune_ec_weight.txt 38 | ``` 39 | 40 | We provide a few examples in the configuration folder: 41 | 42 | `data_path`, choices = ['camera', 'cake']; 43 | 44 | `lrate_mlp`, learning rate of MLP LoRA; 45 | 46 | `lrate_feat`, learning rate of feature map; 47 | 48 | `trank`, tensor decomposition rank; 49 | 50 | `lrank`, MLP LoRA rank; 51 | 52 | `alpha`, MLP LoRA alpha; 53 | 54 | `N_samples`, the number of point in uniform sampling; 55 | 56 | `N_importance`, the number of point in importance sampling; 57 | 58 | More options refer to the `opt.py`. 59 | 60 | ## Citing CodecNeRF 61 | 62 | ``` 63 | @article{kang2024codecnerf, 64 | title={CodecNeRF: Toward Fast Encoding and Decoding, Compact, and High-quality Novel-view Synthesis}, 65 | author={Kang, Gyeongjin and Lee, Younggeun and Park, Eunbyung}, 66 | journal={arXiv preprint arXiv:2404.04913}, 67 | year={2024} 68 | } 69 | ``` 70 | -------------------------------------------------------------------------------- /checkpoints/put_model_here.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/checkpoints/put_model_here.txt -------------------------------------------------------------------------------- /configs/finetune.txt: -------------------------------------------------------------------------------- 1 | expname = finetune 2 | ckptdir = checkpoints 3 | data_path = ./data/camera 4 | camera_path = ./data/One2345_training_pose.json 5 | outdir = ft_out 6 | img_hw = [256, 256] 7 | N_rand = 4096 8 | chunk_size = 8192 9 | lrate_mlp = 1e-3 10 | lrate_feat = 5e-3 11 | trank = 4 12 | lrank = 4 13 | alpha = 4.0 14 | N_samples = 64 15 | N_importance = 64 16 | white_bkgd = True -------------------------------------------------------------------------------- /configs/finetune_ec.txt: -------------------------------------------------------------------------------- 1 | expname = finetune_ec 2 | ckptdir = checkpoints 3 | data_path = ./data/cake 4 | camera_path = ./data/One2345_training_pose.json 5 | outdir = ec_out 6 | img_hw = [256, 256] 7 | N_rand = 4096 8 | chunk_size = 8192 9 | lrate_mlp = 1e-3 10 | lrate_feat = 5e-3 11 | lrate_comp = 1e-5 12 | trank = 4 13 | lrank = 4 14 | alpha = 4.0 15 | N_samples = 64 16 | N_importance = 64 17 | white_bkgd = True -------------------------------------------------------------------------------- /configs/finetune_ec_weight.txt: -------------------------------------------------------------------------------- 1 | expname = finetune_ec_weight 2 | ckptdir = checkpoints 3 | data_path = ./data/camera 4 | camera_path = ./data/One2345_training_pose.json 5 | outdir = ec_weight_out 6 | img_hw = [256, 256] 7 | N_rand = 4096 8 | chunk_size = 8192 9 | lrate_mlp = 1e-3 10 | lrate_feat = 5e-3 11 | lrate_comp = 1e-5 12 | trank = 4 13 | lrank = 4 14 | alpha = 4.0 15 | N_samples = 64 16 | N_importance = 64 17 | white_bkgd = True -------------------------------------------------------------------------------- /configs/finetune_gso.txt: -------------------------------------------------------------------------------- 1 | expname = finetune_gso 2 | ckptdir = checkpoints 3 | data_path = ./data/gso 4 | data_type = gso 5 | outdir = gso_ft_out 6 | img_hw = [256, 256] 7 | N_rand = 4096 8 | chunk_size = 8192 9 | lrate_mlp = 1e-3 10 | lrate_feat = 5e-3 11 | trank = 4 12 | lrank = 4 13 | alpha = 4.0 14 | N_samples = 64 15 | N_importance = 64 16 | white_bkgd = True -------------------------------------------------------------------------------- /configs/finetune_gso_ec.txt: -------------------------------------------------------------------------------- 1 | expname = finetune_gso_ec 2 | ckptdir = checkpoints 3 | data_path = ./data/gso 4 | data_type = gso 5 | outdir = gso_ec_out 6 | img_hw = [256, 256] 7 | N_rand = 4096 8 | chunk_size = 8192 9 | lrate_mlp = 1e-3 10 | lrate_feat = 5e-3 11 | lrate_comp = 1e-5 12 | trank = 4 13 | lrank = 4 14 | alpha = 4.0 15 | N_samples = 64 16 | N_importance = 64 17 | white_bkgd = True -------------------------------------------------------------------------------- /data/cake/view_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0.png -------------------------------------------------------------------------------- /data/cake/view_0_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_0_10.png -------------------------------------------------------------------------------- /data/cake/view_0_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_0_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_0_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_1_10.png -------------------------------------------------------------------------------- /data/cake/view_0_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_0_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_0_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_2_10.png -------------------------------------------------------------------------------- /data/cake/view_0_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_0_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_0_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_3_10.png -------------------------------------------------------------------------------- /data/cake/view_0_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_0_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_0_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_depth.png -------------------------------------------------------------------------------- /data/cake/view_0_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_0_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1.png -------------------------------------------------------------------------------- /data/cake/view_1_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_0_10.png -------------------------------------------------------------------------------- /data/cake/view_1_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_1_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_1_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_1_10.png -------------------------------------------------------------------------------- /data/cake/view_1_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_1_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_1_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_2_10.png -------------------------------------------------------------------------------- /data/cake/view_1_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_1_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_1_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_3_10.png -------------------------------------------------------------------------------- /data/cake/view_1_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_1_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_1_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_depth.png -------------------------------------------------------------------------------- /data/cake/view_1_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_1_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2.png -------------------------------------------------------------------------------- /data/cake/view_2_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_0_10.png -------------------------------------------------------------------------------- /data/cake/view_2_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_2_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_2_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_1_10.png -------------------------------------------------------------------------------- /data/cake/view_2_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_2_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_2_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_2_10.png -------------------------------------------------------------------------------- /data/cake/view_2_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_2_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_2_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_3_10.png -------------------------------------------------------------------------------- /data/cake/view_2_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_2_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_2_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_depth.png -------------------------------------------------------------------------------- /data/cake/view_2_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_2_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3.png -------------------------------------------------------------------------------- /data/cake/view_3_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_0_10.png -------------------------------------------------------------------------------- /data/cake/view_3_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_3_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_3_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_1_10.png -------------------------------------------------------------------------------- /data/cake/view_3_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_3_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_3_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_2_10.png -------------------------------------------------------------------------------- /data/cake/view_3_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_3_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_3_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_3_10.png -------------------------------------------------------------------------------- /data/cake/view_3_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_3_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_3_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_depth.png -------------------------------------------------------------------------------- /data/cake/view_3_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_3_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4.png -------------------------------------------------------------------------------- /data/cake/view_4_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_0_10.png -------------------------------------------------------------------------------- /data/cake/view_4_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_4_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_4_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_1_10.png -------------------------------------------------------------------------------- /data/cake/view_4_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_4_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_4_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_2_10.png -------------------------------------------------------------------------------- /data/cake/view_4_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_4_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_4_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_3_10.png -------------------------------------------------------------------------------- /data/cake/view_4_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_4_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_4_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_depth.png -------------------------------------------------------------------------------- /data/cake/view_4_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_4_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5.png -------------------------------------------------------------------------------- /data/cake/view_5_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_0_10.png -------------------------------------------------------------------------------- /data/cake/view_5_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_5_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_5_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_1_10.png -------------------------------------------------------------------------------- /data/cake/view_5_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_5_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_5_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_2_10.png -------------------------------------------------------------------------------- /data/cake/view_5_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_5_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_5_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_3_10.png -------------------------------------------------------------------------------- /data/cake/view_5_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_5_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_5_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_depth.png -------------------------------------------------------------------------------- /data/cake/view_5_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_5_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6.png -------------------------------------------------------------------------------- /data/cake/view_6_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_0_10.png -------------------------------------------------------------------------------- /data/cake/view_6_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_6_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_6_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_1_10.png -------------------------------------------------------------------------------- /data/cake/view_6_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_6_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_6_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_2_10.png -------------------------------------------------------------------------------- /data/cake/view_6_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_6_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_6_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_3_10.png -------------------------------------------------------------------------------- /data/cake/view_6_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_6_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_6_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_depth.png -------------------------------------------------------------------------------- /data/cake/view_6_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_6_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7.png -------------------------------------------------------------------------------- /data/cake/view_7_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_0_10.png -------------------------------------------------------------------------------- /data/cake/view_7_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_0_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_7_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_7_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_1_10.png -------------------------------------------------------------------------------- /data/cake/view_7_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_1_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_7_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_7_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_2_10.png -------------------------------------------------------------------------------- /data/cake/view_7_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_2_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_7_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_7_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_3_10.png -------------------------------------------------------------------------------- /data/cake/view_7_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_3_10_gt.png -------------------------------------------------------------------------------- /data/cake/view_7_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/cake/view_7_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_depth.png -------------------------------------------------------------------------------- /data/cake/view_7_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/cake/view_7_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0.png -------------------------------------------------------------------------------- /data/camera/view_0_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_0_10.png -------------------------------------------------------------------------------- /data/camera/view_0_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_0_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_0_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_1_10.png -------------------------------------------------------------------------------- /data/camera/view_0_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_0_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_0_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_2_10.png -------------------------------------------------------------------------------- /data/camera/view_0_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_0_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_0_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_3_10.png -------------------------------------------------------------------------------- /data/camera/view_0_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_0_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_0_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_depth.png -------------------------------------------------------------------------------- /data/camera/view_0_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_0_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1.png -------------------------------------------------------------------------------- /data/camera/view_1_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_0_10.png -------------------------------------------------------------------------------- /data/camera/view_1_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_1_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_1_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_1_10.png -------------------------------------------------------------------------------- /data/camera/view_1_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_1_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_1_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_2_10.png -------------------------------------------------------------------------------- /data/camera/view_1_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_1_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_1_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_3_10.png -------------------------------------------------------------------------------- /data/camera/view_1_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_1_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_1_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_depth.png -------------------------------------------------------------------------------- /data/camera/view_1_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_1_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2.png -------------------------------------------------------------------------------- /data/camera/view_2_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_0_10.png -------------------------------------------------------------------------------- /data/camera/view_2_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_2_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_2_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_1_10.png -------------------------------------------------------------------------------- /data/camera/view_2_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_2_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_2_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_2_10.png -------------------------------------------------------------------------------- /data/camera/view_2_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_2_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_2_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_3_10.png -------------------------------------------------------------------------------- /data/camera/view_2_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_2_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_2_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_depth.png -------------------------------------------------------------------------------- /data/camera/view_2_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_2_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3.png -------------------------------------------------------------------------------- /data/camera/view_3_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_0_10.png -------------------------------------------------------------------------------- /data/camera/view_3_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_3_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_3_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_1_10.png -------------------------------------------------------------------------------- /data/camera/view_3_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_3_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_3_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_2_10.png -------------------------------------------------------------------------------- /data/camera/view_3_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_3_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_3_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_3_10.png -------------------------------------------------------------------------------- /data/camera/view_3_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_3_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_3_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_depth.png -------------------------------------------------------------------------------- /data/camera/view_3_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_3_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4.png -------------------------------------------------------------------------------- /data/camera/view_4_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_0_10.png -------------------------------------------------------------------------------- /data/camera/view_4_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_4_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_4_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_1_10.png -------------------------------------------------------------------------------- /data/camera/view_4_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_4_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_4_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_2_10.png -------------------------------------------------------------------------------- /data/camera/view_4_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_4_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_4_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_3_10.png -------------------------------------------------------------------------------- /data/camera/view_4_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_4_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_4_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_depth.png -------------------------------------------------------------------------------- /data/camera/view_4_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_4_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5.png -------------------------------------------------------------------------------- /data/camera/view_5_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_0_10.png -------------------------------------------------------------------------------- /data/camera/view_5_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_5_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_5_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_1_10.png -------------------------------------------------------------------------------- /data/camera/view_5_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_5_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_5_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_2_10.png -------------------------------------------------------------------------------- /data/camera/view_5_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_5_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_5_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_3_10.png -------------------------------------------------------------------------------- /data/camera/view_5_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_5_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_5_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_depth.png -------------------------------------------------------------------------------- /data/camera/view_5_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_5_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6.png -------------------------------------------------------------------------------- /data/camera/view_6_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_0_10.png -------------------------------------------------------------------------------- /data/camera/view_6_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_6_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_6_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_1_10.png -------------------------------------------------------------------------------- /data/camera/view_6_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_6_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_6_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_2_10.png -------------------------------------------------------------------------------- /data/camera/view_6_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_6_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_6_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_3_10.png -------------------------------------------------------------------------------- /data/camera/view_6_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_6_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_6_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_depth.png -------------------------------------------------------------------------------- /data/camera/view_6_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_6_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7.png -------------------------------------------------------------------------------- /data/camera/view_7_0_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_0_10.png -------------------------------------------------------------------------------- /data/camera/view_7_0_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_0_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_7_0_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_0_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_7_1_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_1_10.png -------------------------------------------------------------------------------- /data/camera/view_7_1_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_1_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_7_1_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_1_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_7_2_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_2_10.png -------------------------------------------------------------------------------- /data/camera/view_7_2_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_2_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_7_2_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_2_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_7_3_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_3_10.png -------------------------------------------------------------------------------- /data/camera/view_7_3_10_gt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_3_10_gt.png -------------------------------------------------------------------------------- /data/camera/view_7_3_10_gt_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_3_10_gt_depth_mm.png -------------------------------------------------------------------------------- /data/camera/view_7_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_depth.png -------------------------------------------------------------------------------- /data/camera/view_7_depth_mm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/camera/view_7_depth_mm.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_000.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_001.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_002.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_003.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_004.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_005.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_006.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_007.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_008.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_009.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_010.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_011.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_011.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_012.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_013.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_013.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_014.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_015.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_015.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_016.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_016.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_017.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_017.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_018.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_018.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_019.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_019.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_020.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_021.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_021.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_022.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_022.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_023.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_023.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_024.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_024.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_025.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_025.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_026.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_026.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_027.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_027.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_028.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_028.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_029.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_029.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_030.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_030.png -------------------------------------------------------------------------------- /data/gso/Android_Figure_Orange/r_031.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/Android_Figure_Orange/r_031.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_000.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_001.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_002.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_003.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_004.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_005.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_006.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_007.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_008.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_009.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_010.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_011.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_011.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_012.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_013.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_013.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_014.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_015.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_015.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_016.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_016.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_017.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_017.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_018.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_018.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_019.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_019.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_020.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_021.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_021.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_022.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_022.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_023.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_023.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_024.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_024.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_025.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_025.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_026.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_026.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_027.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_027.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_028.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_028.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_029.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_029.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_030.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_030.png -------------------------------------------------------------------------------- /data/gso/DINNING_ROOM_FURNITURE_SET_1/r_031.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Gynjn/CodecNeRF/616fa21ef04ce8f3f5a09fe63f5d7d80d9990217/data/gso/DINNING_ROOM_FURNITURE_SET_1/r_031.png -------------------------------------------------------------------------------- /dataset/gso_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | import torchvision.transforms as T 6 | import imageio 7 | import cv2 8 | import json 9 | import tqdm 10 | from torch.utils.data import Dataset 11 | 12 | def get_K_from_params(params): 13 | K = np.zeros((3, 3)).astype(np.float32) 14 | K[0][0], K[0][2], K[1][2] = params[:3] 15 | K[1][1] = K[0][0] 16 | K[2][2] = 1. 17 | return K 18 | 19 | def intrinsic_to_fov(K, w=None, h=None): 20 | # Extract the focal lengths from the intrinsic matrix 21 | fx = K[0, 0] 22 | fy = K[1, 1] 23 | 24 | w = K[0, 2]*2 if w is None else w 25 | h = K[1, 2]*2 if h is None else h 26 | 27 | # Calculating field of view 28 | fov_x = 2 * np.arctan2(w, 2 * fx) 29 | fov_y = 2 * np.arctan2(h, 2 * fy) 30 | 31 | return fov_x, fov_y 32 | 33 | class GSODataset(Dataset): 34 | def __init__(self, data_path, camera_path): 35 | super().__init__() 36 | self.data_path = data_path 37 | scenes_name = np.array([f for f in sorted(os.listdir(self.data_path)) if os.path.isdir(f'{self.data_path}/{f}')])[300:] 38 | self.scenes_name = scenes_name 39 | self.img_size = np.array([256, 256]) 40 | 41 | self.camera_path = camera_path 42 | self.blender2opencv = np.array( 43 | [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]] 44 | ) 45 | self.z_near = 0.5 46 | self.z_far = 1.8 47 | # self.z_far = 2.5 48 | self.build_metas() 49 | 50 | def build_metas(self): 51 | 52 | self.scene_infos = {} 53 | 54 | for scene in tqdm.tqdm(self.scenes_name): 55 | 56 | json_info = json.load(open(os.path.join(self.data_path, scene,f'transforms.json'))) 57 | b2c = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]) 58 | scene_info = {'ixts': [], 'c2ws': [], 'w2cs':[], 'img_paths': [], 'fovx': [], 'fovy':[]} 59 | positions = [] 60 | for idx, frame in enumerate(json_info['frames']): 61 | c2w = np.array(frame['transform_matrix']) 62 | c2w = c2w @ b2c 63 | ixt = np.array(frame['intrinsic_matrix']) 64 | fov_x, fov_y = intrinsic_to_fov(ixt) 65 | scene_info['ixts'].append(ixt.astype(np.float32)) 66 | scene_info['c2ws'].append(c2w.astype(np.float32)) 67 | scene_info['w2cs'].append(np.linalg.inv(c2w.astype(np.float32))) 68 | img_path = os.path.join(self.data_path, scene, f'r_{idx:03d}.png') 69 | scene_info['img_paths'].append(img_path) 70 | scene_info['fovx'].append(fov_x) 71 | scene_info['fovy'].append(fov_y) 72 | positions.append(c2w[:3,3]) 73 | 74 | self.scene_infos[scene] = scene_info 75 | 76 | def __len__(self): 77 | return len(self.scene_infos) 78 | 79 | def define_transforms(self): 80 | self.img_transforms = T.Compose( 81 | [T.ToTensor(), T.Normalize((0.0, 0.0, 0.0), (1.0, 1.0, 1.0))] 82 | ) 83 | self.mask_transforms = T.Compose( 84 | [T.ToTensor(), T.Normalize((0.0,), (1.0,))] 85 | ) 86 | 87 | def read_view(self, scene, idx): 88 | img, mask = self.read_image(scene, idx) 89 | ixt, ext = self.read_cam(scene, idx) 90 | return img, mask, ext, ixt 91 | 92 | def read_views(self, scene, src_views, bg_color): 93 | src_ids = src_views 94 | ixts, exts, w2cs, imgs, msks= [], [], [], [], [] 95 | for idx in src_ids: 96 | img, mask = self.read_image(scene, idx, bg_color) 97 | imgs.append(img) 98 | ixt, ext, w2c = self.read_cam(scene, idx) 99 | ixts.append(ixt) 100 | exts.append(ext) 101 | w2cs.append(w2c) 102 | msks.append(mask) 103 | return np.stack(imgs), np.stack(msks), np.stack(exts), np.stack(w2cs), np.stack(ixts) 104 | 105 | def read_cam(self, scene, view_idx): 106 | img_downscale = np.array([256, 256])/512 107 | ext = scene['c2ws'][view_idx] 108 | w2c = scene['w2cs'][view_idx] 109 | ixt = scene['ixts'][view_idx].copy() 110 | ixt[:2] = ixt[:2] * img_downscale.reshape(2,1) 111 | return ixt, ext, w2c 112 | 113 | def read_image(self, scene, view_idx, bg_color): 114 | img_path = scene['img_paths'][view_idx] 115 | img = imageio.imread(img_path) 116 | img = cv2.resize(img, (256, 256)) 117 | mask = (img[...,-1] > 0).astype('uint8') 118 | img = img.astype(np.float32) / 255. 119 | img = (img[..., :3] * img[..., -1:] + bg_color*(1 - img[..., -1:])).astype(np.float32) 120 | return img, mask 121 | 122 | 123 | def __getitem__(self, index): 124 | 125 | scene_name = self.scenes_name[index] 126 | scene_info = self.scene_infos[scene_name] 127 | bg_color = np.ones(3).astype(np.float32) 128 | tar_views = list(range(32)) 129 | all_indices = np.arange(0, 32, 1) 130 | view_indices = [0, 2, 3, 4, 6, 7, 8, 10, 11, 12, 14, 15, 16, 18, 19, 20, 22, 23, 24, 26, 27, 28, 30, 31] 131 | gen_list = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30] 132 | all_indices = np.delete(all_indices, view_indices) 133 | 134 | tar_img, tar_msks, tar_c2ws, tar_w2cs, tar_ixts = self.read_views(scene_info, tar_views, bg_color) 135 | tar_c2ws[:, :3, 3] = tar_c2ws[:, :3, 3]*(2/3) 136 | tar_w2cs[:, :3, 3] = tar_w2cs[:, :3, 3]*(2/3) 137 | 138 | near_far = np.array([0.5, 1.8]).astype(np.float32) 139 | return { 140 | "scene_name": scene_name, 141 | "src_rgbs": torch.from_numpy(tar_img[gen_list]).cuda(), 142 | "src_w2c_mats": torch.from_numpy(tar_w2cs[gen_list]).cuda(), 143 | "src_intrinsics": torch.from_numpy(tar_ixts[gen_list]).cuda(), 144 | "train_rgbs": torch.from_numpy(tar_img[view_indices]).cuda(), 145 | "train_c2w_mats": torch.from_numpy(tar_c2ws[view_indices]).cuda(), 146 | "train_w2c_mats": torch.from_numpy(tar_w2cs[view_indices]).cuda(), 147 | "train_intrinsics": torch.from_numpy(tar_ixts[view_indices]).cuda(), 148 | "all_rgbs": torch.from_numpy(tar_img[all_indices]), 149 | "all_c2w_mats": torch.from_numpy(tar_c2ws[all_indices]).cuda(), 150 | "all_intrinsics": torch.from_numpy(tar_ixts[all_indices]).cuda(), 151 | "depth_range": torch.from_numpy(near_far).cuda(), 152 | } -------------------------------------------------------------------------------- /dataset/obj_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch.utils.data import Dataset 4 | import json 5 | import numpy as np 6 | import torchvision.transforms as T 7 | import imageio 8 | 9 | class OBJDataset(Dataset): 10 | def __init__(self, data_path, camera_path): 11 | super().__init__() 12 | self.data_path = data_path 13 | self.camera_path = camera_path 14 | self.blender2opencv = np.array( 15 | [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]] 16 | ) 17 | self.view_indices = [ 0, 2, 4, 6, 7, 8, 10, 12, 14, 16, 17, 18, 20, 22, 24, 26, 27, 28, 30, 32, 34, 36, 37, 38] 18 | self.gen_indices = [ 0, 2, 4, 6, 10, 12, 14, 16, 20, 22, 24, 26, 30, 32, 34, 36] 19 | self.all_indices = np.arange(0, 40, 1) 20 | self.all_indices = np.delete(self.all_indices, self.view_indices) 21 | 22 | pose_json_path = self.camera_path 23 | with open(pose_json_path, 'r') as f: 24 | meta = json.load(f) 25 | 26 | self.img_ids = list(meta["c2ws"].keys()) 27 | self.img_ids = [f"{image_name}_gt" if image_name[-1].isdigit() and image_name[-3] == '_' and image_name[-2].isdigit() else image_name for image_name in self.img_ids] 28 | self.input_poses = np.array(list(meta["c2ws"].values())) 29 | intrinsic = np.eye(4) 30 | intrinsic[:3, :3] = np.array(meta["intrinsics"]) 31 | self.intrinsics = intrinsic 32 | self.rgb_paths = [os.path.join(data_path, f"{image_name}.png") for image_name in self.img_ids] 33 | self.rgb_paths = np.array(self.rgb_paths) 34 | self.c2ws = [] 35 | self.w2cs = [] 36 | 37 | for idx, img_id in enumerate(self.img_ids): 38 | pose = self.input_poses[idx] 39 | c2w = pose @ self.blender2opencv 40 | self.c2ws.append(c2w) 41 | self.w2cs.append(np.linalg.inv(c2w)) 42 | 43 | self.c2ws = np.stack(self.c2ws, axis=0) 44 | self.w2cs = np.stack(self.w2cs, axis=0) 45 | 46 | self.define_transforms() 47 | 48 | self.z_near = 0.5 49 | self.z_far = 1.8 50 | 51 | def __len__(self): 52 | return 1 53 | 54 | def define_transforms(self): 55 | self.img_transforms = T.Compose( 56 | [T.ToTensor(), T.Normalize((0.0, 0.0, 0.0), (1.0, 1.0, 1.0))] 57 | ) 58 | self.mask_transforms = T.Compose( 59 | [T.ToTensor(), T.Normalize((0.0,), (1.0,))] 60 | ) 61 | 62 | def __getitem__(self, index): 63 | src_rgb_paths = self.rgb_paths[self.gen_indices] 64 | src_w2c_mats = self.w2cs[self.gen_indices] 65 | src_intrinsics = np.array([self.intrinsics] * len(src_rgb_paths)) 66 | 67 | src_rgbs = [] 68 | for rgb_path in src_rgb_paths: 69 | img = imageio.imread(rgb_path) 70 | mask = self.mask_transforms(img[..., 3]) 71 | rgb = self.img_transforms(img[..., :3]) 72 | rgb = rgb * mask + (1.0 - mask) 73 | 74 | h, w = rgb.shape[-2:] 75 | 76 | src_rgbs.append(rgb) 77 | 78 | train_rgb_paths = self.rgb_paths[self.view_indices] 79 | train_c2w_mats = self.c2ws[self.view_indices] 80 | train_w2c_mats = self.w2cs[self.view_indices] 81 | train_intrinsics = np.array([self.intrinsics] * len(train_rgb_paths)) 82 | 83 | train_bbox = [] 84 | train_rgbs = [] 85 | for rgb_path in train_rgb_paths: 86 | img = imageio.imread(rgb_path) 87 | mask = self.mask_transforms(img[..., 3]) 88 | rgb = self.img_transforms(img[..., :3]) 89 | rgb = rgb * mask + (1.0 - mask) 90 | 91 | yy = torch.any(mask, axis=2) 92 | xx = torch.any(mask, axis=1) 93 | ynz = torch.nonzero(yy)[:, 1] 94 | xnz = torch.nonzero(xx)[:, 1] 95 | ymin, ymax = ynz[[0, -1]] 96 | xmin, xmax = xnz[[0, -1]] 97 | bbox = torch.FloatTensor([xmin, ymin, xmax, ymax]) 98 | train_bbox.append(bbox) 99 | 100 | train_rgbs.append(rgb) 101 | 102 | all_rgb_paths = self.rgb_paths[self.all_indices] 103 | all_c2w_mats = self.c2ws[self.all_indices] 104 | all_intrinsics = np.array([self.intrinsics] * len(all_rgb_paths)) 105 | 106 | all_rgbs = [] 107 | for rgb_path in all_rgb_paths: 108 | img = imageio.imread(rgb_path) 109 | mask = self.mask_transforms(img[..., 3]) 110 | rgb = self.img_transforms(img[..., :3]) 111 | rgb = rgb * mask + (1.0 - mask) 112 | 113 | all_rgbs.append(rgb) 114 | 115 | depth_range = np.array([self.z_near, self.z_far]) 116 | 117 | return { 118 | "src_rgbs": torch.stack(src_rgbs).permute([0, 2, 3, 1]).float().cuda(), 119 | "src_w2c_mats": torch.FloatTensor(src_w2c_mats).cuda(), 120 | "src_intrinsics": torch.FloatTensor(src_intrinsics).cuda(), 121 | "train_rgbs": torch.stack(train_rgbs).permute([0, 2, 3, 1]).float().cuda(), 122 | "train_c2w_mats": torch.FloatTensor(train_c2w_mats).cuda(), 123 | "train_w2c_mats": torch.FloatTensor(train_w2c_mats).cuda(), 124 | "train_intrinsics": torch.FloatTensor(train_intrinsics).cuda(), 125 | "all_rgbs": torch.stack(all_rgbs).permute([0, 2, 3, 1]).float(), 126 | "all_c2w_mats": torch.FloatTensor(all_c2w_mats).cuda(), 127 | "all_intrinsics": torch.FloatTensor(all_intrinsics).cuda(), 128 | "depth_range": torch.FloatTensor(depth_range).cuda(), 129 | "train_bbox": torch.stack(train_bbox), 130 | } -------------------------------------------------------------------------------- /entropy_coding/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021-2022, InterDigital Communications, Inc 2 | # All rights reserved. 3 | 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted (subject to the limitations in the disclaimer 6 | # below) provided that the following conditions are met: 7 | 8 | # * Redistributions of source code must retain the above copyright notice, 9 | # this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # * Neither the name of InterDigital Communications, Inc nor the names of its 14 | # contributors may be used to endorse or promote products derived from this 15 | # software without specific prior written permission. 16 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 17 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 18 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 19 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 20 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 21 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 27 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | import math 30 | 31 | from typing import cast 32 | 33 | import torch 34 | import torch.nn as nn 35 | 36 | from torch import Tensor 37 | 38 | from compressai.entropy_models import EntropyBottleneck, GaussianConditional 39 | from .latent_codecs import LatentCodec 40 | from compressai.models.utils import update_registered_buffers 41 | 42 | __all__ = [ 43 | "CompressionModel", 44 | "SimpleVAECompressionModel", 45 | "get_scale_table", 46 | "SCALES_MIN", 47 | "SCALES_MAX", 48 | "SCALES_LEVELS", 49 | ] 50 | 51 | 52 | # From Balle's tensorflow compression examples 53 | SCALES_MIN = 0.11 54 | SCALES_MAX = 256 55 | SCALES_LEVELS = 64 56 | 57 | 58 | def get_scale_table(min=SCALES_MIN, max=SCALES_MAX, levels=SCALES_LEVELS): 59 | """Returns table of logarithmically scales.""" 60 | return torch.exp(torch.linspace(math.log(min), math.log(max), levels)) 61 | 62 | 63 | class CompressionModel(nn.Module): 64 | """Base class for constructing an auto-encoder with any number of 65 | EntropyBottleneck or GaussianConditional modules. 66 | """ 67 | 68 | def load_state_dict(self, state_dict, strict=True): 69 | for name, module in self.named_modules(): 70 | if not any(x.startswith(name) for x in state_dict.keys()): 71 | continue 72 | 73 | if isinstance(module, EntropyBottleneck): 74 | update_registered_buffers( 75 | module, 76 | name, 77 | ["_quantized_cdf", "_offset", "_cdf_length"], 78 | state_dict, 79 | ) 80 | 81 | if isinstance(module, GaussianConditional): 82 | update_registered_buffers( 83 | module, 84 | name, 85 | ["_quantized_cdf", "_offset", "_cdf_length", "scale_table"], 86 | state_dict, 87 | ) 88 | 89 | return nn.Module.load_state_dict(self, state_dict, strict=strict) 90 | 91 | def update(self, scale_table=None, force=False): 92 | """Updates EntropyBottleneck and GaussianConditional CDFs. 93 | 94 | Needs to be called once after training to be able to later perform the 95 | evaluation with an actual entropy coder. 96 | 97 | Args: 98 | scale_table (torch.Tensor): table of scales (i.e. stdev) 99 | for initializing the Gaussian distributions 100 | (default: 64 logarithmically spaced scales from 0.11 to 256) 101 | force (bool): overwrite previous values (default: False) 102 | 103 | Returns: 104 | updated (bool): True if at least one of the modules was updated. 105 | """ 106 | if scale_table is None: 107 | scale_table = get_scale_table() 108 | updated = False 109 | for _, module in self.named_modules(): 110 | if isinstance(module, EntropyBottleneck): 111 | updated |= module.update(force=force) 112 | if isinstance(module, GaussianConditional): 113 | updated |= module.update_scale_table(scale_table, force=force) 114 | return updated 115 | 116 | def aux_loss(self) -> Tensor: 117 | r"""Returns the total auxiliary loss over all ``EntropyBottleneck``\s. 118 | 119 | In contrast to the primary "net" loss used by the "net" 120 | optimizer, the "aux" loss is only used by the "aux" optimizer to 121 | update *only* the ``EntropyBottleneck.quantiles`` parameters. In 122 | fact, the "aux" loss does not depend on image data at all. 123 | 124 | The purpose of the "aux" loss is to determine the range within 125 | which most of the mass of a given distribution is contained, as 126 | well as its median (i.e. 50% probability). That is, for a given 127 | distribution, the "aux" loss converges towards satisfying the 128 | following conditions for some chosen ``tail_mass`` probability: 129 | 130 | * ``cdf(quantiles[0]) = tail_mass / 2`` 131 | * ``cdf(quantiles[1]) = 0.5`` 132 | * ``cdf(quantiles[2]) = 1 - tail_mass / 2`` 133 | 134 | This ensures that the concrete ``_quantized_cdf``\s operate 135 | primarily within a finitely supported region. Any symbols 136 | outside this range must be coded using some alternative method 137 | that does *not* involve the ``_quantized_cdf``\s. Luckily, one 138 | may choose a ``tail_mass`` probability that is sufficiently 139 | small so that this rarely occurs. It is important that we work 140 | with ``_quantized_cdf``\s that have a small finite support; 141 | otherwise, entropy coding runtime performance would suffer. 142 | Thus, ``tail_mass`` should not be too small, either! 143 | """ 144 | loss = sum(m.loss() for m in self.modules() if isinstance(m, EntropyBottleneck)) 145 | return cast(Tensor, loss) 146 | 147 | 148 | class SimpleVAECompressionModel(CompressionModel): 149 | """Simple VAE model with arbitrary latent codec. 150 | 151 | .. code-block:: none 152 | 153 | ┌───┐ y ┌────┐ y_hat ┌───┐ 154 | x ──►──┤g_a├──►──┤ lc ├───►───┤g_s├──►── x_hat 155 | └───┘ └────┘ └───┘ 156 | """ 157 | def __init__(self, args): 158 | super().__init__() 159 | self.entropy_bottleneck = EntropyBottleneck(args.trank) 160 | g_a: nn.Module 161 | g_s: nn.Module 162 | latent_codec: LatentCodec 163 | 164 | def forward(self, x): 165 | x_hat, x_likelihoods = self.entropy_bottleneck(x) 166 | return x_hat, x_likelihoods 167 | 168 | def compress(self, x): 169 | x_hat, x_likelihoods = self.entropy_bottleneck.compress(x) 170 | return x_hat, x_likelihoods 171 | 172 | def decompress(self, strings, shape): 173 | x_hat, x_likelihoods = self.entropy_bottleneck.decompress(strings, shape) 174 | return { 175 | "x_hat": x_hat, 176 | } 177 | -------------------------------------------------------------------------------- /entropy_coding/latent_codecs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021-2022, InterDigital Communications, Inc 2 | # All rights reserved. 3 | 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted (subject to the limitations in the disclaimer 6 | # below) provided that the following conditions are met: 7 | 8 | # * Redistributions of source code must retain the above copyright notice, 9 | # this list of conditions and the following disclaimer. 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # * Neither the name of InterDigital Communications, Inc nor the names of its 14 | # contributors may be used to endorse or promote products derived from this 15 | # software without specific prior written permission. 16 | 17 | # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 18 | # THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 19 | # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT 20 | # NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 21 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 22 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 25 | # OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 26 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 27 | # OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 28 | # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | from typing import Any, Dict, List 31 | 32 | import torch.nn as nn 33 | 34 | from torch import Tensor 35 | 36 | __all__ = [ 37 | "LatentCodec", 38 | ] 39 | 40 | 41 | class _SetDefaultMixin: 42 | """Convenience functions for initializing classes with defaults.""" 43 | 44 | def _setdefault(self, k, v, f): 45 | """Initialize attribute ``k`` with value ``v`` or ``f()``.""" 46 | v = v or f() 47 | setattr(self, k, v) 48 | 49 | # TODO instead of save_direct, override load_state_dict() and state_dict() 50 | def _set_group_defaults(self, group_key, group_dict, defaults, save_direct=False): 51 | """Initialize attribute ``group_key`` with items from 52 | ``group_dict``, using defaults for missing keys. 53 | Ensures ``nn.Module`` attributes are properly registered. 54 | 55 | Args: 56 | - group_key: 57 | Name of attribute. 58 | - group_dict: 59 | Dict of items to initialize ``group_key`` with. 60 | - defaults: 61 | Dict of defaults for items not in ``group_dict``. 62 | - save_direct: 63 | If ``True``, save items directly as attributes of ``self``. 64 | If ``False``, save items in a ``nn.ModuleDict``. 65 | """ 66 | group_dict = group_dict if group_dict is not None else {} 67 | for k, f in defaults.items(): 68 | if k in group_dict: 69 | continue 70 | group_dict[k] = f() 71 | if save_direct: 72 | for k, v in group_dict.items(): 73 | setattr(self, k, v) 74 | else: 75 | group_dict = nn.ModuleDict(group_dict) 76 | setattr(self, group_key, group_dict) 77 | 78 | 79 | class LatentCodec(nn.Module, _SetDefaultMixin): 80 | def forward(self, y: Tensor, *args, **kwargs) -> Dict[str, Any]: 81 | raise NotImplementedError 82 | 83 | def compress(self, y: Tensor, *args, **kwargs) -> Dict[str, Any]: 84 | raise NotImplementedError 85 | 86 | def decompress( 87 | self, strings: List[List[bytes]], shape: Any, *args, **kwargs 88 | ) -> Dict[str, Any]: 89 | raise NotImplementedError -------------------------------------------------------------------------------- /entropy_coding/weight_entropy.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from compressai.entropy_models import EntropyModel 4 | import numpy as np 5 | import torch 6 | 7 | 8 | class WeightEntropyModule(EntropyModel): 9 | """entropy module for network parameters 10 | width * [- (self.n_bins // 2 - 1), ..., -1, 0, 1, 2, ..., self.n_bins // 2 - 1] 11 | e.g.) n_bins = 56, pmf_lengths = 55 12 | 13 | cdf: 1 / (1 + alpha) * slab + alpha / (1 + alpha) * spike 14 | spike: N (0, width / 6) 15 | slab: N(0, sigma) 16 | 17 | quantization interval: width 18 | """ 19 | 20 | def __init__( 21 | self, cdf: Callable, width: float = 5e-3, data_type: str = "float32", **kwargs 22 | ): 23 | super().__init__(**kwargs) 24 | self.cdf = cdf 25 | self.width: float = width 26 | self._tail_mass = 1e-9 27 | # used for compression 28 | self.data_type = data_type 29 | 30 | self.register_buffer("_n_bins", torch.IntTensor()) 31 | self.update(force=True) 32 | 33 | def update(self, force: bool = False) -> bool: 34 | if self._n_bins.numel() > 0 and not force: 35 | return False 36 | 37 | delta = self.width / 2 38 | # accept self.width * 10000 * interval difference at maximum 39 | intervals: torch.Tensor = torch.arange(1, 10001) 40 | upper = self._likelihood_cumulative( 41 | intervals * self.width + delta, stop_gradient=True 42 | ) 43 | lower = self._likelihood_cumulative( 44 | -intervals * self.width - delta, stop_gradient=True 45 | ) 46 | # (upper - lower) - (1 - self._tail_mass) 47 | diff: torch.Tensor = self._tail_mass - lower - (1 - upper) 48 | if not (diff >= 0).any(): 49 | self._n_bins = intervals[-1] 50 | else: 51 | n_bins = intervals[diff.argmax()] 52 | # even value 53 | # self._n_bins = ((n_bins - 1) // 2 + 1) * 2 54 | self._n_bins = (torch.div(n_bins - 1, 2, rounding_mode="trunc") + 1) * 2 55 | self._n_bins = self._n_bins.reshape((1,)) 56 | 57 | # bound = (self._n_bins - 1) // 2 58 | bound = torch.div(self._n_bins - 1, 2, rounding_mode="trunc") 59 | bound = torch.clamp(bound.int(), min=0) 60 | 61 | self._offset = -bound 62 | 63 | pmf_start = -bound 64 | pmf_length = 2 * bound + 1 65 | 66 | max_length = pmf_length.max().item() 67 | device = pmf_start.device 68 | samples = torch.arange(max_length, device=device) 69 | 70 | samples = samples[None, :] + pmf_start[:, None, None] 71 | 72 | half = self.width / 2 73 | 74 | lower = self._likelihood_cumulative( 75 | samples * self.width - half, stop_gradient=True 76 | ) 77 | upper = self._likelihood_cumulative( 78 | samples * self.width + half, stop_gradient=True 79 | ) 80 | pmf = upper - lower 81 | 82 | pmf = pmf[:, 0, :] 83 | tail_mass = lower[:, 0, :1] + (1 - upper[:, 0, -1:]) 84 | 85 | quantized_cdf = self._pmf_to_cdf(pmf, tail_mass, pmf_length, max_length) 86 | self._quantized_cdf = quantized_cdf 87 | self._cdf_length = pmf_length + 2 88 | return True 89 | 90 | def quantize(self, w: torch.Tensor, mode: str, means=None) -> torch.Tensor: 91 | if mode == "noise": 92 | assert self.training 93 | # add uniform noise: [-self.width / 2, self.width / 2] 94 | noise = (torch.rand_like(w) - 0.5) * self.width 95 | return w + noise 96 | 97 | symbols: torch.Tensor = torch.round(w / self.width) 98 | if mode == "symbols": 99 | # bound: torch.Tensor = (self._n_bins - 1) // 2 100 | bound: torch.Tensor = torch.div(self._n_bins - 1, 2, rounding_mode="trunc") 101 | symbols = torch.min(torch.max(symbols, -bound), bound) 102 | return symbols.int() 103 | elif mode == "dequantize": 104 | w_bound: torch.Tensor = (self._n_bins - 1) * self.width / 2 105 | # clamp with (-w_bound, w_bound) 106 | w_hat: torch.Tensor = torch.min( 107 | torch.max(symbols * self.width, -w_bound), w_bound 108 | ) 109 | return (w_hat - w).detach() + w 110 | else: 111 | raise NotImplementedError 112 | 113 | def dequantize( 114 | self, inputs: torch.Tensor, means=None, dtype: torch.dtype = torch.float 115 | ) -> torch.Tensor: 116 | outputs = (inputs * self.width).type(dtype) 117 | return outputs 118 | 119 | # modified from _logits_cumulative 120 | def _likelihood_cumulative( 121 | self, inputs: torch.Tensor, stop_gradient: bool 122 | ) -> torch.Tensor: 123 | if stop_gradient: 124 | with torch.no_grad(): 125 | return self.cdf(inputs) 126 | else: 127 | return self.cdf(inputs) 128 | 129 | def _likelihood(self, inputs: torch.Tensor) -> torch.Tensor: 130 | delta = self.width / 2 131 | v0 = inputs - delta 132 | v1 = inputs + delta 133 | lower = self._likelihood_cumulative(v0, stop_gradient=False) 134 | upper = self._likelihood_cumulative(v1, stop_gradient=False) 135 | likelihood = upper - lower 136 | return likelihood 137 | 138 | def forward(self, x: torch.Tensor, training=None) -> tuple: 139 | if self.width == 0: 140 | outputs = x 141 | likelihood = torch.ones_like(x) * (2 ** -32) 142 | return outputs, likelihood 143 | 144 | if training is None: 145 | training = self.training 146 | 147 | if not torch.jit.is_scripting(): 148 | # x from B x C x ... to C x B x ... 149 | perm = np.arange(len(x.shape)) 150 | perm[0], perm[1] = perm[1], perm[0] 151 | # Compute inverse permutation 152 | inv_perm = np.arange(len(x.shape))[np.argsort(perm)] 153 | else: 154 | # TorchScript in 2D for static inference 155 | # Convert to (channels, ... , batch) format 156 | perm = (1, 2, 3, 0) 157 | inv_perm = (3, 0, 1, 2) 158 | 159 | x = x.permute(*perm).contiguous() 160 | shape = x.size() 161 | values = x.reshape(x.size(0), 1, -1) 162 | 163 | # Add noise or quantize 164 | outputs = self.quantize(values, "dequantize") 165 | outputs_ent = self.quantize(values, "noise") if self.training else outputs 166 | 167 | likelihood = self._likelihood(outputs_ent) 168 | if self.use_likelihood_bound: 169 | likelihood = self.likelihood_lower_bound(likelihood) 170 | 171 | # Convert back to input tensor shape 172 | outputs = outputs.reshape(shape) 173 | outputs = outputs.permute(*inv_perm).contiguous() 174 | 175 | likelihood = likelihood.reshape(shape) 176 | likelihood = likelihood.permute(*inv_perm).contiguous() 177 | 178 | return outputs, likelihood 179 | 180 | @staticmethod 181 | def _build_indexes(size): 182 | dims = len(size) 183 | N = size[0] 184 | C = size[1] 185 | 186 | view_dims = np.ones((dims,), dtype=np.int64) 187 | view_dims[1] = -1 188 | indexes = torch.arange(C).view(*view_dims) 189 | indexes = indexes.int() 190 | 191 | return indexes.repeat(N, 1, *size[2:]) 192 | 193 | def compress(self, x): 194 | if self.width == 0: 195 | strings = list() 196 | for i in range(len(x)): 197 | string = encode_array(x[i].flatten().cpu().numpy(), self.data_type) 198 | strings.append(string) 199 | return strings 200 | 201 | indexes = self._build_indexes(x.size()) 202 | return super().compress(x, indexes) 203 | 204 | def decompress(self, strings, size): 205 | output_size = (len(strings), self._quantized_cdf.size(0), *size) 206 | if self.width == 0: 207 | xs = list() 208 | for string in strings: 209 | x = decode_array(string, self.data_type) 210 | x = torch.from_numpy(x.copy()).to(self._quantized_cdf.device) 211 | xs.append(x) 212 | xs = torch.stack(xs).float().reshape(output_size) 213 | return xs 214 | 215 | indexes = self._build_indexes(output_size).to(self._quantized_cdf.device) 216 | return super().decompress(strings, indexes, torch.float32) 217 | 218 | 219 | def encode_array(x: np.ndarray, data_type: str) -> bytes: 220 | if data_type == "float32": 221 | return x.astype(np.float32).tobytes() 222 | if data_type == "float16": 223 | return x.astype(np.float16).tobytes() 224 | # Zou+, ISM 21 225 | elif data_type == "uint8": 226 | bias = x.min() 227 | x_ = x - bias 228 | scale: float = (255 / x_.max()).astype(np.float32) 229 | arr_qua = np.round(x_ * scale).astype(np.uint8) 230 | return arr_qua.tobytes() + bias.tobytes() + scale.tobytes() 231 | else: 232 | raise NotImplementedError 233 | 234 | 235 | def decode_array(string: bytes, data_type: str) -> np.ndarray: 236 | if data_type == "float32": 237 | return np.frombuffer(string, dtype=np.float32) 238 | if data_type == "float16": 239 | return np.frombuffer(string, dtype=np.float16).astype(np.float32) 240 | # Zou+, ISM 21 241 | elif data_type == "uint8": 242 | arr = np.frombuffer(string[:-8], dtype=np.uint8) 243 | bias = np.frombuffer(string[-8:-4], dtype=np.float32) 244 | scale = np.frombuffer(string[-4:], dtype=np.float32) 245 | return arr / scale + bias 246 | else: 247 | raise NotImplementedError 248 | 249 | 250 | 251 | 252 | from torch import distributions as D 253 | 254 | 255 | class SpikeAndSlabCDF: 256 | def __init__( 257 | self, width: float = 5e-3, sigma: float = 5e-2, alpha: float = 1000 # original 1000 258 | ) -> None: 259 | self.alpha = alpha 260 | 261 | mean = torch.tensor(0.0) 262 | self.slab = D.Normal(mean, torch.tensor(sigma)) 263 | if width != 0: 264 | self.spike = D.Normal(mean, torch.tensor(width / 6)) 265 | else: 266 | self.spike = None 267 | 268 | def __call__(self, x: torch.Tensor) -> torch.Tensor: 269 | cdf_slab = self.slab.cdf(x) 270 | if self.spike is None: 271 | return cdf_slab 272 | else: 273 | cdf_spike = self.spike.cdf(x) 274 | return (cdf_slab + self.alpha * cdf_spike) / (1 + self.alpha) -------------------------------------------------------------------------------- /models/criterion.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import torch 16 | import torch.nn as nn 17 | 18 | TINY_NUMBER = 1e-6 # float32 only has 7 decimal digits precision 19 | 20 | def img2mse(x, y, mask=None): 21 | ''' 22 | :param x: img 1, [(...), 3] 23 | :param y: img 2, [(...), 3] 24 | :param mask: optional, [(...)] 25 | :return: mse score 26 | ''' 27 | if mask is None: 28 | return torch.mean((x - y) * (x - y)) 29 | else: 30 | return torch.sum((x - y) * (x - y) * mask.unsqueeze(-1)) / (torch.sum(mask) * x.shape[-1] + TINY_NUMBER) 31 | 32 | class Criterion(nn.Module): 33 | def __init__(self): 34 | super().__init__() 35 | 36 | def forward(self, outputs, ray_batch, scalars_to_log): 37 | ''' 38 | training criterion 39 | ''' 40 | pred_rgb = outputs['rgb'] 41 | pred_mask = outputs['mask'].float() 42 | gt_rgb = ray_batch['rgb'] 43 | 44 | loss = img2mse(pred_rgb, gt_rgb, pred_mask) 45 | 46 | return loss, scalars_to_log 47 | -------------------------------------------------------------------------------- /models/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch import nn 4 | from network.codec import Codec 5 | import tinycudann as tcnn 6 | import loralib as lora 7 | 8 | 9 | class CodecNerfModel(object): 10 | def __init__(self, args): 11 | self.args = args 12 | device = torch.device('cuda:{}'.format(args.local_rank)) 13 | 14 | self.direction_encoder = tcnn.Encoding( 15 | n_input_dims=3, 16 | encoding_config={ 17 | "otype": "Frequency", 18 | "n_frequencies": 2, 19 | }, 20 | ) 21 | 22 | self.feature_net = Codec(input_dim_2d=3, extractor_channel=64, 23 | input_dim_3d=16*16, dims_3d=(32, 64, 128, 256), 24 | input_dim_tri=32, 25 | feat_dim_tri=64, 26 | spatial_volume_length=1.0, 27 | V = 64, device=device).to(device) 28 | 29 | self.softplus = torch.nn.Softplus() 30 | self.sigmoid = torch.nn.Sigmoid() 31 | 32 | self.start_step = self.load_from_ckpt(args.ckptdir) 33 | 34 | self.net_coarse = nn.Sequential( 35 | lora.Linear(80, 64, args.lrank, args.alpha, bias=False), 36 | nn.ReLU(), 37 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 38 | nn.ReLU(), 39 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 40 | nn.ReLU(), 41 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 42 | nn.ReLU(), 43 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 44 | nn.ReLU(), 45 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 46 | nn.ReLU(), 47 | lora.Linear(64, 4, args.lrank, args.alpha, bias=False), 48 | ).to(device) 49 | lora.mark_only_lora_as_trainable(self.net_coarse) 50 | 51 | self.net_fine = nn.Sequential( 52 | lora.Linear(80, 64, args.lrank, args.alpha, bias=False), 53 | nn.ReLU(), 54 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 55 | nn.ReLU(), 56 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 57 | nn.ReLU(), 58 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 59 | nn.ReLU(), 60 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 61 | nn.ReLU(), 62 | lora.Linear(64, 64, args.lrank, args.alpha, bias=False), 63 | nn.ReLU(), 64 | lora.Linear(64, 4, args.lrank, args.alpha, bias=False), 65 | ).to(device) 66 | lora.mark_only_lora_as_trainable(self.net_fine) 67 | 68 | nn_ckpt = torch.load("checkpoints/nn.pth") 69 | self.net_coarse.load_state_dict(nn_ckpt["nn_coarse"], strict=False) 70 | self.net_fine.load_state_dict(nn_ckpt["nn_fine"], strict=False) 71 | 72 | def encode(self, x, K, poses, w2cs): 73 | """ 74 | Args: 75 | x: input tensor [b, v, h, w, c] 76 | K: intrinsics [b, v, 4, 4] 77 | poses: extrinsics [b, v, 4, 4] 78 | Returns: 79 | Extracted feature maps [b, out_c, h, w] 80 | """ 81 | feat_maps, _, code_mb, _ = self.feature_net(x, K, poses, w2cs) 82 | 83 | return feat_maps, code_mb 84 | 85 | def switch_to_eval(self): 86 | self.net_coarse.eval() 87 | if self.net_fine is not None: 88 | self.net_fine.eval() 89 | 90 | def switch_to_train(self): 91 | self.net_coarse.train() 92 | lora.mark_only_lora_as_trainable(self.net_coarse) 93 | if self.net_fine is not None: 94 | self.net_fine.train() 95 | lora.mark_only_lora_as_trainable(self.net_fine) 96 | 97 | def load_model(self, filename): 98 | to_load = torch.load(filename) 99 | self.feature_net.load_state_dict(to_load['feature_net']) 100 | 101 | # self.net_coarse.load_state_dict(to_load['net_coarse']) 102 | # if self.net_fine is not None and 'net_fine' in to_load.keys(): 103 | # self.net_fine.load_state_dict(to_load['net_fine']) 104 | 105 | 106 | def load_from_ckpt(self, out_folder): 107 | '''Load model from existing checkpoints and return the current step 108 | 109 | Args: 110 | out_folder: the directory that stores ckpts 111 | Returns: 112 | The current starting step 113 | ''' 114 | 115 | # all existing ckpts 116 | ckpts = [] 117 | if os.path.exists(out_folder): 118 | ckpts = [os.path.join(out_folder, f) 119 | for f in sorted(os.listdir(out_folder)) if f.endswith('.pth')] 120 | 121 | fpath = ckpts[0] 122 | self.load_model(fpath) 123 | step = int(fpath[-10:-4]) 124 | print('Reloading from {}, starting at step={}'.format(fpath, step)) 125 | step = 0 126 | 127 | return step 128 | 129 | def get_param(self): 130 | params = [ 131 | {'params': self.net_coarse.parameters(), 'lr': self.args.lrate_mlp}, 132 | {'params': self.net_fine.parameters(), 'lr': self.args.lrate_mlp}, 133 | ] 134 | return params -------------------------------------------------------------------------------- /models/render_image.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from collections import OrderedDict 16 | 17 | import torch 18 | from models.render_ray import render_rays 19 | 20 | 21 | def render_single_image(ray_sampler, 22 | ray_batch, 23 | model, 24 | featmaps, 25 | chunk_size, 26 | N_samples, 27 | inv_uniform=False, 28 | N_importance=0, 29 | det=False, 30 | white_bkgd=False, 31 | render_stride=1): 32 | ''' 33 | Args: 34 | ray_sampler: RaySamplingSingleImage for this view 35 | ray_batch: {'ray_o': [N_rays, 3] , 'ray_d': [N_rays, 3], 'view_dir': [N_rays, 2]} 36 | model: {'net_coarse': , 'net_fine': , ...} 37 | chunk_size: number of rays in a chunk 38 | N_samples: samples along each ray (for both coarse and fine model) 39 | inv_uniform: if True, uniformly sample inverse depth for coarse model 40 | det: if True, use deterministic sampling 41 | white_bkgd: if True, assume background is white 42 | render_stride: stride for rendering 43 | featmaps: feature maps for inference [b, c, h, w] or [b, c, d, h, w] 44 | Return: 45 | {'outputs_coarse': {'rgb': numpy, 'depth': numpy, ...}, 'outputs_fine': {}} 46 | ''' 47 | 48 | all_ret = OrderedDict([('outputs_coarse', OrderedDict()), 49 | ('outputs_fine', OrderedDict())]) 50 | 51 | N_rays = ray_batch['ray_o'].shape[0] 52 | 53 | for i in range(0, N_rays, chunk_size): 54 | chunk = OrderedDict() 55 | for k in ray_batch: 56 | if k in ['intrinsics', 'c2w_mat', 'depth_range', 57 | 'src_rgbs', 'src_intrinsics', 'src_c2w_mats', 58 | 'aabb', 'src_rgbs_multi', 'src_intrinsics_multi', 59 | 'src_c2w_mats_multi', 'src_masks_multi']: 60 | chunk[k] = ray_batch[k] 61 | elif ray_batch[k] is not None: 62 | chunk[k] = ray_batch[k][i:i+chunk_size] 63 | else: 64 | chunk[k] = None 65 | 66 | ret = render_rays(chunk, model, featmaps, 67 | N_samples=N_samples, 68 | inv_uniform=inv_uniform, 69 | N_importance=N_importance, 70 | det=det, 71 | white_bkgd=white_bkgd) 72 | 73 | # handle both coarse and fine outputs 74 | # cache chunk results on cpu 75 | if i == 0: 76 | for k in ret['outputs_coarse']: 77 | all_ret['outputs_coarse'][k] = [] 78 | 79 | if ret['outputs_fine'] is None: 80 | all_ret['outputs_fine'] = None 81 | else: 82 | for k in ret['outputs_fine']: 83 | all_ret['outputs_fine'][k] = [] 84 | 85 | for k in ret['outputs_coarse']: 86 | all_ret['outputs_coarse'][k].append(ret['outputs_coarse'][k].squeeze(0).cpu()) 87 | 88 | if ret['outputs_fine'] is not None: 89 | for k in ret['outputs_fine']: 90 | all_ret['outputs_fine'][k].append(ret['outputs_fine'][k].squeeze(0).cpu()) 91 | 92 | rgb_strided = torch.ones(ray_sampler.H, ray_sampler.W, 3)[::render_stride, ::render_stride, :] 93 | # merge chunk results and reshape 94 | for k in all_ret['outputs_coarse']: 95 | if k == 'random_sigma': 96 | continue 97 | tmp = torch.cat(all_ret['outputs_coarse'][k], dim=0).reshape((rgb_strided.shape[0], 98 | rgb_strided.shape[1], -1)) 99 | all_ret['outputs_coarse'][k] = tmp.squeeze() 100 | 101 | if all_ret['outputs_fine'] is not None: 102 | for k in all_ret['outputs_fine']: 103 | if k == 'random_sigma': 104 | continue 105 | tmp = torch.cat(all_ret['outputs_fine'][k], dim=0).reshape((rgb_strided.shape[0], 106 | rgb_strided.shape[1], -1)) 107 | 108 | all_ret['outputs_fine'][k] = tmp.squeeze() 109 | 110 | return all_ret 111 | -------------------------------------------------------------------------------- /network/attention.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import einops 3 | import numpy as np 4 | 5 | def weight_init(shape, mode, fan_in, fan_out): 6 | if mode == 'xavier_uniform': return np.sqrt(6 / (fan_in + fan_out)) * (torch.rand(*shape) * 2 - 1) 7 | if mode == 'xavier_normal': return np.sqrt(2 / (fan_in + fan_out)) * torch.randn(*shape) 8 | if mode == 'kaiming_uniform': return np.sqrt(3 / fan_in) * (torch.rand(*shape) * 2 - 1) 9 | if mode == 'kaiming_normal': return np.sqrt(1 / fan_in) * torch.randn(*shape) 10 | raise ValueError(f'Invalid init mode "{mode}"') 11 | 12 | class Conv2d(torch.nn.Module): 13 | def __init__(self, 14 | in_channels, out_channels, kernel, bias=True, up=False, down=False, 15 | resample_filter=[1,1], fused_resample=False, init_mode='kaiming_normal', init_weight=1, init_bias=0, 16 | ): 17 | assert not (up and down) 18 | super().__init__() 19 | self.in_channels = in_channels 20 | self.out_channels = out_channels 21 | self.up = up 22 | self.down = down 23 | self.fused_resample = fused_resample 24 | init_kwargs = dict(mode=init_mode, fan_in=in_channels*kernel*kernel, fan_out=out_channels*kernel*kernel) 25 | self.weight = torch.nn.Parameter(weight_init([out_channels, in_channels, kernel, kernel], **init_kwargs) * init_weight) if kernel else None 26 | self.bias = torch.nn.Parameter(weight_init([out_channels], **init_kwargs) * init_bias) if kernel and bias else None 27 | f = torch.as_tensor(resample_filter, dtype=torch.float32) 28 | f = f.ger(f).unsqueeze(0).unsqueeze(1) / f.sum().square() 29 | self.register_buffer('resample_filter', f if up or down else None) 30 | 31 | def forward(self, x, N_views_xa=1): 32 | w = self.weight.to(x.dtype) if self.weight is not None else None 33 | b = self.bias.to(x.dtype) if self.bias is not None else None 34 | f = self.resample_filter.to(x.dtype) if self.resample_filter is not None else None 35 | w_pad = w.shape[-1] // 2 if w is not None else 0 36 | f_pad = (f.shape[-1] - 1) // 2 if f is not None else 0 37 | 38 | if self.fused_resample and self.up and w is not None: 39 | x = torch.nn.functional.conv_transpose2d(x, f.mul(4).tile([self.in_channels, 1, 1, 1]), groups=self.in_channels, stride=2, padding=max(f_pad - w_pad, 0)) 40 | x = torch.nn.functional.conv2d(x, w, padding=max(w_pad - f_pad, 0)) 41 | elif self.fused_resample and self.down and w is not None: 42 | x = torch.nn.functional.conv2d(x, w, padding=w_pad+f_pad) 43 | x = torch.nn.functional.conv2d(x, f.tile([self.out_channels, 1, 1, 1]), groups=self.out_channels, stride=2) 44 | else: 45 | if self.up: 46 | x = torch.nn.functional.conv_transpose2d(x, f.mul(4).tile([self.in_channels, 1, 1, 1]), groups=self.in_channels, stride=2, padding=f_pad) 47 | if self.down: 48 | x = torch.nn.functional.conv2d(x, f.tile([self.in_channels, 1, 1, 1]), groups=self.in_channels, stride=2, padding=f_pad) 49 | if w is not None: 50 | x = torch.nn.functional.conv2d(x, w, padding=w_pad) 51 | if b is not None: 52 | x = x.add_(b.reshape(1, -1, 1, 1)) 53 | return x 54 | 55 | class GroupNorm(torch.nn.Module): 56 | def __init__(self, num_channels, num_groups=32, min_channels_per_group=4, eps=1e-5): 57 | super().__init__() 58 | self.num_groups = min(num_groups, num_channels // min_channels_per_group) 59 | self.eps = eps 60 | self.weight = torch.nn.Parameter(torch.ones(num_channels)) 61 | self.bias = torch.nn.Parameter(torch.zeros(num_channels)) 62 | 63 | def forward(self, x): 64 | x = torch.nn.functional.group_norm(x, num_groups=self.num_groups, weight=self.weight.to(x.dtype), bias=self.bias.to(x.dtype), eps=self.eps) 65 | return x.to(memory_format=torch.channels_last) 66 | 67 | class AttentionOp(torch.autograd.Function): 68 | @staticmethod 69 | def forward(ctx, q, k): 70 | w = torch.einsum('ncq,nck->nqk', q.to(torch.float32), (k / np.sqrt(k.shape[1])).to(torch.float32)).softmax(dim=2).to(q.dtype) 71 | ctx.save_for_backward(q, k, w) 72 | return w 73 | 74 | @staticmethod 75 | def backward(ctx, dw): 76 | q, k, w = ctx.saved_tensors 77 | db = torch._softmax_backward_data(grad_output=dw.to(torch.float32), output=w.to(torch.float32), dim=2, input_dtype=torch.float32) 78 | dq = torch.einsum('nck,nqk->ncq', k.to(torch.float32), db).to(q.dtype) / np.sqrt(k.shape[1]) 79 | dk = torch.einsum('ncq,nqk->nck', q.to(torch.float32), db).to(k.dtype) / np.sqrt(k.shape[1]) 80 | return dq, dk 81 | 82 | 83 | class attentionblock(torch.nn.Module): 84 | def __init__(self, in_ch=256, num_heads=1, 85 | skip_scale=np.sqrt(0.5), eps=1e-5): 86 | 87 | super().__init__() 88 | self.in_ch = in_ch 89 | self.num_heads = num_heads 90 | self.skip_scale = skip_scale 91 | 92 | init_zero = dict(init_mode='xavier_uniform', init_weight=1e-5) 93 | init_attn = dict(init_mode='xavier_uniform', init_weight=np.sqrt(0.2)) 94 | 95 | self.norm2 = GroupNorm(num_channels=in_ch, eps=eps) 96 | self.qkv = Conv2d(in_channels=in_ch, out_channels=in_ch*3, kernel=1, **(init_attn)) 97 | self.proj = Conv2d(in_channels=in_ch, out_channels=in_ch, kernel=1, **init_zero) 98 | 99 | def forward(self, x): 100 | 101 | h_xy, h_xz, h_yz = x # (b, c, h, w) 102 | B, C, H, W = h_xy.shape 103 | in_x = torch.stack((h_xy, h_xz, h_yz), axis=1) # (b, 3, c, h, w) 104 | in_x = in_x.permute(0, 1, 3, 4, 2) # (b, 3, h, w, c) 105 | # (b, 3, h, w, c) -> (b, 3*h, w, c) -> (b, c, 3*h, w) 106 | in_x = in_x.reshape(B, 3 * in_x.shape[2], *in_x.shape[3:]).permute(0, 3, 1, 2) 107 | q, k, v = self.qkv(self.norm2(in_x)).reshape(in_x.shape[0] * self.num_heads, in_x.shape[1] // self.num_heads, 3, -1).unbind(2) 108 | w = AttentionOp.apply(q, k) 109 | a = torch.einsum('nqk,nck->ncq', w, v) 110 | in_x = self.proj(a.reshape(*in_x.shape)).add_(in_x) 111 | in_x = in_x * self.skip_scale 112 | 113 | # (b, c, 3*h, w) -> (b, 3*h, w, c) 114 | in_x = in_x.permute(0, 2, 3, 1) 115 | # (b, 3*h, w, c) -> (b, 3, h, w, c) -> (b, 3, c, h, w) 116 | in_x = in_x.reshape(B, 3, H, W, C).permute(0, 1, 4, 2, 3) 117 | # (b, 3, c, h, w) -> # (b, c, h, w) 118 | 119 | h_xy_out = in_x[:, 0, :, :, :] 120 | h_xz_out = in_x[:, 1, :, :, :] 121 | h_yz_out = in_x[:, 2, :, :, :] 122 | 123 | return h_xy_out, h_xz_out, h_yz_out 124 | -------------------------------------------------------------------------------- /network/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024, Zexin He 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Empty 16 | -------------------------------------------------------------------------------- /network/encoders/dino_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024, Zexin He 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import torch 17 | import torch.nn as nn 18 | from transformers import ViTImageProcessor, ViTModel 19 | 20 | 21 | 22 | class DinoWrapper(nn.Module): 23 | """ 24 | Dino v1 wrapper using huggingface transformer implementation. 25 | """ 26 | def __init__(self, model_name: str, freeze: bool = True): 27 | super().__init__() 28 | self.model, self.processor = self._build_dino(model_name) 29 | if freeze: 30 | self._freeze() 31 | 32 | @torch.compile 33 | def forward_model(self, inputs): 34 | return self.model(**inputs, interpolate_pos_encoding=True) 35 | 36 | def forward(self, image): 37 | # image: [N, C, H, W], on cpu 38 | # RGB image with [0,1] scale and properly sized 39 | inputs = self.processor(images=image, return_tensors="pt", do_rescale=False, do_resize=False).to(self.model.device) 40 | # This resampling of positional embedding uses bicubic interpolation 41 | outputs = self.forward_model(inputs) 42 | last_hidden_states = outputs.last_hidden_state 43 | return last_hidden_states 44 | 45 | def _freeze(self): 46 | self.model.eval() 47 | for name, param in self.model.named_parameters(): 48 | param.requires_grad = False 49 | 50 | @staticmethod 51 | def _build_dino(model_name: str, proxy_error_retries: int = 3, proxy_error_cooldown: int = 5): 52 | import requests 53 | try: 54 | model = ViTModel.from_pretrained(model_name, add_pooling_layer=False) 55 | processor = ViTImageProcessor.from_pretrained(model_name) 56 | return model, processor 57 | except requests.exceptions.ProxyError as err: 58 | if proxy_error_retries > 0: 59 | print(f"Huggingface ProxyError: Retrying ({proxy_error_retries}) in {proxy_error_cooldown} seconds...") 60 | import time 61 | time.sleep(proxy_error_cooldown) 62 | return DinoWrapper._build_dino(model_name, proxy_error_retries - 1, proxy_error_cooldown) 63 | else: 64 | raise err 65 | -------------------------------------------------------------------------------- /network/encoders/dinov2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024, Zexin He 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Empty 16 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/backbones.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | from enum import Enum 7 | from typing import Union 8 | 9 | import torch 10 | 11 | from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name 12 | 13 | 14 | class Weights(Enum): 15 | LVD142M = "LVD142M" 16 | 17 | 18 | def _make_dinov2_model( 19 | *, 20 | arch_name: str = "vit_large", 21 | img_size: int = 518, 22 | patch_size: int = 14, 23 | init_values: float = 1.0, 24 | ffn_layer: str = "mlp", 25 | block_chunks: int = 0, 26 | num_register_tokens: int = 0, 27 | interpolate_antialias: bool = False, 28 | interpolate_offset: float = 0.1, 29 | pretrained: bool = True, 30 | weights: Union[Weights, str] = Weights.LVD142M, 31 | **kwargs, 32 | ): 33 | from ..models import vision_transformer as vits 34 | 35 | if isinstance(weights, str): 36 | try: 37 | weights = Weights[weights] 38 | except KeyError: 39 | raise AssertionError(f"Unsupported weights: {weights}") 40 | 41 | model_base_name = _make_dinov2_model_name(arch_name, patch_size) 42 | vit_kwargs = dict( 43 | img_size=img_size, 44 | patch_size=patch_size, 45 | init_values=init_values, 46 | ffn_layer=ffn_layer, 47 | block_chunks=block_chunks, 48 | num_register_tokens=num_register_tokens, 49 | interpolate_antialias=interpolate_antialias, 50 | interpolate_offset=interpolate_offset, 51 | ) 52 | vit_kwargs.update(**kwargs) 53 | model = vits.__dict__[arch_name](**vit_kwargs) 54 | 55 | if pretrained: 56 | model_full_name = _make_dinov2_model_name(arch_name, patch_size, num_register_tokens) 57 | url = _DINOV2_BASE_URL + f"/{model_base_name}/{model_full_name}_pretrain.pth" 58 | state_dict = torch.hub.load_state_dict_from_url(url, map_location="cpu") 59 | # ********** Modified by Zexin He in 2023-2024 ********** 60 | state_dict = {k: v for k, v in state_dict.items() if 'mask_token' not in k} # DDP concern 61 | if vit_kwargs.get("modulation_dim") is not None: 62 | state_dict = { 63 | k.replace('norm1', 'norm1.norm').replace('norm2', 'norm2.norm'): v 64 | for k, v in state_dict.items() 65 | } 66 | model.load_state_dict(state_dict, strict=False) 67 | else: 68 | model.load_state_dict(state_dict, strict=True) 69 | # ******************************************************** 70 | 71 | return model 72 | 73 | 74 | def dinov2_vits14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 75 | """ 76 | DINOv2 ViT-S/14 model (optionally) pretrained on the LVD-142M dataset. 77 | """ 78 | return _make_dinov2_model(arch_name="vit_small", pretrained=pretrained, weights=weights, **kwargs) 79 | 80 | 81 | def dinov2_vitb14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 82 | """ 83 | DINOv2 ViT-B/14 model (optionally) pretrained on the LVD-142M dataset. 84 | """ 85 | return _make_dinov2_model(arch_name="vit_base", pretrained=pretrained, weights=weights, **kwargs) 86 | 87 | 88 | def dinov2_vitl14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 89 | """ 90 | DINOv2 ViT-L/14 model (optionally) pretrained on the LVD-142M dataset. 91 | """ 92 | return _make_dinov2_model(arch_name="vit_large", pretrained=pretrained, weights=weights, **kwargs) 93 | 94 | 95 | def dinov2_vitg14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 96 | """ 97 | DINOv2 ViT-g/14 model (optionally) pretrained on the LVD-142M dataset. 98 | """ 99 | return _make_dinov2_model( 100 | arch_name="vit_giant2", 101 | ffn_layer="swiglufused", 102 | weights=weights, 103 | pretrained=pretrained, 104 | **kwargs, 105 | ) 106 | 107 | 108 | def dinov2_vits14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 109 | """ 110 | DINOv2 ViT-S/14 model with registers (optionally) pretrained on the LVD-142M dataset. 111 | """ 112 | return _make_dinov2_model( 113 | arch_name="vit_small", 114 | pretrained=pretrained, 115 | weights=weights, 116 | num_register_tokens=4, 117 | interpolate_antialias=True, 118 | interpolate_offset=0.0, 119 | **kwargs, 120 | ) 121 | 122 | 123 | def dinov2_vitb14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 124 | """ 125 | DINOv2 ViT-B/14 model with registers (optionally) pretrained on the LVD-142M dataset. 126 | """ 127 | return _make_dinov2_model( 128 | arch_name="vit_base", 129 | pretrained=pretrained, 130 | weights=weights, 131 | num_register_tokens=4, 132 | interpolate_antialias=True, 133 | interpolate_offset=0.0, 134 | **kwargs, 135 | ) 136 | 137 | 138 | def dinov2_vitl14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 139 | """ 140 | DINOv2 ViT-L/14 model with registers (optionally) pretrained on the LVD-142M dataset. 141 | """ 142 | return _make_dinov2_model( 143 | arch_name="vit_large", 144 | pretrained=pretrained, 145 | weights=weights, 146 | num_register_tokens=4, 147 | interpolate_antialias=True, 148 | interpolate_offset=0.0, 149 | **kwargs, 150 | ) 151 | 152 | 153 | def dinov2_vitg14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs): 154 | """ 155 | DINOv2 ViT-g/14 model with registers (optionally) pretrained on the LVD-142M dataset. 156 | """ 157 | return _make_dinov2_model( 158 | arch_name="vit_giant2", 159 | ffn_layer="swiglufused", 160 | weights=weights, 161 | pretrained=pretrained, 162 | num_register_tokens=4, 163 | interpolate_antialias=True, 164 | interpolate_offset=0.0, 165 | **kwargs, 166 | ) 167 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/classifiers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | from enum import Enum 7 | from typing import Union 8 | 9 | import torch 10 | import torch.nn as nn 11 | 12 | from .backbones import _make_dinov2_model 13 | from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name 14 | 15 | 16 | class Weights(Enum): 17 | IMAGENET1K = "IMAGENET1K" 18 | 19 | 20 | def _make_dinov2_linear_classification_head( 21 | *, 22 | arch_name: str = "vit_large", 23 | patch_size: int = 14, 24 | embed_dim: int = 1024, 25 | layers: int = 4, 26 | pretrained: bool = True, 27 | weights: Union[Weights, str] = Weights.IMAGENET1K, 28 | num_register_tokens: int = 0, 29 | **kwargs, 30 | ): 31 | if layers not in (1, 4): 32 | raise AssertionError(f"Unsupported number of layers: {layers}") 33 | if isinstance(weights, str): 34 | try: 35 | weights = Weights[weights] 36 | except KeyError: 37 | raise AssertionError(f"Unsupported weights: {weights}") 38 | 39 | linear_head = nn.Linear((1 + layers) * embed_dim, 1_000) 40 | 41 | if pretrained: 42 | model_base_name = _make_dinov2_model_name(arch_name, patch_size) 43 | model_full_name = _make_dinov2_model_name(arch_name, patch_size, num_register_tokens) 44 | layers_str = str(layers) if layers == 4 else "" 45 | url = _DINOV2_BASE_URL + f"/{model_base_name}/{model_full_name}_linear{layers_str}_head.pth" 46 | state_dict = torch.hub.load_state_dict_from_url(url, map_location="cpu") 47 | linear_head.load_state_dict(state_dict, strict=True) 48 | 49 | return linear_head 50 | 51 | 52 | class _LinearClassifierWrapper(nn.Module): 53 | def __init__(self, *, backbone: nn.Module, linear_head: nn.Module, layers: int = 4): 54 | super().__init__() 55 | self.backbone = backbone 56 | self.linear_head = linear_head 57 | self.layers = layers 58 | 59 | def forward(self, x): 60 | if self.layers == 1: 61 | x = self.backbone.forward_features(x) 62 | cls_token = x["x_norm_clstoken"] 63 | patch_tokens = x["x_norm_patchtokens"] 64 | # fmt: off 65 | linear_input = torch.cat([ 66 | cls_token, 67 | patch_tokens.mean(dim=1), 68 | ], dim=1) 69 | # fmt: on 70 | elif self.layers == 4: 71 | x = self.backbone.get_intermediate_layers(x, n=4, return_class_token=True) 72 | # fmt: off 73 | linear_input = torch.cat([ 74 | x[0][1], 75 | x[1][1], 76 | x[2][1], 77 | x[3][1], 78 | x[3][0].mean(dim=1), 79 | ], dim=1) 80 | # fmt: on 81 | else: 82 | assert False, f"Unsupported number of layers: {self.layers}" 83 | return self.linear_head(linear_input) 84 | 85 | 86 | def _make_dinov2_linear_classifier( 87 | *, 88 | arch_name: str = "vit_large", 89 | layers: int = 4, 90 | pretrained: bool = True, 91 | weights: Union[Weights, str] = Weights.IMAGENET1K, 92 | num_register_tokens: int = 0, 93 | interpolate_antialias: bool = False, 94 | interpolate_offset: float = 0.1, 95 | **kwargs, 96 | ): 97 | backbone = _make_dinov2_model( 98 | arch_name=arch_name, 99 | pretrained=pretrained, 100 | num_register_tokens=num_register_tokens, 101 | interpolate_antialias=interpolate_antialias, 102 | interpolate_offset=interpolate_offset, 103 | **kwargs, 104 | ) 105 | 106 | embed_dim = backbone.embed_dim 107 | patch_size = backbone.patch_size 108 | linear_head = _make_dinov2_linear_classification_head( 109 | arch_name=arch_name, 110 | patch_size=patch_size, 111 | embed_dim=embed_dim, 112 | layers=layers, 113 | pretrained=pretrained, 114 | weights=weights, 115 | num_register_tokens=num_register_tokens, 116 | ) 117 | 118 | return _LinearClassifierWrapper(backbone=backbone, linear_head=linear_head, layers=layers) 119 | 120 | 121 | def dinov2_vits14_lc( 122 | *, 123 | layers: int = 4, 124 | pretrained: bool = True, 125 | weights: Union[Weights, str] = Weights.IMAGENET1K, 126 | **kwargs, 127 | ): 128 | """ 129 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-S/14 backbone (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 130 | """ 131 | return _make_dinov2_linear_classifier( 132 | arch_name="vit_small", 133 | layers=layers, 134 | pretrained=pretrained, 135 | weights=weights, 136 | **kwargs, 137 | ) 138 | 139 | 140 | def dinov2_vitb14_lc( 141 | *, 142 | layers: int = 4, 143 | pretrained: bool = True, 144 | weights: Union[Weights, str] = Weights.IMAGENET1K, 145 | **kwargs, 146 | ): 147 | """ 148 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-B/14 backbone (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 149 | """ 150 | return _make_dinov2_linear_classifier( 151 | arch_name="vit_base", 152 | layers=layers, 153 | pretrained=pretrained, 154 | weights=weights, 155 | **kwargs, 156 | ) 157 | 158 | 159 | def dinov2_vitl14_lc( 160 | *, 161 | layers: int = 4, 162 | pretrained: bool = True, 163 | weights: Union[Weights, str] = Weights.IMAGENET1K, 164 | **kwargs, 165 | ): 166 | """ 167 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-L/14 backbone (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 168 | """ 169 | return _make_dinov2_linear_classifier( 170 | arch_name="vit_large", 171 | layers=layers, 172 | pretrained=pretrained, 173 | weights=weights, 174 | **kwargs, 175 | ) 176 | 177 | 178 | def dinov2_vitg14_lc( 179 | *, 180 | layers: int = 4, 181 | pretrained: bool = True, 182 | weights: Union[Weights, str] = Weights.IMAGENET1K, 183 | **kwargs, 184 | ): 185 | """ 186 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-g/14 backbone (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 187 | """ 188 | return _make_dinov2_linear_classifier( 189 | arch_name="vit_giant2", 190 | layers=layers, 191 | ffn_layer="swiglufused", 192 | pretrained=pretrained, 193 | weights=weights, 194 | **kwargs, 195 | ) 196 | 197 | 198 | def dinov2_vits14_reg_lc( 199 | *, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.IMAGENET1K, **kwargs 200 | ): 201 | """ 202 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-S/14 backbone with registers (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 203 | """ 204 | return _make_dinov2_linear_classifier( 205 | arch_name="vit_small", 206 | layers=layers, 207 | pretrained=pretrained, 208 | weights=weights, 209 | num_register_tokens=4, 210 | interpolate_antialias=True, 211 | interpolate_offset=0.0, 212 | **kwargs, 213 | ) 214 | 215 | 216 | def dinov2_vitb14_reg_lc( 217 | *, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.IMAGENET1K, **kwargs 218 | ): 219 | """ 220 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-B/14 backbone with registers (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 221 | """ 222 | return _make_dinov2_linear_classifier( 223 | arch_name="vit_base", 224 | layers=layers, 225 | pretrained=pretrained, 226 | weights=weights, 227 | num_register_tokens=4, 228 | interpolate_antialias=True, 229 | interpolate_offset=0.0, 230 | **kwargs, 231 | ) 232 | 233 | 234 | def dinov2_vitl14_reg_lc( 235 | *, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.IMAGENET1K, **kwargs 236 | ): 237 | """ 238 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-L/14 backbone with registers (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 239 | """ 240 | return _make_dinov2_linear_classifier( 241 | arch_name="vit_large", 242 | layers=layers, 243 | pretrained=pretrained, 244 | weights=weights, 245 | num_register_tokens=4, 246 | interpolate_antialias=True, 247 | interpolate_offset=0.0, 248 | **kwargs, 249 | ) 250 | 251 | 252 | def dinov2_vitg14_reg_lc( 253 | *, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.IMAGENET1K, **kwargs 254 | ): 255 | """ 256 | Linear classifier (1 or 4 layers) on top of a DINOv2 ViT-g/14 backbone with registers (optionally) pretrained on the LVD-142M dataset and trained on ImageNet-1k. 257 | """ 258 | return _make_dinov2_linear_classifier( 259 | arch_name="vit_giant2", 260 | layers=layers, 261 | ffn_layer="swiglufused", 262 | pretrained=pretrained, 263 | weights=weights, 264 | num_register_tokens=4, 265 | interpolate_antialias=True, 266 | interpolate_offset=0.0, 267 | **kwargs, 268 | ) 269 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/depth/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | from .decode_heads import BNHead, DPTHead 7 | from .encoder_decoder import DepthEncoderDecoder 8 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/depth/ops.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | import warnings 7 | 8 | import torch.nn.functional as F 9 | 10 | 11 | def resize(input, size=None, scale_factor=None, mode="nearest", align_corners=None, warning=False): 12 | if warning: 13 | if size is not None and align_corners: 14 | input_h, input_w = tuple(int(x) for x in input.shape[2:]) 15 | output_h, output_w = tuple(int(x) for x in size) 16 | if output_h > input_h or output_w > output_h: 17 | if ( 18 | (output_h > 1 and output_w > 1 and input_h > 1 and input_w > 1) 19 | and (output_h - 1) % (input_h - 1) 20 | and (output_w - 1) % (input_w - 1) 21 | ): 22 | warnings.warn( 23 | f"When align_corners={align_corners}, " 24 | "the output would more aligned if " 25 | f"input size {(input_h, input_w)} is `x+1` and " 26 | f"out size {(output_h, output_w)} is `nx+1`" 27 | ) 28 | return F.interpolate(input, size, scale_factor, mode, align_corners) 29 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/depthers.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | from enum import Enum 7 | from functools import partial 8 | from typing import Optional, Tuple, Union 9 | 10 | import torch 11 | 12 | from .backbones import _make_dinov2_model 13 | from .depth import BNHead, DepthEncoderDecoder, DPTHead 14 | from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name, CenterPadding 15 | 16 | 17 | class Weights(Enum): 18 | NYU = "NYU" 19 | KITTI = "KITTI" 20 | 21 | 22 | def _get_depth_range(pretrained: bool, weights: Weights = Weights.NYU) -> Tuple[float, float]: 23 | if not pretrained: # Default 24 | return (0.001, 10.0) 25 | 26 | # Pretrained, set according to the training dataset for the provided weights 27 | if weights == Weights.KITTI: 28 | return (0.001, 80.0) 29 | 30 | if weights == Weights.NYU: 31 | return (0.001, 10.0) 32 | 33 | return (0.001, 10.0) 34 | 35 | 36 | def _make_dinov2_linear_depth_head( 37 | *, 38 | embed_dim: int, 39 | layers: int, 40 | min_depth: float, 41 | max_depth: float, 42 | **kwargs, 43 | ): 44 | if layers not in (1, 4): 45 | raise AssertionError(f"Unsupported number of layers: {layers}") 46 | 47 | if layers == 1: 48 | in_index = [0] 49 | else: 50 | assert layers == 4 51 | in_index = [0, 1, 2, 3] 52 | 53 | return BNHead( 54 | classify=True, 55 | n_bins=256, 56 | bins_strategy="UD", 57 | norm_strategy="linear", 58 | upsample=4, 59 | in_channels=[embed_dim] * len(in_index), 60 | in_index=in_index, 61 | input_transform="resize_concat", 62 | channels=embed_dim * len(in_index) * 2, 63 | align_corners=False, 64 | min_depth=0.001, 65 | max_depth=80, 66 | loss_decode=(), 67 | ) 68 | 69 | 70 | def _make_dinov2_linear_depther( 71 | *, 72 | arch_name: str = "vit_large", 73 | layers: int = 4, 74 | pretrained: bool = True, 75 | weights: Union[Weights, str] = Weights.NYU, 76 | depth_range: Optional[Tuple[float, float]] = None, 77 | **kwargs, 78 | ): 79 | if layers not in (1, 4): 80 | raise AssertionError(f"Unsupported number of layers: {layers}") 81 | if isinstance(weights, str): 82 | try: 83 | weights = Weights[weights] 84 | except KeyError: 85 | raise AssertionError(f"Unsupported weights: {weights}") 86 | 87 | if depth_range is None: 88 | depth_range = _get_depth_range(pretrained, weights) 89 | min_depth, max_depth = depth_range 90 | 91 | backbone = _make_dinov2_model(arch_name=arch_name, pretrained=pretrained, **kwargs) 92 | 93 | embed_dim = backbone.embed_dim 94 | patch_size = backbone.patch_size 95 | model_name = _make_dinov2_model_name(arch_name, patch_size) 96 | linear_depth_head = _make_dinov2_linear_depth_head( 97 | embed_dim=embed_dim, 98 | layers=layers, 99 | min_depth=min_depth, 100 | max_depth=max_depth, 101 | ) 102 | 103 | layer_count = { 104 | "vit_small": 12, 105 | "vit_base": 12, 106 | "vit_large": 24, 107 | "vit_giant2": 40, 108 | }[arch_name] 109 | 110 | if layers == 4: 111 | out_index = { 112 | "vit_small": [2, 5, 8, 11], 113 | "vit_base": [2, 5, 8, 11], 114 | "vit_large": [4, 11, 17, 23], 115 | "vit_giant2": [9, 19, 29, 39], 116 | }[arch_name] 117 | else: 118 | assert layers == 1 119 | out_index = [layer_count - 1] 120 | 121 | model = DepthEncoderDecoder(backbone=backbone, decode_head=linear_depth_head) 122 | model.backbone.forward = partial( 123 | backbone.get_intermediate_layers, 124 | n=out_index, 125 | reshape=True, 126 | return_class_token=True, 127 | norm=False, 128 | ) 129 | model.backbone.register_forward_pre_hook(lambda _, x: CenterPadding(patch_size)(x[0])) 130 | 131 | if pretrained: 132 | layers_str = str(layers) if layers == 4 else "" 133 | weights_str = weights.value.lower() 134 | url = _DINOV2_BASE_URL + f"/{model_name}/{model_name}_{weights_str}_linear{layers_str}_head.pth" 135 | checkpoint = torch.hub.load_state_dict_from_url(url, map_location="cpu") 136 | if "state_dict" in checkpoint: 137 | state_dict = checkpoint["state_dict"] 138 | model.load_state_dict(state_dict, strict=False) 139 | 140 | return model 141 | 142 | 143 | def dinov2_vits14_ld(*, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 144 | return _make_dinov2_linear_depther( 145 | arch_name="vit_small", layers=layers, pretrained=pretrained, weights=weights, **kwargs 146 | ) 147 | 148 | 149 | def dinov2_vitb14_ld(*, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 150 | return _make_dinov2_linear_depther( 151 | arch_name="vit_base", layers=layers, pretrained=pretrained, weights=weights, **kwargs 152 | ) 153 | 154 | 155 | def dinov2_vitl14_ld(*, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 156 | return _make_dinov2_linear_depther( 157 | arch_name="vit_large", layers=layers, pretrained=pretrained, weights=weights, **kwargs 158 | ) 159 | 160 | 161 | def dinov2_vitg14_ld(*, layers: int = 4, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 162 | return _make_dinov2_linear_depther( 163 | arch_name="vit_giant2", layers=layers, ffn_layer="swiglufused", pretrained=pretrained, weights=weights, **kwargs 164 | ) 165 | 166 | 167 | def _make_dinov2_dpt_depth_head(*, embed_dim: int, min_depth: float, max_depth: float): 168 | return DPTHead( 169 | in_channels=[embed_dim] * 4, 170 | channels=256, 171 | embed_dims=embed_dim, 172 | post_process_channels=[embed_dim // 2 ** (3 - i) for i in range(4)], 173 | readout_type="project", 174 | min_depth=min_depth, 175 | max_depth=max_depth, 176 | loss_decode=(), 177 | ) 178 | 179 | 180 | def _make_dinov2_dpt_depther( 181 | *, 182 | arch_name: str = "vit_large", 183 | pretrained: bool = True, 184 | weights: Union[Weights, str] = Weights.NYU, 185 | depth_range: Optional[Tuple[float, float]] = None, 186 | **kwargs, 187 | ): 188 | if isinstance(weights, str): 189 | try: 190 | weights = Weights[weights] 191 | except KeyError: 192 | raise AssertionError(f"Unsupported weights: {weights}") 193 | 194 | if depth_range is None: 195 | depth_range = _get_depth_range(pretrained, weights) 196 | min_depth, max_depth = depth_range 197 | 198 | backbone = _make_dinov2_model(arch_name=arch_name, pretrained=pretrained, **kwargs) 199 | 200 | model_name = _make_dinov2_model_name(arch_name, backbone.patch_size) 201 | dpt_depth_head = _make_dinov2_dpt_depth_head(embed_dim=backbone.embed_dim, min_depth=min_depth, max_depth=max_depth) 202 | 203 | out_index = { 204 | "vit_small": [2, 5, 8, 11], 205 | "vit_base": [2, 5, 8, 11], 206 | "vit_large": [4, 11, 17, 23], 207 | "vit_giant2": [9, 19, 29, 39], 208 | }[arch_name] 209 | 210 | model = DepthEncoderDecoder(backbone=backbone, decode_head=dpt_depth_head) 211 | model.backbone.forward = partial( 212 | backbone.get_intermediate_layers, 213 | n=out_index, 214 | reshape=True, 215 | return_class_token=True, 216 | norm=False, 217 | ) 218 | model.backbone.register_forward_pre_hook(lambda _, x: CenterPadding(backbone.patch_size)(x[0])) 219 | 220 | if pretrained: 221 | weights_str = weights.value.lower() 222 | url = _DINOV2_BASE_URL + f"/{model_name}/{model_name}_{weights_str}_dpt_head.pth" 223 | checkpoint = torch.hub.load_state_dict_from_url(url, map_location="cpu") 224 | if "state_dict" in checkpoint: 225 | state_dict = checkpoint["state_dict"] 226 | model.load_state_dict(state_dict, strict=False) 227 | 228 | return model 229 | 230 | 231 | def dinov2_vits14_dd(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 232 | return _make_dinov2_dpt_depther(arch_name="vit_small", pretrained=pretrained, weights=weights, **kwargs) 233 | 234 | 235 | def dinov2_vitb14_dd(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 236 | return _make_dinov2_dpt_depther(arch_name="vit_base", pretrained=pretrained, weights=weights, **kwargs) 237 | 238 | 239 | def dinov2_vitl14_dd(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 240 | return _make_dinov2_dpt_depther(arch_name="vit_large", pretrained=pretrained, weights=weights, **kwargs) 241 | 242 | 243 | def dinov2_vitg14_dd(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.NYU, **kwargs): 244 | return _make_dinov2_dpt_depther( 245 | arch_name="vit_giant2", ffn_layer="swiglufused", pretrained=pretrained, weights=weights, **kwargs 246 | ) 247 | -------------------------------------------------------------------------------- /network/encoders/dinov2/hub/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | import itertools 7 | import math 8 | 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | 13 | 14 | _DINOV2_BASE_URL = "https://dl.fbaipublicfiles.com/dinov2" 15 | 16 | 17 | def _make_dinov2_model_name(arch_name: str, patch_size: int, num_register_tokens: int = 0) -> str: 18 | compact_arch_name = arch_name.replace("_", "")[:4] 19 | registers_suffix = f"_reg{num_register_tokens}" if num_register_tokens else "" 20 | return f"dinov2_{compact_arch_name}{patch_size}{registers_suffix}" 21 | 22 | 23 | class CenterPadding(nn.Module): 24 | def __init__(self, multiple): 25 | super().__init__() 26 | self.multiple = multiple 27 | 28 | def _get_pad(self, size): 29 | new_size = math.ceil(size / self.multiple) * self.multiple 30 | pad_size = new_size - size 31 | pad_size_left = pad_size // 2 32 | pad_size_right = pad_size - pad_size_left 33 | return pad_size_left, pad_size_right 34 | 35 | @torch.inference_mode() 36 | def forward(self, x): 37 | pads = list(itertools.chain.from_iterable(self._get_pad(m) for m in x.shape[:1:-1])) 38 | output = F.pad(x, pads) 39 | return output 40 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # ****************************************************************************** 7 | # Code modified by Zexin He in 2023-2024. 8 | # Modifications are marked with clearly visible comments 9 | # licensed under the Apache License, Version 2.0. 10 | # ****************************************************************************** 11 | 12 | from .dino_head import DINOHead 13 | from .mlp import Mlp 14 | from .patch_embed import PatchEmbed 15 | from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused 16 | # ********** Modified by Zexin He in 2023-2024 ********** 17 | # Avoid using nested tensor for now, deprecating usage of NestedTensorBlock 18 | from .block import Block, BlockWithModulation 19 | # ******************************************************** 20 | from .attention import MemEffAttention 21 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/attention.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # References: 7 | # https://github.com/facebookresearch/dino/blob/master/vision_transformer.py 8 | # https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py 9 | 10 | import logging 11 | import os 12 | import warnings 13 | 14 | from torch import Tensor 15 | from torch import nn 16 | 17 | 18 | logger = logging.getLogger("dinov2") 19 | 20 | 21 | XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None 22 | try: 23 | if XFORMERS_ENABLED: 24 | from xformers.ops import memory_efficient_attention, unbind 25 | 26 | XFORMERS_AVAILABLE = True 27 | warnings.warn("xFormers is available (Attention)") 28 | else: 29 | warnings.warn("xFormers is disabled (Attention)") 30 | raise ImportError 31 | except ImportError: 32 | XFORMERS_AVAILABLE = False 33 | warnings.warn("xFormers is not available (Attention)") 34 | 35 | 36 | class Attention(nn.Module): 37 | def __init__( 38 | self, 39 | dim: int, 40 | num_heads: int = 8, 41 | qkv_bias: bool = False, 42 | proj_bias: bool = True, 43 | attn_drop: float = 0.0, 44 | proj_drop: float = 0.0, 45 | ) -> None: 46 | super().__init__() 47 | self.num_heads = num_heads 48 | head_dim = dim // num_heads 49 | self.scale = head_dim**-0.5 50 | 51 | self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) 52 | self.attn_drop = nn.Dropout(attn_drop) 53 | self.proj = nn.Linear(dim, dim, bias=proj_bias) 54 | self.proj_drop = nn.Dropout(proj_drop) 55 | 56 | def forward(self, x: Tensor) -> Tensor: 57 | B, N, C = x.shape 58 | qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) 59 | 60 | q, k, v = qkv[0] * self.scale, qkv[1], qkv[2] 61 | attn = q @ k.transpose(-2, -1) 62 | 63 | attn = attn.softmax(dim=-1) 64 | attn = self.attn_drop(attn) 65 | 66 | x = (attn @ v).transpose(1, 2).reshape(B, N, C) 67 | x = self.proj(x) 68 | x = self.proj_drop(x) 69 | return x 70 | 71 | 72 | class MemEffAttention(Attention): 73 | def forward(self, x: Tensor, attn_bias=None) -> Tensor: 74 | if not XFORMERS_AVAILABLE: 75 | if attn_bias is not None: 76 | raise AssertionError("xFormers is required for using nested tensors") 77 | return super().forward(x) 78 | 79 | B, N, C = x.shape 80 | qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) 81 | 82 | q, k, v = unbind(qkv, 2) 83 | 84 | x = memory_efficient_attention(q, k, v, attn_bias=attn_bias) 85 | x = x.reshape([B, N, C]) 86 | 87 | x = self.proj(x) 88 | x = self.proj_drop(x) 89 | return x 90 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/dino_head.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | import torch 7 | import torch.nn as nn 8 | from torch.nn.init import trunc_normal_ 9 | from torch.nn.utils import weight_norm 10 | 11 | 12 | class DINOHead(nn.Module): 13 | def __init__( 14 | self, 15 | in_dim, 16 | out_dim, 17 | use_bn=False, 18 | nlayers=3, 19 | hidden_dim=2048, 20 | bottleneck_dim=256, 21 | mlp_bias=True, 22 | ): 23 | super().__init__() 24 | nlayers = max(nlayers, 1) 25 | self.mlp = _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=hidden_dim, use_bn=use_bn, bias=mlp_bias) 26 | self.apply(self._init_weights) 27 | self.last_layer = weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False)) 28 | self.last_layer.weight_g.data.fill_(1) 29 | 30 | def _init_weights(self, m): 31 | if isinstance(m, nn.Linear): 32 | trunc_normal_(m.weight, std=0.02) 33 | if isinstance(m, nn.Linear) and m.bias is not None: 34 | nn.init.constant_(m.bias, 0) 35 | 36 | def forward(self, x): 37 | x = self.mlp(x) 38 | eps = 1e-6 if x.dtype == torch.float16 else 1e-12 39 | x = nn.functional.normalize(x, dim=-1, p=2, eps=eps) 40 | x = self.last_layer(x) 41 | return x 42 | 43 | 44 | def _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True): 45 | if nlayers == 1: 46 | return nn.Linear(in_dim, bottleneck_dim, bias=bias) 47 | else: 48 | layers = [nn.Linear(in_dim, hidden_dim, bias=bias)] 49 | if use_bn: 50 | layers.append(nn.BatchNorm1d(hidden_dim)) 51 | layers.append(nn.GELU()) 52 | for _ in range(nlayers - 2): 53 | layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias)) 54 | if use_bn: 55 | layers.append(nn.BatchNorm1d(hidden_dim)) 56 | layers.append(nn.GELU()) 57 | layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias)) 58 | return nn.Sequential(*layers) 59 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/drop_path.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # References: 7 | # https://github.com/facebookresearch/dino/blob/master/vision_transformer.py 8 | # https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py 9 | 10 | 11 | from torch import nn 12 | 13 | 14 | def drop_path(x, drop_prob: float = 0.0, training: bool = False): 15 | if drop_prob == 0.0 or not training: 16 | return x 17 | keep_prob = 1 - drop_prob 18 | shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets 19 | random_tensor = x.new_empty(shape).bernoulli_(keep_prob) 20 | if keep_prob > 0.0: 21 | random_tensor.div_(keep_prob) 22 | output = x * random_tensor 23 | return output 24 | 25 | 26 | class DropPath(nn.Module): 27 | """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" 28 | 29 | def __init__(self, drop_prob=None): 30 | super(DropPath, self).__init__() 31 | self.drop_prob = drop_prob 32 | 33 | def forward(self, x): 34 | return drop_path(x, self.drop_prob, self.training) 35 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/layer_scale.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 7 | 8 | from typing import Union 9 | 10 | import torch 11 | from torch import Tensor 12 | from torch import nn 13 | 14 | 15 | class LayerScale(nn.Module): 16 | def __init__( 17 | self, 18 | dim: int, 19 | init_values: Union[float, Tensor] = 1e-5, 20 | inplace: bool = False, 21 | ) -> None: 22 | super().__init__() 23 | self.inplace = inplace 24 | self.gamma = nn.Parameter(init_values * torch.ones(dim)) 25 | 26 | def forward(self, x: Tensor) -> Tensor: 27 | return x.mul_(self.gamma) if self.inplace else x * self.gamma 28 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/mlp.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # References: 7 | # https://github.com/facebookresearch/dino/blob/master/vision_transformer.py 8 | # https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py 9 | 10 | 11 | from typing import Callable, Optional 12 | 13 | from torch import Tensor, nn 14 | 15 | 16 | class Mlp(nn.Module): 17 | def __init__( 18 | self, 19 | in_features: int, 20 | hidden_features: Optional[int] = None, 21 | out_features: Optional[int] = None, 22 | act_layer: Callable[..., nn.Module] = nn.GELU, 23 | drop: float = 0.0, 24 | bias: bool = True, 25 | ) -> None: 26 | super().__init__() 27 | out_features = out_features or in_features 28 | hidden_features = hidden_features or in_features 29 | self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) 30 | self.act = act_layer() 31 | self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) 32 | self.drop = nn.Dropout(drop) 33 | 34 | def forward(self, x: Tensor) -> Tensor: 35 | x = self.fc1(x) 36 | x = self.act(x) 37 | x = self.drop(x) 38 | x = self.fc2(x) 39 | x = self.drop(x) 40 | return x 41 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/patch_embed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | # References: 7 | # https://github.com/facebookresearch/dino/blob/master/vision_transformer.py 8 | # https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py 9 | 10 | from typing import Callable, Optional, Tuple, Union 11 | 12 | from torch import Tensor 13 | import torch.nn as nn 14 | 15 | 16 | def make_2tuple(x): 17 | if isinstance(x, tuple): 18 | assert len(x) == 2 19 | return x 20 | 21 | assert isinstance(x, int) 22 | return (x, x) 23 | 24 | 25 | class PatchEmbed(nn.Module): 26 | """ 27 | 2D image to patch embedding: (B,C,H,W) -> (B,N,D) 28 | 29 | Args: 30 | img_size: Image size. 31 | patch_size: Patch token size. 32 | in_chans: Number of input image channels. 33 | embed_dim: Number of linear projection output channels. 34 | norm_layer: Normalization layer. 35 | """ 36 | 37 | def __init__( 38 | self, 39 | img_size: Union[int, Tuple[int, int]] = 224, 40 | patch_size: Union[int, Tuple[int, int]] = 16, 41 | in_chans: int = 3, 42 | embed_dim: int = 768, 43 | norm_layer: Optional[Callable] = None, 44 | flatten_embedding: bool = True, 45 | ) -> None: 46 | super().__init__() 47 | 48 | image_HW = make_2tuple(img_size) 49 | patch_HW = make_2tuple(patch_size) 50 | patch_grid_size = ( 51 | image_HW[0] // patch_HW[0], 52 | image_HW[1] // patch_HW[1], 53 | ) 54 | 55 | self.img_size = image_HW 56 | self.patch_size = patch_HW 57 | self.patches_resolution = patch_grid_size 58 | self.num_patches = patch_grid_size[0] * patch_grid_size[1] 59 | 60 | self.in_chans = in_chans 61 | self.embed_dim = embed_dim 62 | 63 | self.flatten_embedding = flatten_embedding 64 | 65 | self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) 66 | self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() 67 | 68 | def forward(self, x: Tensor) -> Tensor: 69 | _, _, H, W = x.shape 70 | patch_H, patch_W = self.patch_size 71 | 72 | assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" 73 | assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" 74 | 75 | x = self.proj(x) # B C H W 76 | H, W = x.size(2), x.size(3) 77 | x = x.flatten(2).transpose(1, 2) # B HW C 78 | x = self.norm(x) 79 | if not self.flatten_embedding: 80 | x = x.reshape(-1, H, W, self.embed_dim) # B H W C 81 | return x 82 | 83 | def flops(self) -> float: 84 | Ho, Wo = self.patches_resolution 85 | flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) 86 | if self.norm is not None: 87 | flops += Ho * Wo * self.embed_dim 88 | return flops 89 | -------------------------------------------------------------------------------- /network/encoders/dinov2/layers/swiglu_ffn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | import os 7 | from typing import Callable, Optional 8 | import warnings 9 | 10 | from torch import Tensor, nn 11 | import torch.nn.functional as F 12 | 13 | 14 | class SwiGLUFFN(nn.Module): 15 | def __init__( 16 | self, 17 | in_features: int, 18 | hidden_features: Optional[int] = None, 19 | out_features: Optional[int] = None, 20 | act_layer: Callable[..., nn.Module] = None, 21 | drop: float = 0.0, 22 | bias: bool = True, 23 | ) -> None: 24 | super().__init__() 25 | out_features = out_features or in_features 26 | hidden_features = hidden_features or in_features 27 | self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias) 28 | self.w3 = nn.Linear(hidden_features, out_features, bias=bias) 29 | 30 | def forward(self, x: Tensor) -> Tensor: 31 | x12 = self.w12(x) 32 | x1, x2 = x12.chunk(2, dim=-1) 33 | hidden = F.silu(x1) * x2 34 | return self.w3(hidden) 35 | 36 | 37 | XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None 38 | try: 39 | if XFORMERS_ENABLED: 40 | from xformers.ops import SwiGLU 41 | 42 | XFORMERS_AVAILABLE = True 43 | warnings.warn("xFormers is available (SwiGLU)") 44 | else: 45 | warnings.warn("xFormers is disabled (SwiGLU)") 46 | raise ImportError 47 | except ImportError: 48 | SwiGLU = SwiGLUFFN 49 | XFORMERS_AVAILABLE = False 50 | 51 | warnings.warn("xFormers is not available (SwiGLU)") 52 | 53 | 54 | class SwiGLUFFNFused(SwiGLU): 55 | def __init__( 56 | self, 57 | in_features: int, 58 | hidden_features: Optional[int] = None, 59 | out_features: Optional[int] = None, 60 | act_layer: Callable[..., nn.Module] = None, 61 | drop: float = 0.0, 62 | bias: bool = True, 63 | ) -> None: 64 | out_features = out_features or in_features 65 | hidden_features = hidden_features or in_features 66 | hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8 67 | super().__init__( 68 | in_features=in_features, 69 | hidden_features=hidden_features, 70 | out_features=out_features, 71 | bias=bias, 72 | ) 73 | -------------------------------------------------------------------------------- /network/encoders/dinov2/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the Apache License, Version 2.0 4 | # found in the LICENSE file in the root directory of this source tree. 5 | 6 | import logging 7 | 8 | from . import vision_transformer as vits 9 | 10 | 11 | logger = logging.getLogger("dinov2") 12 | 13 | 14 | def build_model(args, only_teacher=False, img_size=224): 15 | args.arch = args.arch.removesuffix("_memeff") 16 | if "vit" in args.arch: 17 | vit_kwargs = dict( 18 | img_size=img_size, 19 | patch_size=args.patch_size, 20 | init_values=args.layerscale, 21 | ffn_layer=args.ffn_layer, 22 | block_chunks=args.block_chunks, 23 | qkv_bias=args.qkv_bias, 24 | proj_bias=args.proj_bias, 25 | ffn_bias=args.ffn_bias, 26 | num_register_tokens=args.num_register_tokens, 27 | interpolate_offset=args.interpolate_offset, 28 | interpolate_antialias=args.interpolate_antialias, 29 | ) 30 | teacher = vits.__dict__[args.arch](**vit_kwargs) 31 | if only_teacher: 32 | return teacher, teacher.embed_dim 33 | student = vits.__dict__[args.arch]( 34 | **vit_kwargs, 35 | drop_path_rate=args.drop_path_rate, 36 | drop_path_uniform=args.drop_path_uniform, 37 | ) 38 | embed_dim = student.embed_dim 39 | return student, teacher, embed_dim 40 | 41 | 42 | def build_model_from_cfg(cfg, only_teacher=False): 43 | return build_model(cfg.student, only_teacher=only_teacher, img_size=cfg.crops.global_crops_size) 44 | -------------------------------------------------------------------------------- /network/encoders/dinov2_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024, Zexin He 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import torch 17 | import torch.nn as nn 18 | 19 | 20 | 21 | 22 | 23 | class Dinov2Wrapper(nn.Module): 24 | """ 25 | Dino v2 wrapper using original implementation, hacked with modulation. 26 | """ 27 | def __init__(self, model_name: str, modulation_dim: int = None, freeze: bool = True): 28 | super().__init__() 29 | self.modulation_dim = modulation_dim 30 | self.model = self._build_dinov2(model_name, modulation_dim=modulation_dim) 31 | if freeze: 32 | if modulation_dim is not None: 33 | raise ValueError("Modulated Dinov2 requires training, freezing is not allowed.") 34 | self._freeze() 35 | 36 | def _freeze(self): 37 | self.model.eval() 38 | for name, param in self.model.named_parameters(): 39 | param.requires_grad = False 40 | 41 | @staticmethod 42 | def _build_dinov2(model_name: str, modulation_dim: int = None, pretrained: bool = True): 43 | from importlib import import_module 44 | dinov2_hub = import_module(".dinov2.hub.backbones", package=__package__) 45 | model_fn = getattr(dinov2_hub, model_name) 46 | model = model_fn(modulation_dim=modulation_dim, pretrained=pretrained) 47 | return model 48 | 49 | @torch.compile 50 | def forward(self, image: torch.Tensor, mod: torch.Tensor = None): 51 | # image: [N, C, H, W] 52 | # mod: [N, D] or None 53 | # RGB image with [0,1] scale and properly sized 54 | if self.modulation_dim is None: 55 | assert mod is None, "Unexpected modulation input in dinov2 forward." 56 | outs = self.model(image, is_training=True) 57 | else: 58 | assert mod is not None, "Modulation input is required in modulated dinov2 forward." 59 | outs = self.model(image, mod=mod, is_training=True) 60 | ret = torch.cat([ 61 | outs["x_norm_clstoken"].unsqueeze(dim=1), 62 | outs["x_norm_patchtokens"], 63 | ], dim=1) 64 | return outs["x_norm_patchtokens"] #ret 65 | -------------------------------------------------------------------------------- /network/spa_block.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class SpatialUpBlock(nn.Module): 5 | def __init__(self, x_in_dim, out_dim): 6 | super().__init__() 7 | norm_act = lambda c: nn.GroupNorm(8, c) 8 | self.norm = norm_act(x_in_dim) 9 | self.silu = nn.SiLU(True) 10 | self.conv = nn.ConvTranspose3d(x_in_dim, out_dim, kernel_size=3, padding=1, output_padding=1, stride=2) 11 | 12 | def forward(self, x): 13 | return self.conv(self.silu(self.norm(x))) 14 | 15 | 16 | class SpatialBlock(nn.Module): 17 | def __init__(self, x_in_dim, out_dim, stride): 18 | super().__init__() 19 | norm_act = lambda c: nn.GroupNorm(8, c) 20 | self.bn = norm_act(x_in_dim) 21 | self.silu = nn.SiLU(True) 22 | self.conv = nn.Conv3d(x_in_dim, out_dim, 3, stride=stride, padding=1) 23 | 24 | def forward(self, x): 25 | return self.conv(self.silu(self.bn(x))) 26 | 27 | 28 | class Spatial3DNet(nn.Module): 29 | def __init__(self, input_dim=128, dims=(32, 64, 128, 256)): 30 | super().__init__() 31 | d0, d1, d2, d3 = dims 32 | 33 | self.init_conv = nn.Conv3d(input_dim, d0, 3, 1, 1) # 32 34 | self.conv0 = SpatialBlock(d0, d0, stride=1) 35 | 36 | self.conv1 = SpatialBlock(d0, d1, stride=2) 37 | self.conv2_0 = SpatialBlock(d1, d1, stride=1) 38 | self.conv2_1 = SpatialBlock(d1, d1, stride=1) 39 | 40 | self.conv3 = SpatialBlock(d1, d2, stride=2) 41 | self.conv4_0 = SpatialBlock(d2, d2, stride=1) 42 | self.conv4_1 = SpatialBlock(d2, d2, stride=1) 43 | 44 | self.conv5 = SpatialBlock(d2, d3, stride=2) 45 | self.conv6_0 = SpatialBlock(d3, d3, stride=1) 46 | self.conv6_1 = SpatialBlock(d3, d3, stride=1) 47 | 48 | self.conv7 = SpatialUpBlock(d3, d2) 49 | self.conv8 = SpatialUpBlock(d2, d1) 50 | self.conv9 = SpatialUpBlock(d1, d0) 51 | 52 | def forward(self, x): 53 | 54 | x = self.init_conv(x) 55 | conv0 = self.conv0(x) 56 | 57 | x = self.conv1(conv0) 58 | x = self.conv2_0(x) 59 | conv2 = self.conv2_1(x) 60 | 61 | x = self.conv3(conv2) 62 | x = self.conv4_0(x) 63 | conv4 = self.conv4_1(x) 64 | 65 | x = self.conv5(conv4) 66 | x = self.conv6_0(x) 67 | x = self.conv6_1(x) 68 | 69 | x = conv4 + self.conv7(x) 70 | x = conv2 + self.conv8(x) 71 | x = conv0 + self.conv9(x) 72 | return x -------------------------------------------------------------------------------- /network/tri_block.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch.nn.functional as F 3 | 4 | class TriplaneConv(nn.Module): 5 | def __init__( 6 | self, 7 | in_channels: int, 8 | out_channel: int, # but if you are doing 1x1 conv it should be an integer 9 | ker_size=3, 10 | stride=1, 11 | padd=0, 12 | use_norm=None 13 | ): 14 | 15 | super(TriplaneConv, self).__init__() 16 | in_c = in_channels 17 | out_c = out_channel 18 | self.conv_yz = nn.Conv2d(in_c, out_c, ker_size, stride, padd) # Originally out_c 19 | self.conv_xz = nn.Conv2d(in_c, out_c, ker_size, stride, padd) 20 | self.conv_xy = nn.Conv2d(in_c, out_c, ker_size, stride, padd) 21 | 22 | def forward(self, tri_feats: list, tri_noises: list=None, add_noise=False, skip_add=False): 23 | """ 24 | Args: 25 | tri_feats (list): tri-plane feature maps 26 | tri_noises (list, optional): tri-plane noise maps 27 | add_noise (bool, optional): add_tri-lane noise maps 28 | skip_add (bool, optional): skip connection 29 | 30 | Returns: 31 | output: tri-plane feature maps 32 | """ 33 | yz_feat, xz_feat, xy_feat = tri_feats 34 | 35 | if skip_add: 36 | yz_feat_prev = yz_feat 37 | xz_feat_prev = xz_feat 38 | xy_feat_prev = xy_feat 39 | 40 | if add_noise and tri_noises is not None: 41 | yz_feat = yz_feat + tri_noises[0] 42 | xz_feat = xz_feat + tri_noises[1] 43 | xy_feat = xy_feat + tri_noises[2] 44 | 45 | yz_feat = self.conv_yz(yz_feat) 46 | xz_feat = self.conv_xz(xz_feat) 47 | xy_feat = self.conv_xy(xy_feat) 48 | 49 | # skip connect 50 | if skip_add: 51 | yz_feat = yz_feat + yz_feat_prev 52 | xz_feat = xz_feat + xz_feat_prev 53 | xy_feat = xy_feat + xy_feat_prev 54 | 55 | return yz_feat, xz_feat, xy_feat 56 | 57 | 58 | class TriplaneGen(nn.Module): 59 | def __init__(self, n_channels, feat_dim): 60 | 61 | super(TriplaneGen, self).__init__() 62 | self.n_channels = n_channels 63 | self.feat_dim = feat_dim 64 | self.pool_dim = 1 65 | 66 | 67 | self.head_conv = TriplaneConv(self.n_channels, self.feat_dim, 1, 1, 0) 68 | 69 | def forward(self, spatial_volume): 70 | ni = spatial_volume 71 | in_shape = ni.shape[-3:] # 32, 32, 32 72 | yz_feat = F.adaptive_avg_pool3d(ni, (self.pool_dim, in_shape[1], in_shape[2])).squeeze(2) 73 | xz_feat = F.adaptive_avg_pool3d(ni, (in_shape[0], self.pool_dim, in_shape[2])).squeeze(3) 74 | xy_feat = F.adaptive_avg_pool3d(ni, (in_shape[0], in_shape[1], self.pool_dim)).squeeze(4) 75 | 76 | yz_feat, xz_feat, xy_feat = self.head_conv([yz_feat, xz_feat, xy_feat]) 77 | 78 | return xy_feat, xz_feat, yz_feat -------------------------------------------------------------------------------- /network/tri_new.py: -------------------------------------------------------------------------------- 1 | import torch as th 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | 6 | 7 | class TriplaneConv(nn.Module): 8 | def __init__(self, channels, out_channels, kernel_size, padding, is_rollout=True) -> None: 9 | super().__init__() 10 | in_channels = channels * 3 if is_rollout else channels 11 | self.is_rollout = is_rollout 12 | 13 | self.conv_xy = nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding) 14 | self.conv_xz = nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding) 15 | self.conv_yz = nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding) 16 | 17 | def forward(self, featmaps): 18 | # tpl: [B, C, H + D, W + D] 19 | tpl_xy, tpl_xz, tpl_yz = featmaps 20 | H, W = tpl_xy.shape[-2:] 21 | D = tpl_xz.shape[-1] 22 | 23 | if self.is_rollout: 24 | tpl_xy_h = th.cat([tpl_xy, 25 | th.mean(tpl_yz, dim=-1, keepdim=True).transpose(-1, -2).expand_as(tpl_xy), 26 | th.mean(tpl_xz, dim=-1, keepdim=True).expand_as(tpl_xy)], dim=1) # [B, C * 3, H, W] 27 | tpl_xz_h = th.cat([tpl_xz, 28 | th.mean(tpl_xy, dim=-1, keepdim=True).expand_as(tpl_xz), 29 | th.mean(tpl_yz, dim=-2, keepdim=True).expand_as(tpl_xz)], dim=1) # [B, C * 3, H, D] 30 | tpl_yz_h = th.cat([tpl_yz, 31 | th.mean(tpl_xy, dim=-2, keepdim=True).transpose(-1, -2).expand_as(tpl_yz), 32 | th.mean(tpl_xz, dim=-2, keepdim=True).expand_as(tpl_yz)], dim=1) # [B, C * 3, W, D] 33 | else: 34 | tpl_xy_h = tpl_xy 35 | tpl_xz_h = tpl_xz 36 | tpl_yz_h = tpl_yz 37 | 38 | assert tpl_xy_h.shape[-2] == H and tpl_xy_h.shape[-1] == W 39 | assert tpl_xz_h.shape[-2] == H and tpl_xz_h.shape[-1] == D 40 | assert tpl_yz_h.shape[-2] == W and tpl_yz_h.shape[-1] == D 41 | 42 | tpl_xy_h = self.conv_xy(tpl_xy_h) 43 | tpl_xz_h = self.conv_xz(tpl_xz_h) 44 | tpl_yz_h = self.conv_yz(tpl_yz_h) 45 | 46 | return (tpl_xy_h, tpl_xz_h, tpl_yz_h) 47 | 48 | 49 | class TriplaneNorm(nn.Module): 50 | def __init__(self, channels) -> None: 51 | super().__init__() 52 | self.norm_xy = nn.GroupNorm(32, channels) 53 | self.norm_xz = nn.GroupNorm(32, channels) 54 | self.norm_yz = nn.GroupNorm(32, channels) 55 | 56 | def forward(self, featmaps): 57 | tpl_xy, tpl_xz, tpl_yz = featmaps 58 | H, W = tpl_xy.shape[-2:] 59 | D = tpl_xz.shape[-1] 60 | 61 | tpl_xy_h = self.norm_xy(tpl_xy) # [B, C, H, W] 62 | tpl_xz_h = self.norm_xz(tpl_xz) # [B, C, H, D] 63 | tpl_yz_h = self.norm_yz(tpl_yz) # [B, C, W, D] 64 | 65 | assert tpl_xy_h.shape[-2] == H and tpl_xy_h.shape[-1] == W 66 | assert tpl_xz_h.shape[-2] == H and tpl_xz_h.shape[-1] == D 67 | assert tpl_yz_h.shape[-2] == W and tpl_yz_h.shape[-1] == D 68 | 69 | return (tpl_xy_h, tpl_xz_h, tpl_yz_h) 70 | 71 | 72 | class TriplaneSiLU(nn.Module): 73 | def __init__(self) -> None: 74 | super().__init__() 75 | self.silu = nn.SiLU() 76 | 77 | def forward(self, featmaps): 78 | # tpl: [B, C, H + D, W + D] 79 | tpl_xy, tpl_xz, tpl_yz = featmaps 80 | return (self.silu(tpl_xy), self.silu(tpl_xz), self.silu(tpl_yz)) 81 | 82 | 83 | class TriplaneUpsample2x(nn.Module): 84 | def __init__(self) -> None: 85 | super().__init__() 86 | 87 | def forward(self, featmaps): 88 | # tpl: [B, C, H + D, W + D] 89 | tpl_xy, tpl_xz, tpl_yz = featmaps 90 | H, W = tpl_xy.shape[-2:] 91 | D = tpl_xz.shape[-1] 92 | 93 | tpl_xy = F.interpolate(tpl_xy, scale_factor=2, mode='bilinear', align_corners=False) 94 | tpl_xz = F.interpolate(tpl_xz, scale_factor=2, mode='bilinear', align_corners=False) 95 | tpl_yz = F.interpolate(tpl_yz, scale_factor=2, mode='bilinear', align_corners=False) 96 | 97 | assert tpl_xy.shape[-2] == H * 2 and tpl_xy.shape[-1] == W * 2 98 | assert tpl_xz.shape[-2] == H * 2 and tpl_xz.shape[-1] == D * 2 99 | assert tpl_yz.shape[-2] == W * 2 and tpl_yz.shape[-1] == D * 2 100 | 101 | return (tpl_xy, tpl_xz, tpl_yz) 102 | 103 | 104 | class TriplaneResBlock(nn.Module): 105 | 106 | def __init__( 107 | self, 108 | channels, 109 | out_channels, 110 | use_conv=False, 111 | is_rollout=True, 112 | ): 113 | super().__init__() 114 | self.channels = channels 115 | self.out_channels = out_channels 116 | self.use_conv = use_conv 117 | 118 | self.in_layers = nn.Sequential( 119 | TriplaneNorm(channels), 120 | TriplaneSiLU(), 121 | TriplaneConv(channels, self.out_channels, 3, padding=1, is_rollout=is_rollout), 122 | ) 123 | 124 | 125 | self.h_upd = TriplaneUpsample2x() 126 | self.x_upd = TriplaneUpsample2x() 127 | 128 | self.out_layers = nn.Sequential( 129 | TriplaneNorm(self.out_channels), 130 | TriplaneSiLU(), 131 | TriplaneConv(self.out_channels, self.out_channels, 3, padding=1, is_rollout=is_rollout), 132 | TriplaneConv(self.out_channels, self.out_channels, 3, padding=1, is_rollout=is_rollout) 133 | ) 134 | 135 | if self.out_channels == channels: 136 | self.skip_connection = nn.Identity() 137 | elif use_conv: 138 | self.skip_connection = TriplaneConv( 139 | channels, self.out_channels, 3, padding=1, is_rollout=False 140 | ) 141 | else: 142 | self.skip_connection = TriplaneConv(channels, self.out_channels, 1, padding=0, is_rollout=False) 143 | 144 | def forward(self, x): 145 | 146 | h = self.in_layers(x) 147 | h = self.out_layers(h) 148 | 149 | x_skip = self.skip_connection(x) 150 | x_skip_xy, x_skip_xz, x_skip_yz = x_skip 151 | h_xy, h_xz, h_yz = h 152 | return (h_xy + x_skip_xy, h_xz + x_skip_xz, h_yz + x_skip_yz) 153 | 154 | 155 | class TriplaneSR(nn.Module): 156 | 157 | def __init__( 158 | self, 159 | in_channels=128, 160 | out_channels=[64, 64, 32], 161 | use_fp16=False 162 | ): 163 | super().__init__() 164 | 165 | self.in_channels = in_channels 166 | self.out_channels = out_channels 167 | self.dtype = th.float16 if use_fp16 else th.float32 168 | 169 | self.in_conv = TriplaneConv(in_channels, in_channels, 1, padding=0, is_rollout=False) 170 | self.in_res = TriplaneResBlock(in_channels, in_channels) 171 | 172 | # -L version 173 | self.in_res_1 = TriplaneResBlock(in_channels, in_channels) 174 | self.in_res_2 = TriplaneResBlock(in_channels, in_channels) 175 | 176 | self.up1 = TriplaneUpsample2x() 177 | self.res1 = TriplaneResBlock(in_channels, out_channels[0]) 178 | self.res1_1 = TriplaneResBlock(out_channels[0], out_channels[0]) 179 | self.res1_2 = TriplaneResBlock(out_channels[0], out_channels[0]) 180 | 181 | self.out0 = nn.Sequential( 182 | TriplaneNorm(in_channels), 183 | TriplaneSiLU(), 184 | TriplaneConv(in_channels, out_channels[2], 1, padding=0, is_rollout=False), 185 | TriplaneNorm(out_channels[2]), 186 | TriplaneSiLU(), 187 | TriplaneConv(out_channels[2], out_channels[2], 1, padding=0, is_rollout=False), 188 | TriplaneNorm(out_channels[2]), 189 | TriplaneSiLU(), 190 | TriplaneConv(out_channels[2], out_channels[2], 1, padding=0, is_rollout=False) 191 | ) 192 | 193 | self.out1 = nn.Sequential( 194 | TriplaneNorm(out_channels[0]), 195 | TriplaneSiLU(), 196 | TriplaneConv(out_channels[0], out_channels[2], 1, padding=0, is_rollout=False), 197 | TriplaneNorm(out_channels[2]), 198 | TriplaneSiLU(), 199 | TriplaneConv(out_channels[2], out_channels[2], 1, padding=0, is_rollout=False), 200 | TriplaneNorm(out_channels[2]), 201 | TriplaneSiLU(), 202 | TriplaneConv(out_channels[2], out_channels[2], 1, padding=0, is_rollout=False) 203 | ) 204 | 205 | def forward(self, x): 206 | h_triplane0 = self.in_conv(x) 207 | h_triplane0 = self.in_res(h_triplane0) 208 | h_triplane0 = self.in_res_1(h_triplane0) 209 | h_triplane0out = self.in_res_2(h_triplane0) 210 | 211 | h_triplane1 = self.up1(h_triplane0) 212 | h_triplane1 = self.res1(h_triplane1) 213 | h_triplane1 = self.res1_1(h_triplane1) 214 | h_triplane1out = self.res1_2(h_triplane1) 215 | 216 | h_triplane0put = self.out0(h_triplane0out) 217 | h_triplane1put = self.out1(h_triplane1out) 218 | 219 | return [h_triplane0put, h_triplane1put] -------------------------------------------------------------------------------- /network/tri_res.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | def compose_triplane_channelwise(feat_maps): 6 | h_xy, h_xz, h_yz = feat_maps # (H, W), (H, D), (W, D) 7 | assert h_xy.shape[1] == h_xz.shape[1] == h_yz.shape[1] 8 | C, H, W = h_xy.shape[-3:] 9 | D = h_xz.shape[-1] 10 | 11 | newH = max(H, W) 12 | newW = max(W, D) 13 | h_xy = F.pad(h_xy, (0, newW - W, 0, newH - H)) 14 | h_xz = F.pad(h_xz, (0, newW - D, 0, newH - H)) 15 | h_yz = F.pad(h_yz, (0, newW - D, 0, newH - W)) 16 | h = torch.cat([h_xy, h_xz, h_yz], dim=1) # (B, 3C, H, W) 17 | 18 | return h, (H, W, D) 19 | 20 | 21 | def decompose_triplane_channelwise(composed_map, sizes): 22 | H, W, D = sizes 23 | C = composed_map.shape[1] // 3 24 | h_xy = composed_map[:, :C, :H, :W] 25 | h_xz = composed_map[:, C:2*C, :H, :D] 26 | h_yz = composed_map[:, 2*C:, :W, :D] 27 | return h_xy, h_xz, h_yz 28 | 29 | 30 | class TriplaneGroupResnetBlock(nn.Module): 31 | def __init__(self, in_channels, out_channels, up=False, ks=3, input_norm=False, input_act=False): 32 | super().__init__() 33 | in_channels *= 3 34 | out_channels *= 3 35 | 36 | self.in_channels = in_channels 37 | self.out_channels = out_channels 38 | self.up = up 39 | 40 | self.input_norm = input_norm 41 | if input_norm and input_act: 42 | self.in_layers = nn.Sequential( 43 | # nn.GroupNorm(num_groups=3, num_channels=in_channels, eps=1e-6, affine=True), 44 | nn.SiLU(), 45 | nn.Conv2d(in_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2) 46 | ) 47 | elif not input_norm: 48 | if input_act: 49 | self.in_layers = nn.Sequential( 50 | nn.SiLU(), 51 | nn.Conv2d(in_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2) 52 | ) 53 | else: 54 | self.in_layers = nn.Sequential( 55 | nn.Conv2d(in_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2), 56 | nn.SiLU(), 57 | nn.Conv2d(out_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2), 58 | ) 59 | else: 60 | raise NotImplementedError 61 | 62 | self.norm_xy = nn.InstanceNorm2d(out_channels//3, eps=1e-6, affine=True) 63 | self.norm_xz = nn.InstanceNorm2d(out_channels//3, eps=1e-6, affine=True) 64 | self.norm_yz = nn.InstanceNorm2d(out_channels//3, eps=1e-6, affine=True) 65 | 66 | self.out_layers = nn.Sequential( 67 | nn.SiLU(), 68 | nn.Conv2d(out_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2), 69 | nn.SiLU(), 70 | nn.Conv2d(out_channels, out_channels, groups=3, kernel_size=ks, stride=1, padding=(ks - 1)//2), 71 | ) 72 | 73 | if self.in_channels != self.out_channels: 74 | self.shortcut = nn.Conv2d(in_channels, out_channels, groups=3, kernel_size=1, stride=1, padding=0) 75 | else: 76 | self.shortcut = nn.Identity() 77 | 78 | def forward(self, feat_maps): 79 | if self.input_norm: 80 | feat_maps = [self.norm_xy(feat_maps[0]), self.norm_xz(feat_maps[1]), self.norm_yz(feat_maps[2])] 81 | x, (H, W, D) = compose_triplane_channelwise(feat_maps) 82 | 83 | if self.up: 84 | raise NotImplementedError 85 | else: 86 | h = self.in_layers(x) 87 | 88 | h_xy, h_xz, h_yz = decompose_triplane_channelwise(h, (H, W, D)) 89 | h_xy = self.norm_xy(h_xy) 90 | h_xz = self.norm_xz(h_xz) 91 | h_yz = self.norm_yz(h_yz) 92 | h, _ = compose_triplane_channelwise([h_xy, h_xz, h_yz]) 93 | 94 | h = self.out_layers(h) 95 | h = h + self.shortcut(x) 96 | h_maps = decompose_triplane_channelwise(h, (H, W, D)) 97 | return h_maps 98 | -------------------------------------------------------------------------------- /opt.py: -------------------------------------------------------------------------------- 1 | import configargparse 2 | 3 | 4 | def config_parser(): 5 | parser = configargparse.ArgumentParser() 6 | # general 7 | parser.add_argument('--config', is_config_file=True, help='config file path') 8 | parser.add_argument('--expname', type=str, help='experiment name') 9 | parser.add_argument('--ckptdir', type=str, help='checkpoint folder') 10 | parser.add_argument("--local-rank", type=int, default=0, help='rank for distributed training') 11 | parser.add_argument('--seed', type=int, default=42, help='random seed') 12 | parser.add_argument('--iter', type=int, default=2000, help='iteration number') 13 | parser.add_argument('--data_type', type=str, default='obj', help='data type') 14 | 15 | ########## dataset options ########## 16 | ## train and eval dataset 17 | parser.add_argument('--data_path', type=str, help='the dataset to train') 18 | parser.add_argument('--camera_path', type=str, help='the camera data to train') 19 | parser.add_argument('--outdir', type=str, help='output directory') 20 | parser.add_argument('--img_hw', type=int, nargs='+', help='image size option for dataset') 21 | 22 | 23 | parser.add_argument('--N_rand', type=int, default=128, 24 | help='batch size (number of random rays per gradient step)') 25 | parser.add_argument('--chunk_size', type=int, default=128, 26 | help='number of rays processed in parallel, decrease if running out of memory') 27 | 28 | ########### iterations & learning rate options ########## 29 | parser.add_argument('--lrate_mlp', type=float, default=1e-3, help='learning rate for mlp') 30 | parser.add_argument('--lrate_feat', type=float, default=5e-3, help='learning rate for feature map') 31 | parser.add_argument('--lrate_comp', type=float, default=1e-5, help='learning rate for compression model') 32 | 33 | ########## rendering options ########## 34 | parser.add_argument('--N_samples', type=int, default=64, help='number of coarse samples per ray') 35 | parser.add_argument('--N_importance', type=int, default=64, help='number of important samples per ray') 36 | parser.add_argument('--inv_uniform', action='store_true', 37 | help='if True, will uniformly sample inverse depths') 38 | parser.add_argument('--det', action='store_true', help='deterministic sampling for coarse and fine samples') 39 | parser.add_argument('--white_bkgd', action='store_true', 40 | help='apply the trick to avoid fitting to white background') 41 | parser.add_argument('--render_stride', type=int, default=1, 42 | help='render with large stride for validation to save time') 43 | 44 | parser.add_argument('--i_print', type=int, default=100, help='frequency of terminal printout') 45 | parser.add_argument('--trank', type=int, default=4, help='rank for tensor decomposition') 46 | parser.add_argument('--lrank', type=int, default=4, help='rank for lora') 47 | parser.add_argument('--alpha', type=float, default=4.0, help='alpha for lora') 48 | 49 | return parser -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | python finetune.py --config configs/finetune.txt 2 | python finetune_ec.py --config configs/finetune_ec.txt 3 | python finetune_ec_weight.py --config configs/finetune_ec_weight.txt --------------------------------------------------------------------------------