├── XMem ├── model │ ├── __init__.py │ ├── aggregate.py │ ├── losses.py │ ├── group_modules.py │ ├── memory_util.py │ └── cbam.py ├── util │ ├── __init__.py │ ├── load_subset.py │ ├── davis_subset.txt │ ├── tensor_util.py │ ├── palette.py │ ├── log_integrator.py │ └── logger.py ├── dataset │ ├── __init__.py │ ├── reseed.py │ ├── range_transform.py │ ├── util.py │ └── tps.py └── inference │ ├── __init__.py │ ├── data │ ├── __init__.py │ └── mask_mapper.py │ └── interact │ ├── __init__.py │ ├── fbrs │ ├── __init__.py │ ├── model │ │ ├── __init__.py │ │ ├── syncbn │ │ │ ├── __init__.py │ │ │ ├── modules │ │ │ │ ├── __init__.py │ │ │ │ ├── nn │ │ │ │ │ └── __init__.py │ │ │ │ └── functional │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── csrc │ │ │ │ │ ├── ext_lib.cpp │ │ │ │ │ ├── cuda │ │ │ │ │ │ └── ext_lib.h │ │ │ │ │ └── bn.h │ │ │ │ │ └── _csrc.py │ │ │ └── LICENSE │ │ ├── modeling │ │ │ ├── __init__.py │ │ │ ├── resnet.py │ │ │ └── basic_blocks.py │ │ └── ops.py │ ├── utils │ │ ├── __init__.py │ │ ├── cython │ │ │ ├── __init__.py │ │ │ ├── dist_maps.py │ │ │ ├── _get_dist_maps.pyxbld │ │ │ └── _get_dist_maps.pyx │ │ └── misc.py │ └── inference │ │ ├── __init__.py │ │ ├── transforms │ │ ├── __init__.py │ │ ├── limit_longest_side.py │ │ ├── base.py │ │ └── flip.py │ │ ├── evaluation.py │ │ └── predictors │ │ └── brs_losses.py │ ├── s2m │ ├── __init__.py │ ├── s2m_network.py │ └── utils.py │ ├── timer.py │ ├── gui_utils.py │ ├── s2m_controller.py │ └── fbrs_controller.py ├── BundleSDF ├── BundleTrack │ ├── LoFTR │ │ ├── src │ │ │ ├── __init__.py │ │ │ ├── loftr │ │ │ │ ├── __init__.py │ │ │ │ ├── loftr_module │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── fine_preprocess.py │ │ │ │ │ └── linear_attention.py │ │ │ │ ├── backbone │ │ │ │ │ └── __init__.py │ │ │ │ └── utils │ │ │ │ │ ├── cvpr_ds_config.py │ │ │ │ │ ├── position_encoding.py │ │ │ │ │ ├── geometry.py │ │ │ │ │ └── fine_matching.py │ │ │ ├── utils │ │ │ │ ├── dataloader.py │ │ │ │ ├── profiler.py │ │ │ │ └── augment.py │ │ │ └── optimizers │ │ │ │ └── __init__.py │ │ ├── .gitmodules │ │ ├── configs │ │ │ └── loftr │ │ │ │ ├── indoor │ │ │ │ ├── loftr_ot.py │ │ │ │ ├── loftr_ds.py │ │ │ │ ├── buggy_pos_enc │ │ │ │ │ ├── loftr_ot.py │ │ │ │ │ ├── loftr_ds.py │ │ │ │ │ ├── loftr_ds_dense.py │ │ │ │ │ └── loftr_ot_dense.py │ │ │ │ ├── loftr_ot_dense.py │ │ │ │ ├── loftr_ds_dense.py │ │ │ │ └── scannet │ │ │ │ │ ├── loftr_ds_eval.py │ │ │ │ │ └── loftr_ds_eval_new.py │ │ │ │ └── outdoor │ │ │ │ ├── loftr_ot.py │ │ │ │ ├── loftr_ds.py │ │ │ │ ├── buggy_pos_enc │ │ │ │ ├── loftr_ot.py │ │ │ │ ├── loftr_ds.py │ │ │ │ ├── loftr_ot_dense.py │ │ │ │ └── loftr_ds_dense.py │ │ │ │ ├── loftr_ot_dense.py │ │ │ │ └── loftr_ds_dense.py │ │ ├── environment.yaml │ │ ├── requirements.txt │ │ ├── .gitignore │ │ ├── scripts │ │ │ ├── reproduce_test │ │ │ │ ├── indoor_ot.sh │ │ │ │ ├── indoor_ds.sh │ │ │ │ ├── outdoor_ot.sh │ │ │ │ ├── outdoor_ds.sh │ │ │ │ └── indoor_ds_new.sh │ │ │ └── reproduce_train │ │ │ │ ├── indoor_ds.sh │ │ │ │ ├── indoor_ot.sh │ │ │ │ ├── outdoor_ds.sh │ │ │ │ └── outdoor_ot.sh │ │ ├── demo │ │ │ └── run_demo.sh │ │ └── test.py │ ├── src │ │ ├── spdlog │ │ │ ├── version.h │ │ │ ├── fmt │ │ │ │ ├── ostr.h │ │ │ │ ├── fmt.h │ │ │ │ └── bundled │ │ │ │ │ ├── LICENSE.rst │ │ │ │ │ └── locale.h │ │ │ ├── formatter.h │ │ │ ├── details │ │ │ │ ├── null_mutex.h │ │ │ │ ├── log_msg.h │ │ │ │ ├── console_globals.h │ │ │ │ ├── circular_q.h │ │ │ │ └── periodic_worker.h │ │ │ ├── sinks │ │ │ │ ├── msvc_sink.h │ │ │ │ ├── null_sink.h │ │ │ │ ├── ostream_sink.h │ │ │ │ ├── sink.h │ │ │ │ ├── stdout_color_sinks.h │ │ │ │ ├── base_sink.h │ │ │ │ ├── basic_file_sink.h │ │ │ │ ├── dist_sink.h │ │ │ │ ├── syslog_sink.h │ │ │ │ └── stdout_sinks.h │ │ │ ├── async_logger.h │ │ │ └── async.h │ │ ├── cuda │ │ │ ├── CUDACameraUtil.h │ │ │ ├── LossGPU.h │ │ │ ├── Solver │ │ │ │ └── SolverBundlingParameters.h │ │ │ ├── CUDACacheUtil.h │ │ │ ├── cudaUtil.h │ │ │ ├── cuda_ransac.h │ │ │ ├── CUDACache.h │ │ │ └── SBA.h │ │ ├── EigenDenseBaseAddons.h │ │ └── fmt │ │ │ └── format.cc │ ├── .gitignore │ ├── LICENSE │ ├── config_ycbineoat.yml │ ├── config_ho3d.yml │ └── config_behave.yml ├── segmentation_utils.py ├── mycuda │ ├── torch_ngp_grid_encoder │ │ ├── bindings.cpp │ │ ├── LICENSE │ │ └── gridencoder.h │ ├── bindings.cpp │ ├── common.h │ └── setup.py ├── config.yml └── loftr_wrapper.py ├── run.sh ├── requirements.txt ├── capture.sh ├── run_container.sh ├── conv_png_to_jpeg.py ├── mask.py ├── patch_pyrender.py └── README.md /XMem/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/s2m/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/modeling/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from .syncbn import * 2 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/functional/__init__.py: -------------------------------------------------------------------------------- 1 | from .syncbn import batchnorm2d_sync 2 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | cd BundleSDF 2 | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/conda/envs/py38/lib/ 3 | python run_custom.py -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/__init__.py: -------------------------------------------------------------------------------- 1 | from .loftr import LoFTR 2 | from .utils.cvpr_ds_config import default_cfg 3 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/cython/__init__.py: -------------------------------------------------------------------------------- 1 | # noinspection PyUnresolvedReferences 2 | from .dist_maps import get_dist_maps -------------------------------------------------------------------------------- /XMem/dataset/reseed.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import random 3 | 4 | def reseed(seed): 5 | random.seed(seed) 6 | torch.manual_seed(seed) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | progressbar2 2 | gdown 3 | gitpython 4 | git+https://github.com/cheind/py-thin-plate-spline 5 | hickle 6 | tensorboard 7 | numpy -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/loftr_module/__init__.py: -------------------------------------------------------------------------------- 1 | from .transformer import LocalFeatureTransformer 2 | from .fine_preprocess import FinePreprocess 3 | -------------------------------------------------------------------------------- /capture.sh: -------------------------------------------------------------------------------- 1 | python3 rec_con_mask.py 2 | cd XMem 3 | /path/to/conda/base/envs/obj_recon/bin/python eval.py 4 | cd ../input 5 | mv video1 masks 6 | rm -rf JPEGImages 7 | rm -rf Annotations -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/cython/dist_maps.py: -------------------------------------------------------------------------------- 1 | import pyximport; pyximport.install(pyximport=True, language_level=3) 2 | # noinspection PyUnresolvedReferences 3 | from ._get_dist_maps import get_dist_maps -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/SuperGluePretrainedNetwork"] 2 | path = third_party/SuperGluePretrainedNetwork 3 | url = git@github.com:magicleap/SuperGluePretrainedNetwork.git 4 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/loftr_ot.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 4 | 5 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 6 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/loftr_ds.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 4 | 5 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 6 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import SigmoidForPred 2 | from .flip import AddHorizontalFlip 3 | from .zoom_in import ZoomIn 4 | from .limit_longest_side import LimitLongestSide 5 | from .crops import Crops 6 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/buggy_pos_enc/loftr_ot.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 5 | 6 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 7 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/loftr_ot_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 4 | 5 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 6 | 7 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 8 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/buggy_pos_enc/loftr_ds.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 5 | 6 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 7 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/loftr_ds_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 4 | 5 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 6 | 7 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 8 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/cython/_get_dist_maps.pyxbld: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | def make_ext(modname, pyxfilename): 4 | from distutils.extension import Extension 5 | return Extension(modname, [pyxfilename], 6 | include_dirs=[numpy.get_include()], 7 | extra_compile_args=['-O3'], language='c++') 8 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/environment.yaml: -------------------------------------------------------------------------------- 1 | name: loftr 2 | channels: 3 | # - https://dx-mirrors.sensetime.com/anaconda/cloud/pytorch 4 | - pytorch 5 | - conda-forge 6 | - defaults 7 | dependencies: 8 | - python=3.8 9 | - cudatoolkit=10.2 10 | - pytorch=1.8.1 11 | - pip 12 | - pip: 13 | - -r requirements.txt 14 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/buggy_pos_enc/loftr_ds_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 5 | 6 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 7 | 8 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 9 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/buggy_pos_enc/loftr_ot_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 5 | 6 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 7 | 8 | cfg.TRAINER.MSLR_MILESTONES = [3, 6, 9, 12, 17, 20, 23, 26, 29] 9 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/requirements.txt: -------------------------------------------------------------------------------- 1 | opencv_python==4.4.0.46 2 | albumentations==0.5.1 --no-binary=imgaug,albumentations 3 | ray>=1.0.1 4 | einops==0.3.0 5 | kornia==0.4.1 6 | loguru==0.5.3 7 | yacs>=0.1.8 8 | tqdm 9 | autopep8 10 | pylint 11 | ipython 12 | jupyterlab 13 | matplotlib 14 | h5py==3.1.0 15 | pytorch-lightning==1.3.5 16 | joblib>=1.0.1 17 | -------------------------------------------------------------------------------- /run_container.sh: -------------------------------------------------------------------------------- 1 | docker rm -f bundlesdf 2 | DIR=$(pwd)/ 3 | xhost + && docker run --privileged --gpus all --env NVIDIA_DISABLE_REQUIRE=1 -it --network=host --name bundlesdf --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v /home:/home -v /tmp:/tmp -v /mnt:/mnt -v $DIR:$DIR --ipc=host -e DISPLAY=${DISPLAY} -e GIT_INDEX_FILE nvcr.io/nvidian/bundlesdf:latest bash -c "cd $DIR && bash" 4 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/version.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #define SPDLOG_VER_MAJOR 1 9 | #define SPDLOG_VER_MINOR 3 10 | #define SPDLOG_VER_PATCH 1 11 | 12 | #define SPDLOG_VERSION (SPDLOG_VER_MAJOR * 10000 + SPDLOG_VER_MINOR * 100 + SPDLOG_VER_PATCH) 13 | -------------------------------------------------------------------------------- /XMem/dataset/range_transform.py: -------------------------------------------------------------------------------- 1 | import torchvision.transforms as transforms 2 | 3 | im_mean = (124, 116, 104) 4 | 5 | im_normalization = transforms.Normalize( 6 | mean=[0.485, 0.456, 0.406], 7 | std=[0.229, 0.224, 0.225] 8 | ) 9 | 10 | inv_im_trans = transforms.Normalize( 11 | mean=[-0.485/0.229, -0.456/0.224, -0.406/0.225], 12 | std=[1/0.229, 1/0.224, 1/0.225]) 13 | -------------------------------------------------------------------------------- /XMem/dataset/util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def all_to_onehot(masks, labels): 5 | if len(masks.shape) == 3: 6 | Ms = np.zeros((len(labels), masks.shape[0], masks.shape[1], masks.shape[2]), dtype=np.uint8) 7 | else: 8 | Ms = np.zeros((len(labels), masks.shape[0], masks.shape[1]), dtype=np.uint8) 9 | 10 | for ni, l in enumerate(labels): 11 | Ms[ni] = (masks == l).astype(np.uint8) 12 | 13 | return Ms 14 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/fmt/ostr.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | // 8 | // include bundled or external copy of fmtlib's ostream support 9 | // 10 | #if !defined(SPDLOG_FMT_EXTERNAL) 11 | #ifndef FMT_HEADER_ONLY 12 | #define FMT_HEADER_ONLY 13 | #endif 14 | #include "bundled/ostream.h" 15 | #include "fmt.h" 16 | #else 17 | #include 18 | #endif 19 | -------------------------------------------------------------------------------- /XMem/model/aggregate.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | 5 | # Soft aggregation from STM 6 | def aggregate(prob, dim, return_logits=False): 7 | new_prob = torch.cat([ 8 | torch.prod(1-prob, dim=dim, keepdim=True), 9 | prob 10 | ], dim).clamp(1e-7, 1-1e-7) 11 | logits = torch.log((new_prob /(1-new_prob))) 12 | prob = F.softmax(logits, dim=dim) 13 | 14 | if return_logits: 15 | return logits, prob 16 | else: 17 | return prob -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | __pycache__/ 3 | *.pyc 4 | *.DS_Store 5 | *.swp 6 | *.pth 7 | tmp.* 8 | */.ipynb_checkpoints/* 9 | 10 | logs/ 11 | weights/ 12 | dump/ 13 | demo/*.mp4 14 | demo/demo_images/ 15 | src/loftr/utils/superglue.py 16 | 17 | notebooks/QccDayNight.ipynb 18 | notebooks/westlake.ipynb 19 | assets/westlake 20 | assets/qcc_pairs.txt 21 | configs/.petrel* 22 | tools/draw_QccDayNights.py 23 | 24 | scripts/slurm/ 25 | scripts/sbatch_submit.sh 26 | src/utils/client.py 27 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from .resnet_fpn import ResNetFPN_8_2, ResNetFPN_16_4 2 | 3 | 4 | def build_backbone(config): 5 | if config['backbone_type'] == 'ResNetFPN': 6 | if config['resolution'] == (8, 2): 7 | return ResNetFPN_8_2(config['resnetfpn']) 8 | elif config['resolution'] == (16, 4): 9 | return ResNetFPN_16_4(config['resnetfpn']) 10 | else: 11 | raise ValueError(f"LOFTR.BACKBONE_TYPE {config['backbone_type']} not supported.") 12 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/functional/csrc/ext_lib.cpp: -------------------------------------------------------------------------------- 1 | #include "bn.h" 2 | 3 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 4 | m.def("syncbn_sum_sqsum", &syncbn_sum_sqsum, "Sum and Sum^2 computation"); 5 | m.def("syncbn_forward", &syncbn_forward, "SyncBN forward computation"); 6 | m.def("syncbn_backward_xhat", &syncbn_backward_xhat, 7 | "First part of SyncBN backward computation"); 8 | m.def("syncbn_backward", &syncbn_backward, 9 | "Second part of SyncBN backward computation"); 10 | } -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/loftr_ot.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 4 | 5 | cfg.TRAINER.CANONICAL_LR = 8e-3 6 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 7 | cfg.TRAINER.WARMUP_RATIO = 0.1 8 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 9 | 10 | # pose estimation 11 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 12 | 13 | cfg.TRAINER.OPTIMIZER = "adamw" 14 | cfg.TRAINER.ADAMW_DECAY = 0.1 15 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 16 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/loftr_ds.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 4 | 5 | cfg.TRAINER.CANONICAL_LR = 8e-3 6 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 7 | cfg.TRAINER.WARMUP_RATIO = 0.1 8 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 9 | 10 | # pose estimation 11 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 12 | 13 | cfg.TRAINER.OPTIMIZER = "adamw" 14 | cfg.TRAINER.ADAMW_DECAY = 0.1 15 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 16 | -------------------------------------------------------------------------------- /XMem/util/load_subset.py: -------------------------------------------------------------------------------- 1 | """ 2 | load_subset.py - Presents a subset of data 3 | DAVIS - only the training set 4 | YouTubeVOS - I manually filtered some erroneous ones out but I haven't checked all 5 | """ 6 | 7 | 8 | def load_sub_davis(path='util/davis_subset.txt'): 9 | with open(path, mode='r') as f: 10 | subset = set(f.read().splitlines()) 11 | return subset 12 | 13 | def load_sub_yv(path='util/yv_subset.txt'): 14 | with open(path, mode='r') as f: 15 | subset = set(f.read().splitlines()) 16 | return subset 17 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/formatter.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "fmt/fmt.h" 9 | #include "spdlog/details/log_msg.h" 10 | 11 | namespace spdlog { 12 | 13 | class formatter 14 | { 15 | public: 16 | virtual ~formatter() = default; 17 | virtual void format(const details::log_msg &msg, fmt::memory_buffer &dest) = 0; 18 | virtual std::unique_ptr clone() const = 0; 19 | }; 20 | } // namespace spdlog 21 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | *egg* 3 | ros/build/ 4 | ros/devel/ 5 | DATA/ 6 | *.blend* 7 | lfnet/ 8 | Experiments/ 9 | debug/ 10 | src/ThirdParty/bad_slam 11 | src/ThirdParty/dynamic_fusion 12 | src/ThirdParty/libvis 13 | src/ThirdParty/surfel_meshing 14 | **/build/ 15 | *.lib 16 | *.a 17 | *.so 18 | src/ThirdParty/BundleFusion/mLibExternal/ 19 | *.zip 20 | *.tar.gz 21 | **__pycache__** 22 | Features/ 23 | debug (copy)/ 24 | debug_exp9/ 25 | lf-net-release/release 26 | *.pth.tar 27 | masks/ 28 | *.tar.gz 29 | *.pth 30 | !SwiftNet/swiftnet_resnet18_old.pth -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/buggy_pos_enc/loftr_ot.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 5 | 6 | cfg.TRAINER.CANONICAL_LR = 8e-3 7 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 8 | cfg.TRAINER.WARMUP_RATIO = 0.1 9 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 10 | 11 | # pose estimation 12 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 13 | 14 | cfg.TRAINER.OPTIMIZER = "adamw" 15 | cfg.TRAINER.ADAMW_DECAY = 0.1 16 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 17 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/loftr_ot_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 4 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 5 | 6 | cfg.TRAINER.CANONICAL_LR = 8e-3 7 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 8 | cfg.TRAINER.WARMUP_RATIO = 0.1 9 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 10 | 11 | # pose estimation 12 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 13 | 14 | cfg.TRAINER.OPTIMIZER = "adamw" 15 | cfg.TRAINER.ADAMW_DECAY = 0.1 16 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 17 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/buggy_pos_enc/loftr_ds.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 5 | 6 | cfg.TRAINER.CANONICAL_LR = 8e-3 7 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 8 | cfg.TRAINER.WARMUP_RATIO = 0.1 9 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 10 | 11 | # pose estimation 12 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 13 | 14 | cfg.TRAINER.OPTIMIZER = "adamw" 15 | cfg.TRAINER.ADAMW_DECAY = 0.1 16 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 17 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/loftr_ds_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 4 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 5 | 6 | cfg.TRAINER.CANONICAL_LR = 8e-3 7 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 8 | cfg.TRAINER.WARMUP_RATIO = 0.1 9 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 10 | 11 | # pose estimation 12 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 13 | 14 | cfg.TRAINER.OPTIMIZER = "adamw" 15 | cfg.TRAINER.ADAMW_DECAY = 0.1 16 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 17 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/buggy_pos_enc/loftr_ot_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'sinkhorn' 5 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 6 | 7 | cfg.TRAINER.CANONICAL_LR = 8e-3 8 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 9 | cfg.TRAINER.WARMUP_RATIO = 0.1 10 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 11 | 12 | # pose estimation 13 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 14 | 15 | cfg.TRAINER.OPTIMIZER = "adamw" 16 | cfg.TRAINER.ADAMW_DECAY = 0.1 17 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 18 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/outdoor/buggy_pos_enc/loftr_ds_dense.py: -------------------------------------------------------------------------------- 1 | from src.config.default import _CN as cfg 2 | 3 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 4 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 5 | cfg.LOFTR.MATCH_COARSE.SPARSE_SPVS = False 6 | 7 | cfg.TRAINER.CANONICAL_LR = 8e-3 8 | cfg.TRAINER.WARMUP_STEP = 1875 # 3 epochs 9 | cfg.TRAINER.WARMUP_RATIO = 0.1 10 | cfg.TRAINER.MSLR_MILESTONES = [8, 12, 16, 20, 24] 11 | 12 | # pose estimation 13 | cfg.TRAINER.RANSAC_PIXEL_THR = 0.5 14 | 15 | cfg.TRAINER.OPTIMIZER = "adamw" 16 | cfg.TRAINER.ADAMW_DECAY = 0.1 17 | cfg.LOFTR.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.3 18 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/fmt/fmt.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016-2018 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // 9 | // Include a bundled header-only copy of fmtlib or an external one. 10 | // By default spdlog include its own copy. 11 | // 12 | 13 | #if !defined(SPDLOG_FMT_EXTERNAL) 14 | #ifndef FMT_HEADER_ONLY 15 | #define FMT_HEADER_ONLY 16 | #endif 17 | #ifndef FMT_USE_WINDOWS_H 18 | #define FMT_USE_WINDOWS_H 0 19 | #endif 20 | #include "bundled/core.h" 21 | #include "bundled/format.h" 22 | #else // external fmtlib 23 | #include 24 | #include 25 | #endif 26 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/CUDACameraUtil.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef _CUDA_CAMERA_UTIL 3 | #define _CUDA_CAMERA_UTIL 4 | 5 | #include "common.h" 6 | #include 7 | 8 | __inline__ __device__ float2 cameraToDepth(float fx, float fy, float cx, float cy, const float3& pos) 9 | { 10 | return make_float2( 11 | pos.x*fx / pos.z + cx, 12 | pos.y*fy / pos.z + cy); 13 | } 14 | __inline__ __device__ float3 depthToCamera(float fx, float fy, float cx, float cy, const int2& loc, float depth) { 15 | const float x = ((float)loc.x - cx) / fx; 16 | const float y = ((float)loc.y - cy) / fy; 17 | return make_float3(depth*x, depth*y, depth); 18 | } 19 | 20 | #endif //_CUDA_CAMERA_UTIL -------------------------------------------------------------------------------- /BundleSDF/segmentation_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION and its licensors retain all intellectual property 4 | # and proprietary rights in and to this software, related documentation 5 | # and any modifications thereto. Any use, reproduction, disclosure or 6 | # distribution of this software and related documentation without an express 7 | # license agreement from NVIDIA CORPORATION is strictly prohibited. 8 | 9 | import cv2 10 | import os 11 | import numpy as np 12 | 13 | class Segmenter(): 14 | def __int__(self): 15 | return 16 | 17 | def run(self, mask_file=None): 18 | return (cv2.imread(mask_file, -1)>0).astype(np.uint8) 19 | -------------------------------------------------------------------------------- /BundleSDF/mycuda/torch_ngp_grid_encoder/bindings.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION and its licensors retain all intellectual property 5 | * and proprietary rights in and to this software, related documentation 6 | * and any modifications thereto. Any use, reproduction, disclosure or 7 | * distribution of this software and related documentation without an express 8 | * license agreement from NVIDIA CORPORATION is strictly prohibited. 9 | */ 10 | 11 | 12 | #include 13 | 14 | #include "gridencoder.h" 15 | 16 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 17 | m.def("grid_encode_forward", &grid_encode_forward, "grid_encode_forward (CUDA)"); 18 | m.def("grid_encode_backward", &grid_encode_backward, "grid_encode_backward (CUDA)"); 19 | } -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/scannet/loftr_ds_eval.py: -------------------------------------------------------------------------------- 1 | """ A config only for reproducing the ScanNet evaluation results. 2 | 3 | We remove border matches by default, but the originally implemented 4 | `remove_border()` has a bug, leading to only two sides of 5 | all borders are actually removed. However, the [bug fix](https://github.com/zju3dv/LoFTR/commit/e9146c8144dea5f3cbdd98b225f3e147a171c216) 6 | makes the scannet evaluation results worse (auc@10=40.8 => 39.5), which should be 7 | caused by tiny result fluctuation of few image pairs. This config set `BORDER_RM` to 0 8 | to be consistent with the results in our paper. 9 | """ 10 | 11 | from src.config.default import _CN as cfg 12 | 13 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = False 14 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 15 | 16 | cfg.LOFTR.MATCH_COARSE.BORDER_RM = 0 17 | -------------------------------------------------------------------------------- /BundleSDF/mycuda/bindings.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION and its licensors retain all intellectual property 5 | * and proprietary rights in and to this software, related documentation 6 | * and any modifications thereto. Any use, reproduction, disclosure or 7 | * distribution of this software and related documentation without an express 8 | * license agreement from NVIDIA CORPORATION is strictly prohibited. 9 | */ 10 | 11 | 12 | #include 13 | #include "common.h" 14 | 15 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 16 | m.def("sampleRaysUniformOccupiedVoxels", &sampleRaysUniformOccupiedVoxels); 17 | m.def("postprocessOctreeRayTracing", &postprocessOctreeRayTracing); 18 | m.def("rayColorToTextureImageCUDA", &rayColorToTextureImageCUDA); 19 | } -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/LossGPU.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Utils.h" 4 | // #include "CUDACache.h" 5 | #include "SIFTImageManager.h" 6 | 7 | 8 | class OptimizerGpu 9 | { 10 | public: 11 | EIGEN_MAKE_ALIGNED_OPERATOR_NEW 12 | float _w_fm, _w_p2p, _w_rpi, _w_sdf, _w_pm; 13 | std::shared_ptr yml; 14 | std::string _id_str; 15 | 16 | public: 17 | OptimizerGpu(std::shared_ptr yml1); 18 | ~OptimizerGpu(); 19 | void optimizeFrames(const std::vector &global_corres, const std::vector &n_match_per_pair, int n_frames, int H, int W, const std::vector &depths_gpu, const std::vector &colors_gpu, const std::vector &normals_gpu, const std::vector &update_pose_flags, std::vector > &poses, const Eigen::Matrix3f &K); 20 | 21 | }; 22 | -------------------------------------------------------------------------------- /XMem/util/davis_subset.txt: -------------------------------------------------------------------------------- 1 | bear 2 | bmx-bumps 3 | boat 4 | boxing-fisheye 5 | breakdance-flare 6 | bus 7 | car-turn 8 | cat-girl 9 | classic-car 10 | color-run 11 | crossing 12 | dance-jump 13 | dancing 14 | disc-jockey 15 | dog-agility 16 | dog-gooses 17 | dogs-scale 18 | drift-turn 19 | drone 20 | elephant 21 | flamingo 22 | hike 23 | hockey 24 | horsejump-low 25 | kid-football 26 | kite-walk 27 | koala 28 | lady-running 29 | lindy-hop 30 | longboard 31 | lucia 32 | mallard-fly 33 | mallard-water 34 | miami-surf 35 | motocross-bumps 36 | motorbike 37 | night-race 38 | paragliding 39 | planes-water 40 | rallye 41 | rhino 42 | rollerblade 43 | schoolgirls 44 | scooter-board 45 | scooter-gray 46 | sheep 47 | skate-park 48 | snowboard 49 | soccerball 50 | stroller 51 | stunt 52 | surf 53 | swing 54 | tennis 55 | tractor-sand 56 | train 57 | tuk-tuk 58 | upside-down 59 | varanus-cage 60 | walking -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/details/null_mutex.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include 9 | // null, no cost dummy "mutex" and dummy "atomic" int 10 | 11 | namespace spdlog { 12 | namespace details { 13 | struct null_mutex 14 | { 15 | void lock() {} 16 | void unlock() {} 17 | bool try_lock() 18 | { 19 | return true; 20 | } 21 | }; 22 | 23 | struct null_atomic_int 24 | { 25 | int value; 26 | null_atomic_int() = default; 27 | 28 | explicit null_atomic_int(int val) 29 | : value(val) 30 | { 31 | } 32 | 33 | int load(std::memory_order) const 34 | { 35 | return value; 36 | } 37 | 38 | void store(int val) 39 | { 40 | value = val; 41 | } 42 | }; 43 | 44 | } // namespace details 45 | } // namespace spdlog 46 | -------------------------------------------------------------------------------- /XMem/inference/interact/timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | class Timer: 4 | def __init__(self): 5 | self._acc_time = 0 6 | self._paused = True 7 | 8 | def start(self): 9 | if self._paused: 10 | self.last_time = time.time() 11 | self._paused = False 12 | return self 13 | 14 | def pause(self): 15 | self.count() 16 | self._paused = True 17 | return self 18 | 19 | def count(self): 20 | if self._paused: 21 | return self._acc_time 22 | t = time.time() 23 | self._acc_time += t - self.last_time 24 | self.last_time = t 25 | return self._acc_time 26 | 27 | def format(self): 28 | # count = int(self.count()*100) 29 | # return '%02d:%02d:%02d' % (count//6000, (count//100)%60, count%100) 30 | return '%03.2f' % self.count() 31 | 32 | def __str__(self): 33 | return self.format() -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/configs/loftr/indoor/scannet/loftr_ds_eval_new.py: -------------------------------------------------------------------------------- 1 | """ A config only for reproducing the ScanNet evaluation results. 2 | 3 | We remove border matches by default, but the originally implemented 4 | `remove_border()` has a bug, leading to only two sides of 5 | all borders are actually removed. However, the [bug fix](https://github.com/zju3dv/LoFTR/commit/e9146c8144dea5f3cbdd98b225f3e147a171c216) 6 | makes the scannet evaluation results worse (auc@10=40.8 => 39.5), which should be 7 | caused by tiny result fluctuation of few image pairs. This config set `BORDER_RM` to 0 8 | to be consistent with the results in our paper. 9 | 10 | Update: This config is for testing the re-trained model with the pos-enc bug fixed. 11 | """ 12 | 13 | from src.config.default import _CN as cfg 14 | 15 | cfg.LOFTR.COARSE.TEMP_BUG_FIX = True 16 | cfg.LOFTR.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' 17 | 18 | cfg.LOFTR.MATCH_COARSE.BORDER_RM = 0 19 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/transforms/limit_longest_side.py: -------------------------------------------------------------------------------- 1 | from .zoom_in import ZoomIn, get_roi_image_nd 2 | 3 | 4 | class LimitLongestSide(ZoomIn): 5 | def __init__(self, max_size=800): 6 | super().__init__(target_size=max_size, skip_clicks=0) 7 | 8 | def transform(self, image_nd, clicks_lists): 9 | assert image_nd.shape[0] == 1 and len(clicks_lists) == 1 10 | image_max_size = max(image_nd.shape[2:4]) 11 | self.image_changed = False 12 | 13 | if image_max_size <= self.target_size: 14 | return image_nd, clicks_lists 15 | self._input_image = image_nd 16 | 17 | self._object_roi = (0, image_nd.shape[2] - 1, 0, image_nd.shape[3] - 1) 18 | self._roi_image = get_roi_image_nd(image_nd, self._object_roi, self.target_size) 19 | self.image_changed = True 20 | 21 | tclicks_lists = [self._transform_clicks(clicks_lists[0])] 22 | return self._roi_image, tclicks_lists 23 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/transforms/base.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class BaseTransform(object): 5 | def __init__(self): 6 | self.image_changed = False 7 | 8 | def transform(self, image_nd, clicks_lists): 9 | raise NotImplementedError 10 | 11 | def inv_transform(self, prob_map): 12 | raise NotImplementedError 13 | 14 | def reset(self): 15 | raise NotImplementedError 16 | 17 | def get_state(self): 18 | raise NotImplementedError 19 | 20 | def set_state(self, state): 21 | raise NotImplementedError 22 | 23 | 24 | class SigmoidForPred(BaseTransform): 25 | def transform(self, image_nd, clicks_lists): 26 | return image_nd, clicks_lists 27 | 28 | def inv_transform(self, prob_map): 29 | return torch.sigmoid(prob_map) 30 | 31 | def reset(self): 32 | pass 33 | 34 | def get_state(self): 35 | return None 36 | 37 | def set_state(self, state): 38 | pass 39 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_test/indoor_ot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/scannet_test_1500.py" 11 | main_cfg_path="configs/loftr/indoor/buggy_pos_enc/loftr_ot.py" 12 | ckpt_path="weights/indoor_ot.ckpt" 13 | dump_dir="dump/loftr_ot_indoor" 14 | profiler_name="inference" 15 | n_nodes=1 # mannually keep this the same with --nodes 16 | n_gpus_per_node=-1 17 | torch_num_workers=4 18 | batch_size=1 # per gpu 19 | 20 | python -u ./test.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --ckpt_path=${ckpt_path} \ 24 | --dump_dir=${dump_dir} \ 25 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 26 | --batch_size=${batch_size} --num_workers=${torch_num_workers}\ 27 | --profiler_name=${profiler_name} \ 28 | --benchmark 29 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_test/indoor_ds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/scannet_test_1500.py" 11 | main_cfg_path="configs/loftr/indoor/scannet/loftr_ds_eval.py" 12 | ckpt_path="weights/indoor_ds.ckpt" 13 | dump_dir="dump/loftr_ds_indoor" 14 | profiler_name="inference" 15 | n_nodes=1 # mannually keep this the same with --nodes 16 | n_gpus_per_node=-1 17 | torch_num_workers=4 18 | batch_size=1 # per gpu 19 | 20 | python -u ./test.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --ckpt_path=${ckpt_path} \ 24 | --dump_dir=${dump_dir} \ 25 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 26 | --batch_size=${batch_size} --num_workers=${torch_num_workers}\ 27 | --profiler_name=${profiler_name} \ 28 | --benchmark 29 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_test/outdoor_ot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/megadepth_test_1500.py" 11 | main_cfg_path="configs/loftr/outdoor/buggy_pos_enc/loftr_ot.py" 12 | ckpt_path="weights/outdoor_ot.ckpt" 13 | dump_dir="dump/loftr_ot_outdoor" 14 | profiler_name="inference" 15 | n_nodes=1 # mannually keep this the same with --nodes 16 | n_gpus_per_node=-1 17 | torch_num_workers=4 18 | batch_size=1 # per gpu 19 | 20 | python -u ./test.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --ckpt_path=${ckpt_path} \ 24 | --dump_dir=${dump_dir} \ 25 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 26 | --batch_size=${batch_size} --num_workers=${torch_num_workers}\ 27 | --profiler_name=${profiler_name} \ 28 | --benchmark 29 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_test/outdoor_ds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/megadepth_test_1500.py" 11 | main_cfg_path="configs/loftr/outdoor/buggy_pos_enc/loftr_ds.py" 12 | ckpt_path="weights/outdoor_ds.ckpt" 13 | dump_dir="dump/loftr_ds_outdoor" 14 | profiler_name="inference" 15 | n_nodes=1 # mannually keep this the same with --nodes 16 | n_gpus_per_node=-1 17 | torch_num_workers=4 18 | batch_size=1 # per gpu 19 | 20 | python -u ./test.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --ckpt_path=${ckpt_path} \ 24 | --dump_dir=${dump_dir} \ 25 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 26 | --batch_size=${batch_size} --num_workers=${torch_num_workers}\ 27 | --profiler_name=${profiler_name} \ 28 | --benchmark 29 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/utils/dataloader.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | # --- PL-DATAMODULE --- 5 | 6 | def get_local_split(items: list, world_size: int, rank: int, seed: int): 7 | """ The local rank only loads a split of the dataset. """ 8 | n_items = len(items) 9 | items_permute = np.random.RandomState(seed).permutation(items) 10 | if n_items % world_size == 0: 11 | padded_items = items_permute 12 | else: 13 | padding = np.random.RandomState(seed).choice( 14 | items, 15 | world_size - (n_items % world_size), 16 | replace=True) 17 | padded_items = np.concatenate([items_permute, padding]) 18 | assert len(padded_items) % world_size == 0, \ 19 | f'len(padded_items): {len(padded_items)}; world_size: {world_size}; len(padding): {len(padding)}' 20 | n_per_rank = len(padded_items) // world_size 21 | local_items = padded_items[n_per_rank * rank: n_per_rank * (rank+1)] 22 | 23 | return local_items 24 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_test/indoor_ds_new.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | # a indoor_ds model with the pos_enc impl bug fixed. 3 | 4 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 5 | PROJECT_DIR="${SCRIPTPATH}/../../" 6 | 7 | # conda activate loftr 8 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 9 | cd $PROJECT_DIR 10 | 11 | data_cfg_path="configs/data/scannet_test_1500.py" 12 | main_cfg_path="configs/loftr/indoor/scannet/loftr_ds_eval_new.py" 13 | ckpt_path="weights/indoor_ds_new.ckpt" 14 | dump_dir="dump/loftr_ds_indoor_new" 15 | profiler_name="inference" 16 | n_nodes=1 # mannually keep this the same with --nodes 17 | n_gpus_per_node=-1 18 | torch_num_workers=4 19 | batch_size=1 # per gpu 20 | 21 | python -u ./test.py \ 22 | ${data_cfg_path} \ 23 | ${main_cfg_path} \ 24 | --ckpt_path=${ckpt_path} \ 25 | --dump_dir=${dump_dir} \ 26 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 27 | --batch_size=${batch_size} --num_workers=${torch_num_workers}\ 28 | --profiler_name=${profiler_name} \ 29 | --benchmark 30 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/Solver/SolverBundlingParameters.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | struct SolverParameters 6 | { 7 | int depth_association_radius; 8 | unsigned int nNonLinearIterations; // Steps of the non-linear solver 9 | unsigned int nLinIterations; // Steps of the linear solver 10 | 11 | float verifyOptDistThresh; // for verifying local 12 | float verifyOptPercentThresh; 13 | 14 | float highResidualThresh; 15 | float robust_delta; 16 | float sparse_dist_thres = 99999; 17 | float sparse_normal_thres = -1; // cosine of normal dot product 18 | float icp_pose_rot_thres = M_PI*2; 19 | 20 | // dense depth corr 21 | float denseDistThresh; 22 | float denseNormalThresh; 23 | float denseColorThresh; 24 | float denseColorGradientMin; 25 | float denseDepthMin; 26 | float denseDepthMax; 27 | 28 | bool useDenseDepthAllPairwise; // instead of frame-to-frame 29 | unsigned int denseOverlapCheckSubsampleFactor; 30 | 31 | float weightSparse; 32 | float weightDenseDepth; 33 | float weightDenseColor; 34 | bool useDense; 35 | }; 36 | 37 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_train/indoor_ds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/scannet_trainval.py" 11 | main_cfg_path="configs/loftr/indoor/loftr_ds_dense.py" 12 | 13 | n_nodes=1 14 | n_gpus_per_node=4 15 | torch_num_workers=4 16 | batch_size=1 17 | pin_memory=true 18 | exp_name="indoor-ds-bs=$(($n_gpus_per_node * $n_nodes * $batch_size))" 19 | 20 | python -u ./train.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --exp_name=${exp_name} \ 24 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 25 | --batch_size=${batch_size} --num_workers=${torch_num_workers} --pin_memory=${pin_memory} \ 26 | --check_val_every_n_epoch=1 \ 27 | --log_every_n_steps=100 \ 28 | --flush_logs_every_n_steps=100 \ 29 | --limit_val_batches=1. \ 30 | --num_sanity_val_steps=10 \ 31 | --benchmark=True \ 32 | --max_epochs=30 \ 33 | --parallel_load_data 34 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_train/indoor_ot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | data_cfg_path="configs/data/scannet_trainval.py" 11 | main_cfg_path="configs/loftr/indoor/loftr_ot_dense.py" 12 | 13 | n_nodes=1 14 | n_gpus_per_node=4 15 | torch_num_workers=4 16 | batch_size=1 17 | pin_memory=true 18 | exp_name="indoor-ot-bs=$(($n_gpus_per_node * $n_nodes * $batch_size))" 19 | 20 | python -u ./train.py \ 21 | ${data_cfg_path} \ 22 | ${main_cfg_path} \ 23 | --exp_name=${exp_name} \ 24 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 25 | --batch_size=${batch_size} --num_workers=${torch_num_workers} --pin_memory=${pin_memory} \ 26 | --check_val_every_n_epoch=1 \ 27 | --log_every_n_steps=100 \ 28 | --flush_logs_every_n_steps=100 \ 29 | --limit_val_batches=1. \ 30 | --num_sanity_val_steps=10 \ 31 | --benchmark=True \ 32 | --max_epochs=30 \ 33 | --parallel_load_data 34 | -------------------------------------------------------------------------------- /BundleSDF/mycuda/torch_ngp_grid_encoder/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 hawkey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tamaki Kojima 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_train/outdoor_ds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | TRAIN_IMG_SIZE=640 11 | # to reproduced the results in our paper, please use: 12 | # TRAIN_IMG_SIZE=840 13 | data_cfg_path="configs/data/megadepth_trainval_${TRAIN_IMG_SIZE}.py" 14 | main_cfg_path="configs/loftr/outdoor/loftr_ds_dense.py" 15 | 16 | n_nodes=1 17 | n_gpus_per_node=4 18 | torch_num_workers=4 19 | batch_size=1 20 | pin_memory=true 21 | exp_name="outdoor-ds-${TRAIN_IMG_SIZE}-bs=$(($n_gpus_per_node * $n_nodes * $batch_size))" 22 | 23 | python -u ./train.py \ 24 | ${data_cfg_path} \ 25 | ${main_cfg_path} \ 26 | --exp_name=${exp_name} \ 27 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 28 | --batch_size=${batch_size} --num_workers=${torch_num_workers} --pin_memory=${pin_memory} \ 29 | --check_val_every_n_epoch=1 \ 30 | --log_every_n_steps=1 \ 31 | --flush_logs_every_n_steps=1 \ 32 | --limit_val_batches=1. \ 33 | --num_sanity_val_steps=10 \ 34 | --benchmark=True \ 35 | --max_epochs=30 36 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/scripts/reproduce_train/outdoor_ot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | SCRIPTPATH=$(dirname $(readlink -f "$0")) 4 | PROJECT_DIR="${SCRIPTPATH}/../../" 5 | 6 | # conda activate loftr 7 | export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH 8 | cd $PROJECT_DIR 9 | 10 | TRAIN_IMG_SIZE=640 11 | # to reproduced the results in our paper, please use: 12 | # TRAIN_IMG_SIZE=840 13 | data_cfg_path="configs/data/megadepth_trainval_${TRAIN_IMG_SIZE}.py" 14 | main_cfg_path="configs/loftr/outdoor/loftr_ot_dense.py" 15 | 16 | n_nodes=1 17 | n_gpus_per_node=4 18 | torch_num_workers=4 19 | batch_size=1 20 | pin_memory=true 21 | exp_name="outdoor-ot-${TRAIN_IMG_SIZE}-bs=$(($n_gpus_per_node * $n_nodes * $batch_size))" 22 | 23 | python -u ./train.py \ 24 | ${data_cfg_path} \ 25 | ${main_cfg_path} \ 26 | --exp_name=${exp_name} \ 27 | --gpus=${n_gpus_per_node} --num_nodes=${n_nodes} --accelerator="ddp" \ 28 | --batch_size=${batch_size} --num_workers=${torch_num_workers} --pin_memory=${pin_memory} \ 29 | --check_val_every_n_epoch=1 \ 30 | --log_every_n_steps=1 \ 31 | --flush_logs_every_n_steps=1 \ 32 | --limit_val_batches=1. \ 33 | --num_sanity_val_steps=10 \ 34 | --benchmark=True \ 35 | --max_epochs=30 36 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/msvc_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Alexander Dalshov. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #if defined(_WIN32) 13 | 14 | #include "spdlog/details/null_mutex.h" 15 | #include "spdlog/sinks/base_sink.h" 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | 22 | namespace spdlog { 23 | namespace sinks { 24 | /* 25 | * MSVC sink (logging using OutputDebugStringA) 26 | */ 27 | template 28 | class msvc_sink : public base_sink 29 | { 30 | public: 31 | explicit msvc_sink() {} 32 | 33 | protected: 34 | void sink_it_(const details::log_msg &msg) override 35 | { 36 | 37 | fmt::memory_buffer formatted; 38 | sink::formatter_->format(msg, formatted); 39 | OutputDebugStringA(fmt::to_string(formatted).c_str()); 40 | } 41 | 42 | void flush_() override {} 43 | }; 44 | 45 | using msvc_sink_mt = msvc_sink; 46 | using msvc_sink_st = msvc_sink; 47 | 48 | using windebug_sink_mt = msvc_sink_mt; 49 | using windebug_sink_st = msvc_sink_st; 50 | 51 | } // namespace sinks 52 | } // namespace spdlog 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /XMem/dataset/tps.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | import cv2 4 | import thinplate as tps 5 | 6 | cv2.setNumThreads(0) 7 | 8 | def pick_random_points(h, w, n_samples): 9 | y_idx = np.random.choice(np.arange(h), size=n_samples, replace=False) 10 | x_idx = np.random.choice(np.arange(w), size=n_samples, replace=False) 11 | return y_idx/h, x_idx/w 12 | 13 | 14 | def warp_dual_cv(img, mask, c_src, c_dst): 15 | dshape = img.shape 16 | theta = tps.tps_theta_from_points(c_src, c_dst, reduced=True) 17 | grid = tps.tps_grid(theta, c_dst, dshape) 18 | mapx, mapy = tps.tps_grid_to_remap(grid, img.shape) 19 | return cv2.remap(img, mapx, mapy, cv2.INTER_LINEAR), cv2.remap(mask, mapx, mapy, cv2.INTER_NEAREST) 20 | 21 | 22 | def random_tps_warp(img, mask, scale, n_ctrl_pts=12): 23 | """ 24 | Apply a random TPS warp of the input image and mask 25 | Uses randomness from numpy 26 | """ 27 | img = np.asarray(img) 28 | mask = np.asarray(mask) 29 | 30 | h, w = mask.shape 31 | points = pick_random_points(h, w, n_ctrl_pts) 32 | c_src = np.stack(points, 1) 33 | c_dst = c_src + np.random.normal(scale=scale, size=c_src.shape) 34 | warp_im, warp_gt = warp_dual_cv(img, mask, c_src, c_dst) 35 | 36 | return Image.fromarray(warp_im), Image.fromarray(warp_gt) 37 | 38 | -------------------------------------------------------------------------------- /conv_png_to_jpeg.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from PIL import Image 4 | 5 | # Define the source and destination paths 6 | source_folder = 'input/rgb' 7 | destination_folder = os.path.join('input/JPEGImages', 'video1') 8 | 9 | # Create the destination folder if it doesn't exist 10 | os.makedirs(destination_folder, exist_ok=True) 11 | 12 | # Function to convert PNG to JPG 13 | def convert_png_to_jpg(source_path, destination_path): 14 | with Image.open(source_path) as img: 15 | rgb_img = img.convert('RGB') 16 | destination_path = destination_path.replace('.png', '.jpg') 17 | rgb_img.save(destination_path, 'JPEG') 18 | 19 | # Copy the contents of the source folder to the destination folder 20 | for item in os.listdir(source_folder): 21 | source_item = os.path.join(source_folder, item) 22 | destination_item = os.path.join(destination_folder, item) 23 | 24 | if os.path.isdir(source_item): 25 | shutil.copytree(source_item, destination_item, dirs_exist_ok=True) 26 | else: 27 | if source_item.lower().endswith('.png'): 28 | convert_png_to_jpg(source_item, destination_item) 29 | else: 30 | shutil.copy2(source_item, destination_item) 31 | 32 | print(f'Contents of {source_folder} copied to {destination_folder}, and PNG files converted to JPG') 33 | -------------------------------------------------------------------------------- /mask.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import numpy as np 4 | 5 | points = [] 6 | 7 | def select_points(event, x, y, flags, param): 8 | if event == cv2.EVENT_LBUTTONDOWN: 9 | points.append((x, y)) 10 | cv2.circle(image_display, (x, y), 3, (0, 255, 0), -1) 11 | cv2.imshow("Image", image_display) 12 | 13 | def create_mask(image, points): 14 | mask = np.zeros(image.shape[:2], dtype=np.uint8) 15 | points_array = np.array(points, dtype=np.int32) 16 | cv2.fillPoly(mask, [points_array], 255) 17 | return mask 18 | 19 | 20 | image_path = './input/JPEGImages/video1/1668813025164826994.jpg' 21 | image = cv2.imread(image_path) 22 | image_display = image.copy() 23 | 24 | cv2.namedWindow("Image") 25 | cv2.setMouseCallback("Image", select_points) 26 | 27 | print("Click on the image to select points. Press Enter when done.") 28 | 29 | while True: 30 | cv2.imshow("Image", image_display) 31 | key = cv2.waitKey(1) & 0xFF 32 | if key == 13: # Enter key 33 | break 34 | 35 | mask = create_mask(image, points) 36 | # Make folder if not exist 37 | os.makedirs('./input/Annotations/video1', exist_ok=True) 38 | 39 | # Save the mask image 40 | cv2.imwrite('./input/Annotations/video1/1668813025164826994.png', mask) 41 | 42 | cv2.imshow('Mask', mask) 43 | cv2.waitKey(0) 44 | cv2.destroyAllWindows() 45 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/functional/csrc/cuda/ext_lib.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | CUDA SyncBN code 4 | 5 | *****************************************************************************/ 6 | #pragma once 7 | #include 8 | #include 9 | 10 | /// Sync-BN 11 | std::vector syncbn_sum_sqsum_cuda(const at::Tensor& x); 12 | at::Tensor syncbn_forward_cuda(const at::Tensor& x, const at::Tensor& weight, 13 | const at::Tensor& bias, const at::Tensor& mean, 14 | const at::Tensor& var, bool affine, float eps); 15 | std::vector syncbn_backward_xhat_cuda(const at::Tensor& dz, 16 | const at::Tensor& x, 17 | const at::Tensor& mean, 18 | const at::Tensor& var, 19 | float eps); 20 | std::vector syncbn_backward_cuda( 21 | const at::Tensor& dz, const at::Tensor& x, const at::Tensor& weight, 22 | const at::Tensor& bias, const at::Tensor& mean, const at::Tensor& var, 23 | const at::Tensor& sum_dz, const at::Tensor& sum_dz_xhat, bool affine, 24 | float eps); 25 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/demo/run_demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | # set -x 4 | 5 | if [ ! -f utils.py ]; then 6 | echo "Downloading utils.py from the SuperGlue repo." 7 | echo "We cannot provide this file directly due to its strict licence." 8 | wget https://raw.githubusercontent.com/magicleap/SuperGluePretrainedNetwork/master/models/utils.py 9 | fi 10 | 11 | # Use webcam 0 as input source. 12 | input=0 13 | # or use a pre-recorded video given the path. 14 | # input=/home/sunjiaming/Downloads/scannet_test/$scene_name.mp4 15 | 16 | # Toggle indoor/outdoor model here. 17 | model_ckpt=../weights/indoor_ds.ckpt 18 | # model_ckpt=../weights/outdoor_ds.ckpt 19 | 20 | # Optionally assign the GPU ID. 21 | # export CUDA_VISIBLE_DEVICES=0 22 | 23 | # echo "Running LoFTR demo.." 24 | # eval "$(conda shell.bash hook)" 25 | # conda activate loftr 26 | # python demo_loftr.py --weight $model_ckpt --input $input 27 | # To save the input video and output match visualizations. 28 | # python demo_loftr.py --weight $model_ckpt --input $input --save_video --save_input 29 | 30 | # Running on remote GPU servers with no GUI. 31 | # Save images first. 32 | # python demo_loftr.py --weight $model_ckpt --input $input --no_display --output_dir="./demo_images/" 33 | # Then convert them to a video. 34 | # ffmpeg -framerate 15 -pattern_type glob -i '*.png' -c:v libx264 -r 30 -pix_fmt yuv420p out.mp4 35 | -------------------------------------------------------------------------------- /XMem/inference/interact/gui_utils.py: -------------------------------------------------------------------------------- 1 | from PySide6.QtCore import Qt 2 | from PySide6.QtWidgets import (QBoxLayout, QHBoxLayout, QLabel, QSpinBox, QVBoxLayout, QProgressBar) 3 | 4 | 5 | def create_parameter_box(min_val, max_val, text, step=1, callback=None): 6 | layout = QHBoxLayout() 7 | 8 | dial = QSpinBox() 9 | dial.setMaximumHeight(28) 10 | dial.setMaximumWidth(150) 11 | dial.setMinimum(min_val) 12 | dial.setMaximum(max_val) 13 | dial.setAlignment(Qt.AlignmentFlag.AlignRight) 14 | dial.setSingleStep(step) 15 | dial.valueChanged.connect(callback) 16 | 17 | label = QLabel(text) 18 | label.setAlignment(Qt.AlignmentFlag.AlignRight) 19 | 20 | layout.addWidget(label) 21 | layout.addWidget(dial) 22 | 23 | return dial, layout 24 | 25 | 26 | def create_gauge(text): 27 | layout = QHBoxLayout() 28 | 29 | gauge = QProgressBar() 30 | gauge.setMaximumHeight(28) 31 | gauge.setMaximumWidth(200) 32 | gauge.setAlignment(Qt.AlignmentFlag.AlignCenter) 33 | 34 | label = QLabel(text) 35 | label.setAlignment(Qt.AlignmentFlag.AlignRight) 36 | 37 | layout.addWidget(label) 38 | layout.addWidget(gauge) 39 | 40 | return gauge, layout 41 | 42 | 43 | def apply_to_all_children_widget(layout, func): 44 | # deliberately non-recursive 45 | for i in range(layout.count()): 46 | func(layout.itemAt(i).widget()) -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/utils/profiler.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from pytorch_lightning.profiler import SimpleProfiler, PassThroughProfiler 3 | from contextlib import contextmanager 4 | from pytorch_lightning.utilities import rank_zero_only 5 | 6 | 7 | class InferenceProfiler(SimpleProfiler): 8 | """ 9 | This profiler records duration of actions with cuda.synchronize() 10 | Use this in test time. 11 | """ 12 | 13 | def __init__(self): 14 | super().__init__() 15 | self.start = rank_zero_only(self.start) 16 | self.stop = rank_zero_only(self.stop) 17 | self.summary = rank_zero_only(self.summary) 18 | 19 | @contextmanager 20 | def profile(self, action_name: str) -> None: 21 | try: 22 | torch.cuda.synchronize() 23 | self.start(action_name) 24 | yield action_name 25 | finally: 26 | torch.cuda.synchronize() 27 | self.stop(action_name) 28 | 29 | 30 | def build_profiler(name): 31 | if name == 'inference': 32 | return InferenceProfiler() 33 | elif name == 'pytorch': 34 | from pytorch_lightning.profiler import PyTorchProfiler 35 | return PyTorchProfiler(use_cuda=True, profile_memory=True, row_limit=100) 36 | elif name is None: 37 | return PassThroughProfiler() 38 | else: 39 | raise ValueError(f'Invalid profiler: {name}') 40 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/null_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "spdlog/details/null_mutex.h" 13 | #include "spdlog/sinks/base_sink.h" 14 | 15 | #include 16 | 17 | namespace spdlog { 18 | namespace sinks { 19 | 20 | template 21 | class null_sink : public base_sink 22 | { 23 | protected: 24 | void sink_it_(const details::log_msg &) override {} 25 | void flush_() override {} 26 | }; 27 | 28 | using null_sink_mt = null_sink; 29 | using null_sink_st = null_sink; 30 | 31 | } // namespace sinks 32 | 33 | template 34 | inline std::shared_ptr null_logger_mt(const std::string &logger_name) 35 | { 36 | auto null_logger = Factory::template create(logger_name); 37 | null_logger->set_level(level::off); 38 | return null_logger; 39 | } 40 | 41 | template 42 | inline std::shared_ptr null_logger_st(const std::string &logger_name) 43 | { 44 | auto null_logger = Factory::template create(logger_name); 45 | null_logger->set_level(level::off); 46 | return null_logger; 47 | } 48 | 49 | } // namespace spdlog 50 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/transforms/flip.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ..clicker import Click 4 | from .base import BaseTransform 5 | 6 | 7 | class AddHorizontalFlip(BaseTransform): 8 | def transform(self, image_nd, clicks_lists): 9 | assert len(image_nd.shape) == 4 10 | image_nd = torch.cat([image_nd, torch.flip(image_nd, dims=[3])], dim=0) 11 | 12 | image_width = image_nd.shape[3] 13 | clicks_lists_flipped = [] 14 | for clicks_list in clicks_lists: 15 | clicks_list_flipped = [Click(is_positive=click.is_positive, 16 | coords=(click.coords[0], image_width - click.coords[1] - 1)) 17 | for click in clicks_list] 18 | clicks_lists_flipped.append(clicks_list_flipped) 19 | clicks_lists = clicks_lists + clicks_lists_flipped 20 | 21 | return image_nd, clicks_lists 22 | 23 | def inv_transform(self, prob_map): 24 | assert len(prob_map.shape) == 4 and prob_map.shape[0] % 2 == 0 25 | num_maps = prob_map.shape[0] // 2 26 | prob_map, prob_map_flipped = prob_map[:num_maps], prob_map[num_maps:] 27 | 28 | return 0.5 * (prob_map + torch.flip(prob_map_flipped, dims=[3])) 29 | 30 | def get_state(self): 31 | return None 32 | 33 | def set_state(self, state): 34 | pass 35 | 36 | def reset(self): 37 | pass 38 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/fmt/bundled/LICENSE.rst: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 - 2016, Victor Zverovich 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /BundleSDF/mycuda/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION and its licensors retain all intellectual property 5 | * and proprietary rights in and to this software, related documentation 6 | * and any modifications thereto. Any use, reproduction, disclosure or 7 | * distribution of this software and related documentation without an express 8 | * license agreement from NVIDIA CORPORATION is strictly prohibited. 9 | */ 10 | 11 | 12 | #pragma once 13 | 14 | #include 15 | #include 16 | 17 | #define CHECK_CUDA(x) AT_ASSERTM(x.is_cuda(), #x " must be a CUDA tensor") 18 | #define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x " must be contiguous") 19 | #define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) 20 | 21 | 22 | inline int divCeil(int a, int b) 23 | { 24 | return (a+b-1)/b; 25 | }; 26 | 27 | 28 | at::Tensor sampleRaysUniformOccupiedVoxels(const at::Tensor z_in_out, const at::Tensor z_sampled, at::Tensor z_vals); 29 | at::Tensor postprocessOctreeRayTracing(const at::Tensor ray_index, const at::Tensor depth_in_out, const at::Tensor unique_intersect_ray_ids, const at::Tensor start_poss, const int max_intersections, const int N_rays); 30 | void rayColorToTextureImageCUDA(const at::Tensor &F, const at::Tensor &V, const at::Tensor &hit_locations, const at::Tensor &hit_face_ids, const at::Tensor &uvs_tex, at::Tensor &uvs); 31 | -------------------------------------------------------------------------------- /XMem/util/tensor_util.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | 3 | 4 | def compute_tensor_iu(seg, gt): 5 | intersection = (seg & gt).float().sum() 6 | union = (seg | gt).float().sum() 7 | 8 | return intersection, union 9 | 10 | def compute_tensor_iou(seg, gt): 11 | intersection, union = compute_tensor_iu(seg, gt) 12 | iou = (intersection + 1e-6) / (union + 1e-6) 13 | 14 | return iou 15 | 16 | # STM 17 | def pad_divide_by(in_img, d): 18 | h, w = in_img.shape[-2:] 19 | 20 | if h % d > 0: 21 | new_h = h + d - h % d 22 | else: 23 | new_h = h 24 | if w % d > 0: 25 | new_w = w + d - w % d 26 | else: 27 | new_w = w 28 | lh, uh = int((new_h-h) / 2), int(new_h-h) - int((new_h-h) / 2) 29 | lw, uw = int((new_w-w) / 2), int(new_w-w) - int((new_w-w) / 2) 30 | pad_array = (int(lw), int(uw), int(lh), int(uh)) 31 | out = F.pad(in_img, pad_array) 32 | return out, pad_array 33 | 34 | def unpad(img, pad): 35 | if len(img.shape) == 4: 36 | if pad[2]+pad[3] > 0: 37 | img = img[:,:,pad[2]:-pad[3],:] 38 | if pad[0]+pad[1] > 0: 39 | img = img[:,:,:,pad[0]:-pad[1]] 40 | elif len(img.shape) == 3: 41 | if pad[2]+pad[3] > 0: 42 | img = img[:,pad[2]:-pad[3],:] 43 | if pad[0]+pad[1] > 0: 44 | img = img[:,:,pad[0]:-pad[1]] 45 | else: 46 | raise NotImplementedError 47 | return img -------------------------------------------------------------------------------- /BundleSDF/mycuda/torch_ngp_grid_encoder/gridencoder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION and its licensors retain all intellectual property 5 | * and proprietary rights in and to this software, related documentation 6 | * and any modifications thereto. Any use, reproduction, disclosure or 7 | * distribution of this software and related documentation without an express 8 | * license agreement from NVIDIA CORPORATION is strictly prohibited. 9 | */ 10 | 11 | 12 | #ifndef _HASH_ENCODE_H 13 | #define _HASH_ENCODE_H 14 | 15 | #include 16 | #include 17 | 18 | // inputs: [B, D], float, in [0, 1] 19 | // embeddings: [sO, C], float 20 | // offsets: [L + 1], uint32_t 21 | // outputs: [B, L * C], float 22 | // H: base resolution 23 | void grid_encode_forward(const at::Tensor inputs, const at::Tensor embeddings, const at::Tensor offsets, at::Tensor outputs, const uint32_t B, const uint32_t D, const uint32_t C, const uint32_t L, const float S, const uint32_t H, const bool calc_grad_inputs, at::Tensor dy_dx, const uint32_t gridtype, const bool align_corners); 24 | void grid_encode_backward(const at::Tensor grad, const at::Tensor inputs, const at::Tensor embeddings, const at::Tensor offsets, at::Tensor grad_embeddings, const uint32_t B, const uint32_t D, const uint32_t C, const uint32_t L, const float S, const uint32_t H, const bool calc_grad_inputs, const at::Tensor dy_dx, at::Tensor grad_inputs, const uint32_t gridtype, const bool align_corners); 25 | 26 | #endif -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/details/log_msg.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/common.h" 9 | #include "spdlog/details/os.h" 10 | 11 | #include 12 | #include 13 | 14 | namespace spdlog { 15 | namespace details { 16 | struct log_msg 17 | { 18 | 19 | log_msg(source_loc loc, const std::string *loggers_name, level::level_enum lvl, string_view_t view) 20 | : logger_name(loggers_name) 21 | , level(lvl) 22 | #ifndef SPDLOG_NO_DATETIME 23 | , time(os::now()) 24 | #endif 25 | 26 | #ifndef SPDLOG_NO_THREAD_ID 27 | , thread_id(os::thread_id()) 28 | #endif 29 | , source(loc) 30 | , payload(view) 31 | { 32 | } 33 | 34 | log_msg(const std::string *loggers_name, level::level_enum lvl, string_view_t view) 35 | : log_msg(source_loc{}, loggers_name, lvl, view) 36 | { 37 | } 38 | 39 | log_msg(const log_msg &other) = default; 40 | 41 | const std::string *logger_name{nullptr}; 42 | level::level_enum level{level::off}; 43 | log_clock::time_point time; 44 | size_t thread_id{0}; 45 | size_t msg_id{0}; 46 | 47 | // wrapping the formatted text with color (updated by pattern_formatter). 48 | mutable size_t color_range_start{0}; 49 | mutable size_t color_range_end{0}; 50 | 51 | source_loc source; 52 | const string_view_t payload; 53 | }; 54 | } // namespace details 55 | } // namespace spdlog 56 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/ostream_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "spdlog/details/null_mutex.h" 13 | #include "spdlog/sinks/base_sink.h" 14 | 15 | #include 16 | #include 17 | 18 | namespace spdlog { 19 | namespace sinks { 20 | template 21 | class ostream_sink final : public base_sink 22 | { 23 | public: 24 | explicit ostream_sink(std::ostream &os, bool force_flush = false) 25 | : ostream_(os) 26 | , force_flush_(force_flush) 27 | { 28 | } 29 | ostream_sink(const ostream_sink &) = delete; 30 | ostream_sink &operator=(const ostream_sink &) = delete; 31 | 32 | protected: 33 | void sink_it_(const details::log_msg &msg) override 34 | { 35 | fmt::memory_buffer formatted; 36 | sink::formatter_->format(msg, formatted); 37 | ostream_.write(formatted.data(), static_cast(formatted.size())); 38 | if (force_flush_) 39 | { 40 | ostream_.flush(); 41 | } 42 | } 43 | 44 | void flush_() override 45 | { 46 | ostream_.flush(); 47 | } 48 | 49 | std::ostream &ostream_; 50 | bool force_flush_; 51 | }; 52 | 53 | using ostream_sink_mt = ostream_sink; 54 | using ostream_sink_st = ostream_sink; 55 | 56 | } // namespace sinks 57 | } // namespace spdlog 58 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/details/console_globals.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | // 3 | // Copyright(c) 2018 Gabi Melman. 4 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 5 | // 6 | 7 | #include "spdlog/details/null_mutex.h" 8 | #include 9 | #include 10 | 11 | #ifdef _WIN32 12 | 13 | #ifndef NOMINMAX 14 | #define NOMINMAX // prevent windows redefining min/max 15 | #endif 16 | 17 | #ifndef WIN32_LEAN_AND_MEAN 18 | #define WIN32_LEAN_AND_MEAN 19 | #endif 20 | 21 | #include 22 | #endif 23 | 24 | namespace spdlog { 25 | namespace details { 26 | struct console_stdout 27 | { 28 | static std::FILE *stream() 29 | { 30 | return stdout; 31 | } 32 | #ifdef _WIN32 33 | static HANDLE handle() 34 | { 35 | return ::GetStdHandle(STD_OUTPUT_HANDLE); 36 | } 37 | #endif 38 | }; 39 | 40 | struct console_stderr 41 | { 42 | static std::FILE *stream() 43 | { 44 | return stderr; 45 | } 46 | #ifdef _WIN32 47 | static HANDLE handle() 48 | { 49 | return ::GetStdHandle(STD_ERROR_HANDLE); 50 | } 51 | #endif 52 | }; 53 | 54 | struct console_mutex 55 | { 56 | using mutex_t = std::mutex; 57 | static mutex_t &mutex() 58 | { 59 | static mutex_t s_mutex; 60 | return s_mutex; 61 | } 62 | }; 63 | 64 | struct console_nullmutex 65 | { 66 | using mutex_t = null_mutex; 67 | static mutex_t &mutex() 68 | { 69 | static mutex_t s_mutex; 70 | return s_mutex; 71 | } 72 | }; 73 | } // namespace details 74 | } // namespace spdlog 75 | -------------------------------------------------------------------------------- /BundleSDF/mycuda/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION and its licensors retain all intellectual property 4 | # and proprietary rights in and to this software, related documentation 5 | # and any modifications thereto. Any use, reproduction, disclosure or 6 | # distribution of this software and related documentation without an express 7 | # license agreement from NVIDIA CORPORATION is strictly prohibited. 8 | 9 | 10 | from setuptools import setup 11 | import os,sys 12 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 13 | from torch.utils.cpp_extension import load 14 | 15 | code_dir = os.path.dirname(os.path.realpath(__file__)) 16 | 17 | 18 | nvcc_flags = ['-Xcompiler', '-O3', '-std=c++14', '-U__CUDA_NO_HALF_OPERATORS__', '-U__CUDA_NO_HALF_CONVERSIONS__', '-U__CUDA_NO_HALF2_OPERATORS__'] 19 | c_flags = ['-O3', '-std=c++14'] 20 | 21 | setup( 22 | name='common', 23 | extra_cflags=c_flags, 24 | extra_cuda_cflags=nvcc_flags, 25 | ext_modules=[ 26 | CUDAExtension('common', [ 27 | 'bindings.cpp', 28 | 'common.cu', 29 | ],extra_compile_args={'gcc': c_flags, 'nvcc': nvcc_flags}), 30 | CUDAExtension('gridencoder', [ 31 | f"{code_dir}/torch_ngp_grid_encoder/gridencoder.cu", 32 | f"{code_dir}/torch_ngp_grid_encoder/bindings.cpp", 33 | ],extra_compile_args={'gcc': c_flags, 'nvcc': nvcc_flags}), 34 | ], 35 | include_dirs=[ 36 | "/usr/local/include/eigen3", 37 | "/usr/include/eigen3", 38 | ], 39 | cmdclass={ 40 | 'build_ext': BuildExtension 41 | }) 42 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.optim.lr_scheduler import MultiStepLR, CosineAnnealingLR, ExponentialLR 3 | 4 | 5 | def build_optimizer(model, config): 6 | name = config.TRAINER.OPTIMIZER 7 | lr = config.TRAINER.TRUE_LR 8 | 9 | if name == "adam": 10 | return torch.optim.Adam(model.parameters(), lr=lr, weight_decay=config.TRAINER.ADAM_DECAY) 11 | elif name == "adamw": 12 | return torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=config.TRAINER.ADAMW_DECAY) 13 | else: 14 | raise ValueError(f"TRAINER.OPTIMIZER = {name} is not a valid optimizer!") 15 | 16 | 17 | def build_scheduler(config, optimizer): 18 | """ 19 | Returns: 20 | scheduler (dict):{ 21 | 'scheduler': lr_scheduler, 22 | 'interval': 'step', # or 'epoch' 23 | 'monitor': 'val_f1', (optional) 24 | 'frequency': x, (optional) 25 | } 26 | """ 27 | scheduler = {'interval': config.TRAINER.SCHEDULER_INTERVAL} 28 | name = config.TRAINER.SCHEDULER 29 | 30 | if name == 'MultiStepLR': 31 | scheduler.update( 32 | {'scheduler': MultiStepLR(optimizer, config.TRAINER.MSLR_MILESTONES, gamma=config.TRAINER.MSLR_GAMMA)}) 33 | elif name == 'CosineAnnealing': 34 | scheduler.update( 35 | {'scheduler': CosineAnnealingLR(optimizer, config.TRAINER.COSA_TMAX)}) 36 | elif name == 'ExponentialLR': 37 | scheduler.update( 38 | {'scheduler': ExponentialLR(optimizer, config.TRAINER.ELR_GAMMA)}) 39 | else: 40 | raise NotImplementedError() 41 | 42 | return scheduler 43 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/modeling/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from .resnetv1b import resnet34_v1b, resnet50_v1s, resnet101_v1s, resnet152_v1s 3 | 4 | 5 | class ResNetBackbone(torch.nn.Module): 6 | def __init__(self, backbone='resnet50', pretrained_base=True, dilated=True, **kwargs): 7 | super(ResNetBackbone, self).__init__() 8 | 9 | if backbone == 'resnet34': 10 | pretrained = resnet34_v1b(pretrained=pretrained_base, dilated=dilated, **kwargs) 11 | elif backbone == 'resnet50': 12 | pretrained = resnet50_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs) 13 | elif backbone == 'resnet101': 14 | pretrained = resnet101_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs) 15 | elif backbone == 'resnet152': 16 | pretrained = resnet152_v1s(pretrained=pretrained_base, dilated=dilated, **kwargs) 17 | else: 18 | raise RuntimeError(f'unknown backbone: {backbone}') 19 | 20 | self.conv1 = pretrained.conv1 21 | self.bn1 = pretrained.bn1 22 | self.relu = pretrained.relu 23 | self.maxpool = pretrained.maxpool 24 | self.layer1 = pretrained.layer1 25 | self.layer2 = pretrained.layer2 26 | self.layer3 = pretrained.layer3 27 | self.layer4 = pretrained.layer4 28 | 29 | def forward(self, x): 30 | x = self.conv1(x) 31 | x = self.bn1(x) 32 | x = self.relu(x) 33 | x = self.maxpool(x) 34 | c1 = self.layer1(x) 35 | c2 = self.layer2(c1) 36 | c3 = self.layer3(c2) 37 | c4 = self.layer4(c3) 38 | 39 | return c1, c2, c3, c4 40 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/CUDACacheUtil.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef CUDA_CACHE_UTIL 3 | #define CUDA_CACHE_UTIL 4 | 5 | #include 6 | #include 7 | 8 | 9 | struct CUDACachedFrame { 10 | void alloc(unsigned int width, unsigned int height) { 11 | cudaMalloc(&d_depthDownsampled, sizeof(float) * width * height); 12 | //cudaMalloc(&d_colorDownsampled, sizeof(uchar4) * width * height)); 13 | cudaMalloc(&d_cameraposDownsampled, sizeof(float4) * width * height); 14 | cudaMalloc(&d_num_valid_points, sizeof(int)); 15 | cudaMemset(d_num_valid_points, 0, sizeof(int)); 16 | 17 | cudaMalloc(&d_intensityDownsampled, sizeof(float) * width * height); 18 | cudaMalloc(&d_intensityDerivsDownsampled, sizeof(float2) * width * height); 19 | cudaMalloc(&d_normalsDownsampled, sizeof(float4) * width * height); 20 | } 21 | void free() { 22 | cudaFree(d_depthDownsampled); 23 | cudaFree(d_num_valid_points); 24 | //cudaFree(d_colorDownsampled); 25 | cudaFree(d_cameraposDownsampled); 26 | 27 | cudaFree(d_intensityDownsampled); 28 | cudaFree(d_intensityDerivsDownsampled); 29 | cudaFree(d_normalsDownsampled); 30 | } 31 | 32 | int* d_num_valid_points; 33 | float* d_depthDownsampled; 34 | //uchar4* d_colorDownsampled; 35 | float4* d_cameraposDownsampled; 36 | 37 | //for dense color term 38 | float* d_intensityDownsampled; //this could be packed with intensityDerivaties to a float4 dunno about the read there 39 | float2* d_intensityDerivsDownsampled; //TODO could have energy over intensity gradient instead of intensity 40 | float4* d_normalsDownsampled; 41 | }; 42 | 43 | #endif //CUDA_CACHE_UTIL -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/EigenDenseBaseAddons.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION and its licensors retain all intellectual property 5 | * and proprietary rights in and to this software, related documentation 6 | * and any modifications thereto. Any use, reproduction, disclosure or 7 | * distribution of this software and related documentation without an express 8 | * license agreement from NVIDIA CORPORATION is strictly prohibited. 9 | */ 10 | 11 | 12 | /** 13 | * @file EigenDenseBaseAddons.h 14 | * https://stackoverflow.com/questions/18382457/eigen-and-boostserialize 15 | */ 16 | #ifndef EIGEN_DENSE_BASE_ADDONS_H_ 17 | #define EIGEN_DENSE_BASE_ADDONS_H_ 18 | 19 | friend class boost::serialization::access; 20 | template 21 | void save(Archive & ar, const unsigned int version) const { 22 | derived().eval(); 23 | const Index rows = derived().rows(), cols = derived().cols(); 24 | ar & rows; 25 | ar & cols; 26 | for (Index j = 0; j < cols; ++j ) 27 | for (Index i = 0; i < rows; ++i ) 28 | ar & derived().coeff(i, j); 29 | } 30 | 31 | template 32 | void load(Archive & ar, const unsigned int version) { 33 | Index rows, cols; 34 | ar & rows; 35 | ar & cols; 36 | if (rows != derived().rows() || cols != derived().cols() ) 37 | derived().resize(rows, cols); 38 | ar & boost::serialization::make_array(derived().data(), derived().size()); 39 | } 40 | 41 | template 42 | void serialize(Archive & ar, const unsigned int file_version) { 43 | boost::serialization::split_member(ar, *this, file_version); 44 | } 45 | 46 | #endif // EIGEN_DENSE_BASE_ADDONS_H_ -------------------------------------------------------------------------------- /XMem/inference/interact/s2m_controller.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from ..interact.s2m.s2m_network import deeplabv3plus_resnet50 as S2M 4 | 5 | from util.tensor_util import pad_divide_by, unpad 6 | 7 | 8 | class S2MController: 9 | """ 10 | A controller for Scribble-to-Mask (for user interaction, not for DAVIS) 11 | Takes the image, previous mask, and scribbles to produce a new mask 12 | ignore_class is usually 255 13 | 0 is NOT the ignore class -- it is the label for the background 14 | """ 15 | def __init__(self, s2m_net:S2M, num_objects, ignore_class, device='cuda:0'): 16 | self.s2m_net = s2m_net 17 | self.num_objects = num_objects 18 | self.ignore_class = ignore_class 19 | self.device = device 20 | 21 | def interact(self, image, prev_mask, scr_mask): 22 | print(self.device) 23 | image = image.to(self.device, non_blocking=True) 24 | prev_mask = prev_mask.unsqueeze(0) 25 | 26 | h, w = image.shape[-2:] 27 | unaggre_mask = torch.zeros((self.num_objects, h, w), dtype=torch.float32, device=image.device) 28 | 29 | for ki in range(1, self.num_objects+1): 30 | p_srb = (scr_mask==ki).astype(np.uint8) 31 | n_srb = ((scr_mask!=ki) * (scr_mask!=self.ignore_class)).astype(np.uint8) 32 | 33 | Rs = torch.from_numpy(np.stack([p_srb, n_srb], 0)).unsqueeze(0).float().to(image.device) 34 | 35 | inputs = torch.cat([image, (prev_mask==ki).float().unsqueeze(0), Rs], 1) 36 | inputs, pads = pad_divide_by(inputs, 16) 37 | 38 | unaggre_mask[ki-1] = unpad(torch.sigmoid(self.s2m_net(inputs)), pads) 39 | 40 | return unaggre_mask -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/details/log_msg.h" 9 | #include "spdlog/details/pattern_formatter.h" 10 | #include "spdlog/formatter.h" 11 | 12 | namespace spdlog { 13 | namespace sinks { 14 | class sink 15 | { 16 | public: 17 | sink() 18 | : level_(level::trace) 19 | , formatter_(new pattern_formatter()) 20 | { 21 | } 22 | 23 | explicit sink(std::unique_ptr formatter) 24 | : level_(level::trace) 25 | , formatter_(std::move(formatter)) 26 | { 27 | } 28 | 29 | virtual ~sink() = default; 30 | virtual void log(const details::log_msg &msg) = 0; 31 | virtual void flush() = 0; 32 | virtual void set_pattern(const std::string &pattern) = 0; 33 | virtual void set_formatter(std::unique_ptr sink_formatter) = 0; 34 | 35 | bool should_log(level::level_enum msg_level) const 36 | { 37 | return msg_level >= level_.load(std::memory_order_relaxed); 38 | } 39 | 40 | void set_level(level::level_enum log_level) 41 | { 42 | level_.store(log_level); 43 | } 44 | 45 | level::level_enum level() const 46 | { 47 | return static_cast(level_.load(std::memory_order_relaxed)); 48 | } 49 | 50 | protected: 51 | // sink log level - default is all 52 | level_t level_; 53 | 54 | // sink formatter - default is full format 55 | std::unique_ptr formatter_; 56 | }; 57 | 58 | } // namespace sinks 59 | } // namespace spdlog 60 | -------------------------------------------------------------------------------- /patch_pyrender.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import inspect 4 | import pyrender 5 | import pyrender.platforms.pyglet_platform as pyglet_mod 6 | import pyrender.viewer as viewer_mod 7 | 8 | def patch_pyglet(): 9 | pyglet_file = inspect.getfile(pyglet_mod) 10 | print(f"Patching {pyglet_file}") 11 | with open(pyglet_file, 'r') as f: 12 | content = f.read() 13 | 14 | old_except = "except pyglet.window.NoSuchConfigException as e:" 15 | new_except = "except (pyglet.window.NoSuchConfigException, pyglet.gl.ContextException):" 16 | 17 | if old_except in content: 18 | content = content.replace(old_except, new_except) 19 | else: 20 | print("Warning: Could not find except block in pyglet_platform.py") 21 | 22 | with open(pyglet_file, 'w') as f: 23 | f.write(content) 24 | 25 | def patch_viewer(): 26 | viewer_file = inspect.getfile(viewer_mod) 27 | print(f"Patching {viewer_file}") 28 | with open(viewer_file, 'r') as f: 29 | content = f.read() 30 | 31 | old_except = "except pyglet.window.NoSuchConfigException:" 32 | new_except = "except (pyglet.window.NoSuchConfigException, pyglet.gl.ContextException):" 33 | 34 | if old_except in content: 35 | content = content.replace(old_except, new_except) 36 | else: 37 | print("Warning: Could not find except block in viewer.py") 38 | 39 | with open(viewer_file, 'w') as f: 40 | f.write(content) 41 | 42 | if __name__ == "__main__": 43 | try: 44 | patch_pyglet() 45 | patch_viewer() 46 | print("Patched pyrender successfully") 47 | except Exception as e: 48 | print(f"Error patching pyrender: {e}") 49 | sys.exit(1) 50 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/utils/cvpr_ds_config.py: -------------------------------------------------------------------------------- 1 | from yacs.config import CfgNode as CN 2 | 3 | 4 | def lower_config(yacs_cfg): 5 | if not isinstance(yacs_cfg, CN): 6 | return yacs_cfg 7 | return {k.lower(): lower_config(v) for k, v in yacs_cfg.items()} 8 | 9 | 10 | _CN = CN() 11 | _CN.BACKBONE_TYPE = 'ResNetFPN' 12 | _CN.RESOLUTION = (8, 2) # options: [(8, 2), (16, 4)] 13 | _CN.FINE_WINDOW_SIZE = 5 # window_size in fine_level, must be odd 14 | _CN.FINE_CONCAT_COARSE_FEAT = True 15 | 16 | # 1. LoFTR-backbone (local feature CNN) config 17 | _CN.RESNETFPN = CN() 18 | _CN.RESNETFPN.INITIAL_DIM = 128 19 | _CN.RESNETFPN.BLOCK_DIMS = [128, 196, 256] # s1, s2, s3 20 | 21 | # 2. LoFTR-coarse module config 22 | _CN.COARSE = CN() 23 | _CN.COARSE.D_MODEL = 256 24 | _CN.COARSE.D_FFN = 256 25 | _CN.COARSE.NHEAD = 8 26 | _CN.COARSE.LAYER_NAMES = ['self', 'cross'] * 4 27 | _CN.COARSE.ATTENTION = 'linear' # options: ['linear', 'full'] 28 | _CN.COARSE.TEMP_BUG_FIX = False 29 | 30 | # 3. Coarse-Matching config 31 | _CN.MATCH_COARSE = CN() 32 | _CN.MATCH_COARSE.THR = 0.2 33 | _CN.MATCH_COARSE.BORDER_RM = 2 34 | _CN.MATCH_COARSE.MATCH_TYPE = 'dual_softmax' # options: ['dual_softmax, 'sinkhorn'] 35 | _CN.MATCH_COARSE.DSMAX_TEMPERATURE = 0.1 36 | _CN.MATCH_COARSE.SKH_ITERS = 3 37 | _CN.MATCH_COARSE.SKH_INIT_BIN_SCORE = 1.0 38 | _CN.MATCH_COARSE.SKH_PREFILTER = True 39 | _CN.MATCH_COARSE.TRAIN_COARSE_PERCENT = 0.4 # training tricks: save GPU memory 40 | _CN.MATCH_COARSE.TRAIN_PAD_NUM_GT_MIN = 200 # training tricks: avoid DDP deadlock 41 | 42 | # 4. LoFTR-fine module config 43 | _CN.FINE = CN() 44 | _CN.FINE.D_MODEL = 128 45 | _CN.FINE.D_FFN = 128 46 | _CN.FINE.NHEAD = 8 47 | _CN.FINE.LAYER_NAMES = ['self', 'cross'] * 1 48 | _CN.FINE.ATTENTION = 'linear' 49 | 50 | default_cfg = lower_config(_CN) 51 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/utils/augment.py: -------------------------------------------------------------------------------- 1 | import albumentations as A 2 | 3 | 4 | class DarkAug(object): 5 | """ 6 | Extreme dark augmentation aiming at Aachen Day-Night 7 | """ 8 | 9 | def __init__(self) -> None: 10 | self.augmentor = A.Compose([ 11 | A.RandomBrightnessContrast(p=0.75, brightness_limit=(-0.6, 0.0), contrast_limit=(-0.5, 0.3)), 12 | A.Blur(p=0.1, blur_limit=(3, 9)), 13 | A.MotionBlur(p=0.2, blur_limit=(3, 25)), 14 | A.RandomGamma(p=0.1, gamma_limit=(15, 65)), 15 | A.HueSaturationValue(p=0.1, val_shift_limit=(-100, -40)) 16 | ], p=0.75) 17 | 18 | def __call__(self, x): 19 | return self.augmentor(image=x)['image'] 20 | 21 | 22 | class MobileAug(object): 23 | """ 24 | Random augmentations aiming at images of mobile/handhold devices. 25 | """ 26 | 27 | def __init__(self): 28 | self.augmentor = A.Compose([ 29 | A.MotionBlur(p=0.25), 30 | A.ColorJitter(p=0.5), 31 | A.RandomRain(p=0.1), # random occlusion 32 | A.RandomSunFlare(p=0.1), 33 | A.JpegCompression(p=0.25), 34 | A.ISONoise(p=0.25) 35 | ], p=1.0) 36 | 37 | def __call__(self, x): 38 | return self.augmentor(image=x)['image'] 39 | 40 | 41 | def build_augmentor(method=None, **kwargs): 42 | if method is not None: 43 | raise NotImplementedError('Using of augmentation functions are not supported yet!') 44 | if method == 'dark': 45 | return DarkAug() 46 | elif method == 'mobile': 47 | return MobileAug() 48 | elif method is None: 49 | return None 50 | else: 51 | raise ValueError(f'Invalid augmentation method: {method}') 52 | 53 | 54 | if __name__ == '__main__': 55 | augmentor = build_augmentor('FDA') 56 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/functional/_csrc.py: -------------------------------------------------------------------------------- 1 | """ 2 | /*****************************************************************************/ 3 | 4 | Extension module loader 5 | 6 | code referenced from : https://github.com/facebookresearch/maskrcnn-benchmark 7 | 8 | /*****************************************************************************/ 9 | """ 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import glob 15 | import os.path 16 | 17 | import torch 18 | 19 | try: 20 | from torch.utils.cpp_extension import load 21 | from torch.utils.cpp_extension import CUDA_HOME 22 | except ImportError: 23 | raise ImportError( 24 | "The cpp layer extensions requires PyTorch 0.4 or higher") 25 | 26 | 27 | def _load_C_extensions(): 28 | this_dir = os.path.dirname(os.path.abspath(__file__)) 29 | this_dir = os.path.join(this_dir, "csrc") 30 | 31 | main_file = glob.glob(os.path.join(this_dir, "*.cpp")) 32 | sources_cpu = glob.glob(os.path.join(this_dir, "cpu", "*.cpp")) 33 | sources_cuda = glob.glob(os.path.join(this_dir, "cuda", "*.cu")) 34 | 35 | sources = main_file + sources_cpu 36 | 37 | extra_cflags = [] 38 | extra_cuda_cflags = [] 39 | if torch.cuda.is_available() and CUDA_HOME is not None: 40 | sources.extend(sources_cuda) 41 | extra_cflags = ["-O3", "-DWITH_CUDA"] 42 | extra_cuda_cflags = ["--expt-extended-lambda"] 43 | sources = [os.path.join(this_dir, s) for s in sources] 44 | extra_include_paths = [this_dir] 45 | return load( 46 | name="ext_lib", 47 | sources=sources, 48 | extra_cflags=extra_cflags, 49 | extra_include_paths=extra_include_paths, 50 | extra_cuda_cflags=extra_cuda_cflags, 51 | ) 52 | 53 | 54 | _backend = _load_C_extensions() 55 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/details/circular_q.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2018 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | // cirucal q view of std::vector. 7 | #pragma once 8 | 9 | #include 10 | 11 | namespace spdlog { 12 | namespace details { 13 | template 14 | class circular_q 15 | { 16 | public: 17 | using item_type = T; 18 | 19 | explicit circular_q(size_t max_items) 20 | : max_items_(max_items + 1) // one item is reserved as marker for full q 21 | , v_(max_items_) 22 | { 23 | } 24 | 25 | // push back, overrun (oldest) item if no room left 26 | void push_back(T &&item) 27 | { 28 | v_[tail_] = std::move(item); 29 | tail_ = (tail_ + 1) % max_items_; 30 | 31 | if (tail_ == head_) // overrun last item if full 32 | { 33 | head_ = (head_ + 1) % max_items_; 34 | ++overrun_counter_; 35 | } 36 | } 37 | 38 | // Pop item from front. 39 | // If there are no elements in the container, the behavior is undefined. 40 | void pop_front(T &popped_item) 41 | { 42 | popped_item = std::move(v_[head_]); 43 | head_ = (head_ + 1) % max_items_; 44 | } 45 | 46 | bool empty() 47 | { 48 | return tail_ == head_; 49 | } 50 | 51 | bool full() 52 | { 53 | // head is ahead of the tail by 1 54 | return ((tail_ + 1) % max_items_) == head_; 55 | } 56 | 57 | size_t overrun_counter() const 58 | { 59 | return overrun_counter_; 60 | } 61 | 62 | private: 63 | size_t max_items_; 64 | typename std::vector::size_type head_ = 0; 65 | typename std::vector::size_type tail_ = 0; 66 | 67 | std::vector v_; 68 | 69 | size_t overrun_counter_ = 0; 70 | }; 71 | } // namespace details 72 | } // namespace spdlog 73 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/misc.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import torch 4 | import numpy as np 5 | 6 | 7 | def get_dims_with_exclusion(dim, exclude=None): 8 | dims = list(range(dim)) 9 | if exclude is not None: 10 | dims.remove(exclude) 11 | 12 | return dims 13 | 14 | 15 | def get_unique_labels(mask): 16 | return np.nonzero(np.bincount(mask.flatten() + 1))[0] - 1 17 | 18 | 19 | def get_bbox_from_mask(mask): 20 | rows = np.any(mask, axis=1) 21 | cols = np.any(mask, axis=0) 22 | rmin, rmax = np.where(rows)[0][[0, -1]] 23 | cmin, cmax = np.where(cols)[0][[0, -1]] 24 | 25 | return rmin, rmax, cmin, cmax 26 | 27 | 28 | def expand_bbox(bbox, expand_ratio, min_crop_size=None): 29 | rmin, rmax, cmin, cmax = bbox 30 | rcenter = 0.5 * (rmin + rmax) 31 | ccenter = 0.5 * (cmin + cmax) 32 | height = expand_ratio * (rmax - rmin + 1) 33 | width = expand_ratio * (cmax - cmin + 1) 34 | if min_crop_size is not None: 35 | height = max(height, min_crop_size) 36 | width = max(width, min_crop_size) 37 | 38 | rmin = int(round(rcenter - 0.5 * height)) 39 | rmax = int(round(rcenter + 0.5 * height)) 40 | cmin = int(round(ccenter - 0.5 * width)) 41 | cmax = int(round(ccenter + 0.5 * width)) 42 | 43 | return rmin, rmax, cmin, cmax 44 | 45 | 46 | def clamp_bbox(bbox, rmin, rmax, cmin, cmax): 47 | return (max(rmin, bbox[0]), min(rmax, bbox[1]), 48 | max(cmin, bbox[2]), min(cmax, bbox[3])) 49 | 50 | 51 | def get_bbox_iou(b1, b2): 52 | h_iou = get_segments_iou(b1[:2], b2[:2]) 53 | w_iou = get_segments_iou(b1[2:4], b2[2:4]) 54 | return h_iou * w_iou 55 | 56 | 57 | def get_segments_iou(s1, s2): 58 | a, b = s1 59 | c, d = s2 60 | intersection = max(0, min(b, d) - max(a, c) + 1) 61 | union = max(1e-6, max(b, d) - min(a, c) + 1) 62 | return intersection / union 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Object_Reconstruction 2 | 3 | ## 1) Data Download 4 | 5 | ### Download Pretrained Weights for Segmentation Network 6 | 7 | [XMem-s012.pth](https://drive.google.com/file/d/1LJ6U3NmI9MoUKG27mzqlgP1ioHWq-a5e/view?usp=sharing) 8 | 9 | place them under `./XMem/saves/` 10 | 11 | ### Download Pretrained Weights for LoFTR 12 | 13 | [outdoor_ds.ckpt](https://drive.google.com/drive/folders/11b1-Wzxcn7LpmTgHPqlC3H1ZzGsB6j6R?usp=sharing) 14 | 15 | place them under `./BundleSDF/BundleTrack/LoFTR/weights/` 16 | 17 | 18 | ## 2) Docker Setup 19 | 20 | To set up the Docker environment, run the following command: 21 | 22 | ```bash 23 | docker build --network host -t nvcr.io/nvidian/bundlesdf . 24 | ``` 25 | ## 3) Methodology 26 | ### Capture Data from RealSense Depth Camera 27 | 28 | To capture data from a RealSense depth camera, run the `rec_con_mask.py` script. 29 | 30 |
1)Install [librealsense SDK](https://github.com/IntelRealSense/librealsense) 31 |
2)Install dependencies for Xmem and Realsense 32 |
conda environment (optional) 33 | ```bash 34 | conda create -n obj_recon python=3.10 35 | ``` 36 | ```bash 37 | pip install -r requirements.txt 38 | ``` 39 | 40 | #### Prepare Input Directory 41 | 42 | Run 43 | ```bash 44 | conda info --base 45 | ``` 46 | to get conda base directory. Copy the path and paste it in `capture.sh`. Then, run the following. 47 | 48 | ```bash 49 | sudo bash capture.sh 50 | ``` 51 |
Note : (Don't run in Docker Container)
52 | 53 | Press `Enter` to start recording RGB and depth frames. Then, create a boundary by selecting points on the window to create a mask for the first image. 54 | 55 | ### Steps to Run 56 | 57 | 1. Run the container: 58 | 59 | ```bash 60 | bash run_container.sh 61 | ``` 62 | 63 | 2. Build the project: 64 | 65 | ```bash 66 | bash build.sh 67 | ``` 68 | 69 | 3. Run the project: 70 | 71 | ```bash 72 | bash run.sh 73 | ``` 74 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LICENSE: -------------------------------------------------------------------------------- 1 | Authors: Bowen Wen 2 | Contact: wenbowenxjtu@gmail.com 3 | Created in 2021 4 | 5 | Copyright (c) Rutgers University, 2021 All rights reserved. 6 | 7 | Bowen Wen and Kostas Bekris. "BundleTrack: 6D Pose Tracking for Novel Objects 8 | without Instance or Category-Level 3D Models." 9 | In IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS). 2021. 10 | 11 | Redistribution and use in source and binary forms, with or without 12 | modification, are permitted provided that the following conditions are met: 13 | * Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | * Redistributions in binary form must reproduce the above copyright 16 | notice, this list of conditions and the following disclaimer in the 17 | documentation and/or other materials provided with the distribution. 18 | * Neither the name of the Bowen Wen, Kostas Bekris, Rutgers University, 19 | nor the names of its contributors may be used to 20 | endorse or promote products derived from this software without 21 | specific prior written permission. 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND 23 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY 26 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 27 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 31 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/stdout_color_sinks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2018 spdlog 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #ifdef _WIN32 13 | #include "spdlog/sinks/wincolor_sink.h" 14 | #else 15 | #include "spdlog/sinks/ansicolor_sink.h" 16 | #endif 17 | 18 | namespace spdlog { 19 | namespace sinks { 20 | #ifdef _WIN32 21 | using stdout_color_sink_mt = wincolor_stdout_sink_mt; 22 | using stdout_color_sink_st = wincolor_stdout_sink_st; 23 | using stderr_color_sink_mt = wincolor_stderr_sink_mt; 24 | using stderr_color_sink_st = wincolor_stderr_sink_st; 25 | #else 26 | using stdout_color_sink_mt = ansicolor_stdout_sink_mt; 27 | using stdout_color_sink_st = ansicolor_stdout_sink_st; 28 | using stderr_color_sink_mt = ansicolor_stderr_sink_mt; 29 | using stderr_color_sink_st = ansicolor_stderr_sink_st; 30 | #endif 31 | } // namespace sinks 32 | 33 | template 34 | inline std::shared_ptr stdout_color_mt(const std::string &logger_name) 35 | { 36 | return Factory::template create(logger_name); 37 | } 38 | 39 | template 40 | inline std::shared_ptr stdout_color_st(const std::string &logger_name) 41 | { 42 | return Factory::template create(logger_name); 43 | } 44 | 45 | template 46 | inline std::shared_ptr stderr_color_mt(const std::string &logger_name) 47 | { 48 | return Factory::template create(logger_name); 49 | } 50 | 51 | template 52 | inline std::shared_ptr stderr_color_st(const std::string &logger_name) 53 | { 54 | return Factory::template create(logger_name); 55 | } 56 | } // namespace spdlog 57 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/evaluation.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | 3 | import numpy as np 4 | import torch 5 | 6 | from ..inference import utils 7 | from ..inference.clicker import Clicker 8 | 9 | try: 10 | get_ipython() 11 | from tqdm import tqdm_notebook as tqdm 12 | except NameError: 13 | from tqdm import tqdm 14 | 15 | 16 | def evaluate_dataset(dataset, predictor, oracle_eval=False, **kwargs): 17 | all_ious = [] 18 | 19 | start_time = time() 20 | for index in tqdm(range(len(dataset)), leave=False): 21 | sample = dataset.get_sample(index) 22 | item = dataset[index] 23 | 24 | if oracle_eval: 25 | gt_mask = torch.tensor(sample['instances_mask'], dtype=torch.float32) 26 | gt_mask = gt_mask.unsqueeze(0).unsqueeze(0) 27 | predictor.opt_functor.mask_loss.set_gt_mask(gt_mask) 28 | _, sample_ious, _ = evaluate_sample(item['images'], sample['instances_mask'], predictor, **kwargs) 29 | all_ious.append(sample_ious) 30 | end_time = time() 31 | elapsed_time = end_time - start_time 32 | 33 | return all_ious, elapsed_time 34 | 35 | 36 | def evaluate_sample(image_nd, instances_mask, predictor, max_iou_thr, 37 | pred_thr=0.49, max_clicks=20): 38 | clicker = Clicker(gt_mask=instances_mask) 39 | pred_mask = np.zeros_like(instances_mask) 40 | ious_list = [] 41 | 42 | with torch.no_grad(): 43 | predictor.set_input_image(image_nd) 44 | 45 | for click_number in range(max_clicks): 46 | clicker.make_next_click(pred_mask) 47 | pred_probs = predictor.get_prediction(clicker) 48 | pred_mask = pred_probs > pred_thr 49 | 50 | iou = utils.get_iou(instances_mask, pred_mask) 51 | ious_list.append(iou) 52 | 53 | if iou >= max_iou_thr: 54 | break 55 | 56 | return clicker.clicks_list, np.array(ious_list, dtype=np.float32), pred_probs 57 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs_controller.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from .fbrs.controller import InteractiveController 3 | from .fbrs.inference import utils 4 | 5 | 6 | class FBRSController: 7 | def __init__(self, checkpoint_path, device='cuda:0', max_size=800): 8 | model = utils.load_is_model(checkpoint_path, device, cpu_dist_maps=True, norm_radius=260) 9 | 10 | # Predictor params 11 | zoomin_params = { 12 | 'skip_clicks': 1, 13 | 'target_size': 480, 14 | 'expansion_ratio': 1.4, 15 | } 16 | 17 | predictor_params = { 18 | 'brs_mode': 'f-BRS-B', 19 | 'prob_thresh': 0.5, 20 | 'zoom_in_params': zoomin_params, 21 | 'predictor_params': { 22 | 'net_clicks_limit': 8, 23 | 'max_size': 800, 24 | }, 25 | 'brs_opt_func_params': {'min_iou_diff': 1e-3}, 26 | 'lbfgs_params': {'maxfun': 20} 27 | } 28 | 29 | self.controller = InteractiveController(model, device, predictor_params) 30 | self.anchored = False 31 | self.device = device 32 | 33 | def unanchor(self): 34 | self.anchored = False 35 | 36 | def interact(self, image, x, y, is_positive): 37 | image = image.to(self.device, non_blocking=True) 38 | if not self.anchored: 39 | self.controller.set_image(image) 40 | self.controller.reset_predictor() 41 | self.anchored = True 42 | 43 | self.controller.add_click(x, y, is_positive) 44 | # return self.controller.result_mask 45 | # return self.controller.probs_history[-1][1] 46 | return (self.controller.probs_history[-1][1]>0.5).float() 47 | 48 | def undo(self): 49 | self.controller.undo_click() 50 | if len(self.controller.probs_history) == 0: 51 | return None 52 | else: 53 | return (self.controller.probs_history[-1][1]>0.5).float() -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/cudaUtil.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include "common.h" 7 | 8 | 9 | // https://stackoverflow.com/questions/17399119/how-do-i-use-atomicmax-on-floating-point-values-in-cuda 10 | __inline__ __device__ float atomicMax(float* address, float val) 11 | { 12 | int* address_as_i = (int*) address; 13 | int old = *address_as_i, assumed; 14 | do { 15 | assumed = old; 16 | old = atomicCAS(address_as_i, assumed, __float_as_int(fmaxf(val, __int_as_float(assumed)))); 17 | } while (assumed != old); 18 | return __int_as_float(old); 19 | }; 20 | 21 | 22 | __inline__ __device__ float atomicMin(float* address, float val) 23 | { 24 | int* address_as_i = (int*) address; 25 | int old = *address_as_i, assumed; 26 | do { 27 | assumed = old; 28 | old = atomicCAS(address_as_i, assumed, __float_as_int(fminf(val, __int_as_float(assumed)))); 29 | } while (assumed != old); 30 | return __int_as_float(old); 31 | }; 32 | 33 | 34 | 35 | __inline__ __device__ float warpReduce(float val) { 36 | int offset = 32 >> 1; 37 | while (offset > 0) { 38 | val = val + __shfl_down_sync(FULL_MASK, val, offset, 32); 39 | offset = offset >> 1; 40 | } 41 | return val; 42 | } 43 | 44 | /** 45 | * @brief : From g2o 46 | * @e: squared error 47 | */ 48 | __inline__ __device__ void huberLoss(const float e, const float delta, float3 &rho) 49 | { 50 | float dsqr = delta * delta; 51 | if (e <= dsqr) 52 | { // inlier 53 | rho.x = e; 54 | rho.y = 1.; 55 | rho.z = 0.; 56 | } 57 | else 58 | { // outlier 59 | double sqrte = sqrt(e); // absolut value of the error 60 | rho.x = 2 * sqrte * delta - dsqr; // rho(e) = 2 * delta * e^(1/2) - delta^2 61 | rho.y = delta / sqrte; // rho'(e) = delta / sqrt(e) 62 | rho.z = -0.5 * rho.y / e; // rho''(e) = -1 / (2*e^(3/2)) = -1/2 * (delta/e) / e 63 | } 64 | } 65 | 66 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/base_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | // 8 | // base sink templated over a mutex (either dummy or real) 9 | // concrete implementation should override the sink_it_() and flush_() methods. 10 | // locking is taken care of in this class - no locking needed by the 11 | // implementers.. 12 | // 13 | 14 | #include "spdlog/common.h" 15 | #include "spdlog/details/log_msg.h" 16 | #include "spdlog/formatter.h" 17 | #include "spdlog/sinks/sink.h" 18 | 19 | namespace spdlog { 20 | namespace sinks { 21 | template 22 | class base_sink : public sink 23 | { 24 | public: 25 | base_sink() = default; 26 | base_sink(const base_sink &) = delete; 27 | base_sink &operator=(const base_sink &) = delete; 28 | 29 | void log(const details::log_msg &msg) final 30 | { 31 | std::lock_guard lock(mutex_); 32 | sink_it_(msg); 33 | } 34 | 35 | void flush() final 36 | { 37 | std::lock_guard lock(mutex_); 38 | flush_(); 39 | } 40 | 41 | void set_pattern(const std::string &pattern) final 42 | { 43 | std::lock_guard lock(mutex_); 44 | set_pattern_(pattern); 45 | } 46 | 47 | void set_formatter(std::unique_ptr sink_formatter) final 48 | { 49 | std::lock_guard lock(mutex_); 50 | set_formatter_(std::move(sink_formatter)); 51 | } 52 | 53 | protected: 54 | virtual void sink_it_(const details::log_msg &msg) = 0; 55 | virtual void flush_() = 0; 56 | 57 | virtual void set_pattern_(const std::string &pattern) 58 | { 59 | set_formatter_(details::make_unique(pattern)); 60 | } 61 | 62 | virtual void set_formatter_(std::unique_ptr sink_formatter) 63 | { 64 | formatter_ = std::move(sink_formatter); 65 | } 66 | Mutex mutex_; 67 | }; 68 | } // namespace sinks 69 | } // namespace spdlog 70 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/basic_file_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015-2018 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "spdlog/details/file_helper.h" 13 | #include "spdlog/details/null_mutex.h" 14 | #include "spdlog/sinks/base_sink.h" 15 | 16 | #include 17 | #include 18 | 19 | namespace spdlog { 20 | namespace sinks { 21 | /* 22 | * Trivial file sink with single file as target 23 | */ 24 | template 25 | class basic_file_sink final : public base_sink 26 | { 27 | public: 28 | explicit basic_file_sink(const filename_t &filename, bool truncate = false) 29 | { 30 | file_helper_.open(filename, truncate); 31 | } 32 | 33 | protected: 34 | void sink_it_(const details::log_msg &msg) override 35 | { 36 | fmt::memory_buffer formatted; 37 | sink::formatter_->format(msg, formatted); 38 | file_helper_.write(formatted); 39 | } 40 | 41 | void flush_() override 42 | { 43 | file_helper_.flush(); 44 | } 45 | 46 | private: 47 | details::file_helper file_helper_; 48 | }; 49 | 50 | using basic_file_sink_mt = basic_file_sink; 51 | using basic_file_sink_st = basic_file_sink; 52 | 53 | } // namespace sinks 54 | 55 | // 56 | // factory functions 57 | // 58 | template 59 | inline std::shared_ptr basic_logger_mt(const std::string &logger_name, const filename_t &filename, bool truncate = false) 60 | { 61 | return Factory::template create(logger_name, filename, truncate); 62 | } 63 | 64 | template 65 | inline std::shared_ptr basic_logger_st(const std::string &logger_name, const filename_t &filename, bool truncate = false) 66 | { 67 | return Factory::template create(logger_name, filename, truncate); 68 | } 69 | 70 | } // namespace spdlog 71 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/utils/position_encoding.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch import nn 4 | 5 | 6 | class PositionEncodingSine(nn.Module): 7 | """ 8 | This is a sinusoidal position encoding that generalized to 2-dimensional images 9 | """ 10 | 11 | def __init__(self, d_model, max_shape=(256, 256), temp_bug_fix=True): 12 | """ 13 | Args: 14 | max_shape (tuple): for 1/8 featmap, the max length of 256 corresponds to 2048 pixels 15 | temp_bug_fix (bool): As noted in this [issue](https://github.com/zju3dv/LoFTR/issues/41), 16 | the original implementation of LoFTR includes a bug in the pos-enc impl, which has little impact 17 | on the final performance. For now, we keep both impls for backward compatability. 18 | We will remove the buggy impl after re-training all variants of our released models. 19 | """ 20 | super().__init__() 21 | 22 | pe = torch.zeros((d_model, *max_shape)) 23 | y_position = torch.ones(max_shape).cumsum(0).float().unsqueeze(0) 24 | x_position = torch.ones(max_shape).cumsum(1).float().unsqueeze(0) 25 | if temp_bug_fix: 26 | div_term = torch.exp(torch.arange(0, d_model//2, 2).float() * (-math.log(10000.0) / (d_model//2))) 27 | else: # a buggy implementation (for backward compatability only) 28 | div_term = torch.exp(torch.arange(0, d_model//2, 2).float() * (-math.log(10000.0) / d_model//2)) 29 | div_term = div_term[:, None, None] # [C//4, 1, 1] 30 | pe[0::4, :, :] = torch.sin(x_position * div_term) 31 | pe[1::4, :, :] = torch.cos(x_position * div_term) 32 | pe[2::4, :, :] = torch.sin(y_position * div_term) 33 | pe[3::4, :, :] = torch.cos(y_position * div_term) 34 | 35 | self.register_buffer('pe', pe.unsqueeze(0), persistent=False) # [1, C, H, W] 36 | 37 | def forward(self, x): 38 | """ 39 | Args: 40 | x: [N, C, H, W] 41 | """ 42 | return x + self.pe[:, :, :x.size(2), :x.size(3)] 43 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/details/periodic_worker.h: -------------------------------------------------------------------------------- 1 | 2 | // 3 | // Copyright(c) 2018 Gabi Melman. 4 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 5 | // 6 | 7 | #pragma once 8 | 9 | // periodic worker thread - periodically executes the given callback function. 10 | // 11 | // RAII over the owned thread: 12 | // creates the thread on construction. 13 | // stops and joins the thread on destruction (if the thread is executing a callback, wait for it to finish first). 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | namespace spdlog { 21 | namespace details { 22 | 23 | class periodic_worker 24 | { 25 | public: 26 | periodic_worker(const std::function &callback_fun, std::chrono::seconds interval) 27 | { 28 | active_ = (interval > std::chrono::seconds::zero()); 29 | if (!active_) 30 | { 31 | return; 32 | } 33 | 34 | worker_thread_ = std::thread([this, callback_fun, interval]() { 35 | for (;;) 36 | { 37 | std::unique_lock lock(this->mutex_); 38 | if (this->cv_.wait_for(lock, interval, [this] { return !this->active_; })) 39 | { 40 | return; // active_ == false, so exit this thread 41 | } 42 | callback_fun(); 43 | } 44 | }); 45 | } 46 | 47 | periodic_worker(const periodic_worker &) = delete; 48 | periodic_worker &operator=(const periodic_worker &) = delete; 49 | 50 | // stop the worker thread and join it 51 | ~periodic_worker() 52 | { 53 | if (worker_thread_.joinable()) 54 | { 55 | { 56 | std::lock_guard lock(mutex_); 57 | active_ = false; 58 | } 59 | cv_.notify_one(); 60 | worker_thread_.join(); 61 | } 62 | } 63 | 64 | private: 65 | bool active_; 66 | std::thread worker_thread_; 67 | std::mutex mutex_; 68 | std::condition_variable cv_; 69 | }; 70 | } // namespace details 71 | } // namespace spdlog 72 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/inference/predictors/brs_losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ...model.losses import SigmoidBinaryCrossEntropyLoss 4 | 5 | 6 | class BRSMaskLoss(torch.nn.Module): 7 | def __init__(self, eps=1e-5): 8 | super().__init__() 9 | self._eps = eps 10 | 11 | def forward(self, result, pos_mask, neg_mask): 12 | pos_diff = (1 - result) * pos_mask 13 | pos_target = torch.sum(pos_diff ** 2) 14 | pos_target = pos_target / (torch.sum(pos_mask) + self._eps) 15 | 16 | neg_diff = result * neg_mask 17 | neg_target = torch.sum(neg_diff ** 2) 18 | neg_target = neg_target / (torch.sum(neg_mask) + self._eps) 19 | 20 | loss = pos_target + neg_target 21 | 22 | with torch.no_grad(): 23 | f_max_pos = torch.max(torch.abs(pos_diff)).item() 24 | f_max_neg = torch.max(torch.abs(neg_diff)).item() 25 | 26 | return loss, f_max_pos, f_max_neg 27 | 28 | 29 | class OracleMaskLoss(torch.nn.Module): 30 | def __init__(self): 31 | super().__init__() 32 | self.gt_mask = None 33 | self.loss = SigmoidBinaryCrossEntropyLoss(from_sigmoid=True) 34 | self.predictor = None 35 | self.history = [] 36 | 37 | def set_gt_mask(self, gt_mask): 38 | self.gt_mask = gt_mask 39 | self.history = [] 40 | 41 | def forward(self, result, pos_mask, neg_mask): 42 | gt_mask = self.gt_mask.to(result.device) 43 | if self.predictor.object_roi is not None: 44 | r1, r2, c1, c2 = self.predictor.object_roi[:4] 45 | gt_mask = gt_mask[:, :, r1:r2 + 1, c1:c2 + 1] 46 | gt_mask = torch.nn.functional.interpolate(gt_mask, result.size()[2:], mode='bilinear', align_corners=True) 47 | 48 | if result.shape[0] == 2: 49 | gt_mask_flipped = torch.flip(gt_mask, dims=[3]) 50 | gt_mask = torch.cat([gt_mask, gt_mask_flipped], dim=0) 51 | 52 | loss = self.loss(result, gt_mask) 53 | self.history.append(loss.detach().cpu().numpy()[0]) 54 | 55 | if len(self.history) > 5 and abs(self.history[-5] - self.history[-1]) < 1e-5: 56 | return 0, 0, 0 57 | 58 | return loss, 1.0, 1.0 59 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/utils/cython/_get_dist_maps.pyx: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | cimport cython 3 | cimport numpy as np 4 | from libc.stdlib cimport malloc, free 5 | 6 | ctypedef struct qnode: 7 | int row 8 | int col 9 | int layer 10 | int orig_row 11 | int orig_col 12 | 13 | @cython.infer_types(True) 14 | @cython.boundscheck(False) 15 | @cython.wraparound(False) 16 | @cython.nonecheck(False) 17 | def get_dist_maps(np.ndarray[np.float32_t, ndim=2, mode="c"] points, 18 | int height, int width, float norm_delimeter): 19 | cdef np.ndarray[np.float32_t, ndim=3, mode="c"] dist_maps = \ 20 | np.full((2, height, width), 1e6, dtype=np.float32, order="C") 21 | 22 | cdef int *dxy = [-1, 0, 0, -1, 0, 1, 1, 0] 23 | cdef int i, j, x, y, dx, dy 24 | cdef qnode v 25 | cdef qnode *q = malloc((4 * height * width + 1) * sizeof(qnode)) 26 | cdef int qhead = 0, qtail = -1 27 | cdef float ndist 28 | 29 | for i in range(points.shape[0]): 30 | x, y = round(points[i, 0]), round(points[i, 1]) 31 | if x >= 0: 32 | qtail += 1 33 | q[qtail].row = x 34 | q[qtail].col = y 35 | q[qtail].orig_row = x 36 | q[qtail].orig_col = y 37 | if i >= points.shape[0] / 2: 38 | q[qtail].layer = 1 39 | else: 40 | q[qtail].layer = 0 41 | dist_maps[q[qtail].layer, x, y] = 0 42 | 43 | while qtail - qhead + 1 > 0: 44 | v = q[qhead] 45 | qhead += 1 46 | 47 | for k in range(4): 48 | x = v.row + dxy[2 * k] 49 | y = v.col + dxy[2 * k + 1] 50 | 51 | ndist = ((x - v.orig_row)/norm_delimeter) ** 2 + ((y - v.orig_col)/norm_delimeter) ** 2 52 | if (x >= 0 and y >= 0 and x < height and y < width and 53 | dist_maps[v.layer, x, y] > ndist): 54 | qtail += 1 55 | q[qtail].orig_col = v.orig_col 56 | q[qtail].orig_row = v.orig_row 57 | q[qtail].layer = v.layer 58 | q[qtail].row = x 59 | q[qtail].col = y 60 | dist_maps[v.layer, x, y] = ndist 61 | 62 | free(q) 63 | return dist_maps 64 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/utils/geometry.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | @torch.no_grad() 5 | def warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1): 6 | """ Warp kpts0 from I0 to I1 with depth, K and Rt 7 | Also check covisibility and depth consistency. 8 | Depth is consistent if relative error < 0.2 (hard-coded). 9 | 10 | Args: 11 | kpts0 (torch.Tensor): [N, L, 2] - , 12 | depth0 (torch.Tensor): [N, H, W], 13 | depth1 (torch.Tensor): [N, H, W], 14 | T_0to1 (torch.Tensor): [N, 3, 4], 15 | K0 (torch.Tensor): [N, 3, 3], 16 | K1 (torch.Tensor): [N, 3, 3], 17 | Returns: 18 | calculable_mask (torch.Tensor): [N, L] 19 | warped_keypoints0 (torch.Tensor): [N, L, 2] 20 | """ 21 | kpts0_long = kpts0.round().long() 22 | 23 | # Sample depth, get calculable_mask on depth != 0 24 | kpts0_depth = torch.stack( 25 | [depth0[i, kpts0_long[i, :, 1], kpts0_long[i, :, 0]] for i in range(kpts0.shape[0])], dim=0 26 | ) # (N, L) 27 | nonzero_mask = kpts0_depth != 0 28 | 29 | # Unproject 30 | kpts0_h = torch.cat([kpts0, torch.ones_like(kpts0[:, :, [0]])], dim=-1) * kpts0_depth[..., None] # (N, L, 3) 31 | kpts0_cam = K0.inverse() @ kpts0_h.transpose(2, 1) # (N, 3, L) 32 | 33 | # Rigid Transform 34 | w_kpts0_cam = T_0to1[:, :3, :3] @ kpts0_cam + T_0to1[:, :3, [3]] # (N, 3, L) 35 | w_kpts0_depth_computed = w_kpts0_cam[:, 2, :] 36 | 37 | # Project 38 | w_kpts0_h = (K1 @ w_kpts0_cam).transpose(2, 1) # (N, L, 3) 39 | w_kpts0 = w_kpts0_h[:, :, :2] / (w_kpts0_h[:, :, [2]] + 1e-4) # (N, L, 2), +1e-4 to avoid zero depth 40 | 41 | # Covisible Check 42 | h, w = depth1.shape[1:3] 43 | covisible_mask = (w_kpts0[:, :, 0] > 0) * (w_kpts0[:, :, 0] < w-1) * \ 44 | (w_kpts0[:, :, 1] > 0) * (w_kpts0[:, :, 1] < h-1) 45 | w_kpts0_long = w_kpts0.long() 46 | w_kpts0_long[~covisible_mask, :] = 0 47 | 48 | w_kpts0_depth = torch.stack( 49 | [depth1[i, w_kpts0_long[i, :, 1], w_kpts0_long[i, :, 0]] for i in range(w_kpts0_long.shape[0])], dim=0 50 | ) # (N, L) 51 | consistent_mask = ((w_kpts0_depth - w_kpts0_depth_computed) / w_kpts0_depth).abs() < 0.2 52 | valid_mask = nonzero_mask * covisible_mask * consistent_mask 53 | 54 | return valid_mask, w_kpts0 55 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/cuda_ransac.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 Nghia Ho. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are 5 | permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of 8 | conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 11 | of conditions and the following disclaimer in the documentation and/or other materials 12 | provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY NGHIA HO ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BY NGHIA HO OR 17 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 22 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those of the 25 | authors and should not be interpreted as representing official policies, either expressed 26 | or implied, of Nghia Ho. 27 | */ 28 | 29 | #pragma once 30 | 31 | #include 32 | #include 33 | #include "common.h" 34 | #include "cuda_SimpleMatrixUtil.h" 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | 43 | void ransacMultiPairGPU(const std::vector &ptsA, const std::vector &ptsB, const std::vector &normalsA, const std::vector &normalsB, std::vector uvA, std::vector uvB, const std::vector confs, const std::vector &n_pts, const int n_trials, const float dist_thres, const float cos_normal_angle, const std::vector &max_transs, const std::vector &max_rots, std::vector> &inlier_ids); 44 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/syncbn/modules/functional/csrc/bn.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | 3 | SyncBN 4 | 5 | *****************************************************************************/ 6 | #pragma once 7 | 8 | #ifdef WITH_CUDA 9 | #include "cuda/ext_lib.h" 10 | #endif 11 | 12 | /// SyncBN 13 | 14 | std::vector syncbn_sum_sqsum(const at::Tensor& x) { 15 | if (x.is_cuda()) { 16 | #ifdef WITH_CUDA 17 | return syncbn_sum_sqsum_cuda(x); 18 | #else 19 | AT_ERROR("Not compiled with GPU support"); 20 | #endif 21 | } else { 22 | AT_ERROR("CPU implementation not supported"); 23 | } 24 | } 25 | 26 | at::Tensor syncbn_forward(const at::Tensor& x, const at::Tensor& weight, 27 | const at::Tensor& bias, const at::Tensor& mean, 28 | const at::Tensor& var, bool affine, float eps) { 29 | if (x.is_cuda()) { 30 | #ifdef WITH_CUDA 31 | return syncbn_forward_cuda(x, weight, bias, mean, var, affine, eps); 32 | #else 33 | AT_ERROR("Not compiled with GPU support"); 34 | #endif 35 | } else { 36 | AT_ERROR("CPU implementation not supported"); 37 | } 38 | } 39 | 40 | std::vector syncbn_backward_xhat(const at::Tensor& dz, 41 | const at::Tensor& x, 42 | const at::Tensor& mean, 43 | const at::Tensor& var, float eps) { 44 | if (dz.is_cuda()) { 45 | #ifdef WITH_CUDA 46 | return syncbn_backward_xhat_cuda(dz, x, mean, var, eps); 47 | #else 48 | AT_ERROR("Not compiled with GPU support"); 49 | #endif 50 | } else { 51 | AT_ERROR("CPU implementation not supported"); 52 | } 53 | } 54 | 55 | std::vector syncbn_backward( 56 | const at::Tensor& dz, const at::Tensor& x, const at::Tensor& weight, 57 | const at::Tensor& bias, const at::Tensor& mean, const at::Tensor& var, 58 | const at::Tensor& sum_dz, const at::Tensor& sum_dz_xhat, bool affine, 59 | float eps) { 60 | if (dz.is_cuda()) { 61 | #ifdef WITH_CUDA 62 | return syncbn_backward_cuda(dz, x, weight, bias, mean, var, sum_dz, 63 | sum_dz_xhat, affine, eps); 64 | #else 65 | AT_ERROR("Not compiled with GPU support"); 66 | #endif 67 | } else { 68 | AT_ERROR("CPU implementation not supported"); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /XMem/inference/data/mask_mapper.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from dataset.util import all_to_onehot 5 | 6 | 7 | class MaskMapper: 8 | """ 9 | This class is used to convert a indexed-mask to a one-hot representation. 10 | It also takes care of remapping non-continuous indices 11 | It has two modes: 12 | 1. Default. Only masks with new indices are supposed to go into the remapper. 13 | This is also the case for YouTubeVOS. 14 | i.e., regions with index 0 are not "background", but "don't care". 15 | 16 | 2. Exhaustive. Regions with index 0 are considered "background". 17 | Every single pixel is considered to be "labeled". 18 | """ 19 | def __init__(self): 20 | self.labels = [] 21 | self.remappings = {} 22 | 23 | # if coherent, no mapping is required 24 | self.coherent = True 25 | 26 | def convert_mask(self, mask, exhaustive=False): 27 | # mask is in index representation, H*W numpy array 28 | labels = np.unique(mask).astype(np.uint8) 29 | labels = labels[labels!=0].tolist() 30 | 31 | new_labels = list(set(labels) - set(self.labels)) 32 | if not exhaustive: 33 | assert len(new_labels) == len(labels), 'Old labels found in non-exhaustive mode' 34 | 35 | # add new remappings 36 | for i, l in enumerate(new_labels): 37 | self.remappings[l] = i+len(self.labels)+1 38 | if self.coherent and i+len(self.labels)+1 != l: 39 | self.coherent = False 40 | 41 | if exhaustive: 42 | new_mapped_labels = range(1, len(self.labels)+len(new_labels)+1) 43 | else: 44 | if self.coherent: 45 | new_mapped_labels = new_labels 46 | else: 47 | new_mapped_labels = range(len(self.labels)+1, len(self.labels)+len(new_labels)+1) 48 | 49 | self.labels.extend(new_labels) 50 | mask = torch.from_numpy(all_to_onehot(mask, self.labels)).float() 51 | 52 | # mask num_objects*H*W 53 | return mask, new_mapped_labels 54 | 55 | 56 | def remap_index_mask(self, mask): 57 | # mask is in index representation, H*W numpy array 58 | if self.coherent: 59 | return mask 60 | 61 | new_mask = np.zeros_like(mask) 62 | for l, i in self.remappings.items(): 63 | new_mask[mask==i] = l 64 | return new_mask -------------------------------------------------------------------------------- /XMem/util/palette.py: -------------------------------------------------------------------------------- 1 | davis_palette = b'\x00\x00\x00\x80\x00\x00\x00\x80\x00\x80\x80\x00\x00\x00\x80\x80\x00\x80\x00\x80\x80\x80\x80\x80@\x00\x00\xc0\x00\x00@\x80\x00\xc0\x80\x00@\x00\x80\xc0\x00\x80@\x80\x80\xc0\x80\x80\x00@\x00\x80@\x00\x00\xc0\x00\x80\xc0\x00\x00@\x80\x80@\x80\x00\xc0\x80\x80\xc0\x80@@\x00\xc0@\x00@\xc0\x00\xc0\xc0\x00@@\x80\xc0@\x80@\xc0\x80\xc0\xc0\x80\x00\x00@\x80\x00@\x00\x80@\x80\x80@\x00\x00\xc0\x80\x00\xc0\x00\x80\xc0\x80\x80\xc0@\x00@\xc0\x00@@\x80@\xc0\x80@@\x00\xc0\xc0\x00\xc0@\x80\xc0\xc0\x80\xc0\x00@@\x80@@\x00\xc0@\x80\xc0@\x00@\xc0\x80@\xc0\x00\xc0\xc0\x80\xc0\xc0@@@\xc0@@@\xc0@\xc0\xc0@@@\xc0\xc0@\xc0@\xc0\xc0\xc0\xc0\xc0 \x00\x00\xa0\x00\x00 \x80\x00\xa0\x80\x00 \x00\x80\xa0\x00\x80 \x80\x80\xa0\x80\x80`\x00\x00\xe0\x00\x00`\x80\x00\xe0\x80\x00`\x00\x80\xe0\x00\x80`\x80\x80\xe0\x80\x80 @\x00\xa0@\x00 \xc0\x00\xa0\xc0\x00 @\x80\xa0@\x80 \xc0\x80\xa0\xc0\x80`@\x00\xe0@\x00`\xc0\x00\xe0\xc0\x00`@\x80\xe0@\x80`\xc0\x80\xe0\xc0\x80 \x00@\xa0\x00@ \x80@\xa0\x80@ \x00\xc0\xa0\x00\xc0 \x80\xc0\xa0\x80\xc0`\x00@\xe0\x00@`\x80@\xe0\x80@`\x00\xc0\xe0\x00\xc0`\x80\xc0\xe0\x80\xc0 @@\xa0@@ \xc0@\xa0\xc0@ @\xc0\xa0@\xc0 \xc0\xc0\xa0\xc0\xc0`@@\xe0@@`\xc0@\xe0\xc0@`@\xc0\xe0@\xc0`\xc0\xc0\xe0\xc0\xc0\x00 \x00\x80 \x00\x00\xa0\x00\x80\xa0\x00\x00 \x80\x80 \x80\x00\xa0\x80\x80\xa0\x80@ \x00\xc0 \x00@\xa0\x00\xc0\xa0\x00@ \x80\xc0 \x80@\xa0\x80\xc0\xa0\x80\x00`\x00\x80`\x00\x00\xe0\x00\x80\xe0\x00\x00`\x80\x80`\x80\x00\xe0\x80\x80\xe0\x80@`\x00\xc0`\x00@\xe0\x00\xc0\xe0\x00@`\x80\xc0`\x80@\xe0\x80\xc0\xe0\x80\x00 @\x80 @\x00\xa0@\x80\xa0@\x00 \xc0\x80 \xc0\x00\xa0\xc0\x80\xa0\xc0@ @\xc0 @@\xa0@\xc0\xa0@@ \xc0\xc0 \xc0@\xa0\xc0\xc0\xa0\xc0\x00`@\x80`@\x00\xe0@\x80\xe0@\x00`\xc0\x80`\xc0\x00\xe0\xc0\x80\xe0\xc0@`@\xc0`@@\xe0@\xc0\xe0@@`\xc0\xc0`\xc0@\xe0\xc0\xc0\xe0\xc0 \x00\xa0 \x00 \xa0\x00\xa0\xa0\x00 \x80\xa0 \x80 \xa0\x80\xa0\xa0\x80` \x00\xe0 \x00`\xa0\x00\xe0\xa0\x00` \x80\xe0 \x80`\xa0\x80\xe0\xa0\x80 `\x00\xa0`\x00 \xe0\x00\xa0\xe0\x00 `\x80\xa0`\x80 \xe0\x80\xa0\xe0\x80``\x00\xe0`\x00`\xe0\x00\xe0\xe0\x00``\x80\xe0`\x80`\xe0\x80\xe0\xe0\x80 @\xa0 @ \xa0@\xa0\xa0@ \xc0\xa0 \xc0 \xa0\xc0\xa0\xa0\xc0` @\xe0 @`\xa0@\xe0\xa0@` \xc0\xe0 \xc0`\xa0\xc0\xe0\xa0\xc0 `@\xa0`@ \xe0@\xa0\xe0@ `\xc0\xa0`\xc0 \xe0\xc0\xa0\xe0\xc0``@\xe0`@`\xe0@\xe0\xe0@``\xc0\xe0`\xc0`\xe0\xc0\xe0\xe0\xc0' 2 | 3 | youtube_palette = b'\x00\x00\x00\xec_g\xf9\x91W\xfa\xc8c\x99\xc7\x94b\xb3\xb2f\x99\xcc\xc5\x94\xc5\xabyg\xff\xff\xffes~\x0b\x0b\x0b\x0c\x0c\x0c\r\r\r\x0e\x0e\x0e\x0f\x0f\x0f' 4 | -------------------------------------------------------------------------------- /XMem/model/losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | from collections import defaultdict 6 | 7 | 8 | def dice_loss(input_mask, cls_gt): 9 | num_objects = input_mask.shape[1] 10 | losses = [] 11 | for i in range(num_objects): 12 | mask = input_mask[:,i].flatten(start_dim=1) 13 | # background not in mask, so we add one to cls_gt 14 | gt = (cls_gt==(i+1)).float().flatten(start_dim=1) 15 | numerator = 2 * (mask * gt).sum(-1) 16 | denominator = mask.sum(-1) + gt.sum(-1) 17 | loss = 1 - (numerator + 1) / (denominator + 1) 18 | losses.append(loss) 19 | return torch.cat(losses).mean() 20 | 21 | 22 | # https://stackoverflow.com/questions/63735255/how-do-i-compute-bootstrapped-cross-entropy-loss-in-pytorch 23 | class BootstrappedCE(nn.Module): 24 | def __init__(self, start_warm, end_warm, top_p=0.15): 25 | super().__init__() 26 | 27 | self.start_warm = start_warm 28 | self.end_warm = end_warm 29 | self.top_p = top_p 30 | 31 | def forward(self, input, target, it): 32 | if it < self.start_warm: 33 | return F.cross_entropy(input, target), 1.0 34 | 35 | raw_loss = F.cross_entropy(input, target, reduction='none').view(-1) 36 | num_pixels = raw_loss.numel() 37 | 38 | if it > self.end_warm: 39 | this_p = self.top_p 40 | else: 41 | this_p = self.top_p + (1-self.top_p)*((self.end_warm-it)/(self.end_warm-self.start_warm)) 42 | loss, _ = torch.topk(raw_loss, int(num_pixels * this_p), sorted=False) 43 | return loss.mean(), this_p 44 | 45 | 46 | class LossComputer: 47 | def __init__(self, config): 48 | super().__init__() 49 | self.config = config 50 | self.bce = BootstrappedCE(config['start_warm'], config['end_warm']) 51 | 52 | def compute(self, data, num_objects, it): 53 | losses = defaultdict(int) 54 | 55 | b, t = data['rgb'].shape[:2] 56 | 57 | losses['total_loss'] = 0 58 | for ti in range(1, t): 59 | for bi in range(b): 60 | loss, p = self.bce(data[f'logits_{ti}'][bi:bi+1, :num_objects[bi]+1], data['cls_gt'][bi:bi+1,ti,0], it) 61 | losses['p'] += p / b / (t-1) 62 | losses[f'ce_loss_{ti}'] += loss / b 63 | 64 | losses['total_loss'] += losses['ce_loss_%d'%ti] 65 | losses[f'dice_loss_{ti}'] = dice_loss(data[f'masks_{ti}'], data['cls_gt'][:,ti,0]) 66 | losses['total_loss'] += losses[f'dice_loss_{ti}'] 67 | 68 | return losses 69 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/async_logger.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Very fast asynchronous logger (millions of logs per second on an average 9 | // desktop) 10 | // Uses pre allocated lockfree queue for maximum throughput even under large 11 | // number of threads. 12 | // Creates a single back thread to pop messages from the queue and log them. 13 | // 14 | // Upon each log write the logger: 15 | // 1. Checks if its log level is enough to log the message 16 | // 2. Push a new copy of the message to a queue (or block the caller until 17 | // space is available in the queue) 18 | // 3. will throw spdlog_ex upon log exceptions 19 | // Upon destruction, logs all remaining messages in the queue before 20 | // destructing.. 21 | 22 | #include "spdlog/common.h" 23 | #include "spdlog/logger.h" 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace spdlog { 30 | 31 | // Async overflow policy - block by default. 32 | enum class async_overflow_policy 33 | { 34 | block, // Block until message can be enqueued 35 | overrun_oldest // Discard oldest message in the queue if full when trying to 36 | // add new item. 37 | }; 38 | 39 | namespace details { 40 | class thread_pool; 41 | } 42 | 43 | class async_logger final : public std::enable_shared_from_this, public logger 44 | { 45 | friend class details::thread_pool; 46 | 47 | public: 48 | template 49 | async_logger(std::string logger_name, It begin, It end, std::weak_ptr tp, 50 | async_overflow_policy overflow_policy = async_overflow_policy::block); 51 | 52 | async_logger(std::string logger_name, sinks_init_list sinks_list, std::weak_ptr tp, 53 | async_overflow_policy overflow_policy = async_overflow_policy::block); 54 | 55 | async_logger(std::string logger_name, sink_ptr single_sink, std::weak_ptr tp, 56 | async_overflow_policy overflow_policy = async_overflow_policy::block); 57 | 58 | std::shared_ptr clone(std::string new_name) override; 59 | 60 | protected: 61 | void sink_it_(details::log_msg &msg) override; 62 | void flush_() override; 63 | 64 | void backend_log_(const details::log_msg &incoming_log_msg); 65 | void backend_flush_(); 66 | 67 | private: 68 | std::weak_ptr thread_pool_; 69 | async_overflow_policy overflow_policy_; 70 | }; 71 | } // namespace spdlog 72 | 73 | #include "details/async_logger_impl.h" 74 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/CUDACache.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "CUDACacheUtil.h" 4 | #include "CUDAImageUtil.h" 5 | #include 6 | 7 | class CUDACache { 8 | public: 9 | EIGEN_MAKE_ALIGNED_OPERATOR_NEW 10 | 11 | CUDACache(unsigned int widthDepthInput, unsigned int heightDepthInput, unsigned int widthDownSampled, unsigned int heightDownSampled, unsigned int maxNumImages, const Eigen::Matrix4f& inputIntrinsics); 12 | ~CUDACache(); 13 | void alloc(); 14 | void free(); 15 | 16 | void storeFrame(unsigned int inputDepthWidth, unsigned int inputDepthHeight, const float* d_depth, const uchar4* d_color, const float4 *d_normals); 17 | 18 | void reset() { 19 | m_currentFrame = 0; 20 | } 21 | 22 | const std::vector& getCacheFrames() const { return m_cache; } 23 | const CUDACachedFrame* getCacheFramesGPU() const { return d_cache; } 24 | 25 | void copyCacheFrameFrom(CUDACache* other, unsigned int frameFrom); 26 | //! for invalid (global) frames don't need to copy 27 | void incrementCache() { 28 | m_currentFrame++; 29 | } 30 | 31 | unsigned int getWidth() const { return m_width; } 32 | unsigned int getHeight() const { return m_height; } 33 | 34 | const Eigen::Matrix4f& getIntrinsics() const { return m_intrinsics; } 35 | const Eigen::Matrix4f& getIntrinsicsInv() const { return m_intrinsicsInv; } 36 | 37 | unsigned int getNumFrames() const { return m_currentFrame; } 38 | 39 | //!debugging only 40 | std::vector& getCachedFramesDEBUG() { return m_cache; } 41 | void setCurrentFrame(unsigned int c) { m_currentFrame = c; } 42 | void setIntrinsics(const Eigen::Matrix4f& inputIntrinsics, const Eigen::Matrix4f& intrinsics) { 43 | m_inputIntrinsics = inputIntrinsics; m_inputIntrinsicsInv = inputIntrinsics.inverse(); 44 | m_intrinsics = intrinsics; m_intrinsicsInv = intrinsics.inverse(); 45 | } 46 | 47 | private: 48 | unsigned int m_width; 49 | unsigned int m_height; 50 | Eigen::Matrix4f m_intrinsics; 51 | Eigen::Matrix4f m_intrinsicsInv; 52 | 53 | unsigned int m_currentFrame; 54 | unsigned int m_maxNumImages; 55 | 56 | std::vector < CUDACachedFrame > m_cache; 57 | CUDACachedFrame* d_cache; 58 | 59 | //for hi-res compute 60 | float* d_filterHelper; 61 | float4* d_helperCamPos, *d_helperNormals; //TODO ANGIE 62 | unsigned int m_inputDepthWidth; 63 | unsigned int m_inputDepthHeight; 64 | Eigen::Matrix4f m_inputIntrinsics; 65 | Eigen::Matrix4f m_inputIntrinsicsInv; 66 | 67 | float* d_intensityHelper; 68 | float m_filterIntensitySigma; 69 | float m_filterDepthSigmaD; 70 | float m_filterDepthSigmaR; 71 | }; 72 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/dist_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2015 David Schury, Gabi Melman 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "base_sink.h" 13 | #include "spdlog/details/log_msg.h" 14 | #include "spdlog/details/null_mutex.h" 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | // Distribution sink (mux). Stores a vector of sinks which get called when log 22 | // is called 23 | 24 | namespace spdlog { 25 | namespace sinks { 26 | 27 | template 28 | class dist_sink : public base_sink 29 | { 30 | public: 31 | dist_sink() = default; 32 | dist_sink(const dist_sink &) = delete; 33 | dist_sink &operator=(const dist_sink &) = delete; 34 | 35 | void add_sink(std::shared_ptr sink) 36 | { 37 | std::lock_guard lock(base_sink::mutex_); 38 | sinks_.push_back(sink); 39 | } 40 | 41 | void remove_sink(std::shared_ptr sink) 42 | { 43 | std::lock_guard lock(base_sink::mutex_); 44 | sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sink), sinks_.end()); 45 | } 46 | 47 | void set_sinks(std::vector> sinks) 48 | { 49 | std::lock_guard lock(base_sink::mutex_); 50 | sinks_ = std::move(sinks); 51 | } 52 | 53 | protected: 54 | void sink_it_(const details::log_msg &msg) override 55 | { 56 | 57 | for (auto &sink : sinks_) 58 | { 59 | if (sink->should_log(msg.level)) 60 | { 61 | sink->log(msg); 62 | } 63 | } 64 | } 65 | 66 | void flush_() override 67 | { 68 | for (auto &sink : sinks_) 69 | { 70 | sink->flush(); 71 | } 72 | } 73 | 74 | void set_pattern_(const std::string &pattern) override 75 | { 76 | set_formatter_(details::make_unique(pattern)); 77 | } 78 | 79 | void set_formatter_(std::unique_ptr sink_formatter) override 80 | { 81 | base_sink::formatter_ = std::move(sink_formatter); 82 | for (auto &sink : sinks_) 83 | { 84 | sink->set_formatter(base_sink::formatter_->clone()); 85 | } 86 | } 87 | std::vector> sinks_; 88 | }; 89 | 90 | using dist_sink_mt = dist_sink; 91 | using dist_sink_st = dist_sink; 92 | 93 | } // namespace sinks 94 | } // namespace spdlog 95 | -------------------------------------------------------------------------------- /XMem/util/log_integrator.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integrate numerical values for some iterations 3 | Typically used for loss computation / logging to tensorboard 4 | Call finalize and create a new Integrator when you want to display/log 5 | """ 6 | 7 | import torch 8 | 9 | 10 | class Integrator: 11 | def __init__(self, logger, distributed=True, local_rank=0, world_size=1): 12 | self.values = {} 13 | self.counts = {} 14 | self.hooks = [] # List is used here to maintain insertion order 15 | 16 | self.logger = logger 17 | 18 | self.distributed = distributed 19 | self.local_rank = local_rank 20 | self.world_size = world_size 21 | 22 | def add_tensor(self, key, tensor): 23 | if key not in self.values: 24 | self.counts[key] = 1 25 | if type(tensor) == float or type(tensor) == int: 26 | self.values[key] = tensor 27 | else: 28 | self.values[key] = tensor.mean().item() 29 | else: 30 | self.counts[key] += 1 31 | if type(tensor) == float or type(tensor) == int: 32 | self.values[key] += tensor 33 | else: 34 | self.values[key] += tensor.mean().item() 35 | 36 | def add_dict(self, tensor_dict): 37 | for k, v in tensor_dict.items(): 38 | self.add_tensor(k, v) 39 | 40 | def add_hook(self, hook): 41 | """ 42 | Adds a custom hook, i.e. compute new metrics using values in the dict 43 | The hook takes the dict as argument, and returns a (k, v) tuple 44 | e.g. for computing IoU 45 | """ 46 | if type(hook) == list: 47 | self.hooks.extend(hook) 48 | else: 49 | self.hooks.append(hook) 50 | 51 | def reset_except_hooks(self): 52 | self.values = {} 53 | self.counts = {} 54 | 55 | # Average and output the metrics 56 | def finalize(self, prefix, it, f=None): 57 | 58 | for hook in self.hooks: 59 | k, v = hook(self.values) 60 | self.add_tensor(k, v) 61 | 62 | for k, v in self.values.items(): 63 | 64 | if k[:4] == 'hide': 65 | continue 66 | 67 | avg = v / self.counts[k] 68 | 69 | if self.distributed: 70 | # Inplace operation 71 | avg = torch.tensor(avg).cuda() 72 | torch.distributed.reduce(avg, dst=0) 73 | 74 | if self.local_rank == 0: 75 | avg = (avg/self.world_size).cpu().item() 76 | self.logger.log_metrics(prefix, k, avg, it, f) 77 | else: 78 | # Simple does it 79 | self.logger.log_metrics(prefix, k, avg, it, f) 80 | 81 | -------------------------------------------------------------------------------- /XMem/inference/interact/s2m/s2m_network.py: -------------------------------------------------------------------------------- 1 | # Credit: https://github.com/VainF/DeepLabV3Plus-Pytorch 2 | 3 | from .utils import IntermediateLayerGetter 4 | from ._deeplab import DeepLabHead, DeepLabHeadV3Plus, DeepLabV3 5 | from . import s2m_resnet 6 | 7 | def _segm_resnet(name, backbone_name, num_classes, output_stride, pretrained_backbone): 8 | 9 | if output_stride==8: 10 | replace_stride_with_dilation=[False, True, True] 11 | aspp_dilate = [12, 24, 36] 12 | else: 13 | replace_stride_with_dilation=[False, False, True] 14 | aspp_dilate = [6, 12, 18] 15 | 16 | backbone = s2m_resnet.__dict__[backbone_name]( 17 | pretrained=pretrained_backbone, 18 | replace_stride_with_dilation=replace_stride_with_dilation) 19 | 20 | inplanes = 2048 21 | low_level_planes = 256 22 | 23 | if name=='deeplabv3plus': 24 | return_layers = {'layer4': 'out', 'layer1': 'low_level'} 25 | classifier = DeepLabHeadV3Plus(inplanes, low_level_planes, num_classes, aspp_dilate) 26 | elif name=='deeplabv3': 27 | return_layers = {'layer4': 'out'} 28 | classifier = DeepLabHead(inplanes , num_classes, aspp_dilate) 29 | backbone = IntermediateLayerGetter(backbone, return_layers=return_layers) 30 | 31 | model = DeepLabV3(backbone, classifier) 32 | return model 33 | 34 | def _load_model(arch_type, backbone, num_classes, output_stride, pretrained_backbone): 35 | 36 | if backbone.startswith('resnet'): 37 | model = _segm_resnet(arch_type, backbone, num_classes, output_stride=output_stride, pretrained_backbone=pretrained_backbone) 38 | else: 39 | raise NotImplementedError 40 | return model 41 | 42 | 43 | # Deeplab v3 44 | def deeplabv3_resnet50(num_classes=1, output_stride=16, pretrained_backbone=False): 45 | """Constructs a DeepLabV3 model with a ResNet-50 backbone. 46 | 47 | Args: 48 | num_classes (int): number of classes. 49 | output_stride (int): output stride for deeplab. 50 | pretrained_backbone (bool): If True, use the pretrained backbone. 51 | """ 52 | return _load_model('deeplabv3', 'resnet50', num_classes, output_stride=output_stride, pretrained_backbone=pretrained_backbone) 53 | 54 | 55 | # Deeplab v3+ 56 | def deeplabv3plus_resnet50(num_classes=1, output_stride=16, pretrained_backbone=False): 57 | """Constructs a DeepLabV3 model with a ResNet-50 backbone. 58 | 59 | Args: 60 | num_classes (int): number of classes. 61 | output_stride (int): output stride for deeplab. 62 | pretrained_backbone (bool): If True, use the pretrained backbone. 63 | """ 64 | return _load_model('deeplabv3plus', 'resnet50', num_classes, output_stride=output_stride, pretrained_backbone=pretrained_backbone) 65 | 66 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/loftr_module/fine_preprocess.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from einops.einops import rearrange, repeat 5 | 6 | 7 | class FinePreprocess(nn.Module): 8 | def __init__(self, config): 9 | super().__init__() 10 | 11 | self.config = config 12 | self.cat_c_feat = config['fine_concat_coarse_feat'] 13 | self.W = self.config['fine_window_size'] 14 | 15 | d_model_c = self.config['coarse']['d_model'] 16 | d_model_f = self.config['fine']['d_model'] 17 | self.d_model_f = d_model_f 18 | if self.cat_c_feat: 19 | self.down_proj = nn.Linear(d_model_c, d_model_f, bias=True) 20 | self.merge_feat = nn.Linear(2*d_model_f, d_model_f, bias=True) 21 | 22 | self._reset_parameters() 23 | 24 | def _reset_parameters(self): 25 | for p in self.parameters(): 26 | if p.dim() > 1: 27 | nn.init.kaiming_normal_(p, mode="fan_out", nonlinearity="relu") 28 | 29 | def forward(self, feat_f0, feat_f1, feat_c0, feat_c1, data): 30 | W = self.W 31 | stride = data['hw0_f'][0] // data['hw0_c'][0] 32 | 33 | data.update({'W': W}) 34 | if data['b_ids'].shape[0] == 0: 35 | feat0 = torch.empty(0, self.W**2, self.d_model_f, device=feat_f0.device) 36 | feat1 = torch.empty(0, self.W**2, self.d_model_f, device=feat_f0.device) 37 | return feat0, feat1 38 | 39 | # 1. unfold(crop) all local windows 40 | feat_f0_unfold = F.unfold(feat_f0, kernel_size=(W, W), stride=stride, padding=W//2) 41 | feat_f0_unfold = rearrange(feat_f0_unfold, 'n (c ww) l -> n l ww c', ww=W**2) 42 | feat_f1_unfold = F.unfold(feat_f1, kernel_size=(W, W), stride=stride, padding=W//2) 43 | feat_f1_unfold = rearrange(feat_f1_unfold, 'n (c ww) l -> n l ww c', ww=W**2) 44 | 45 | # 2. select only the predicted matches 46 | feat_f0_unfold = feat_f0_unfold[data['b_ids'], data['i_ids']] # [n, ww, cf] 47 | feat_f1_unfold = feat_f1_unfold[data['b_ids'], data['j_ids']] 48 | 49 | # option: use coarse-level loftr feature as context: concat and linear 50 | if self.cat_c_feat: 51 | feat_c_win = self.down_proj(torch.cat([feat_c0[data['b_ids'], data['i_ids']], 52 | feat_c1[data['b_ids'], data['j_ids']]], 0)) # [2n, c] 53 | feat_cf_win = self.merge_feat(torch.cat([ 54 | torch.cat([feat_f0_unfold, feat_f1_unfold], 0), # [2n, ww, cf] 55 | repeat(feat_c_win, 'n c -> n ww c', ww=W**2), # [2n, ww, cf] 56 | ], -1)) 57 | feat_f0_unfold, feat_f1_unfold = torch.chunk(feat_cf_win, 2, dim=0) 58 | 59 | return feat_f0_unfold, feat_f1_unfold 60 | -------------------------------------------------------------------------------- /XMem/model/group_modules.py: -------------------------------------------------------------------------------- 1 | """ 2 | Group-specific modules 3 | They handle features that also depends on the mask. 4 | Features are typically of shape 5 | batch_size * num_objects * num_channels * H * W 6 | 7 | All of them are permutation equivariant w.r.t. to the num_objects dimension 8 | """ 9 | 10 | import torch 11 | import torch.nn as nn 12 | import torch.nn.functional as F 13 | 14 | 15 | def interpolate_groups(g, ratio, mode, align_corners): 16 | batch_size, num_objects = g.shape[:2] 17 | g = F.interpolate(g.flatten(start_dim=0, end_dim=1), 18 | scale_factor=ratio, mode=mode, align_corners=align_corners) 19 | g = g.view(batch_size, num_objects, *g.shape[1:]) 20 | return g 21 | 22 | def upsample_groups(g, ratio=2, mode='bilinear', align_corners=False): 23 | return interpolate_groups(g, ratio, mode, align_corners) 24 | 25 | def downsample_groups(g, ratio=1/2, mode='area', align_corners=None): 26 | return interpolate_groups(g, ratio, mode, align_corners) 27 | 28 | 29 | class GConv2D(nn.Conv2d): 30 | def forward(self, g): 31 | batch_size, num_objects = g.shape[:2] 32 | g = super().forward(g.flatten(start_dim=0, end_dim=1)) 33 | return g.view(batch_size, num_objects, *g.shape[1:]) 34 | 35 | 36 | class GroupResBlock(nn.Module): 37 | def __init__(self, in_dim, out_dim): 38 | super().__init__() 39 | 40 | if in_dim == out_dim: 41 | self.downsample = None 42 | else: 43 | self.downsample = GConv2D(in_dim, out_dim, kernel_size=3, padding=1) 44 | 45 | self.conv1 = GConv2D(in_dim, out_dim, kernel_size=3, padding=1) 46 | self.conv2 = GConv2D(out_dim, out_dim, kernel_size=3, padding=1) 47 | 48 | def forward(self, g): 49 | out_g = self.conv1(F.relu(g)) 50 | out_g = self.conv2(F.relu(out_g)) 51 | 52 | if self.downsample is not None: 53 | g = self.downsample(g) 54 | 55 | return out_g + g 56 | 57 | 58 | class MainToGroupDistributor(nn.Module): 59 | def __init__(self, x_transform=None, method='cat', reverse_order=False): 60 | super().__init__() 61 | 62 | self.x_transform = x_transform 63 | self.method = method 64 | self.reverse_order = reverse_order 65 | 66 | def forward(self, x, g): 67 | num_objects = g.shape[1] 68 | 69 | if self.x_transform is not None: 70 | x = self.x_transform(x) 71 | 72 | if self.method == 'cat': 73 | if self.reverse_order: 74 | g = torch.cat([g, x.unsqueeze(1).expand(-1,num_objects,-1,-1,-1)], 2) 75 | else: 76 | g = torch.cat([x.unsqueeze(1).expand(-1,num_objects,-1,-1,-1), g], 2) 77 | elif self.method == 'add': 78 | g = x.unsqueeze(1).expand(-1,num_objects,-1,-1,-1) + g 79 | else: 80 | raise NotImplementedError 81 | 82 | return g 83 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/test.py: -------------------------------------------------------------------------------- 1 | import pytorch_lightning as pl 2 | import argparse 3 | import pprint 4 | from loguru import logger as loguru_logger 5 | 6 | from src.config.default import get_cfg_defaults 7 | from src.utils.profiler import build_profiler 8 | 9 | from src.lightning.data import MultiSceneDataModule 10 | from src.lightning.lightning_loftr import PL_LoFTR 11 | 12 | 13 | def parse_args(): 14 | # init a costum parser which will be added into pl.Trainer parser 15 | # check documentation: https://pytorch-lightning.readthedocs.io/en/latest/common/trainer.html#trainer-flags 16 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 17 | parser.add_argument( 18 | 'data_cfg_path', type=str, help='data config path') 19 | parser.add_argument( 20 | 'main_cfg_path', type=str, help='main config path') 21 | parser.add_argument( 22 | '--ckpt_path', type=str, default="weights/indoor_ds.ckpt", help='path to the checkpoint') 23 | parser.add_argument( 24 | '--dump_dir', type=str, default=None, help="if set, the matching results will be dump to dump_dir") 25 | parser.add_argument( 26 | '--profiler_name', type=str, default=None, help='options: [inference, pytorch], or leave it unset') 27 | parser.add_argument( 28 | '--batch_size', type=int, default=1, help='batch_size per gpu') 29 | parser.add_argument( 30 | '--num_workers', type=int, default=2) 31 | parser.add_argument( 32 | '--thr', type=float, default=None, help='modify the coarse-level matching threshold.') 33 | 34 | parser = pl.Trainer.add_argparse_args(parser) 35 | return parser.parse_args() 36 | 37 | 38 | if __name__ == '__main__': 39 | # parse arguments 40 | args = parse_args() 41 | pprint.pprint(vars(args)) 42 | 43 | # init default-cfg and merge it with the main- and data-cfg 44 | config = get_cfg_defaults() 45 | config.merge_from_file(args.main_cfg_path) 46 | config.merge_from_file(args.data_cfg_path) 47 | pl.seed_everything(config.TRAINER.SEED) # reproducibility 48 | 49 | # tune when testing 50 | if args.thr is not None: 51 | config.LOFTR.MATCH_COARSE.THR = args.thr 52 | 53 | loguru_logger.info(f"Args and config initialized!") 54 | 55 | # lightning module 56 | profiler = build_profiler(args.profiler_name) 57 | model = PL_LoFTR(config, pretrained_ckpt=args.ckpt_path, profiler=profiler, dump_dir=args.dump_dir) 58 | loguru_logger.info(f"LoFTR-lightning initialized!") 59 | 60 | # lightning data 61 | data_module = MultiSceneDataModule(args, config) 62 | loguru_logger.info(f"DataModule initialized!") 63 | 64 | # lightning trainer 65 | trainer = pl.Trainer.from_argparse_args(args, replace_sampler_ddp=False, logger=False) 66 | 67 | loguru_logger.info(f"Start testing!") 68 | trainer.test(model, datamodule=data_module, verbose=False) 69 | -------------------------------------------------------------------------------- /XMem/model/memory_util.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | from typing import Optional 5 | 6 | 7 | def get_similarity(mk, ms, qk, qe): 8 | # used for training/inference and memory reading/memory potentiation 9 | # mk: B x CK x [N] - Memory keys 10 | # ms: B x 1 x [N] - Memory shrinkage 11 | # qk: B x CK x [HW/P] - Query keys 12 | # qe: B x CK x [HW/P] - Query selection 13 | # Dimensions in [] are flattened 14 | CK = mk.shape[1] 15 | mk = mk.flatten(start_dim=2) 16 | ms = ms.flatten(start_dim=1).unsqueeze(2) if ms is not None else None 17 | qk = qk.flatten(start_dim=2) 18 | qe = qe.flatten(start_dim=2) if qe is not None else None 19 | 20 | if qe is not None: 21 | # See appendix for derivation 22 | # or you can just trust me ヽ(ー_ー )ノ 23 | mk = mk.transpose(1, 2) 24 | a_sq = (mk.pow(2) @ qe) 25 | two_ab = 2 * (mk @ (qk * qe)) 26 | b_sq = (qe * qk.pow(2)).sum(1, keepdim=True) 27 | similarity = (-a_sq+two_ab-b_sq) 28 | else: 29 | # similar to STCN if we don't have the selection term 30 | a_sq = mk.pow(2).sum(1).unsqueeze(2) 31 | two_ab = 2 * (mk.transpose(1, 2) @ qk) 32 | similarity = (-a_sq+two_ab) 33 | 34 | if ms is not None: 35 | similarity = similarity * ms / math.sqrt(CK) # B*N*HW 36 | else: 37 | similarity = similarity / math.sqrt(CK) # B*N*HW 38 | 39 | return similarity 40 | 41 | def do_softmax(similarity, top_k: Optional[int]=None, inplace=False, return_usage=False): 42 | # normalize similarity with top-k softmax 43 | # similarity: B x N x [HW/P] 44 | # use inplace with care 45 | if top_k is not None: 46 | values, indices = torch.topk(similarity, k=top_k, dim=1) 47 | 48 | x_exp = values.exp_() 49 | x_exp /= torch.sum(x_exp, dim=1, keepdim=True) 50 | if inplace: 51 | similarity.zero_().scatter_(1, indices, x_exp) # B*N*HW 52 | affinity = similarity 53 | else: 54 | affinity = torch.zeros_like(similarity).scatter_(1, indices, x_exp) # B*N*HW 55 | else: 56 | maxes = torch.max(similarity, dim=1, keepdim=True)[0] 57 | x_exp = torch.exp(similarity - maxes) 58 | x_exp_sum = torch.sum(x_exp, dim=1, keepdim=True) 59 | affinity = x_exp / x_exp_sum 60 | indices = None 61 | 62 | if return_usage: 63 | return affinity, affinity.sum(dim=2) 64 | 65 | return affinity 66 | 67 | def get_affinity(mk, ms, qk, qe): 68 | # shorthand used in training with no top-k 69 | similarity = get_similarity(mk, ms, qk, qe) 70 | affinity = do_softmax(similarity) 71 | return affinity 72 | 73 | def readout(affinity, mv): 74 | B, CV, T, H, W = mv.shape 75 | 76 | mo = mv.view(B, CV, T*H*W) 77 | mem = torch.bmm(mo, affinity) 78 | mem = mem.view(B, CV, H, W) 79 | 80 | return mem 81 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/modeling/basic_blocks.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | from ...model import ops 4 | 5 | 6 | class ConvHead(nn.Module): 7 | def __init__(self, out_channels, in_channels=32, num_layers=1, 8 | kernel_size=3, padding=1, 9 | norm_layer=nn.BatchNorm2d): 10 | super(ConvHead, self).__init__() 11 | convhead = [] 12 | 13 | for i in range(num_layers): 14 | convhead.extend([ 15 | nn.Conv2d(in_channels, in_channels, kernel_size, padding=padding), 16 | nn.ReLU(), 17 | norm_layer(in_channels) if norm_layer is not None else nn.Identity() 18 | ]) 19 | convhead.append(nn.Conv2d(in_channels, out_channels, 1, padding=0)) 20 | 21 | self.convhead = nn.Sequential(*convhead) 22 | 23 | def forward(self, *inputs): 24 | return self.convhead(inputs[0]) 25 | 26 | 27 | class SepConvHead(nn.Module): 28 | def __init__(self, num_outputs, in_channels, mid_channels, num_layers=1, 29 | kernel_size=3, padding=1, dropout_ratio=0.0, dropout_indx=0, 30 | norm_layer=nn.BatchNorm2d): 31 | super(SepConvHead, self).__init__() 32 | 33 | sepconvhead = [] 34 | 35 | for i in range(num_layers): 36 | sepconvhead.append( 37 | SeparableConv2d(in_channels=in_channels if i == 0 else mid_channels, 38 | out_channels=mid_channels, 39 | dw_kernel=kernel_size, dw_padding=padding, 40 | norm_layer=norm_layer, activation='relu') 41 | ) 42 | if dropout_ratio > 0 and dropout_indx == i: 43 | sepconvhead.append(nn.Dropout(dropout_ratio)) 44 | 45 | sepconvhead.append( 46 | nn.Conv2d(in_channels=mid_channels, out_channels=num_outputs, kernel_size=1, padding=0) 47 | ) 48 | 49 | self.layers = nn.Sequential(*sepconvhead) 50 | 51 | def forward(self, *inputs): 52 | x = inputs[0] 53 | 54 | return self.layers(x) 55 | 56 | 57 | class SeparableConv2d(nn.Module): 58 | def __init__(self, in_channels, out_channels, dw_kernel, dw_padding, dw_stride=1, 59 | activation=None, use_bias=False, norm_layer=None): 60 | super(SeparableConv2d, self).__init__() 61 | _activation = ops.select_activation_function(activation) 62 | self.body = nn.Sequential( 63 | nn.Conv2d(in_channels, in_channels, kernel_size=dw_kernel, stride=dw_stride, 64 | padding=dw_padding, bias=use_bias, groups=in_channels), 65 | nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, bias=use_bias), 66 | norm_layer(out_channels) if norm_layer is not None else nn.Identity(), 67 | _activation() 68 | ) 69 | 70 | def forward(self, x): 71 | return self.body(x) 72 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/config_ycbineoat.yml: -------------------------------------------------------------------------------- 1 | data_dir: /home/bowen/debug/ycbineoat 2 | model_name: textured_simple 3 | model_dir: "" 4 | debug_dir: /home/bowen/debug/BundleTrack/ 5 | init_pose_dir: "" 6 | SPDLOG: 2 7 | port: '5555' 8 | seg_port: '1111' 9 | nerf_port: "9999" 10 | 11 | segmentation: 12 | ob_scales: [0.3,0.3,0.3] 13 | tolerance: 0.03 14 | bg_dist: 0.01 15 | bg_dir: /home/bowen/debug/2022-09-07-16-49-22/bg.ply 16 | 17 | depth_processing: 18 | zfar: 2.0 19 | erode: 20 | radius: 1 21 | diff: 0.001 22 | ratio: 0.8 #if ratio larger than this, depth set to 0 23 | bilateral_filter: 24 | radius: 2 25 | sigma_D: 2 26 | sigma_R: 100000 27 | outlier_removal: 28 | num: 30 29 | std_mul: 1 30 | edge_normal_thres: 10 #deg between normal and ray 31 | denoise_cloud: False 32 | 33 | visible_angle: 70 # Angle between normal and point to camera origin within XXX is regarded as visible 34 | 35 | bundle: 36 | num_iter_outter: 7 37 | num_iter_inner: 5 38 | window_size: 5 #exclude keyframes, include new frame 39 | max_BA_frames: 10 40 | subset_selection_method: normal_orientation_nearest 41 | depth_association_radius: 5 # 0: findDenseCorr; 1: findDenseCorrNearestNeighbor3D 42 | non_neighbor_max_rot: 180 43 | non_neighbor_min_visible: 0.1 # ratio of pixel visible 44 | icp_pose_rot_thres: 180 # Rotation larger than XX deg is ignored for icp 45 | w_rpi: 0 46 | w_p2p: 1 47 | w_fm: 1 48 | w_sdf: 0 49 | w_pm: 0 50 | robust_delta: 0.005 51 | min_fm_edges_newframe: 15 52 | image_downscale: [4] 53 | feature_edge_dist_thres: 0.01 54 | feature_edge_normal_thres: 30 # Normal angle should be within this range 55 | 56 | keyframe: 57 | min_interval: 1 58 | min_feat_num: 0 59 | min_trans: 0 60 | min_rot: 5 61 | min_visible: 1 62 | 63 | 64 | sift: 65 | scales: [2,4,8] 66 | max_match_per_query: 5 67 | nOctaveLayers: 3 68 | contrastThreshold: 0.01 69 | edgeThreshold: 50 70 | sigma: 1.6 71 | 72 | feature_corres: 73 | mutual: True 74 | map_points: False 75 | max_dist_no_neighbor: 0.02 76 | max_normal_no_neighbor: 45 77 | max_dist_neighbor: 0.03 78 | max_normal_neighbor: 45 79 | suppression_patch_size: 5 80 | max_view_normal_angle: 180 81 | min_match_with_ref: 5 82 | resize: 400 83 | 84 | ransac: 85 | max_iter: 2000 86 | num_sample: 3 87 | inlier_dist: 0.015 88 | inlier_normal_angle: 45 89 | desired_succ_rate: 0.99 90 | max_trans_neighbor: 0.03 #ransac model estimated pose shouldnt be too far 91 | max_rot_deg_neighbor: 30 92 | max_trans_no_neighbor: 0.02 93 | max_rot_no_neighbor: 10 94 | epipolar_thres: 1 95 | min_match_after_ransac: 5 96 | 97 | p2p: 98 | projective: false 99 | max_dist: 0.02 100 | max_normal_angle: 45 101 | 102 | sdf_edge: 103 | max_dist: 0.02 104 | 105 | shape: 106 | res: 0.005 107 | xrange: [-0.2,0.2] 108 | yrange: [-0.2,0.2] 109 | zrange: [-0.2,0.2] 110 | max_weight: 100 111 | truncate_dist: 0.005 112 | 113 | 114 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/utils/fine_matching.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torch.nn as nn 4 | 5 | from kornia.geometry.subpix import dsnt 6 | from kornia.utils.grid import create_meshgrid 7 | 8 | 9 | class FineMatching(nn.Module): 10 | """FineMatching with s2d paradigm""" 11 | 12 | def __init__(self): 13 | super().__init__() 14 | 15 | def forward(self, feat_f0, feat_f1, data): 16 | """ 17 | Args: 18 | feat0 (torch.Tensor): [M, WW, C] 19 | feat1 (torch.Tensor): [M, WW, C] 20 | data (dict) 21 | Update: 22 | data (dict):{ 23 | 'expec_f' (torch.Tensor): [M, 3], 24 | 'mkpts0_f' (torch.Tensor): [M, 2], 25 | 'mkpts1_f' (torch.Tensor): [M, 2]} 26 | """ 27 | M, WW, C = feat_f0.shape 28 | W = int(math.sqrt(WW)) 29 | scale = data['hw0_i'][0] / data['hw0_f'][0] 30 | self.M, self.W, self.WW, self.C, self.scale = M, W, WW, C, scale 31 | 32 | # corner case: if no coarse matches found 33 | if M == 0: 34 | assert self.training == False, "M is always >0, when training, see coarse_matching.py" 35 | # logger.warning('No matches found in coarse-level.') 36 | data.update({ 37 | 'expec_f': torch.empty(0, 3, device=feat_f0.device), 38 | 'mkpts0_f': data['mkpts0_c'], 39 | 'mkpts1_f': data['mkpts1_c'], 40 | }) 41 | return 42 | 43 | feat_f0_picked = feat_f0_picked = feat_f0[:, WW//2, :] 44 | sim_matrix = torch.einsum('mc,mrc->mr', feat_f0_picked, feat_f1) 45 | softmax_temp = 1. / C**.5 46 | heatmap = torch.softmax(softmax_temp * sim_matrix, dim=1).view(-1, W, W) 47 | 48 | # compute coordinates from heatmap 49 | coords_normalized = dsnt.spatial_expectation2d(heatmap[None], True)[0] # [M, 2] 50 | grid_normalized = create_meshgrid(W, W, True, heatmap.device).reshape(1, -1, 2) # [1, WW, 2] 51 | 52 | # compute std over 53 | var = torch.sum(grid_normalized**2 * heatmap.view(-1, WW, 1), dim=1) - coords_normalized**2 # [M, 2] 54 | std = torch.sum(torch.sqrt(torch.clamp(var, min=1e-10)), -1) # [M] clamp needed for numerical stability 55 | 56 | # for fine-level supervision 57 | data.update({'expec_f': torch.cat([coords_normalized, std.unsqueeze(1)], -1)}) 58 | 59 | # compute absolute kpt coords 60 | self.get_fine_match(coords_normalized, data) 61 | 62 | @torch.no_grad() 63 | def get_fine_match(self, coords_normed, data): 64 | W, WW, C, scale = self.W, self.WW, self.C, self.scale 65 | 66 | # mkpts0_f and mkpts1_f 67 | mkpts0_f = data['mkpts0_c'] 68 | scale1 = scale * data['scale1'][data['b_ids']] if 'scale0' in data else scale 69 | mkpts1_f = data['mkpts1_c'] + (coords_normed * (W // 2) * scale1)[:len(data['mconf'])] 70 | 71 | data.update({ 72 | "mkpts0_f": mkpts0_f, 73 | "mkpts1_f": mkpts1_f 74 | }) 75 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/fmt/bundled/locale.h: -------------------------------------------------------------------------------- 1 | // Formatting library for C++ - std::locale support 2 | // 3 | // Copyright (c) 2012 - present, Victor Zverovich 4 | // All rights reserved. 5 | // 6 | // For the license information refer to format.h. 7 | 8 | #ifndef FMT_LOCALE_H_ 9 | #define FMT_LOCALE_H_ 10 | 11 | #include "format.h" 12 | #include 13 | 14 | FMT_BEGIN_NAMESPACE 15 | 16 | namespace internal { 17 | template 18 | typename buffer_context::type::iterator vformat_to( 19 | const std::locale &loc, basic_buffer &buf, 20 | basic_string_view format_str, 21 | basic_format_args::type> args) { 22 | typedef back_insert_range > range; 23 | return vformat_to>( 24 | buf, to_string_view(format_str), args, internal::locale_ref(loc)); 25 | } 26 | 27 | template 28 | std::basic_string vformat( 29 | const std::locale &loc, basic_string_view format_str, 30 | basic_format_args::type> args) { 31 | basic_memory_buffer buffer; 32 | internal::vformat_to(loc, buffer, format_str, args); 33 | return fmt::to_string(buffer); 34 | } 35 | } 36 | 37 | template 38 | inline std::basic_string vformat( 39 | const std::locale &loc, const S &format_str, 40 | basic_format_args::type> args) { 41 | return internal::vformat(loc, to_string_view(format_str), args); 42 | } 43 | 44 | template 45 | inline std::basic_string format( 46 | const std::locale &loc, const S &format_str, const Args &... args) { 47 | return internal::vformat( 48 | loc, to_string_view(format_str), 49 | *internal::checked_args(format_str, args...)); 50 | } 51 | 52 | template 53 | inline typename std::enable_if::value, 54 | OutputIt>::type 55 | vformat_to(OutputIt out, const std::locale &loc, const String &format_str, 56 | typename format_args_t::type args) { 57 | typedef output_range range; 58 | return vformat_to>( 59 | range(out), to_string_view(format_str), args, internal::locale_ref(loc)); 60 | } 61 | 62 | template 63 | inline typename std::enable_if< 64 | internal::is_string::value && 65 | internal::is_output_iterator::value, OutputIt>::type 66 | format_to(OutputIt out, const std::locale &loc, const S &format_str, 67 | const Args &... args) { 68 | internal::check_format_string(format_str); 69 | typedef typename format_context_t::type context; 70 | format_arg_store as{args...}; 71 | return vformat_to(out, loc, to_string_view(format_str), 72 | basic_format_args(as)); 73 | } 74 | 75 | FMT_END_NAMESPACE 76 | 77 | #endif // FMT_LOCALE_H_ 78 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/config_ho3d.yml: -------------------------------------------------------------------------------- 1 | data_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/HO3D_v3/evaluation/SM1 2 | model_name: textured_simple 3 | model_dir: "" 4 | debug_dir: /home/bowen/debug/BundleTrack/ 5 | init_pose_dir: "" 6 | SPDLOG: 2 7 | port: '5555' 8 | seg_port: '1111' 9 | nerf_port: "9999" 10 | 11 | segmentation: 12 | ob_scales: [0.3,0.3,0.3] 13 | tolerance: 0.03 14 | 15 | depth_processing: 16 | zfar: 1.0 17 | erode: 18 | radius: 1 19 | diff: 0.001 20 | ratio: 0.8 #if ratio larger than this, depth set to 0 21 | bilateral_filter: 22 | radius: 2 23 | sigma_D: 2 24 | sigma_R: 100000 25 | outlier_removal: 26 | num: 30 27 | std_mul: 3 28 | edge_normal_thres: 10 #deg between normal and ray 29 | denoise_cloud: False 30 | percentile: 100 31 | 32 | visible_angle: 70 # Angle between normal and point to camera origin within XXX is regarded as visible 33 | 34 | bundle: 35 | num_iter_outter: 7 36 | num_iter_inner: 5 37 | window_size: 5 #exclude keyframes, include new frame 38 | max_BA_frames: 10 39 | subset_selection_method: normal_orientation_nearest 40 | depth_association_radius: 5 # 0: findDenseCorr; 1: findDenseCorrNearestNeighbor3D 41 | non_neighbor_max_rot: 90 42 | non_neighbor_min_visible: 0.1 # ratio of pixel visible 43 | icp_pose_rot_thres: 60 # Rotation larger than XX deg is ignored for icp 44 | w_rpi: 0 45 | w_p2p: 1 # Used in loss.cpp 46 | w_fm: 1 47 | w_sdf: 0 48 | w_pm: 0 49 | robust_delta: 0.005 50 | min_fm_edges_newframe: 15 51 | image_downscale: [4] 52 | feature_edge_dist_thres: 0.01 53 | feature_edge_normal_thres: 30 # Normal angle should be within this range 54 | max_optimized_feature_loss: 0.03 55 | 56 | keyframe: 57 | min_interval: 1 58 | min_feat_num: 0 59 | min_trans: 0 60 | min_rot: 5 61 | min_visible: 1 62 | 63 | 64 | sift: 65 | scales: [2,4,8] 66 | max_match_per_query: 5 67 | nOctaveLayers: 3 68 | contrastThreshold: 0.01 69 | edgeThreshold: 50 70 | sigma: 1.6 71 | 72 | feature_corres: 73 | mutual: True 74 | map_points: False 75 | max_dist_no_neighbor: 999 76 | max_normal_no_neighbor: 180 77 | max_dist_neighbor: 0.02 78 | max_normal_neighbor: 30 79 | suppression_patch_size: 5 80 | max_view_normal_angle: 180 81 | min_match_with_ref: 5 82 | resize: 400 83 | rematch_after_nerf: False 84 | 85 | ransac: 86 | max_iter: 2000 87 | num_sample: 3 88 | inlier_dist: 0.005 89 | inlier_normal_angle: 30 90 | desired_succ_rate: 0.99 91 | max_trans_neighbor: 0.02 #ransac model estimated pose shouldnt be too far 92 | max_rot_deg_neighbor: 30 93 | max_trans_no_neighbor: 999 94 | max_rot_no_neighbor: 180 95 | epipolar_thres: 1 96 | min_match_after_ransac: 5 97 | 98 | p2p: 99 | projective: false 100 | max_dist: 0.01 101 | max_normal_angle: 20 102 | 103 | sdf_edge: 104 | max_dist: 0.02 105 | 106 | shape: 107 | res: 0.005 108 | xrange: [-0.2,0.2] 109 | yrange: [-0.2,0.2] 110 | zrange: [-0.2,0.2] 111 | max_weight: 100 112 | truncate_dist: 0.005 113 | 114 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/config_behave.yml: -------------------------------------------------------------------------------- 1 | data_dir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/DATASET/behave/BOWEN_ADDON/Date03_Sub03_boxmedium.2.color 2 | model_name: textured_simple 3 | model_dir: "" 4 | debug_dir: /home/bowen/debug/BundleTrack/ 5 | init_pose_dir: "" 6 | SPDLOG: 2 7 | port: '5555' 8 | seg_port: '1111' 9 | nerf_port: "9999" 10 | image_down_scale: 3 11 | 12 | segmentation: 13 | ob_scales: [0.3,0.3,0.3] 14 | tolerance: 0.03 15 | 16 | depth_processing: 17 | zfar: 3.5 18 | erode: 19 | radius: 1 20 | diff: 0.001 21 | ratio: 0.8 #if ratio larger than this, depth set to 0 22 | bilateral_filter: 23 | radius: 2 24 | sigma_D: 2 25 | sigma_R: 100000 26 | outlier_removal: 27 | num: 30 28 | std_mul: 3 29 | edge_normal_thres: 10 #deg between normal and ray 30 | denoise_cloud: False 31 | percentile: 100 32 | 33 | visible_angle: 70 # Angle between normal and point to camera origin within XXX is regarded as visible 34 | 35 | bundle: 36 | num_iter_outter: 7 37 | num_iter_inner: 5 38 | window_size: 5 #exclude keyframes, include new frame 39 | max_BA_frames: 10 40 | subset_selection_method: normal_orientation_nearest 41 | depth_association_radius: 5 # 0: findDenseCorr; 1: findDenseCorrNearestNeighbor3D 42 | non_neighbor_max_rot: 90 43 | non_neighbor_min_visible: 0.1 # ratio of pixel visible 44 | icp_pose_rot_thres: 60 # Rotation larger than XX deg is ignored for icp 45 | w_rpi: 0 46 | w_p2p: 1 47 | w_fm: 1 48 | w_sdf: 0 49 | w_pm: 0 50 | robust_delta: 0.005 51 | min_fm_edges_newframe: 15 52 | image_downscale: [4] 53 | feature_edge_dist_thres: 0.01 54 | feature_edge_normal_thres: 30 # Normal angle should be within this range 55 | max_optimized_feature_loss: 0.05 56 | 57 | keyframe: 58 | min_interval: 1 59 | min_feat_num: 0 60 | min_trans: 0 61 | min_rot: 5 62 | min_visible: 1 63 | 64 | 65 | sift: 66 | scales: [2,4,8] 67 | max_match_per_query: 5 68 | nOctaveLayers: 3 69 | contrastThreshold: 0.01 70 | edgeThreshold: 50 71 | sigma: 1.6 72 | 73 | feature_corres: 74 | mutual: True 75 | map_points: False 76 | max_dist_no_neighbor: 999 77 | max_normal_no_neighbor: 180 78 | max_dist_neighbor: 0.1 79 | max_normal_neighbor: 30 80 | suppression_patch_size: 5 81 | max_view_normal_angle: 180 82 | min_match_with_ref: 15 83 | resize: 400 84 | rematch_after_nerf: False 85 | 86 | ransac: 87 | max_iter: 2000 88 | num_sample: 3 89 | inlier_dist: 0.01 90 | inlier_normal_angle: 20 91 | desired_succ_rate: 0.99 92 | max_trans_neighbor: 0.1 #ransac model estimated pose shouldnt be too far 93 | max_rot_deg_neighbor: 30 94 | max_trans_no_neighbor: 999 95 | max_rot_no_neighbor: 180 96 | epipolar_thres: 1 97 | min_match_after_ransac: 5 98 | 99 | p2p: 100 | projective: false 101 | max_dist: 0.02 102 | max_normal_angle: 45 103 | 104 | sdf_edge: 105 | max_dist: 0.02 106 | 107 | shape: 108 | res: 0.005 109 | xrange: [-0.2,0.2] 110 | yrange: [-0.2,0.2] 111 | zrange: [-0.2,0.2] 112 | max_weight: 100 113 | truncate_dist: 0.005 114 | 115 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/LoFTR/src/loftr/loftr_module/linear_attention.py: -------------------------------------------------------------------------------- 1 | """ 2 | Linear Transformer proposed in "Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention" 3 | Modified from: https://github.com/idiap/fast-transformers/blob/master/fast_transformers/attention/linear_attention.py 4 | """ 5 | 6 | import torch 7 | from torch.nn import Module, Dropout 8 | 9 | 10 | def elu_feature_map(x): 11 | return torch.nn.functional.elu(x) + 1 12 | 13 | 14 | class LinearAttention(Module): 15 | def __init__(self, eps=1e-6): 16 | super().__init__() 17 | self.feature_map = elu_feature_map 18 | self.eps = eps 19 | 20 | def forward(self, queries, keys, values, q_mask=None, kv_mask=None): 21 | """ Multi-Head linear attention proposed in "Transformers are RNNs" 22 | Args: 23 | queries: [N, L, H, D] 24 | keys: [N, S, H, D] 25 | values: [N, S, H, D] 26 | q_mask: [N, L] 27 | kv_mask: [N, S] 28 | Returns: 29 | queried_values: (N, L, H, D) 30 | """ 31 | Q = self.feature_map(queries) 32 | K = self.feature_map(keys) 33 | 34 | # set padded position to zero 35 | if q_mask is not None: 36 | Q = Q * q_mask[:, :, None, None] 37 | if kv_mask is not None: 38 | K = K * kv_mask[:, :, None, None] 39 | values = values * kv_mask[:, :, None, None] 40 | 41 | v_length = values.size(1) 42 | values = values / v_length # prevent fp16 overflow 43 | KV = torch.einsum("nshd,nshv->nhdv", K, values) # (S,D)' @ S,V 44 | Z = 1 / (torch.einsum("nlhd,nhd->nlh", Q, K.sum(dim=1)) + self.eps) 45 | queried_values = torch.einsum("nlhd,nhdv,nlh->nlhv", Q, KV, Z) * v_length 46 | 47 | return queried_values.contiguous() 48 | 49 | 50 | class FullAttention(Module): 51 | def __init__(self, use_dropout=False, attention_dropout=0.1): 52 | super().__init__() 53 | self.use_dropout = use_dropout 54 | self.dropout = Dropout(attention_dropout) 55 | 56 | def forward(self, queries, keys, values, q_mask=None, kv_mask=None): 57 | """ Multi-head scaled dot-product attention, a.k.a full attention. 58 | Args: 59 | queries: [N, L, H, D] 60 | keys: [N, S, H, D] 61 | values: [N, S, H, D] 62 | q_mask: [N, L] 63 | kv_mask: [N, S] 64 | Returns: 65 | queried_values: (N, L, H, D) 66 | """ 67 | 68 | # Compute the unnormalized attention and apply the masks 69 | QK = torch.einsum("nlhd,nshd->nlsh", queries, keys) 70 | if kv_mask is not None: 71 | QK.masked_fill_(~(q_mask[:, :, None, None] * kv_mask[:, None, :, None]), float('-inf')) 72 | 73 | # Compute the attention and the weighted average 74 | softmax_temp = 1. / queries.size(3)**.5 # sqrt(D) 75 | A = torch.softmax(softmax_temp * QK, dim=2) 76 | if self.use_dropout: 77 | A = self.dropout(A) 78 | 79 | queried_values = torch.einsum("nlsh,nshd->nlhd", A, values) 80 | 81 | return queried_values.contiguous() 82 | -------------------------------------------------------------------------------- /BundleSDF/config.yml: -------------------------------------------------------------------------------- 1 | notes: '' 2 | n_step: 500 3 | netdepth: 8 4 | netwidth: 256 5 | netdepth_fine: 8 6 | netwidth_fine: 256 7 | N_rand: 2048 # Batch number of rays 8 | first_frame_ray_in_batch: 0 9 | lrate: 0.01 10 | lrate_pose: 0.01 11 | pose_optimize_start: 0 12 | decay_rate: 0.1 13 | chunk: 99999999999 14 | netchunk: 6553600 15 | no_batching: 0 16 | amp: true 17 | 18 | N_samples: 128 #number of coarse samples per ray 19 | N_samples_around_depth: 64 20 | N_importance: 0 21 | N_importance_iter: 1 22 | perturb: 1 23 | use_viewdirs: 1 24 | i_embed: 1 #set 1 for hashed embedding, 0 for default positional encoding, 2 for spherical; 3 for octree grid 25 | i_embed_views: 2 #set 1 for hashed embedding, 0 for default positional encoding, 2 for spherical 26 | multires: 8 #log2 of max freq for positional encoding (3D location) 27 | multires_views: 3 #log2 of max freq for positional encoding (2D direction) 28 | feature_grid_dim: 2 29 | raw_noise_std: 0 30 | white_bkgd: 0 31 | gradient_max_norm: 0.1 32 | gradient_pose_max_norm: 0.1 33 | 34 | # logging/saving options 35 | i_print: 999999 36 | i_img: 999999 37 | i_weights: 999999 38 | i_mesh: 999999 39 | i_nerf_normals: 999999 40 | i_save_ray: 999999 41 | i_pose: 999999 42 | save_octree_clouds: True 43 | 44 | finest_res: 128 45 | base_res: 16 46 | num_levels: 4 47 | log2_hashmap_size: 22 48 | datadir: /mnt/9a72c439-d0a7-45e8-8d20-d7a235d02763/github/bundlesdf/data/bundlesdf_bundlesdf_e03000196 49 | n_train_image: 300 50 | use_octree: 1 51 | first_frame_weight: 10 52 | denoise_depth_use_octree_cloud: true 53 | octree_embed_base_voxel_size: 0.02 54 | octree_smallest_voxel_size: 0.02 # This determines the smallest feature vox size 55 | octree_raytracing_voxel_size: 0.02 56 | octree_dilate_size: 0.02 # meters 57 | down_scale_ratio: 1 58 | bounding_box: [[-1,-1,-1], [1,1,1]] 59 | farthest_pose_sampling: 0 # Sampling train images. This replace uniform skip 60 | use_mask: 1 61 | dilate_mask_size: 0 62 | rays_valid_depth_only: true 63 | near: 0.1 64 | far: 2 65 | rgb_weight: 10 66 | depth_weight: 0 67 | trunc: 0.01 #length of the truncation region in meters 68 | trunc_start: 0.01 69 | sdf_lambda: 5 70 | neg_trunc_ratio: 1 # -trunc distance ratio compared to +trunc 71 | trunc_decay_type: '' 72 | sdf_loss_type: l2 73 | fs_weight: 100 74 | empty_weight: 0.01 75 | fs_rgb_weight: 0 76 | trunc_weight: 6000 77 | sparse_loss_weight: 0 78 | tv_loss_weight: 0 79 | frame_features: 0 #number of channels of the learnable per-frame features 80 | optimize_poses: 1 #optimize a pose refinement for the initial poses 81 | pose_reg_weight: 0 82 | point_cloud_loss_weight: 0 83 | point_cloud_loss_normal_weight: 0 84 | eikonal_weight: 0 85 | normal_loss_weight: 0 86 | feature_reg_weight: 0.1 87 | share_coarse_fine: 1 88 | mode: sdf 89 | fs_sdf: 0.001 # Uncertain free space 90 | crop: 0 91 | mesh_resolution: 0.002 92 | max_trans: 0.02 # meters 93 | max_rot: 20 # deg 94 | 95 | continual: True 96 | 97 | ######### dbscan 98 | dbscan_eps: 0.06 99 | dbscan_eps_min_samples: 1 100 | 101 | ####### bundlenerf 102 | sync_max_delay: 0 # 0 for strict sync 103 | -------------------------------------------------------------------------------- /BundleSDF/loftr_wrapper.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION and its licensors retain all intellectual property 4 | # and proprietary rights in and to this software, related documentation 5 | # and any modifications thereto. Any use, reproduction, disclosure or 6 | # distribution of this software and related documentation without an express 7 | # license agreement from NVIDIA CORPORATION is strictly prohibited. 8 | 9 | 10 | import os,zmq,pdb,sys,time,torchvision 11 | code_dir = os.path.dirname(os.path.realpath(__file__)) 12 | import argparse 13 | import cv2 14 | import torch,imageio 15 | from BundleTrack.LoFTR.src.loftr import * 16 | from Utils import * 17 | 18 | 19 | class LoftrRunner: 20 | def __init__(self): 21 | default_cfg['match_coarse']['thr'] = 0.2 22 | print("default_cfg",default_cfg) 23 | self.matcher = LoFTR(config=default_cfg) 24 | self.matcher.load_state_dict(torch.load(f'{code_dir}/BundleTrack/LoFTR/weights/outdoor_ds.ckpt')['state_dict']) 25 | self.matcher = self.matcher.eval().cuda() 26 | 27 | 28 | @torch.no_grad() 29 | def predict(self, rgbAs:np.ndarray, rgbBs:np.ndarray): 30 | ''' 31 | @rgbAs: (N,H,W,C) 32 | ''' 33 | image0 = torch.from_numpy(rgbAs).permute(0,3,1,2).float().cuda() 34 | image1 = torch.from_numpy(rgbBs).permute(0,3,1,2).float().cuda() 35 | if image0.shape[-1]==3: 36 | image0 = torchvision.transforms.functional.rgb_to_grayscale(image0) 37 | image1 = torchvision.transforms.functional.rgb_to_grayscale(image1) 38 | image0 = image0/255.0 39 | image1 = image1/255.0 40 | last_data = {'image0': image0, 'image1': image1} 41 | logging.info(f"image0: {last_data['image0'].shape}") 42 | 43 | batch_size = 16 44 | ret_keys = ['mkpts0_f','mkpts1_f','mconf','m_bids'] 45 | with torch.cuda.amp.autocast(enabled=True): 46 | i_b = 0 47 | for b in range(0,len(last_data['image0']),batch_size): 48 | tmp = {'image0': last_data['image0'][b:b+batch_size], 'image1': last_data['image1'][b:b+batch_size]} 49 | with torch.no_grad(): 50 | self.matcher(tmp) 51 | tmp['m_bids'] += i_b 52 | for k in ret_keys: 53 | if k not in last_data: 54 | last_data[k] = [] 55 | last_data[k].append(tmp[k]) 56 | i_b += len(tmp['image0']) 57 | 58 | logging.info("net forward") 59 | 60 | for k in ret_keys: 61 | last_data[k] = torch.cat(last_data[k],dim=0) 62 | 63 | total_n_matches = len(last_data['mkpts0_f']) 64 | mkpts0 = last_data['mkpts0_f'].cpu().numpy() 65 | mkpts1 = last_data['mkpts1_f'].cpu().numpy() 66 | mconf = last_data['mconf'].cpu().numpy() 67 | pair_ids = last_data['m_bids'].cpu().numpy() 68 | logging.info(f"mconf, {mconf.min()} {mconf.max()}") 69 | logging.info(f'pair_ids {pair_ids.shape}') 70 | corres = np.concatenate((mkpts0.reshape(-1,2),mkpts1.reshape(-1,2),mconf.reshape(-1,1)),axis=-1).reshape(-1,5).astype(np.float32) 71 | 72 | logging.info(f'corres: {corres.shape}') 73 | corres_tmp = [] 74 | for i in range(len(rgbAs)): 75 | cur_corres = corres[pair_ids==i] 76 | corres_tmp.append(cur_corres) 77 | corres = corres_tmp 78 | 79 | del last_data, image0, image1 80 | torch.cuda.empty_cache() 81 | 82 | return corres -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/fmt/format.cc: -------------------------------------------------------------------------------- 1 | // Formatting library for C++ 2 | // 3 | // Copyright (c) 2012 - 2016, Victor Zverovich 4 | // All rights reserved. 5 | // 6 | // For the license information refer to format.h. 7 | 8 | #include "fmt/format-inl.h" 9 | 10 | FMT_BEGIN_NAMESPACE 11 | namespace detail { 12 | 13 | template 14 | int format_float(char* buf, std::size_t size, const char* format, int precision, 15 | T value) { 16 | #ifdef FMT_FUZZ 17 | if (precision > 100000) 18 | throw std::runtime_error( 19 | "fuzz mode - avoid large allocation inside snprintf"); 20 | #endif 21 | // Suppress the warning about nonliteral format string. 22 | int (*snprintf_ptr)(char*, size_t, const char*, ...) = FMT_SNPRINTF; 23 | return precision < 0 ? snprintf_ptr(buf, size, format, value) 24 | : snprintf_ptr(buf, size, format, precision, value); 25 | } 26 | 27 | template FMT_API dragonbox::decimal_fp dragonbox::to_decimal(float x) 28 | FMT_NOEXCEPT; 29 | template FMT_API dragonbox::decimal_fp dragonbox::to_decimal(double x) 30 | FMT_NOEXCEPT; 31 | } // namespace detail 32 | 33 | // Workaround a bug in MSVC2013 that prevents instantiation of format_float. 34 | int (*instantiate_format_float)(double, int, detail::float_specs, 35 | detail::buffer&) = detail::format_float; 36 | 37 | #ifndef FMT_STATIC_THOUSANDS_SEPARATOR 38 | template FMT_API detail::locale_ref::locale_ref(const std::locale& loc); 39 | template FMT_API std::locale detail::locale_ref::get() const; 40 | #endif 41 | 42 | // Explicit instantiations for char. 43 | 44 | template FMT_API auto detail::thousands_sep_impl(locale_ref) 45 | -> thousands_sep_result; 46 | template FMT_API char detail::decimal_point_impl(locale_ref); 47 | 48 | template FMT_API void detail::buffer::append(const char*, const char*); 49 | 50 | // DEPRECATED! 51 | // There is no correspondent extern template in format.h because of 52 | // incompatibility between clang and gcc (#2377). 53 | template FMT_API void detail::vformat_to( 54 | detail::buffer&, string_view, 55 | basic_format_args, detail::locale_ref); 56 | 57 | template FMT_API int detail::snprintf_float(double, int, detail::float_specs, 58 | detail::buffer&); 59 | template FMT_API int detail::snprintf_float(long double, int, 60 | detail::float_specs, 61 | detail::buffer&); 62 | template FMT_API int detail::format_float(double, int, detail::float_specs, 63 | detail::buffer&); 64 | template FMT_API int detail::format_float(long double, int, detail::float_specs, 65 | detail::buffer&); 66 | 67 | // Explicit instantiations for wchar_t. 68 | 69 | template FMT_API auto detail::thousands_sep_impl(locale_ref) 70 | -> thousands_sep_result; 71 | template FMT_API wchar_t detail::decimal_point_impl(locale_ref); 72 | 73 | template FMT_API void detail::buffer::append(const wchar_t*, 74 | const wchar_t*); 75 | 76 | template struct detail::basic_data; 77 | 78 | FMT_END_NAMESPACE 79 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/cuda/SBA.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | 4 | #include "SIFTImageManager.h" 5 | #include "Solver/CUDASolverBundling.h" 6 | #include "yaml-cpp/yaml.h" 7 | 8 | struct JacobianBlock { 9 | Eigen::Vector3f data[6]; 10 | }; 11 | 12 | class SBA 13 | { 14 | public: 15 | SBA(int maxNumImages, int maxNumResiduals, int max_corr_per_image, const std::vector &update_pose_flags, std::shared_ptr yml1); 16 | ~SBA(); 17 | void init(unsigned int maxImages, unsigned int maxNumResiduals, unsigned int max_corr_per_image, const std::vector &update_pose_flags); 18 | 19 | //return if removed res 20 | bool align(const std::vector &global_corres, const std::vector &n_match_per_pair, int n_images, const CUDACache* cudaCache, float4x4* d_transforms, bool useVerify, bool isLocal, bool recordConvergence, bool isStart, bool isEnd, bool isScanDoneOpt, unsigned int revalidateIdx); 21 | 22 | float getMaxResidual() const { return m_maxResidual; } 23 | const std::vector& getLinearConvergenceAnalysis() const { return m_solver->getLinearConvergenceAnalysis(); } 24 | bool useVerification() const { return m_bVerify; } 25 | 26 | void evaluateSolverTimings() { 27 | m_solver->evaluateTimings(); 28 | } 29 | //void setLocalWeights(const std::vector& weightsSparse, const std::vector& weightsDenseDepth, const std::vector& weightsDenseColor) { 30 | // m_localWeightsSparse = weightsSparse; 31 | // m_localWeightsDenseDepth = weightsDenseDepth; 32 | // m_localWeightsDenseColor = weightsDenseColor; 33 | //} 34 | void setGlobalWeights(const std::vector& weightsSparse, const std::vector& weightsDenseDepth, const std::vector& weightsDenseColor, bool useGlobalDenseOpt) { 35 | m_globalWeightsMutex.lock(); 36 | m_globalWeightsSparse = weightsSparse; 37 | m_globalWeightsDenseDepth = weightsDenseDepth; 38 | m_globalWeightsDenseColor = weightsDenseColor; 39 | m_bUseGlobalDenseOpt = useGlobalDenseOpt; 40 | m_globalWeightsMutex.unlock(); 41 | } 42 | 43 | 44 | public: 45 | 46 | bool alignCUDA(const CUDACache* cudaCache, bool useDensePairwise, 47 | const std::vector& weightsSparse, const std::vector& weightsDenseDepth, const std::vector& weightsDenseColor, bool isStart, bool isEnd, unsigned int revalidateIdx); 48 | 49 | float3* d_xRot; 50 | float3* d_xTrans; 51 | unsigned int m_numCorrespondences; 52 | EntryJ* d_correspondences; 53 | int* d_validImages; 54 | 55 | 56 | std::vector _global_corres; 57 | int _n_images; 58 | 59 | //dense opt params 60 | bool m_bUseLocalDense; 61 | bool m_bUseGlobalDenseOpt; 62 | std::vector m_localWeightsSparse; 63 | std::vector m_localWeightsDenseDepth; 64 | std::vector m_localWeightsDenseColor; 65 | std::vector m_globalWeightsSparse; 66 | std::vector m_globalWeightsDenseDepth; 67 | std::vector m_globalWeightsDenseColor; 68 | std::mutex m_globalWeightsMutex; 69 | 70 | CUDASolverBundling* m_solver; 71 | 72 | bool m_bUseComprehensiveFrameInvalidation; 73 | 74 | //record residual removal 75 | float m_maxResidual; 76 | //for gpu solver 77 | bool m_bVerify; 78 | 79 | std::vector< std::vector > m_recordedConvergence; 80 | std::shared_ptr yml; 81 | 82 | }; 83 | 84 | -------------------------------------------------------------------------------- /XMem/model/cbam.py: -------------------------------------------------------------------------------- 1 | # Modified from https://github.com/Jongchan/attention-module/blob/master/MODELS/cbam.py 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | 7 | class BasicConv(nn.Module): 8 | def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True): 9 | super(BasicConv, self).__init__() 10 | self.out_channels = out_planes 11 | self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias) 12 | 13 | def forward(self, x): 14 | x = self.conv(x) 15 | return x 16 | 17 | class Flatten(nn.Module): 18 | def forward(self, x): 19 | return x.view(x.size(0), -1) 20 | 21 | class ChannelGate(nn.Module): 22 | def __init__(self, gate_channels, reduction_ratio=16, pool_types=['avg', 'max']): 23 | super(ChannelGate, self).__init__() 24 | self.gate_channels = gate_channels 25 | self.mlp = nn.Sequential( 26 | Flatten(), 27 | nn.Linear(gate_channels, gate_channels // reduction_ratio), 28 | nn.ReLU(), 29 | nn.Linear(gate_channels // reduction_ratio, gate_channels) 30 | ) 31 | self.pool_types = pool_types 32 | def forward(self, x): 33 | channel_att_sum = None 34 | for pool_type in self.pool_types: 35 | if pool_type=='avg': 36 | avg_pool = F.avg_pool2d( x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3))) 37 | channel_att_raw = self.mlp( avg_pool ) 38 | elif pool_type=='max': 39 | max_pool = F.max_pool2d( x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3))) 40 | channel_att_raw = self.mlp( max_pool ) 41 | 42 | if channel_att_sum is None: 43 | channel_att_sum = channel_att_raw 44 | else: 45 | channel_att_sum = channel_att_sum + channel_att_raw 46 | 47 | scale = torch.sigmoid( channel_att_sum ).unsqueeze(2).unsqueeze(3).expand_as(x) 48 | return x * scale 49 | 50 | class ChannelPool(nn.Module): 51 | def forward(self, x): 52 | return torch.cat( (torch.max(x,1)[0].unsqueeze(1), torch.mean(x,1).unsqueeze(1)), dim=1 ) 53 | 54 | class SpatialGate(nn.Module): 55 | def __init__(self): 56 | super(SpatialGate, self).__init__() 57 | kernel_size = 7 58 | self.compress = ChannelPool() 59 | self.spatial = BasicConv(2, 1, kernel_size, stride=1, padding=(kernel_size-1) // 2) 60 | def forward(self, x): 61 | x_compress = self.compress(x) 62 | x_out = self.spatial(x_compress) 63 | scale = torch.sigmoid(x_out) # broadcasting 64 | return x * scale 65 | 66 | class CBAM(nn.Module): 67 | def __init__(self, gate_channels, reduction_ratio=16, pool_types=['avg', 'max'], no_spatial=False): 68 | super(CBAM, self).__init__() 69 | self.ChannelGate = ChannelGate(gate_channels, reduction_ratio, pool_types) 70 | self.no_spatial=no_spatial 71 | if not no_spatial: 72 | self.SpatialGate = SpatialGate() 73 | def forward(self, x): 74 | x_out = self.ChannelGate(x) 75 | if not self.no_spatial: 76 | x_out = self.SpatialGate(x_out) 77 | return x_out 78 | -------------------------------------------------------------------------------- /XMem/inference/interact/s2m/utils.py: -------------------------------------------------------------------------------- 1 | # Credit: https://github.com/VainF/DeepLabV3Plus-Pytorch 2 | 3 | import torch 4 | import torch.nn as nn 5 | import numpy as np 6 | import torch.nn.functional as F 7 | from collections import OrderedDict 8 | 9 | class _SimpleSegmentationModel(nn.Module): 10 | def __init__(self, backbone, classifier): 11 | super(_SimpleSegmentationModel, self).__init__() 12 | self.backbone = backbone 13 | self.classifier = classifier 14 | 15 | def forward(self, x): 16 | input_shape = x.shape[-2:] 17 | features = self.backbone(x) 18 | x = self.classifier(features) 19 | x = F.interpolate(x, size=input_shape, mode='bilinear', align_corners=False) 20 | return x 21 | 22 | 23 | class IntermediateLayerGetter(nn.ModuleDict): 24 | """ 25 | Module wrapper that returns intermediate layers from a model 26 | 27 | It has a strong assumption that the modules have been registered 28 | into the model in the same order as they are used. 29 | This means that one should **not** reuse the same nn.Module 30 | twice in the forward if you want this to work. 31 | 32 | Additionally, it is only able to query submodules that are directly 33 | assigned to the model. So if `model` is passed, `model.feature1` can 34 | be returned, but not `model.feature1.layer2`. 35 | 36 | Arguments: 37 | model (nn.Module): model on which we will extract the features 38 | return_layers (Dict[name, new_name]): a dict containing the names 39 | of the modules for which the activations will be returned as 40 | the key of the dict, and the value of the dict is the name 41 | of the returned activation (which the user can specify). 42 | 43 | Examples:: 44 | 45 | >>> m = torchvision.models.resnet18(pretrained=True) 46 | >>> # extract layer1 and layer3, giving as names `feat1` and feat2` 47 | >>> new_m = torchvision.models._utils.IntermediateLayerGetter(m, 48 | >>> {'layer1': 'feat1', 'layer3': 'feat2'}) 49 | >>> out = new_m(torch.rand(1, 3, 224, 224)) 50 | >>> print([(k, v.shape) for k, v in out.items()]) 51 | >>> [('feat1', torch.Size([1, 64, 56, 56])), 52 | >>> ('feat2', torch.Size([1, 256, 14, 14]))] 53 | """ 54 | def __init__(self, model, return_layers): 55 | if not set(return_layers).issubset([name for name, _ in model.named_children()]): 56 | raise ValueError("return_layers are not present in model") 57 | 58 | orig_return_layers = return_layers 59 | return_layers = {k: v for k, v in return_layers.items()} 60 | layers = OrderedDict() 61 | for name, module in model.named_children(): 62 | layers[name] = module 63 | if name in return_layers: 64 | del return_layers[name] 65 | if not return_layers: 66 | break 67 | 68 | super(IntermediateLayerGetter, self).__init__(layers) 69 | self.return_layers = orig_return_layers 70 | 71 | def forward(self, x): 72 | out = OrderedDict() 73 | for name, module in self.named_children(): 74 | x = module(x) 75 | if name in self.return_layers: 76 | out_name = self.return_layers[name] 77 | out[out_name] = x 78 | return out 79 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/syslog_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "spdlog/sinks/base_sink.h" 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | namespace spdlog { 19 | namespace sinks { 20 | /** 21 | * Sink that write to syslog using the `syscall()` library call. 22 | * 23 | * Locking is not needed, as `syslog()` itself is thread-safe. 24 | */ 25 | template 26 | class syslog_sink : public base_sink 27 | { 28 | public: 29 | // 30 | explicit syslog_sink(std::string ident = "", int syslog_option = 0, int syslog_facility = LOG_USER) 31 | : ident_(std::move(ident)) 32 | { 33 | priorities_[static_cast(level::trace)] = LOG_DEBUG; 34 | priorities_[static_cast(level::debug)] = LOG_DEBUG; 35 | priorities_[static_cast(level::info)] = LOG_INFO; 36 | priorities_[static_cast(level::warn)] = LOG_WARNING; 37 | priorities_[static_cast(level::err)] = LOG_ERR; 38 | priorities_[static_cast(level::critical)] = LOG_CRIT; 39 | priorities_[static_cast(level::off)] = LOG_INFO; 40 | 41 | // set ident to be program name if empty 42 | ::openlog(ident_.empty() ? nullptr : ident_.c_str(), syslog_option, syslog_facility); 43 | } 44 | 45 | ~syslog_sink() override 46 | { 47 | ::closelog(); 48 | } 49 | 50 | syslog_sink(const syslog_sink &) = delete; 51 | syslog_sink &operator=(const syslog_sink &) = delete; 52 | 53 | protected: 54 | void sink_it_(const details::log_msg &msg) override 55 | { 56 | ::syslog(syslog_prio_from_level(msg), "%s", fmt::to_string(msg.payload).c_str()); 57 | } 58 | 59 | void flush_() override {} 60 | 61 | private: 62 | std::array priorities_; 63 | // must store the ident because the man says openlog might use the pointer as 64 | // is and not a string copy 65 | const std::string ident_; 66 | 67 | // 68 | // Simply maps spdlog's log level to syslog priority level. 69 | // 70 | int syslog_prio_from_level(const details::log_msg &msg) const 71 | { 72 | return priorities_[static_cast(msg.level)]; 73 | } 74 | }; 75 | 76 | using syslog_sink_mt = syslog_sink; 77 | using syslog_sink_st = syslog_sink; 78 | } // namespace sinks 79 | 80 | // Create and register a syslog logger 81 | template 82 | inline std::shared_ptr syslog_logger_mt( 83 | const std::string &logger_name, const std::string &syslog_ident = "", int syslog_option = 0, int syslog_facility = (1 << 3)) 84 | { 85 | return Factory::template create(logger_name, syslog_ident, syslog_option, syslog_facility); 86 | } 87 | 88 | template 89 | inline std::shared_ptr syslog_logger_st( 90 | const std::string &logger_name, const std::string &syslog_ident = "", int syslog_option = 0, int syslog_facility = (1 << 3)) 91 | { 92 | return Factory::template create(logger_name, syslog_ident, syslog_option, syslog_facility); 93 | } 94 | } // namespace spdlog 95 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/sinks/stdout_sinks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #ifndef SPDLOG_H 9 | #include "spdlog/spdlog.h" 10 | #endif 11 | 12 | #include "spdlog/details/console_globals.h" 13 | #include "spdlog/details/null_mutex.h" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace spdlog { 20 | 21 | namespace sinks { 22 | 23 | template 24 | class stdout_sink final : public sink 25 | { 26 | public: 27 | using mutex_t = typename ConsoleMutex::mutex_t; 28 | stdout_sink() 29 | : mutex_(ConsoleMutex::mutex()) 30 | , file_(TargetStream::stream()) 31 | { 32 | } 33 | ~stdout_sink() override = default; 34 | 35 | stdout_sink(const stdout_sink &other) = delete; 36 | stdout_sink &operator=(const stdout_sink &other) = delete; 37 | 38 | void log(const details::log_msg &msg) override 39 | { 40 | std::lock_guard lock(mutex_); 41 | fmt::memory_buffer formatted; 42 | formatter_->format(msg, formatted); 43 | fwrite(formatted.data(), sizeof(char), formatted.size(), file_); 44 | fflush(TargetStream::stream()); 45 | } 46 | 47 | void flush() override 48 | { 49 | std::lock_guard lock(mutex_); 50 | fflush(file_); 51 | } 52 | 53 | void set_pattern(const std::string &pattern) override 54 | { 55 | std::lock_guard lock(mutex_); 56 | formatter_ = std::unique_ptr(new pattern_formatter(pattern)); 57 | } 58 | 59 | void set_formatter(std::unique_ptr sink_formatter) override 60 | { 61 | std::lock_guard lock(mutex_); 62 | formatter_ = std::move(sink_formatter); 63 | } 64 | 65 | private: 66 | mutex_t &mutex_; 67 | FILE *file_; 68 | }; 69 | 70 | using stdout_sink_mt = stdout_sink; 71 | using stdout_sink_st = stdout_sink; 72 | 73 | using stderr_sink_mt = stdout_sink; 74 | using stderr_sink_st = stdout_sink; 75 | 76 | } // namespace sinks 77 | 78 | // factory methods 79 | template 80 | inline std::shared_ptr stdout_logger_mt(const std::string &logger_name) 81 | { 82 | return Factory::template create(logger_name); 83 | } 84 | 85 | template 86 | inline std::shared_ptr stdout_logger_st(const std::string &logger_name) 87 | { 88 | return Factory::template create(logger_name); 89 | } 90 | 91 | template 92 | inline std::shared_ptr stderr_logger_mt(const std::string &logger_name) 93 | { 94 | return Factory::template create(logger_name); 95 | } 96 | 97 | template 98 | inline std::shared_ptr stderr_logger_st(const std::string &logger_name) 99 | { 100 | return Factory::template create(logger_name); 101 | } 102 | } // namespace spdlog 103 | -------------------------------------------------------------------------------- /BundleSDF/BundleTrack/src/spdlog/async.h: -------------------------------------------------------------------------------- 1 | 2 | // 3 | // Copyright(c) 2018 Gabi Melman. 4 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 5 | // 6 | 7 | #pragma once 8 | 9 | // 10 | // Async logging using global thread pool 11 | // All loggers created here share same global thread pool. 12 | // Each log message is pushed to a queue along withe a shared pointer to the 13 | // logger. 14 | // If a logger deleted while having pending messages in the queue, it's actual 15 | // destruction will defer 16 | // until all its messages are processed by the thread pool. 17 | // This is because each message in the queue holds a shared_ptr to the 18 | // originating logger. 19 | 20 | #include "spdlog/async_logger.h" 21 | #include "spdlog/details/registry.h" 22 | #include "spdlog/details/thread_pool.h" 23 | 24 | #include 25 | #include 26 | 27 | namespace spdlog { 28 | 29 | namespace details { 30 | static const size_t default_async_q_size = 8192; 31 | } 32 | 33 | // async logger factory - creates async loggers backed with thread pool. 34 | // if a global thread pool doesn't already exist, create it with default queue 35 | // size of 8192 items and single thread. 36 | template 37 | struct async_factory_impl 38 | { 39 | template 40 | static std::shared_ptr create(std::string logger_name, SinkArgs &&... args) 41 | { 42 | auto ®istry_inst = details::registry::instance(); 43 | 44 | // create global thread pool if not already exists.. 45 | std::lock_guard tp_lock(registry_inst.tp_mutex()); 46 | auto tp = registry_inst.get_tp(); 47 | if (tp == nullptr) 48 | { 49 | tp = std::make_shared(details::default_async_q_size, 1); 50 | registry_inst.set_tp(tp); 51 | } 52 | 53 | auto sink = std::make_shared(std::forward(args)...); 54 | auto new_logger = std::make_shared(std::move(logger_name), std::move(sink), std::move(tp), OverflowPolicy); 55 | registry_inst.initialize_logger(new_logger); 56 | return new_logger; 57 | } 58 | }; 59 | 60 | using async_factory = async_factory_impl; 61 | using async_factory_nonblock = async_factory_impl; 62 | 63 | template 64 | inline std::shared_ptr create_async(std::string logger_name, SinkArgs &&... sink_args) 65 | { 66 | return async_factory::create(std::move(logger_name), std::forward(sink_args)...); 67 | } 68 | 69 | template 70 | inline std::shared_ptr create_async_nb(std::string logger_name, SinkArgs &&... sink_args) 71 | { 72 | return async_factory_nonblock::create(std::move(logger_name), std::forward(sink_args)...); 73 | } 74 | 75 | // set global thread pool. 76 | inline void init_thread_pool(size_t q_size, size_t thread_count) 77 | { 78 | auto tp = std::make_shared(q_size, thread_count); 79 | details::registry::instance().set_tp(std::move(tp)); 80 | } 81 | 82 | // get the global thread pool. 83 | inline std::shared_ptr thread_pool() 84 | { 85 | return details::registry::instance().get_tp(); 86 | } 87 | } // namespace spdlog 88 | -------------------------------------------------------------------------------- /XMem/util/logger.py: -------------------------------------------------------------------------------- 1 | """ 2 | Dumps things to tensorboard and console 3 | """ 4 | 5 | import os 6 | import warnings 7 | 8 | import torchvision.transforms as transforms 9 | from torch.utils.tensorboard import SummaryWriter 10 | 11 | 12 | def tensor_to_numpy(image): 13 | image_np = (image.numpy() * 255).astype('uint8') 14 | return image_np 15 | 16 | def detach_to_cpu(x): 17 | return x.detach().cpu() 18 | 19 | def fix_width_trunc(x): 20 | return ('{:.9s}'.format('{:0.9f}'.format(x))) 21 | 22 | class TensorboardLogger: 23 | def __init__(self, short_id, id, git_info): 24 | self.short_id = short_id 25 | if self.short_id == 'NULL': 26 | self.short_id = 'DEBUG' 27 | 28 | if id is None: 29 | self.no_log = True 30 | warnings.warn('Logging has been disbaled.') 31 | else: 32 | self.no_log = False 33 | 34 | self.inv_im_trans = transforms.Normalize( 35 | mean=[-0.485/0.229, -0.456/0.224, -0.406/0.225], 36 | std=[1/0.229, 1/0.224, 1/0.225]) 37 | 38 | self.inv_seg_trans = transforms.Normalize( 39 | mean=[-0.5/0.5], 40 | std=[1/0.5]) 41 | 42 | log_path = os.path.join('.', 'saves', '%s' % id) 43 | self.logger = SummaryWriter(log_path) 44 | 45 | self.log_string('git', git_info) 46 | 47 | def log_scalar(self, tag, x, step): 48 | if self.no_log: 49 | warnings.warn('Logging has been disabled.') 50 | return 51 | self.logger.add_scalar(tag, x, step) 52 | 53 | def log_metrics(self, l1_tag, l2_tag, val, step, f=None): 54 | tag = l1_tag + '/' + l2_tag 55 | text = '{:s} - It {:6d} [{:5s}] [{:13}]: {:s}'.format(self.short_id, step, l1_tag.upper(), l2_tag, fix_width_trunc(val)) 56 | print(text) 57 | if f is not None: 58 | f.write(text + '\n') 59 | f.flush() 60 | self.log_scalar(tag, val, step) 61 | 62 | def log_im(self, tag, x, step): 63 | if self.no_log: 64 | warnings.warn('Logging has been disabled.') 65 | return 66 | x = detach_to_cpu(x) 67 | x = self.inv_im_trans(x) 68 | x = tensor_to_numpy(x) 69 | self.logger.add_image(tag, x, step) 70 | 71 | def log_cv2(self, tag, x, step): 72 | if self.no_log: 73 | warnings.warn('Logging has been disabled.') 74 | return 75 | x = x.transpose((2, 0, 1)) 76 | self.logger.add_image(tag, x, step) 77 | 78 | def log_seg(self, tag, x, step): 79 | if self.no_log: 80 | warnings.warn('Logging has been disabled.') 81 | return 82 | x = detach_to_cpu(x) 83 | x = self.inv_seg_trans(x) 84 | x = tensor_to_numpy(x) 85 | self.logger.add_image(tag, x, step) 86 | 87 | def log_gray(self, tag, x, step): 88 | if self.no_log: 89 | warnings.warn('Logging has been disabled.') 90 | return 91 | x = detach_to_cpu(x) 92 | x = tensor_to_numpy(x) 93 | self.logger.add_image(tag, x, step) 94 | 95 | def log_string(self, tag, x): 96 | print(tag, x) 97 | if self.no_log: 98 | warnings.warn('Logging has been disabled.') 99 | return 100 | self.logger.add_text(tag, x) 101 | -------------------------------------------------------------------------------- /XMem/inference/interact/fbrs/model/ops.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn as nn 3 | import numpy as np 4 | 5 | from . import initializer as initializer 6 | from ..utils.cython import get_dist_maps 7 | 8 | 9 | def select_activation_function(activation): 10 | if isinstance(activation, str): 11 | if activation.lower() == 'relu': 12 | return nn.ReLU 13 | elif activation.lower() == 'softplus': 14 | return nn.Softplus 15 | else: 16 | raise ValueError(f"Unknown activation type {activation}") 17 | elif isinstance(activation, nn.Module): 18 | return activation 19 | else: 20 | raise ValueError(f"Unknown activation type {activation}") 21 | 22 | 23 | class BilinearConvTranspose2d(nn.ConvTranspose2d): 24 | def __init__(self, in_channels, out_channels, scale, groups=1): 25 | kernel_size = 2 * scale - scale % 2 26 | self.scale = scale 27 | 28 | super().__init__( 29 | in_channels, out_channels, 30 | kernel_size=kernel_size, 31 | stride=scale, 32 | padding=1, 33 | groups=groups, 34 | bias=False) 35 | 36 | self.apply(initializer.Bilinear(scale=scale, in_channels=in_channels, groups=groups)) 37 | 38 | 39 | class DistMaps(nn.Module): 40 | def __init__(self, norm_radius, spatial_scale=1.0, cpu_mode=False): 41 | super(DistMaps, self).__init__() 42 | self.spatial_scale = spatial_scale 43 | self.norm_radius = norm_radius 44 | self.cpu_mode = cpu_mode 45 | 46 | def get_coord_features(self, points, batchsize, rows, cols): 47 | if self.cpu_mode: 48 | coords = [] 49 | for i in range(batchsize): 50 | norm_delimeter = self.spatial_scale * self.norm_radius 51 | coords.append(get_dist_maps(points[i].cpu().float().numpy(), rows, cols, 52 | norm_delimeter)) 53 | coords = torch.from_numpy(np.stack(coords, axis=0)).to(points.device).float() 54 | else: 55 | num_points = points.shape[1] // 2 56 | points = points.view(-1, 2) 57 | invalid_points = torch.max(points, dim=1, keepdim=False)[0] < 0 58 | row_array = torch.arange(start=0, end=rows, step=1, dtype=torch.float32, device=points.device) 59 | col_array = torch.arange(start=0, end=cols, step=1, dtype=torch.float32, device=points.device) 60 | 61 | coord_rows, coord_cols = torch.meshgrid(row_array, col_array) 62 | coords = torch.stack((coord_rows, coord_cols), dim=0).unsqueeze(0).repeat(points.size(0), 1, 1, 1) 63 | 64 | add_xy = (points * self.spatial_scale).view(points.size(0), points.size(1), 1, 1) 65 | coords.add_(-add_xy) 66 | coords.div_(self.norm_radius * self.spatial_scale) 67 | coords.mul_(coords) 68 | 69 | coords[:, 0] += coords[:, 1] 70 | coords = coords[:, :1] 71 | 72 | coords[invalid_points, :, :, :] = 1e6 73 | 74 | coords = coords.view(-1, num_points, 1, rows, cols) 75 | coords = coords.min(dim=1)[0] # -> (bs * num_masks * 2) x 1 x h x w 76 | coords = coords.view(-1, 2, rows, cols) 77 | 78 | coords.sqrt_().mul_(2).tanh_() 79 | 80 | return coords 81 | 82 | def forward(self, x, coords): 83 | return self.get_coord_features(coords, x.shape[0], x.shape[2], x.shape[3]) 84 | --------------------------------------------------------------------------------