├── LICENSE.md ├── Makefile ├── README.md ├── docker └── Dockerfile ├── docs ├── .nojekyll ├── README.html ├── _modules │ ├── index.html │ └── kp2d │ │ ├── datasets │ │ └── patches_dataset.html │ │ ├── evaluation │ │ ├── descriptor_evaluation.html │ │ ├── detector_evaluation.html │ │ └── evaluate.html │ │ ├── networks │ │ ├── inlier_net.html │ │ └── keypoint_net.html │ │ └── utils │ │ ├── image.html │ │ └── keypoints.html ├── _sources │ ├── README.rst.txt │ ├── datasets │ │ ├── datasets.patches_dataset.rst.txt │ │ └── datasets.rst.txt │ ├── evaluation │ │ ├── evaluation.descriptor_evaluation.rst.txt │ │ ├── evaluation.detector_evaluation.rst.txt │ │ ├── evaluation.evaluate.rst.txt │ │ └── evaluation.rst.txt │ ├── index.rst.txt │ ├── networks │ │ ├── networks.inlier_net.rst.txt │ │ ├── networks.keypoint_net.rst.txt │ │ └── networks.rst.txt │ └── utils │ │ ├── utils.image.rst.txt │ │ ├── utils.keypoints.rst.txt │ │ └── utils.rst.txt ├── _static │ ├── basic.css │ ├── css │ │ ├── badge_only.css │ │ └── theme.css │ ├── custom.css │ ├── doctools.js │ ├── documentation_options.js │ ├── file.png │ ├── fonts │ │ ├── Inconsolata-Bold.ttf │ │ ├── Inconsolata-Regular.ttf │ │ ├── Inconsolata.ttf │ │ ├── Lato-Bold.ttf │ │ ├── Lato-Regular.ttf │ │ ├── Lato │ │ │ ├── lato-bold.eot │ │ │ ├── lato-bold.ttf │ │ │ ├── lato-bold.woff │ │ │ ├── lato-bold.woff2 │ │ │ ├── lato-bolditalic.eot │ │ │ ├── lato-bolditalic.ttf │ │ │ ├── lato-bolditalic.woff │ │ │ ├── lato-bolditalic.woff2 │ │ │ ├── lato-italic.eot │ │ │ ├── lato-italic.ttf │ │ │ ├── lato-italic.woff │ │ │ ├── lato-italic.woff2 │ │ │ ├── lato-regular.eot │ │ │ ├── lato-regular.ttf │ │ │ ├── lato-regular.woff │ │ │ └── lato-regular.woff2 │ │ ├── RobotoSlab-Bold.ttf │ │ ├── RobotoSlab-Regular.ttf │ │ ├── RobotoSlab │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ └── roboto-slab-v7-regular.woff2 │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.svg │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ └── fontawesome-webfont.woff2 │ ├── jquery-3.5.1.js │ ├── jquery.js │ ├── js │ │ ├── modernizr.min.js │ │ └── theme.js │ ├── language_data.js │ ├── logo.png │ ├── minus.png │ ├── plus.png │ ├── pygments.css │ ├── searchtools.js │ ├── underscore-1.3.1.js │ └── underscore.js ├── datasets │ ├── datasets.html │ └── datasets.patches_dataset.html ├── evaluation │ ├── evaluation.descriptor_evaluation.html │ ├── evaluation.detector_evaluation.html │ ├── evaluation.evaluate.html │ └── evaluation.html ├── genindex.html ├── index.html ├── networks │ ├── networks.html │ ├── networks.inlier_net.html │ └── networks.keypoint_net.html ├── objects.inv ├── py-modindex.html ├── search.html ├── searchindex.js └── utils │ ├── utils.html │ ├── utils.image.html │ └── utils.keypoints.html ├── kp2d ├── configs │ ├── base_config.py │ ├── v0.yaml │ ├── v1.yaml │ ├── v2.yaml │ ├── v2_resnet.yaml │ ├── v3.yaml │ └── v4.yaml ├── datasets │ ├── __init__.py │ ├── augmentations.py │ ├── coco.py │ └── patches_dataset.py ├── evaluation │ ├── descriptor_evaluation.py │ ├── detector_evaluation.py │ └── evaluate.py ├── models │ └── KeypointNetwithIOLoss.py ├── networks │ ├── __init__.py │ ├── inlier_net.py │ ├── keypoint_net.py │ └── keypoint_resnet.py └── utils │ ├── __init__.py │ ├── config.py │ ├── horovod.py │ ├── image.py │ ├── keypoints.py │ ├── logging.py │ └── wandb.py ├── media ├── gifs │ ├── compressed_h2.gif │ ├── compressed_v2.gif │ ├── compressed_w2.gif │ ├── h1.gif │ ├── v1.gif │ └── w1.gif └── imgs │ ├── diagram_architecture.png │ ├── l1.png │ ├── l2.png │ ├── p1.png │ ├── p2.png │ ├── r1.png │ └── r2.png └── scripts ├── eval_keypoint_net.py ├── train_keypoint_net.py └── train_keypoint_net_utils.py /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Toyota Research Institute (TRI) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT ?= kp2d 2 | WORKSPACE ?= /workspace/$(PROJECT) 3 | DOCKER_IMAGE ?= ${PROJECT}:latest 4 | 5 | SHMSIZE ?= 444G 6 | WANDB_MODE ?= run 7 | DOCKER_OPTS := \ 8 | --name ${PROJECT} \ 9 | --rm -it \ 10 | --shm-size=${SHMSIZE} \ 11 | -e AWS_DEFAULT_REGION \ 12 | -e AWS_ACCESS_KEY_ID \ 13 | -e AWS_SECRET_ACCESS_KEY \ 14 | -e WANDB_API_KEY \ 15 | -e WANDB_ENTITY \ 16 | -e WANDB_MODE \ 17 | -e HOST_HOSTNAME= \ 18 | -e OMP_NUM_THREADS=1 -e KMP_AFFINITY="granularity=fine,compact,1,0" \ 19 | -e OMPI_ALLOW_RUN_AS_ROOT=1 \ 20 | -e OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 \ 21 | -e NCCL_DEBUG=VERSION \ 22 | -e DISPLAY=${DISPLAY} \ 23 | -e XAUTHORITY \ 24 | -e NVIDIA_DRIVER_CAPABILITIES=all \ 25 | -v ~/.aws:/root/.aws \ 26 | -v /root/.ssh:/root/.ssh \ 27 | -v ~/.cache:/root/.cache \ 28 | -v /data:/data \ 29 | -v /dev/null:/dev/raw1394 \ 30 | -v /tmp:/tmp \ 31 | -v /tmp/.X11-unix/X0:/tmp/.X11-unix/X0 \ 32 | -v /var/run/docker.sock:/var/run/docker.sock \ 33 | -v ${PWD}:${WORKSPACE} \ 34 | -w ${WORKSPACE} \ 35 | --privileged \ 36 | --ipc=host \ 37 | --network=host 38 | 39 | NGPUS=$(shell nvidia-smi -L | wc -l) 40 | MPI_CMD=mpirun \ 41 | -allow-run-as-root \ 42 | -np ${NGPUS} \ 43 | -H localhost:${NGPUS} \ 44 | -x MASTER_ADDR=127.0.0.1 \ 45 | -x MASTER_PORT=23457 \ 46 | -x HOROVOD_TIMELINE \ 47 | -x OMP_NUM_THREADS=1 \ 48 | -x KMP_AFFINITY='granularity=fine,compact,1,0' \ 49 | -bind-to none -map-by slot -x NCCL_DEBUG=INFO -x NCCL_MIN_NRINGS=4 \ 50 | --report-bindings 51 | 52 | .PHONY: all clean docker-build 53 | 54 | all: clean 55 | 56 | clean: 57 | find . -name "*.pyc" | xargs rm -f && \ 58 | find . -name "__pycache__" | xargs rm -rf 59 | 60 | docker-build: 61 | docker build \ 62 | -f docker/Dockerfile \ 63 | -t ${DOCKER_IMAGE} . 64 | 65 | docker-start: docker-build 66 | nvidia-docker run ${DOCKER_OPTS} ${DOCKER_IMAGE} bash 67 | 68 | docker-run: docker-build 69 | nvidia-docker run ${DOCKER_OPTS} ${DOCKER_IMAGE} \ 70 | bash -c "${COMMAND}" 71 | 72 | docker-run-mpi: docker-build 73 | nvidia-docker run ${DOCKER_OPTS} ${DOCKER_IMAGE} \ 74 | bash -c "${MPI_CMD} ${COMMAND}" 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Neural Outlier Rejection for Self-Supervised Keypoint Learning 2 | 3 | ## Overview 4 | ![](media/imgs/diagram_architecture.png) 5 | - **IO-Net:** A novel proxy task for the self-supervision of keypoint description. 6 | - **KeyPointNet:** An improved keypoint-network architecture that is especially amenable to robust keypoint detection and description. 7 | 8 | [**[Full paper]**](https://openreview.net/pdf?id=Skx82ySYPH) 9 | 10 | ### Setting up your environment 11 | 12 | You need a machine with recent Nvidia drivers and a GPU. We recommend using docker (see [nvidia-docker2](https://github.com/NVIDIA/nvidia-docker) instructions) to have a reproducible environment. To setup your environment, type in a terminal (only tested in Ubuntu 18.04 and with Pytorch 1.6): 13 | 14 | ```bash 15 | git clone https://github.com/TRI-ML/KP2D.git 16 | cd KP2D 17 | # if you want to use docker (recommended) 18 | make docker-build 19 | ``` 20 | 21 | We will list below all commands as if run directly inside our container. To run any of the commands in a container, you can either start the container in interactive mode with `make docker-start` to land in a shell where you can type those commands, or you can do it in one step: 22 | 23 | ```bash 24 | # single GPU 25 | make docker-run COMMAND="some-command" 26 | # multi-GPU 27 | make docker-run-mpi COMMAND="some-command" 28 | ``` 29 | 30 | If you want to use features related to [Weights & Biases (WANDB)](https://www.wandb.com/) (for experiment management/visualization), then you should create associated accounts and configure your shell with the following environment variables: 31 | 32 | export WANDB_ENTITY="something" 33 | export WANDB_API_KEY="something" 34 | To enable WANDB logging and AWS checkpoint syncing, you can then set the corresponding configuration parameters in `configs/.yaml` (cf. [configs/base_config.py](configs/base_config.py) for defaults and docs): 35 | 36 | ``` 37 | wandb: 38 | dry_run: True # Wandb dry-run (not logging) 39 | name: '' # Wandb run name 40 | project: os.environ.get("WANDB_PROJECT", "") # Wandb project 41 | entity: os.environ.get("WANDB_ENTITY", "") # Wandb entity 42 | tags: [] # Wandb tags 43 | dir: '' # Wandb save folder 44 | ``` 45 | 46 | ### Data 47 | 48 | Download the HPatches dataset for evaluation: 49 | 50 | ```bash 51 | cd /data/datasets/kp2d/ 52 | wget http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz 53 | tar -xvf hpatches-sequences-release.tar.gz 54 | mv hpatches-sequences-release HPatches 55 | ``` 56 | 57 | Download the COCO dataset for training: 58 | ```bash 59 | mkdir -p /data/datasets/kp2d/coco/ && cd /data/datasets/kp2d/coco/ 60 | wget http://images.cocodataset.org/zips/train2017.zip 61 | unzip train2017.zip 62 | ``` 63 | 64 | ### Training 65 | 66 | To train a model run: 67 | 68 | ```bash 69 | make docker-run COMMAND="python scripts/train_keypoint_net.py kp2d/configs/v4.yaml" 70 | ``` 71 | 72 | To train on multiple GPUs, simply replace `docker-run` with `docker-run-mpi`. Note that we provide the `v0-v4.yaml` config files, one for each version of our model as presented in the ablative analysis of our paper. For evaluating the pre-trained models corresponding to each config file please see hte following section. 73 | 74 | 75 | ### Pre-trained models: 76 | 77 | Download the pre-trained models from [here](https://tri-ml-public.s3.amazonaws.com/github/kp2d/models/pretrained_models.tar.gz) and place them in `/data/models/kp2d/` 78 | 79 | To evaluate any of the models, simply run: 80 | 81 | ```bash 82 | make docker-run COMMAND="python scripts/eval_keypoint_net.py --pretrained_model /data/models/kp2d/v4.ckpt --input /data/datasets/kp2d/HPatches/" 83 | ``` 84 | 85 | Evaluation for **`(320, 240)`**: 86 | 87 | | Model | Repeatability | Localization | C1 | C3 | C5 | MScore | 88 | |---|---|---|---|---|---|---| 89 | | V0*| 0.644 | 1.087 | 0.459 | 0.816 | 0.888 | 0.518 | 90 | | V1*| 0.678 | 0.98 | 0.453 | 0.828 | 0.905 | 0.552 | 91 | | V2*| 0.679 | 0.942 | 0.534 | 0.86 | 0.914 | 0.573 | 92 | | V3| 0.685 | 0.885 | 0.602 | 0.836 | 0.886 | 0.52 | 93 | | V4| 0.687 | 0.892 | 0.593 | 0.867 | 0.91 | 0.546 | 94 | 95 | 96 | Evaluation for **`(640, 480)`**: 97 | 98 | | Model | Repeatability | Localization | C1 | C3 | C5 | MScore | 99 | |---|---|---|---|---|---|---| 100 | | V0*| 0.633 | 1.157 | 0.45 | 0.81 | 0.89 | 0.486 | 101 | | V1*| 0.673 | 1.049 | 0.464 | 0.817 | 0.895 | 0.519 | 102 | | V2*| 0.68 | 1.008 | 0.51 | 0.855 | 0.921 | 0.544 | 103 | | V3| 0.682 | 0.972 | 0.55 | 0.812 | 0.883 | 0.486 | 104 | | V4| 0.684 | 0.972 | 0.566 | 0.84 | 0.9 | 0.511 | 105 | 106 | *-these models were trained again after submission - the numbers deviate slightly from the paper, however the same trends can be observed. 107 | 108 | 109 | ### Over-fitting Examples 110 | 111 | These examples show the model over-fitting on single images. For each image, we show the original frame with detected keypoints (left), the score map (center) and the random crop used for training (right). As training progresses, the model learns to detect salient regions in the images. 112 | 113 | 114 | - **Toy example:** 115 |

116 | Target Frame 117 | Heatmap 118 | Source Frame 119 |

120 | 121 | - **TRI example:** 122 |

123 | Target Frame 124 | Heatmap 125 | Source Frame 126 |

127 | 128 | ### Qualatitive Results 129 | 130 | - **Illumination Cases:** 131 | 132 |

133 | Illumination case(1) 134 | Illumination case(2) 135 |

136 | 137 | - **Perspective Cases:** 138 |

139 | Perspective case(1) 140 | Perspective case(2) 141 |

142 | 143 | - **Rotation Cases:** 144 |

145 | Rotation case(1) 146 | Rotation case(2) 147 |

148 | 149 | ### License 150 | 151 | The source code is released under the [MIT license](LICENSE.md). 152 | 153 | 154 | ### Citation 155 | Please use the following citation when referencing our work: 156 | ``` 157 | @inproceedings{ 158 | tang2020neural, 159 | title={Neural Outlier Rejection for Self-Supervised Keypoint Learning}, 160 | author={Jiexiong Tang and Hanme Kim and Vitor Guizilini and Sudeep Pillai and Rares Ambrus}, 161 | booktitle={International Conference on Learning Representations}, 162 | year={2020} 163 | } 164 | ``` -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-devel-ubuntu18.04 2 | 3 | ENV PROJECT=kp2d 4 | ENV PYTORCH_VERSION=1.6.0 5 | ENV TORCHVISION_VERSION=0.7.0 6 | ENV CUDNN_VERSION=7.6.5.32-1+cuda10.1 7 | ENV NCCL_VERSION=2.7.8-1+cuda10.1 8 | ENV HOROVOD_VERSION=0.19.5 9 | 10 | # Python 2.7 or 3.6 is supported by Ubuntu Bionic out of the box 11 | ARG python=3.6 12 | ENV PYTHON_VERSION=${python} 13 | ENV DEBIAN_FRONTEND=noninteractive 14 | 15 | # Set default shell to /bin/bash 16 | SHELL ["/bin/bash", "-cu"] 17 | 18 | RUN apt-get clean && apt-get update && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \ 19 | build-essential \ 20 | cmake \ 21 | g++-4.8 \ 22 | git \ 23 | curl \ 24 | docker.io \ 25 | vim \ 26 | wget \ 27 | ca-certificates \ 28 | libcudnn7=${CUDNN_VERSION} \ 29 | libnccl2=${NCCL_VERSION} \ 30 | libnccl-dev=${NCCL_VERSION} \ 31 | libjpeg-dev \ 32 | libpng-dev \ 33 | python${PYTHON_VERSION} \ 34 | python${PYTHON_VERSION}-dev \ 35 | python3-tk \ 36 | librdmacm1 \ 37 | libibverbs1 \ 38 | ibverbs-providers \ 39 | libgtk2.0-dev \ 40 | unzip \ 41 | bzip2 \ 42 | htop \ 43 | gnuplot \ 44 | ffmpeg 45 | 46 | # Install Open MPI 47 | RUN mkdir /tmp/openmpi && \ 48 | cd /tmp/openmpi && \ 49 | wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz && \ 50 | tar zxf openmpi-4.0.0.tar.gz && \ 51 | cd openmpi-4.0.0 && \ 52 | ./configure --enable-orterun-prefix-by-default && \ 53 | make -j $(nproc) all && \ 54 | make install && \ 55 | ldconfig && \ 56 | rm -rf /tmp/openmpi 57 | 58 | # Install OpenSSH for MPI to communicate between containers 59 | RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \ 60 | mkdir -p /var/run/sshd 61 | 62 | # Allow OpenSSH to talk to containers without asking for confirmation 63 | RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \ 64 | echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \ 65 | mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config 66 | 67 | # Instal Python and pip 68 | RUN if [[ "${PYTHON_VERSION}" == "3.6" ]]; then \ 69 | apt-get install -y python${PYTHON_VERSION}-distutils; \ 70 | fi 71 | 72 | RUN ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python 73 | 74 | RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ 75 | python get-pip.py && \ 76 | rm get-pip.py 77 | 78 | # Install PyTorch 79 | RUN pip install future typing numpy awscli packaging 80 | RUN PYTAGS=$(python -c "from packaging import tags; tag = list(tags.sys_tags())[0]; print(f'{tag.interpreter}-{tag.abi}')") && \ 81 | pip install https://download.pytorch.org/whl/cu101/torch-${PYTORCH_VERSION}%2Bcu101-${PYTAGS}-linux_x86_64.whl \ 82 | https://download.pytorch.org/whl/cu101/torchvision-${TORCHVISION_VERSION}%2Bcu101-${PYTAGS}-linux_x86_64.whl 83 | 84 | RUN pip install numpy \ 85 | keras \ 86 | h5py 87 | 88 | RUN pip install keras h5py pandas \ 89 | torch==${PYTORCH_VERSION} \ 90 | torchvision==${TORCHVISION_VERSION} && ldconfig 91 | 92 | RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs && \ 93 | HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_WITH_PYTORCH=1 \ 94 | pip install --no-cache-dir horovod==${HOROVOD_VERSION} && \ 95 | ldconfig 96 | 97 | 98 | RUN pip install awscli wandb tensorboardx tqdm termcolor path.py pillow opencv-python-headless matplotlib jupyter boto3 pycuda 99 | RUN pip install cython==0.29.10 yacs 100 | 101 | # Configure environment variables - default working directory is "/workspace" 102 | ENV PYTHONPATH="/workspace" 103 | WORKDIR /workspace 104 | 105 | RUN mkdir -p /workspace/experiments 106 | RUN mkdir -p /workspace/${PROJECT} 107 | WORKDIR /workspace/${PROJECT} 108 | 109 | # Expose Port for jupyter (8888) 110 | EXPOSE 8888 111 | 112 | # Copy project source last (to avoid cache busting) 113 | WORKDIR /workspace/${PROJECT} 114 | COPY . /workspace/${PROJECT} 115 | ENV PYTHONPATH="/workspace/${PROJECT}:$PYTHONPATH" 116 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/.nojekyll -------------------------------------------------------------------------------- /docs/_modules/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Overview: module code — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
45 | 46 | 103 | 104 |
105 | 106 | 107 | 113 | 114 | 115 |
116 | 117 |
118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 |
136 | 137 |
    138 | 139 |
  • Docs »
    • 140 | 141 |
  • Overview: module code
    • 142 | 143 | 144 |
  • 145 | 146 |
    • 147 | 148 |
149 | 150 | 151 |
152 |
153 |
154 | 168 | 169 |
170 |
171 | 172 | 173 |
174 | 175 |
176 |

177 | © Copyright 2020, Toyota Research Institute (TRI) 178 | 179 |

180 |
181 | Built with Sphinx using a theme provided by Read the Docs. 182 | 183 |
184 | 185 |
186 |
187 | 188 |
189 | 190 |
191 | 192 | 193 | 194 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | -------------------------------------------------------------------------------- /docs/_modules/kp2d/utils/keypoints.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | kp2d.utils.keypoints — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 |
45 | 46 | 103 | 104 |
105 | 106 | 107 | 113 | 114 | 115 |
116 | 117 |
118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 |
136 | 137 |
    138 | 139 |
  • Docs »
    • 140 | 141 |
  • Module code »
    • 142 | 143 |
  • kp2d.utils.keypoints
    • 144 | 145 | 146 |
  • 147 | 148 |
    • 149 | 150 |
151 | 152 | 153 |
154 |
155 |
156 |
157 | 158 |

Source code for kp2d.utils.keypoints

159 | # Copyright 2020 Toyota Research Institute.  All rights reserved.
160 | 
161 | import cv2
162 | import numpy as np
163 | import torch
164 | 
165 | 
166 | 
[docs]def warp_keypoints(keypoints, H):
167 |     """Warp keypoints given a homography
168 | 
169 |     Parameters
170 |     ----------
171 |     keypoints: numpy.ndarray (N,2)
172 |         Keypoint vector.
173 |     H: numpy.ndarray (3,3)
174 |         Homography.
175 | 
176 |     Returns
177 |     -------
178 |     warped_keypoints: numpy.ndarray (N,2)
179 |         Warped keypoints vector.
180 |     """
181 |     num_points = keypoints.shape[0]
182 |     homogeneous_points = np.concatenate([keypoints, np.ones((num_points, 1))], axis=1)
183 |     warped_points = np.dot(homogeneous_points, np.transpose(H))
184 |     return warped_points[:, :2] / warped_points[:, 2:]

185 |
186 | 187 |
188 | 189 |
190 |
191 | 192 | 193 |
194 | 195 |
196 |

197 | © Copyright 2020, Toyota Research Institute (TRI) 198 | 199 |

200 |
201 | Built with Sphinx using a theme provided by Read the Docs. 202 | 203 |
204 | 205 |
206 |
207 | 208 |
209 | 210 |
211 | 212 | 213 | 214 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | -------------------------------------------------------------------------------- /docs/_sources/README.rst.txt: -------------------------------------------------------------------------------- 1 | Neural Outlier Rejection for Self-Supervised Keypoint Learning 2 | ============================================================== 3 | 4 | Overview 5 | -------- 6 | 7 | **IO-Net:** A novel proxy task for the self-supervision of 8 | keypoint description. 9 | 10 | **KeyPointNet:** An improved keypoint-network 11 | architecture that is especially amenable to robust keypoint detection 12 | and description. 13 | 14 | `**[Full paper]** `__ 15 | 16 | Inference and evaluation 17 | ------------------------ 18 | 19 | Setting up your environment 20 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 21 | 22 | You need a machine with recent Nvidia drivers and a GPU. We recommend 23 | using docker (see 24 | `nvidia-docker2 `__ 25 | instructions) to have a reproducible environment. To setup your 26 | environment, type in a terminal (only tested in Ubuntu 18.04): 27 | 28 | .. code:: bash 29 | 30 | git clone https://github.com/TRI-ML/KP2D.git 31 | cd KP2D 32 | # if you want to use docker (recommended) 33 | make docker-build 34 | 35 | We will list below all commands as if run directly inside our container. 36 | To run any of the commands in a container, you can either start the 37 | container in interactive mode with ``make docker-start`` to land in a 38 | shell where you can type those commands, or you can do it in one step: 39 | 40 | .. code:: bash 41 | 42 | make docker-run COMMAND="some-command" 43 | 44 | Data 45 | ~~~~ 46 | 47 | Download HPatches data: 48 | 49 | .. code:: bash 50 | 51 | cd /data/datasets/kp2d/ 52 | wget http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz 53 | tar -xvf hpatches-sequences-release.tar.gz 54 | mv hpatches-release HPatches 55 | 56 | Pre-trained models: 57 | ~~~~~~~~~~~~~~~~~~~ 58 | 59 | Download the pre-trained models from 60 | `here `__ 61 | and place them in ``/data/models/kp2d/`` 62 | 63 | To evaluate any of the models, simply run: 64 | 65 | .. code:: bash 66 | 67 | make docker-run COMMAND="python scripts/eval_keypoint_net.py --pretrained_model /data/models/kp2d/v4.ckpt --input /data/datasets/kp2d/HPatches/" 68 | 69 | Evaluation for **(320, 240)**: 70 | 71 | +-------+---------------+--------------+-------+-------+-------+--------+ 72 | | Model | Repeatability | Localization | C1 | C3 | C5 | MScore | 73 | +=======+===============+==============+=======+=======+=======+========+ 74 | | V0* | 0.644 | 1.087 | 0.459 | 0.816 | 0.888 | 0.518 | 75 | +-------+---------------+--------------+-------+-------+-------+--------+ 76 | | V1* | 0.678 | 0.980 | 0.453 | 0.828 | 0.905 | 0.552 | 77 | +-------+---------------+--------------+-------+-------+-------+--------+ 78 | | V2* | 0.679 | 0.942 | 0.534 | 0.860 | 0.914 | 0.573 | 79 | +-------+---------------+--------------+-------+-------+-------+--------+ 80 | | V3 | 0.685 | 0.885 | 0.602 | 0.836 | 0.886 | 0.520 | 81 | +-------+---------------+--------------+-------+-------+-------+--------+ 82 | | V4 | 0.687 | 0.892 | 0.593 | 0.867 | 0.910 | 0.546 | 83 | +-------+---------------+--------------+-------+-------+-------+--------+ 84 | 85 | Evaluation for **(640, 480)**: 86 | 87 | +-------+---------------+--------------+-------+-------+-------+--------+ 88 | | Model | Repeatability | Localization | C1 | C3 | C5 | MScore | 89 | +=======+===============+==============+=======+=======+=======+========+ 90 | | V0* | 0.633 | 1.157 | 0.45 | 0.810 | 0.890 | 0.486 | 91 | +-------+---------------+--------------+-------+-------+-------+--------+ 92 | | V1* | 0.673 | 1.049 | 0.464 | 0.817 | 0.895 | 0.519 | 93 | +-------+---------------+--------------+-------+-------+-------+--------+ 94 | | V2* | 0.68 | 1.008 | 0.510 | 0.855 | 0.921 | 0.544 | 95 | +-------+---------------+--------------+-------+-------+-------+--------+ 96 | | V3 | 0.682 | 0.972 | 0.550 | 0.812 | 0.883 | 0.486 | 97 | +-------+---------------+--------------+-------+-------+-------+--------+ 98 | | V4 | 0.684 | 0.972 | 0.566 | 0.84 | 0.900 | 0.511 | 99 | +-------+---------------+--------------+-------+-------+-------+--------+ 100 | 101 | \* - these models were trained again after submission - the numbers 102 | deviate slightly from the paper, however the same trends can be 103 | observed. 104 | 105 | License 106 | ------- 107 | 108 | The source code is released under the `MIT license `__. 109 | 110 | Citation 111 | -------- 112 | 113 | Please use the following citation when referencing our work: 114 | 115 | :: 116 | 117 | @inproceedings{ 118 | tang2020neural, 119 | title={Neural Outlier Rejection for Self-Supervised Keypoint Learning}, 120 | author={Jiexiong Tang and Hanme Kim and Vitor Guizilini and Sudeep Pillai and Rares Ambrus}, 121 | booktitle={International Conference on Learning Representations}, 122 | year={2020} 123 | } 124 | 125 | .. |image0| image:: media/imgs/diagram_architecture.png 126 | -------------------------------------------------------------------------------- /docs/_sources/datasets/datasets.patches_dataset.rst.txt: -------------------------------------------------------------------------------- 1 | patches_dataset 2 | =============== 3 | 4 | .. automodule:: kp2d.datasets.patches_dataset 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/datasets/datasets.rst.txt: -------------------------------------------------------------------------------- 1 | datasets 2 | ======== 3 | 4 | .. toctree:: 5 | datasets.patches_dataset 6 | -------------------------------------------------------------------------------- /docs/_sources/evaluation/evaluation.descriptor_evaluation.rst.txt: -------------------------------------------------------------------------------- 1 | descriptor_evaluation 2 | ===================== 3 | 4 | .. automodule:: kp2d.evaluation.descriptor_evaluation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/evaluation/evaluation.detector_evaluation.rst.txt: -------------------------------------------------------------------------------- 1 | detector_evaluation 2 | =================== 3 | 4 | .. automodule:: kp2d.evaluation.detector_evaluation 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/evaluation/evaluation.evaluate.rst.txt: -------------------------------------------------------------------------------- 1 | evaluate 2 | ======== 3 | 4 | .. automodule:: kp2d.evaluation.evaluate 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/evaluation/evaluation.rst.txt: -------------------------------------------------------------------------------- 1 | evaluation 2 | ========== 3 | 4 | .. toctree:: 5 | evaluation.detector_evaluation 6 | evaluation.evaluate 7 | evaluation.descriptor_evaluation 8 | -------------------------------------------------------------------------------- /docs/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. PackNet-SfM documentation master file, created by 2 | sphinx-quickstart on Thu Apr 23 07:39:57 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include:: README.rst 7 | 8 | .. toctree:: 9 | :maxdepth: 5 10 | :caption: Contents 11 | 12 | datasets/datasets.rst 13 | evaluation/evaluation.rst 14 | networks/networks.rst 15 | utils/utils.rst 16 | 17 | .. toctree:: 18 | :glob: 19 | :maxdepth: 1 20 | :caption: Contact 21 | 22 | Toyota Research Institute 23 | KP2D GitHub 24 | 25 | Indices and tables 26 | ================== 27 | 28 | * :ref:`genindex` 29 | * :ref:`modindex` 30 | * :ref:`search` 31 | -------------------------------------------------------------------------------- /docs/_sources/networks/networks.inlier_net.rst.txt: -------------------------------------------------------------------------------- 1 | inlier_net 2 | ========== 3 | 4 | .. automodule:: kp2d.networks.inlier_net 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/networks/networks.keypoint_net.rst.txt: -------------------------------------------------------------------------------- 1 | keypoint_net 2 | ============ 3 | 4 | .. automodule:: kp2d.networks.keypoint_net 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/networks/networks.rst.txt: -------------------------------------------------------------------------------- 1 | networks 2 | ======== 3 | 4 | .. toctree:: 5 | networks.keypoint_net 6 | networks.inlier_net 7 | -------------------------------------------------------------------------------- /docs/_sources/utils/utils.image.rst.txt: -------------------------------------------------------------------------------- 1 | image 2 | ===== 3 | 4 | .. automodule:: kp2d.utils.image 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/utils/utils.keypoints.rst.txt: -------------------------------------------------------------------------------- 1 | keypoints 2 | ========= 3 | 4 | .. automodule:: kp2d.utils.keypoints 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/_sources/utils/utils.rst.txt: -------------------------------------------------------------------------------- 1 | utils 2 | ===== 3 | 4 | .. toctree:: 5 | utils.keypoints 6 | utils.image 7 | -------------------------------------------------------------------------------- /docs/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} 2 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | .wy-side-nav-search, .wy-nav-top { 2 | background: #ffffff; 3 | } 4 | .wy-nav-side { 5 | background: #222222; 6 | } 7 | .wy-menu > .caption > span.caption-text { 8 | color: #bb0000; 9 | } 10 | 11 | -------------------------------------------------------------------------------- /docs/_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '1.0', 4 | LANGUAGE: 'None', 5 | COLLAPSE_INDEX: false, 6 | BUILDER: 'html', 7 | FILE_SUFFIX: '.html', 8 | LINK_SUFFIX: '.html', 9 | HAS_SOURCE: true, 10 | SOURCELINK_SUFFIX: '.txt', 11 | NAVIGATION_WITH_KEYS: false 12 | }; -------------------------------------------------------------------------------- /docs/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/file.png -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Inconsolata-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Inconsolata.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Inconsolata.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab-Bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab-Regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/_static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/_static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/_static/js/theme.js: -------------------------------------------------------------------------------- 1 | /* sphinx_rtd_theme version 0.4.3 | MIT license */ 2 | /* Built 20190212 16:02 */ 3 | require=function r(s,a,l){function c(e,n){if(!a[e]){if(!s[e]){var i="function"==typeof require&&require;if(!n&&i)return i(e,!0);if(u)return u(e,!0);var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}var o=a[e]={exports:{}};s[e][0].call(o.exports,function(n){return c(s[e][1][n]||n)},o,o.exports,r,s,a,l)}return a[e].exports}for(var u="function"==typeof require&&require,n=0;n"),i("table.docutils.footnote").wrap("
"),i("table.docutils.citation").wrap("
"),i(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var e=i(this);expand=i(''),expand.on("click",function(n){return t.toggleCurrent(e),n.stopPropagation(),!1}),e.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}0this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var r=0,n=["ms","moz","webkit","o"],e=0;e 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | datasets — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 108 | 109 |
110 | 111 | 112 | 118 | 119 | 120 |
121 | 122 |
123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 |
141 | 142 |
    143 | 144 |
  • Docs »
    • 145 | 146 |
  • datasets
    • 147 | 148 | 149 |
  • 150 | 151 | 152 | View page source 153 | 154 | 155 |
    • 156 | 157 |
158 | 159 | 160 |
161 |
162 |
163 |
164 | 165 |
166 |

datasets

167 |
168 | 171 |
172 |
173 | 174 | 175 |
176 | 177 |
178 |
179 | 180 | 188 | 189 | 190 |
191 | 192 |
193 |

194 | © Copyright 2020, Toyota Research Institute (TRI) 195 | 196 |

197 |
198 | Built with Sphinx using a theme provided by Read the Docs. 199 | 200 |
201 | 202 |
203 |
204 | 205 |
206 | 207 |
208 | 209 | 210 | 211 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | -------------------------------------------------------------------------------- /docs/datasets/datasets.patches_dataset.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | patches_dataset — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 108 | 109 |
110 | 111 | 112 | 118 | 119 | 120 |
121 | 122 |
123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 |
141 | 142 |
    143 | 144 |
  • Docs »
    • 145 | 146 |
  • datasets »
    • 147 | 148 |
  • patches_dataset
    • 149 | 150 | 151 |
  • 152 | 153 | 154 | View page source 155 | 156 | 157 |
    • 158 | 159 |
160 | 161 | 162 |
163 |
164 |
165 |
166 | 167 |
168 |

patches_dataset

169 |
170 |
171 | class kp2d.datasets.patches_dataset.PatchesDataset(root_dir, use_color=True, data_transform=None, output_shape=None, type='all')[source]
172 |

Bases: torch.utils.data.dataset.Dataset

173 |
174 |
175 | scale_homography(homography, original_scale, new_scale, pre)[source]
176 |
177 | 178 |
179 | 180 |
181 | 182 | 183 |
184 | 185 |
186 |
187 | 188 | 196 | 197 | 198 |
199 | 200 |
201 |

202 | © Copyright 2020, Toyota Research Institute (TRI) 203 | 204 |

205 |
206 | Built with Sphinx using a theme provided by Read the Docs. 207 | 208 |
209 | 210 |
211 |
212 | 213 |
214 | 215 |
216 | 217 | 218 | 219 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | -------------------------------------------------------------------------------- /docs/evaluation/evaluation.detector_evaluation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | detector_evaluation — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 110 | 111 |
112 | 113 | 114 | 120 | 121 | 122 |
123 | 124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 |
143 | 144 |
    145 | 146 |
  • Docs »
    • 147 | 148 |
  • evaluation »
    • 149 | 150 |
  • detector_evaluation
    • 151 | 152 | 153 |
  • 154 | 155 | 156 | View page source 157 | 158 | 159 |
    • 160 | 161 |
162 | 163 | 164 |
165 |
166 |
167 |
168 | 169 |
170 |

detector_evaluation

171 |
172 |
173 | kp2d.evaluation.detector_evaluation.compute_repeatability(data, keep_k_points=300, distance_thresh=3)[source]
174 |

Compute the repeatability metric between 2 sets of keypoints inside data.

175 |
176 |
Parameters
177 |
    178 |

  • data (dict) –

    Input dictionary containing: 179 | image_shape: tuple (H,W)

    180 |
    181 |

    Original image shape.

    182 |
    183 |
    184 |
    homography: numpy.ndarray (3,3)

    Ground truth homography.

    185 |
    186 |
    prob: numpy.ndarray (N,3)

    Keypoint vector, consisting of (x,y,probability).

    187 |
    188 |
    warped_prob: numpy.ndarray (N,3)

    Warped keypoint vector, consisting of (x,y,probability).

    189 |
    190 |
    191 |

    • 192 |

  • keep_k_points (int) – Number of keypoints to select, based on probability.

    • 193 |

  • distance_thresh (int) – Distance threshold in pixels for a corresponding keypoint to be considered a correct match.

    • 194 |
195 |
196 |
Returns
197 |

    198 |

  • N1 (int) – Number of true keypoints in the first image.

    • 199 |

  • N2 (int) – Number of true keypoints in the second image.

    • 200 |

  • repeatability (float) – Keypoint repeatability metric.

    • 201 |

  • loc_err (float) – Keypoint localization error.

    • 202 |
203 |

204 |
205 |
206 |
207 | 208 |
209 | 210 | 211 |
212 | 213 |
214 |
215 | 216 | 224 | 225 | 226 |
227 | 228 |
229 |

230 | © Copyright 2020, Toyota Research Institute (TRI) 231 | 232 |

233 |
234 | Built with Sphinx using a theme provided by Read the Docs. 235 | 236 |
237 | 238 |
239 |
240 | 241 |
242 | 243 |
244 | 245 | 246 | 247 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | -------------------------------------------------------------------------------- /docs/evaluation/evaluation.evaluate.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | evaluate — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 110 | 111 |
112 | 113 | 114 | 120 | 121 | 122 |
123 | 124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 |
143 | 144 | 162 | 163 | 164 |
165 |
166 |
167 |
168 | 169 |
170 |

evaluate

171 |
172 |
173 | kp2d.evaluation.evaluate.evaluate_keypoint_net(data_loader, keypoint_net, output_shape=320, 240, top_k=300, use_color=True)[source]
174 |

Keypoint net evaluation script.

175 |
176 |
Parameters
177 |
    178 |

  • data_loader (torch.utils.data.DataLoader) – Dataset loader.

    • 179 |

  • keypoint_net (torch.nn.module) – Keypoint network.

    • 180 |

  • output_shape (tuple) – Original image shape.

    • 181 |

  • top_k (int) – Number of keypoints to use to compute metrics, selected based on probability.

    • 182 |

  • use_color (bool) – Use color or grayscale images.

    • 183 |
184 |
185 |
186 |
187 | 188 |
189 | 190 | 191 |
192 | 193 |
194 |
195 | 196 | 204 | 205 | 206 |
207 | 208 |
209 |

210 | © Copyright 2020, Toyota Research Institute (TRI) 211 | 212 |

213 |
214 | Built with Sphinx using a theme provided by Read the Docs. 215 | 216 |
217 | 218 |
219 |
220 | 221 |
222 | 223 |
224 | 225 | 226 | 227 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /docs/evaluation/evaluation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | evaluation — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 110 | 111 |
112 | 113 | 114 | 120 | 121 | 122 |
123 | 124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 |
143 | 144 |
    145 | 146 |
  • Docs »
    • 147 | 148 |
  • evaluation
    • 149 | 150 | 151 |
  • 152 | 153 | 154 | View page source 155 | 156 | 157 |
    • 158 | 159 |
160 | 161 | 162 |
163 |
164 |
165 |
166 | 167 |
168 |

evaluation

169 |
170 | 175 |
176 |
177 | 178 | 179 |
180 | 181 |
182 |
183 | 184 | 192 | 193 | 194 |
195 | 196 |
197 |

198 | © Copyright 2020, Toyota Research Institute (TRI) 199 | 200 |

201 |
202 | Built with Sphinx using a theme provided by Read the Docs. 203 | 204 |
205 | 206 |
207 |
208 | 209 |
210 | 211 |
212 | 213 | 214 | 215 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | -------------------------------------------------------------------------------- /docs/networks/networks.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | networks — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 109 | 110 |
111 | 112 | 113 | 119 | 120 | 121 |
122 | 123 |
124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 |
142 | 143 |
    144 | 145 |
  • Docs »
    • 146 | 147 |
  • networks
    • 148 | 149 | 150 |
  • 151 | 152 | 153 | View page source 154 | 155 | 156 |
    • 157 | 158 |
159 | 160 | 161 |
162 |
163 |
164 |
165 | 166 |
167 |

networks

168 |
169 | 173 |
174 |
175 | 176 | 177 |
178 | 179 |
180 |
181 | 182 | 190 | 191 | 192 |
193 | 194 |
195 |

196 | © Copyright 2020, Toyota Research Institute (TRI) 197 | 198 |

199 |
200 | Built with Sphinx using a theme provided by Read the Docs. 201 | 202 |
203 | 204 |
205 |
206 | 207 |
208 | 209 |
210 | 211 | 212 | 213 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | -------------------------------------------------------------------------------- /docs/networks/networks.inlier_net.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | inlier_net — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 109 | 110 |
111 | 112 | 113 | 119 | 120 | 121 |
122 | 123 |
124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 |
142 | 143 |
    144 | 145 |
  • Docs »
    • 146 | 147 |
  • networks »
    • 148 | 149 |
  • inlier_net
    • 150 | 151 | 152 |
  • 153 | 154 | 155 | View page source 156 | 157 | 158 |
    • 159 | 160 |
161 | 162 | 163 |
164 |
165 |
166 |
167 | 168 |
169 |

inlier_net

170 |
171 |
172 | class kp2d.networks.inlier_net.InlierNet(blocks)[source]
173 |

Bases: torch.nn.modules.module.Module

174 |
175 |
176 | forward(inputs)[source]
177 |

Defines the computation performed at every call.

178 |

Should be overridden by all subclasses.

179 |
180 |

Note

181 |

Although the recipe for forward pass needs to be defined within 182 | this function, one should call the Module instance afterwards 183 | instead of this since the former takes care of running the 184 | registered hooks while the latter silently ignores them.

185 |
186 |
187 | 188 |
189 | 190 |
191 | 192 | 193 |
194 | 195 |
196 |
197 | 198 | 206 | 207 | 208 |
209 | 210 |
211 |

212 | © Copyright 2020, Toyota Research Institute (TRI) 213 | 214 |

215 |
216 | Built with Sphinx using a theme provided by Read the Docs. 217 | 218 |
219 | 220 |
221 |
222 | 223 |
224 | 225 |
226 | 227 | 228 | 229 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | -------------------------------------------------------------------------------- /docs/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/docs/objects.inv -------------------------------------------------------------------------------- /docs/py-modindex.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Python Module Index — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
48 | 49 | 106 | 107 |
108 | 109 | 110 | 116 | 117 | 118 |
119 | 120 |
121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 |
139 | 140 |
    141 | 142 |
  • Docs »
    • 143 | 144 |
  • 145 | 146 | 147 |
  • 148 | 149 |
    • 150 | 151 |
152 | 153 | 154 |
155 |
156 |
157 |
158 | 159 | 160 |

Python Module Index

161 | 162 |
163 | k 164 |
165 | 166 | 167 | 168 | 170 | 171 | 173 | 176 | 177 | 178 | 181 | 182 | 183 | 186 | 187 | 188 | 191 | 192 | 193 | 196 | 197 | 198 | 201 | 202 | 203 | 206 | 207 | 208 | 211 | 212 | 213 | 216 |
 
169 | k
174 | kp2d 175 |
    179 | kp2d.datasets.patches_dataset 180 |
    184 | kp2d.evaluation.descriptor_evaluation 185 |
    189 | kp2d.evaluation.detector_evaluation 190 |
    194 | kp2d.evaluation.evaluate 195 |
    199 | kp2d.networks.inlier_net 200 |
    204 | kp2d.networks.keypoint_net 205 |
    209 | kp2d.utils.image 210 |
    214 | kp2d.utils.keypoints 215 |
217 | 218 | 219 |
220 | 221 |
222 |
223 | 224 | 225 |
226 | 227 |
228 |

229 | © Copyright 2020, Toyota Research Institute (TRI) 230 | 231 |

232 |
233 | Built with Sphinx using a theme provided by Read the Docs. 234 | 235 |
236 | 237 |
238 |
239 | 240 |
241 | 242 |
243 | 244 | 245 | 246 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | -------------------------------------------------------------------------------- /docs/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Search — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 |
46 | 47 | 104 | 105 |
106 | 107 | 108 | 114 | 115 | 116 |
117 | 118 |
119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 |
137 | 138 |
    139 | 140 |
  • Docs »
    • 141 | 142 |
  • 143 | 144 | 145 |
  • 146 | 147 | 148 | 149 |
    • 150 | 151 |
152 | 153 | 154 |
155 |
156 |
157 |
158 | 159 | 167 | 168 | 169 |
170 | 171 |
172 | 173 |
174 | 175 |
176 |
177 | 178 | 179 |
180 | 181 |
182 |

183 | © Copyright 2020, Toyota Research Institute (TRI) 184 | 185 |

186 |
187 | Built with Sphinx using a theme provided by Read the Docs. 188 | 189 |
190 | 191 |
192 |
193 | 194 |
195 | 196 |
197 | 198 | 199 | 200 | 205 | 206 | 207 | 208 | 209 | 210 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | -------------------------------------------------------------------------------- /docs/searchindex.js: -------------------------------------------------------------------------------- 1 | Search.setIndex({docnames:["README","datasets/datasets","datasets/datasets.patches_dataset","evaluation/evaluation","evaluation/evaluation.descriptor_evaluation","evaluation/evaluation.detector_evaluation","evaluation/evaluation.evaluate","index","networks/networks","networks/networks.inlier_net","networks/networks.keypoint_net","utils/utils","utils/utils.image","utils/utils.keypoints"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["README.rst","datasets/datasets.rst","datasets/datasets.patches_dataset.rst","evaluation/evaluation.rst","evaluation/evaluation.descriptor_evaluation.rst","evaluation/evaluation.detector_evaluation.rst","evaluation/evaluation.evaluate.rst","index.rst","networks/networks.rst","networks/networks.inlier_net.rst","networks/networks.keypoint_net.rst","utils/utils.rst","utils/utils.image.rst","utils/utils.keypoints.rst"],objects:{"kp2d.datasets":{patches_dataset:[2,0,0,"-"]},"kp2d.datasets.patches_dataset":{PatchesDataset:[2,1,1,""]},"kp2d.datasets.patches_dataset.PatchesDataset":{scale_homography:[2,2,1,""]},"kp2d.evaluation":{descriptor_evaluation:[4,0,0,"-"],detector_evaluation:[5,0,0,"-"],evaluate:[6,0,0,"-"]},"kp2d.evaluation.descriptor_evaluation":{compute_homography:[4,3,1,""],compute_matching_score:[4,3,1,""],keep_shared_points:[4,3,1,""],select_k_best:[4,3,1,""]},"kp2d.evaluation.detector_evaluation":{compute_repeatability:[5,3,1,""]},"kp2d.evaluation.evaluate":{evaluate_keypoint_net:[6,3,1,""]},"kp2d.networks":{inlier_net:[9,0,0,"-"],keypoint_net:[10,0,0,"-"]},"kp2d.networks.inlier_net":{InlierNet:[9,1,1,""]},"kp2d.networks.inlier_net.InlierNet":{forward:[9,2,1,""]},"kp2d.networks.keypoint_net":{KeypointNet:[10,1,1,""]},"kp2d.networks.keypoint_net.KeypointNet":{forward:[10,2,1,""]},"kp2d.utils":{image:[12,0,0,"-"],keypoints:[13,0,0,"-"]},"kp2d.utils.image":{image_grid:[12,3,1,""],meshgrid:[12,3,1,""],to_color_normalized:[12,3,1,""],to_gray_normalized:[12,3,1,""]},"kp2d.utils.keypoints":{warp_keypoints:[13,3,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:function"},terms:{"class":[2,9,10],"float":[4,5],"function":9,"int":[4,5,6,12],"return":[4,5,10,12,13],"true":[2,5,6,10,12],"while":9,The:[0,7],Use:[4,6,10],after:[0,7],afterward:9,again:[0,7],all:[0,2,7,9],although:9,ambru:[0,7],amen:[0,7],ani:[0,7],architectur:[0,7],associ:4,author:[0,7],b3hw:12,base:[2,4,5,6,9,10],batch:[10,12],below:[0,7],between:[4,5],bhw:12,block:9,booktitl:[0,7],bool:[6,10,12],border:10,build:[0,7],call:9,can:[0,7],care:9,cell:10,ckpt:[0,7],clone:[0,7],code:[0,7],color:[6,10],com:[0,7],command:[0,7],comput:[4,5,6,9],compute_homographi:4,compute_matching_scor:4,compute_repeat:5,confer:[0,7],consid:5,consist:[4,5],contact:7,contain:[0,4,5,7],content:7,convert:12,coord:10,coordin:[4,10,12],correct:[4,5],correctness1:4,correctness3:4,correctness5:4,correspond:[4,5,12],creat:12,data:[2,4,5,6],data_load:6,data_transform:2,dataload:6,dataset:[0,2,6,7],defin:9,desc:4,descript:[0,7],descriptor:[4,10],descriptor_evalu:[3,7],desns:10,detect:[0,7,10],detector_evalu:[3,7],deviat:[0,7],devic:12,dict:[4,5,10],dictionari:[4,5],dimens:12,directli:[0,7],distanc:5,distance_thresh:5,do_cross:10,do_upsampl:10,docker2:[0,7],docker:[0,7],download:[0,7],driver:[0,7],dropout:10,dtype:12,either:[0,7],error:5,especi:[0,7],eval_keypoint_net:[0,7],evalu:[4,5],evaluate_keypoint_net:6,everi:9,extra:10,fals:12,feat:10,filter:4,first:5,follow:[0,7],former:9,forward:[9,10],from:[0,4,7],full:[0,7],git:[0,7],github:[0,7],given:[12,13],gpu:[0,7],grayscal:[6,10,12],grid:12,ground:[4,5],guizilini:[0,7],h_out:10,hanm:[0,7],has:4,have:[0,7],height:12,here:[0,7],homographi:[2,4,5,13],hook:9,howev:[0,7],hpatch:[0,7],http:[0,7],icvl:[0,7],ignor:9,imag:[4,5,6,7,10,11],image_grid:12,image_shap:[4,5],improv:[0,7],index:7,inlier_net:[7,8],inliernet:9,inproceed:[0,7],input:[0,4,5,7,9,10,12],insid:[0,4,5,7],instanc:9,instead:9,institut:7,instruct:[0,7],integ:12,interact:[0,7],intern:[0,7],jiexiong:[0,7],keep:4,keep_k_point:[4,5],keep_shared_point:4,keypoint:[4,5,6,10,11],keypoint_net:[6,7,8],keypointnet:[0,7,10],kim:[0,7],kp2d:[0,2,4,5,6,7,9,10,12,13],kwarg:10,land:[0,7],last:4,latter:9,list:[0,4,7],loader:6,loc_err:5,local:[0,5,7],machin:[0,7],make:[0,7],map:[4,10],match:[4,5],mesh:12,meshgrid:12,metric:[4,5,6],mit:[0,7],mode:[0,7],modul:[6,7,9,10],most:4,mscore:[0,7],ndarrai:[4,5,13],need:[0,7,9],net:[0,6,7],network:[0,6,7,9,10],new_scal:2,none:2,normal:12,normalized_imag:12,novel:[0,7],num_point:4,number:[0,4,5,6,7],numpi:[4,5,13],nvidia:[0,7],observ:[0,7],onc:4,one:[0,7,9],ones:12,onli:[0,4,7],origin:[4,5,6],original_scal:2,our:[0,7],output_shap:[2,6],outsid:10,overridden:9,page:7,paper:[0,7],paramet:[4,5,6,10,12,13],pass:9,patches_dataset:[1,7],patchesdataset:2,perform:[9,12],pillai:[0,7],pixel:5,place:[0,7],pleas:[0,7],point:4,pre:2,predict:10,preserv:12,pretrained_model:[0,7],prob:[4,5],proba:4,probabl:[4,5,6],process:10,proxi:[0,7],python:[0,7],rare:[0,7],recent:[0,7],recip:9,recommend:[0,7],referenc:[0,7],regist:9,releas:[0,7],repeat:[0,5,7],represent:[0,7],reproduc:[0,7],research:7,robust:[0,7],root_dir:2,run:[0,7,9],same:[0,7],scale_homographi:2,score:[4,10],script:[0,6,7],search:7,second:5,see:[0,7],select:[4,5,6],select_k_best:4,selected_descriptor:4,selected_point:4,sequenc:[0,7],set:[4,5],setup:[0,7],shape:[4,5,6,12],shell:[0,7],should:9,silent:9,simpli:[0,7],sinc:9,size:12,slightli:[0,7],some:[0,7],sourc:[0,2,4,5,6,7,9,10,12,13],start:[0,7],step:[0,7],still:4,str:12,strip:4,subclass:9,submiss:[0,7],sudeep:[0,7],take:9,tang2020neur:[0,7],tang:[0,7],tar:[0,7],task:[0,7],tensor:[10,12],termin:[0,7],test:[0,7],them:[0,7,9],thi:9,those:[0,7],threshold:5,titl:[0,7],to_color_norm:12,to_gray_norm:12,top_k:6,torch:[2,6,9,10,12],toyota:7,trend:[0,7],tri:[0,7],truth:[4,5],tupl:[4,5,6],two:4,type:[0,2,4,7,12,13],ubuntu:[0,7],under:[0,7],upsampl:10,use:[0,6,7],use_color:[2,6,10],using:[0,7],util:[2,6,7,12,13],vbalnt:[0,7],vector:[4,5,13],vitor:[0,7],w_out:10,want:[0,7],warp:[4,5,13],warp_keypoint:13,warped_desc:4,warped_keypoint:13,warped_prob:[4,5],were:[0,7],wget:[0,7],when:[0,7],where:[0,4,7],width:12,with_drop:10,within:9,work:[0,7],xvf:[0,7],year:[0,7],you:[0,7]},titles:["Neural Outlier Rejection for Self-Supervised Keypoint Learning","datasets","patches_dataset","evaluation","descriptor_evaluation","detector_evaluation","evaluate","Neural Outlier Rejection for Self-Supervised Keypoint Learning","networks","inlier_net","keypoint_net","utils","image","keypoints"],titleterms:{citat:[0,7],data:[0,7],dataset:1,descriptor_evalu:4,detector_evalu:5,environ:[0,7],evalu:[0,3,6,7],imag:12,indic:7,infer:[0,7],inlier_net:9,keypoint:[0,7,13],keypoint_net:10,learn:[0,7],licens:[0,7],model:[0,7],network:8,neural:[0,7],outlier:[0,7],overview:[0,7],patches_dataset:2,pre:[0,7],reject:[0,7],self:[0,7],set:[0,7],supervis:[0,7],tabl:7,train:[0,7],util:11,your:[0,7]}}) -------------------------------------------------------------------------------- /docs/utils/utils.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | utils — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 109 | 110 |
111 | 112 | 113 | 119 | 120 | 121 |
122 | 123 |
124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 |
142 | 143 |
    144 | 145 |
  • Docs »
    • 146 | 147 |
  • utils
    • 148 | 149 | 150 |
  • 151 | 152 | 153 | View page source 154 | 155 | 156 |
    • 157 | 158 |
159 | 160 | 161 |
162 |
163 |
164 |
165 | 166 |
167 |

utils

168 |
169 | 173 |
174 |
175 | 176 | 177 |
178 | 179 |
180 |
181 | 182 | 190 | 191 | 192 |
193 | 194 |
195 |

196 | © Copyright 2020, Toyota Research Institute (TRI) 197 | 198 |

199 |
200 | Built with Sphinx using a theme provided by Read the Docs. 201 | 202 |
203 | 204 |
205 |
206 | 207 |
208 | 209 |
210 | 211 | 212 | 213 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | -------------------------------------------------------------------------------- /docs/utils/utils.keypoints.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | keypoints — PackNet-SfM 1.0 documentation 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | 109 | 110 |
111 | 112 | 113 | 119 | 120 | 121 |
122 | 123 |
124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 |
142 | 143 |
    144 | 145 |
  • Docs »
    • 146 | 147 |
  • utils »
    • 148 | 149 |
  • keypoints
    • 150 | 151 | 152 |
  • 153 | 154 | 155 | View page source 156 | 157 | 158 |
    • 159 | 160 |
161 | 162 | 163 |
164 |
165 |
166 |
167 | 168 |
169 |

keypoints

170 |
171 |
172 | kp2d.utils.keypoints.warp_keypoints(keypoints, H)[source]
173 |

Warp keypoints given a homography

174 |
175 |
Parameters
176 |
    177 |

  • keypoints (numpy.ndarray (N,2)) – Keypoint vector.

    • 178 |

  • H (numpy.ndarray (3,3)) – Homography.

    • 179 |
180 |
181 |
Returns
182 |

warped_keypoints – Warped keypoints vector.

183 |
184 |
Return type
185 |

numpy.ndarray (N,2)

186 |
187 |
188 |
189 | 190 |
191 | 192 | 193 |
194 | 195 |
196 |
197 | 198 | 206 | 207 | 208 |
209 | 210 |
211 |

212 | © Copyright 2020, Toyota Research Institute (TRI) 213 | 214 |

215 |
216 | Built with Sphinx using a theme provided by Read the Docs. 217 | 218 |
219 | 220 |
221 |
222 | 223 |
224 | 225 |
226 | 227 | 228 | 229 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | -------------------------------------------------------------------------------- /kp2d/configs/base_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | """Default kp2d configuration parameters (overridable in configs/*.yaml) 4 | """ 5 | 6 | import os 7 | from yacs.config import CfgNode as CN 8 | 9 | ######################################################################################################################## 10 | cfg = CN() 11 | cfg.name = '' # Run name 12 | cfg.debug = True # Debugging flag 13 | ######################################################################################################################## 14 | ### ARCH 15 | ######################################################################################################################## 16 | cfg.arch = CN() 17 | cfg.arch.seed = 42 # Random seed for Pytorch/Numpy initialization 18 | cfg.arch.epochs = 50 # Maximum number of epochs 19 | ######################################################################################################################## 20 | ### WANDB 21 | ######################################################################################################################## 22 | cfg.wandb = CN() 23 | cfg.wandb.dry_run = True # Wandb dry-run (not logging) 24 | cfg.wandb.name = '' # Wandb run name 25 | cfg.wandb.project = os.environ.get("WANDB_PROJECT", "") # Wandb project 26 | cfg.wandb.entity = os.environ.get("WANDB_ENTITY", "") # Wandb entity 27 | cfg.wandb.tags = [] # Wandb tags 28 | cfg.wandb.dir = '' # Wandb save folder 29 | ######################################################################################################################## 30 | ### MODEL 31 | ######################################################################################################################## 32 | cfg.model = CN() 33 | cfg.model.checkpoint_path = '/data/experiments/kp2d/' # Checkpoint path for model saving 34 | cfg.model.save_checkpoint = True 35 | ######################################################################################################################## 36 | ### MODEL.SCHEDULER 37 | ######################################################################################################################## 38 | cfg.model.scheduler = CN() 39 | cfg.model.scheduler.decay = 0.5 # Scheduler decay rate 40 | cfg.model.scheduler.lr_epoch_divide_frequency = 40 # Schedule number of epochs when to decay the initial learning rate by decay rate 41 | ######################################################################################################################## 42 | ### MODEL.OPTIMIZER 43 | ######################################################################################################################## 44 | cfg.model.optimizer = CN() 45 | cfg.model.optimizer.learning_rate = 0.001 46 | cfg.model.optimizer.weight_decay = 0.0 47 | ######################################################################################################################## 48 | ### MODEL.PARAMS 49 | ######################################################################################################################## 50 | cfg.model.params = CN() 51 | cfg.model.params.keypoint_loss_weight = 1.0 # Keypoint loss weight 52 | cfg.model.params.descriptor_loss_weight = 1.0 # Descriptor loss weight 53 | cfg.model.params.score_loss_weight = 1.0 # Score loss weight 54 | cfg.model.params.use_color = True # Use color or grayscale images 55 | cfg.model.params.with_io = True # Use IONet 56 | cfg.model.params.do_upsample = True # Upsample descriptors 57 | cfg.model.params.do_cross = True # Use cross-border keypoints 58 | cfg.model.params.descriptor_loss = True # Use hardest negative mining descriptor loss 59 | cfg.model.params.keypoint_net_type = 'KeypointNet' # Type of keypoint network. Supported ['KeypointNet', 'KeypointResnet'] 60 | ######################################################################################################################## 61 | ### DATASETS 62 | ######################################################################################################################## 63 | cfg.datasets = CN() 64 | ######################################################################################################################## 65 | ### DATASETS.AUGMENTATION 66 | ######################################################################################################################## 67 | cfg.datasets.augmentation = CN() 68 | cfg.datasets.augmentation.image_shape = (240, 320) # Image shape 69 | cfg.datasets.augmentation.jittering = (0.5, 0.5, 0.2, 0.05) # Color jittering values 70 | ######################################################################################################################## 71 | ### DATASETS.TRAIN 72 | ######################################################################################################################## 73 | cfg.datasets.train = CN() 74 | cfg.datasets.train.batch_size = 8 # Training batch size 75 | cfg.datasets.train.num_workers = 16 # Training number of workers 76 | cfg.datasets.train.path = '/data/datasets/kp2d/coco/train2017/' # Training data path (COCO dataset) 77 | cfg.datasets.train.repeat = 1 # Number of times training dataset is repeated per epoch 78 | ######################################################################################################################## 79 | ### DATASETS.VAL 80 | ######################################################################################################################## 81 | cfg.datasets.val = CN() 82 | cfg.datasets.val.path = '/data/datasets/kp2d/HPatches/' # Validation data path (HPatches) 83 | ######################################################################################################################## 84 | ### THESE SHOULD NOT BE CHANGED 85 | ######################################################################################################################## 86 | cfg.config = '' # Run configuration file 87 | cfg.default = '' # Run default configuration file 88 | cfg.wandb.url = '' # Wandb URL 89 | ######################################################################################################################## 90 | 91 | def get_cfg_defaults(): 92 | return cfg.clone() 93 | -------------------------------------------------------------------------------- /kp2d/configs/v0.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v0 model: 4 | # - IONet 5 | # - descriptor upsampling 6 | # - cross-border keypoints 7 | # + descriptor loss 8 | 9 | model: 10 | params: 11 | use_color: True 12 | with_io: False 13 | do_upsample: False 14 | do_cross: False 15 | descriptor_loss: True 16 | 17 | wandb: 18 | dry_run: True 19 | project: "" -------------------------------------------------------------------------------- /kp2d/configs/v1.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v1 model: 4 | # - IONet 5 | # - descriptor upsampling 6 | # + cross-border keypoints 7 | # + descriptor loss 8 | 9 | model: 10 | params: 11 | use_color: True 12 | with_io: False 13 | do_upsample: False 14 | do_cross: True 15 | descriptor_loss: True 16 | 17 | wandb: 18 | dry_run: True 19 | project: "" -------------------------------------------------------------------------------- /kp2d/configs/v2.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v2 model: 4 | # - IONet 5 | # + descriptor upsampling 6 | # + cross-border keypoints 7 | # + descriptor loss 8 | 9 | model: 10 | params: 11 | use_color: True 12 | with_io: False 13 | do_upsample: True 14 | do_cross: True 15 | descriptor_loss: True 16 | 17 | wandb: 18 | dry_run: True 19 | project: "" 20 | -------------------------------------------------------------------------------- /kp2d/configs/v2_resnet.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v2 model: 4 | # - IONet 5 | # + descriptor upsampling 6 | # + cross-border keypoints 7 | # + descriptor loss 8 | # Note: this uses a ResNet-based backbone. 9 | 10 | datasets: 11 | augmentation: 12 | image_shape: (256, 320) 13 | 14 | model: 15 | params: 16 | use_color: True 17 | with_io: False 18 | do_upsample: True 19 | do_cross: True 20 | descriptor_loss: True 21 | keypoint_net_type: 'KeypointResnet' 22 | 23 | datasets: 24 | augmentation: 25 | image_shape: (256,320) 26 | jittering: (0.2, 0.2, 0.2, 0.05) # Color jittering values 27 | train: 28 | batch_size: 8 29 | 30 | wandb: 31 | dry_run: False 32 | project: "kp2d" -------------------------------------------------------------------------------- /kp2d/configs/v3.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v3 model: 4 | # + IONet 5 | # + descriptor upsampling 6 | # + cross-border keypoints 7 | # - descriptor loss 8 | 9 | model: 10 | params: 11 | use_color: True 12 | with_io: True 13 | do_upsample: True 14 | do_cross: True 15 | descriptor_loss: False 16 | 17 | wandb: 18 | dry_run: True 19 | project: "" -------------------------------------------------------------------------------- /kp2d/configs/v4.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | # Config for training the v4 model: 4 | # + IONet 5 | # + descriptor upsampling 6 | # + cross-border keypoints 7 | # + descriptor loss 8 | 9 | model: 10 | params: 11 | use_color: True 12 | with_io: True 13 | do_upsample: True 14 | do_cross: True 15 | descriptor_loss: True 16 | 17 | wandb: 18 | dry_run: True 19 | project: "" 20 | -------------------------------------------------------------------------------- /kp2d/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/kp2d/datasets/__init__.py -------------------------------------------------------------------------------- /kp2d/datasets/coco.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import glob 4 | 5 | from PIL import Image 6 | from torch.utils.data import Dataset 7 | 8 | 9 | class COCOLoader(Dataset): 10 | """ 11 | Coco dataset class. 12 | 13 | Parameters 14 | ---------- 15 | root_dir : str 16 | Path to the dataset 17 | data_transform : Function 18 | Transformations applied to the sample 19 | """ 20 | def __init__(self, root_dir, data_transform=None): 21 | 22 | super().__init__() 23 | self.root_dir = root_dir 24 | 25 | self.files=[] 26 | 27 | for filename in glob.glob(root_dir + '/*.jpg'): 28 | self.files.append(filename) 29 | self.data_transform = data_transform 30 | 31 | def __len__(self): 32 | return len(self.files) 33 | 34 | def _read_rgb_file(self, filename): 35 | return Image.open(filename) 36 | 37 | def __getitem__(self, idx): 38 | 39 | filename = self.files[idx] 40 | image = self._read_rgb_file(filename) 41 | 42 | if image.mode == 'L': 43 | image_new = Image.new("RGB", image.size) 44 | image_new.paste(image) 45 | sample = {'image': image_new, 'idx': idx} 46 | else: 47 | sample = {'image': image, 'idx': idx} 48 | 49 | if self.data_transform: 50 | sample = self.data_transform(sample) 51 | 52 | return sample 53 | -------------------------------------------------------------------------------- /kp2d/datasets/patches_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | from pathlib import Path 4 | 5 | import cv2 6 | import numpy as np 7 | import torch 8 | import torchvision.transforms as transforms 9 | from torch.utils.data import Dataset 10 | 11 | 12 | class PatchesDataset(Dataset): 13 | """ 14 | HPatches dataset class. 15 | Note: output_shape = (output_width, output_height) 16 | Note: this returns Pytorch tensors, resized to output_shape (if specified) 17 | Note: the homography will be adjusted according to output_shape. 18 | 19 | Parameters 20 | ---------- 21 | root_dir : str 22 | Path to the dataset 23 | use_color : bool 24 | Return color images or convert to grayscale. 25 | data_transform : Function 26 | Transformations applied to the sample 27 | output_shape: tuple 28 | If specified, the images and homographies will be resized to the desired shape. 29 | type: str 30 | Dataset subset to return from ['i', 'v', 'all']: 31 | i - illumination sequences 32 | v - viewpoint sequences 33 | all - all sequences 34 | """ 35 | def __init__(self, root_dir, use_color=True, data_transform=None, output_shape=None, type='all'): 36 | 37 | super().__init__() 38 | self.type = type 39 | self.root_dir = root_dir 40 | self.data_transform = data_transform 41 | self.output_shape = output_shape 42 | self.use_color = use_color 43 | base_path = Path(root_dir) 44 | folder_paths = [x for x in base_path.iterdir() if x.is_dir()] 45 | image_paths = [] 46 | warped_image_paths = [] 47 | homographies = [] 48 | for path in folder_paths: 49 | if self.type == 'i' and path.stem[0] != 'i': 50 | continue 51 | if self.type == 'v' and path.stem[0] != 'v': 52 | continue 53 | num_images = 5 54 | file_ext = '.ppm' 55 | for i in range(2, 2 + num_images): 56 | image_paths.append(str(Path(path, "1" + file_ext))) 57 | warped_image_paths.append(str(Path(path, str(i) + file_ext))) 58 | homographies.append(np.loadtxt(str(Path(path, "H_1_" + str(i))))) 59 | self.files = {'image_paths': image_paths, 'warped_image_paths': warped_image_paths, 'homography': homographies} 60 | 61 | @staticmethod 62 | def scale_homography(homography, original_scale, new_scale, pre): 63 | scales = np.divide(new_scale, original_scale) 64 | if pre: 65 | s = np.diag(np.append(scales, 1.)) 66 | homography = np.matmul(s, homography) 67 | else: 68 | sinv = np.diag(np.append(1. / scales, 1.)) 69 | homography = np.matmul(homography, sinv) 70 | return homography 71 | 72 | def __len__(self): 73 | return len(self.files['image_paths']) 74 | 75 | def __getitem__(self, idx): 76 | 77 | def _read_image(path): 78 | img = cv2.imread(path, cv2.IMREAD_COLOR) 79 | if self.use_color: 80 | return img 81 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 82 | return gray 83 | 84 | image = _read_image(self.files['image_paths'][idx]) 85 | 86 | warped_image = _read_image(self.files['warped_image_paths'][idx]) 87 | homography = np.array(self.files['homography'][idx]) 88 | sample = {'image': image, 'warped_image': warped_image, 'homography': homography, 'index' : idx} 89 | 90 | # Apply transformations 91 | if self.output_shape is not None: 92 | sample['homography'] = self.scale_homography(sample['homography'], 93 | sample['image'].shape[:2][::-1], 94 | self.output_shape, 95 | pre=False) 96 | sample['homography'] = self.scale_homography(sample['homography'], 97 | sample['warped_image'].shape[:2][::-1], 98 | self.output_shape, 99 | pre=True) 100 | 101 | for key in ['image', 'warped_image']: 102 | sample[key] = cv2.resize(sample[key], self.output_shape) 103 | if self.use_color is False: 104 | sample[key] = np.expand_dims(sample[key], axis=2) 105 | 106 | transform = transforms.ToTensor() 107 | for key in ['image', 'warped_image']: 108 | sample[key] = transform(sample[key]).type('torch.FloatTensor') 109 | return sample 110 | -------------------------------------------------------------------------------- /kp2d/evaluation/detector_evaluation.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | # Adapted from: https://github.com/rpautrat/SuperPoint/blob/master/superpoint/evaluations/detector_evaluation.py 3 | 4 | import random 5 | from glob import glob 6 | from os import path as osp 7 | 8 | import cv2 9 | import numpy as np 10 | 11 | from kp2d.utils.keypoints import warp_keypoints 12 | 13 | 14 | def compute_repeatability(data, keep_k_points=300, distance_thresh=3): 15 | """ 16 | Compute the repeatability metric between 2 sets of keypoints inside data. 17 | 18 | Parameters 19 | ---------- 20 | data: dict 21 | Input dictionary containing: 22 | image_shape: tuple (H,W) 23 | Original image shape. 24 | homography: numpy.ndarray (3,3) 25 | Ground truth homography. 26 | prob: numpy.ndarray (N,3) 27 | Keypoint vector, consisting of (x,y,probability). 28 | warped_prob: numpy.ndarray (N,3) 29 | Warped keypoint vector, consisting of (x,y,probability). 30 | keep_k_points: int 31 | Number of keypoints to select, based on probability. 32 | distance_thresh: int 33 | Distance threshold in pixels for a corresponding keypoint to be considered a correct match. 34 | 35 | Returns 36 | ------- 37 | N1: int 38 | Number of true keypoints in the first image. 39 | N2: int 40 | Number of true keypoints in the second image. 41 | repeatability: float 42 | Keypoint repeatability metric. 43 | loc_err: float 44 | Keypoint localization error. 45 | """ 46 | def filter_keypoints(points, shape): 47 | """ Keep only the points whose coordinates are inside the dimensions of shape. """ 48 | mask = (points[:, 0] >= 0) & (points[:, 0] < shape[0]) &\ 49 | (points[:, 1] >= 0) & (points[:, 1] < shape[1]) 50 | return points[mask, :] 51 | 52 | def keep_true_keypoints(points, H, shape): 53 | """ Keep only the points whose warped coordinates by H are still inside shape. """ 54 | warped_points = warp_keypoints(points[:,:2], H) 55 | mask = (warped_points[:, 0] >= 0) & (warped_points[:, 0] < shape[0]) &\ 56 | (warped_points[:, 1] >= 0) & (warped_points[:, 1] < shape[1]) 57 | return points[mask, :] 58 | 59 | 60 | def select_k_best(points, k): 61 | """ Select the k most probable points (and strip their probability). 62 | points has shape (num_points, 3) where the last coordinate is the probability. """ 63 | sorted_prob = points[points[:, 2].argsort(), :2] 64 | start = min(k, points.shape[0]) 65 | return sorted_prob[-start:, :] 66 | 67 | H = data['homography'] 68 | shape = data['image_shape'] 69 | 70 | # Filter out predictions 71 | keypoints = data['prob'][:,:2] 72 | warped_keypoints = data['warped_prob'] 73 | warped_keypoints = keep_true_keypoints(warped_keypoints, np.linalg.inv(H), shape) 74 | 75 | # Warp the original keypoints with the true homography 76 | true_warped_keypoints = warp_keypoints(keypoints, H) 77 | true_warped_keypoints = np.stack([true_warped_keypoints[:, 0], true_warped_keypoints[:, 1], data['prob'][:, 2]], axis=-1) 78 | true_warped_keypoints = filter_keypoints(true_warped_keypoints, shape) 79 | 80 | # Keep only the keep_k_points best predictions 81 | warped_keypoints = select_k_best(warped_keypoints, keep_k_points) 82 | true_warped_keypoints = select_k_best(true_warped_keypoints, keep_k_points) 83 | 84 | # Compute the repeatability 85 | N1 = true_warped_keypoints.shape[0] 86 | N2 = warped_keypoints.shape[0] 87 | true_warped_keypoints = np.expand_dims(true_warped_keypoints, 1) 88 | warped_keypoints = np.expand_dims(warped_keypoints, 0) 89 | # shapes are broadcasted to N1 x N2 x 2: 90 | norm = np.linalg.norm(true_warped_keypoints - warped_keypoints, ord=None, axis=2) 91 | count1 = 0 92 | count2 = 0 93 | le1 = 0 94 | le2 = 0 95 | if N2 != 0: 96 | min1 = np.min(norm, axis=1) 97 | correct1 = (min1 <= distance_thresh) 98 | count1 = np.sum(correct1) 99 | le1 = min1[correct1].sum() 100 | if N1 != 0: 101 | min2 = np.min(norm, axis=0) 102 | correct2 = (min2 <= distance_thresh) 103 | count2 = np.sum(correct2) 104 | le2 = min2[correct2].sum() 105 | if N1 + N2 > 0: 106 | repeatability = (count1 + count2) / (N1 + N2) 107 | loc_err = (le1 + le2) / (count1 + count2) 108 | else: 109 | repeatability = -1 110 | loc_err = -1 111 | 112 | return N1, N2, repeatability, loc_err 113 | -------------------------------------------------------------------------------- /kp2d/evaluation/evaluate.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import numpy as np 4 | import torch 5 | import torchvision.transforms as transforms 6 | from tqdm import tqdm 7 | 8 | from kp2d.evaluation.descriptor_evaluation import (compute_homography, 9 | compute_matching_score) 10 | from kp2d.evaluation.detector_evaluation import compute_repeatability 11 | from kp2d.utils.image import to_color_normalized, to_gray_normalized 12 | 13 | 14 | def evaluate_keypoint_net(data_loader, keypoint_net, output_shape=(320, 240), top_k=300, use_color=True): 15 | """Keypoint net evaluation script. 16 | 17 | Parameters 18 | ---------- 19 | data_loader: torch.utils.data.DataLoader 20 | Dataset loader. 21 | keypoint_net: torch.nn.module 22 | Keypoint network. 23 | output_shape: tuple 24 | Original image shape. 25 | top_k: int 26 | Number of keypoints to use to compute metrics, selected based on probability. 27 | use_color: bool 28 | Use color or grayscale images. 29 | """ 30 | keypoint_net.eval() 31 | keypoint_net.training = False 32 | 33 | conf_threshold = 0.0 34 | localization_err, repeatability = [], [] 35 | correctness1, correctness3, correctness5, MScore = [], [], [], [] 36 | 37 | with torch.no_grad(): 38 | for i, sample in tqdm(enumerate(data_loader), desc="evaluate_keypoint_net"): 39 | if use_color: 40 | image = to_color_normalized(sample['image'].cuda()) 41 | warped_image = to_color_normalized(sample['warped_image'].cuda()) 42 | else: 43 | image = to_gray_normalized(sample['image'].cuda()) 44 | warped_image = to_gray_normalized(sample['warped_image'].cuda()) 45 | 46 | score_1, coord_1, desc1 = keypoint_net(image) 47 | score_2, coord_2, desc2 = keypoint_net(warped_image) 48 | B, C, Hc, Wc = desc1.shape 49 | 50 | # Scores & Descriptors 51 | score_1 = torch.cat([coord_1, score_1], dim=1).view(3, -1).t().cpu().numpy() 52 | score_2 = torch.cat([coord_2, score_2], dim=1).view(3, -1).t().cpu().numpy() 53 | desc1 = desc1.view(C, Hc, Wc).view(C, -1).t().cpu().numpy() 54 | desc2 = desc2.view(C, Hc, Wc).view(C, -1).t().cpu().numpy() 55 | 56 | # Filter based on confidence threshold 57 | desc1 = desc1[score_1[:, 2] > conf_threshold, :] 58 | desc2 = desc2[score_2[:, 2] > conf_threshold, :] 59 | score_1 = score_1[score_1[:, 2] > conf_threshold, :] 60 | score_2 = score_2[score_2[:, 2] > conf_threshold, :] 61 | 62 | # Prepare data for eval 63 | data = {'image': sample['image'].numpy().squeeze(), 64 | 'image_shape' : output_shape, 65 | 'warped_image': sample['warped_image'].numpy().squeeze(), 66 | 'homography': sample['homography'].squeeze().numpy(), 67 | 'prob': score_1, 68 | 'warped_prob': score_2, 69 | 'desc': desc1, 70 | 'warped_desc': desc2} 71 | 72 | # Compute repeatabilty and localization error 73 | _, _, rep, loc_err = compute_repeatability(data, keep_k_points=top_k, distance_thresh=3) 74 | repeatability.append(rep) 75 | localization_err.append(loc_err) 76 | 77 | # Compute correctness 78 | c1, c2, c3 = compute_homography(data, keep_k_points=top_k) 79 | correctness1.append(c1) 80 | correctness3.append(c2) 81 | correctness5.append(c3) 82 | 83 | # Compute matching score 84 | mscore = compute_matching_score(data, keep_k_points=top_k) 85 | MScore.append(mscore) 86 | 87 | return np.mean(repeatability), np.mean(localization_err), \ 88 | np.mean(correctness1), np.mean(correctness3), np.mean(correctness5), np.mean(MScore) 89 | -------------------------------------------------------------------------------- /kp2d/networks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/kp2d/networks/__init__.py -------------------------------------------------------------------------------- /kp2d/networks/inlier_net.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import torch 4 | import torch.nn.functional as F 5 | 6 | # Slightly modified version of 1d-CNN from https://arxiv.org/abs/1905.04132. 7 | # More details: https://github.com/vislearn/ngransac 8 | # Code adapted from https://github.com/vislearn/ngransac/blob/master/network.py 9 | 10 | class InlierNet(torch.nn.Module): 11 | def __init__(self, blocks): 12 | super(InlierNet, self).__init__() 13 | 14 | self.res_blocks = [] 15 | self.bn_momentum = 0.1 16 | self.p_in = torch.nn.Sequential(torch.nn.Conv2d(5, 128, 1, 1, 0, bias=False), 17 | torch.nn.BatchNorm2d(128, momentum=0.9)) 18 | for i in range(0, blocks): 19 | self.res_blocks.append(( 20 | torch.nn.Conv2d(128, 128, 1, 1, 0), 21 | torch.nn.BatchNorm2d(128, momentum=self.bn_momentum), 22 | torch.nn.Conv2d(128, 128, 1, 1, 0), 23 | torch.nn.BatchNorm2d(128, momentum=self.bn_momentum), 24 | )) 25 | 26 | # register list of residual block with the module 27 | for i, r in enumerate(self.res_blocks): 28 | super(InlierNet, self).add_module(str(i) + 's0', r[0]) 29 | super(InlierNet, self).add_module(str(i) + 's1', r[1]) 30 | super(InlierNet, self).add_module(str(i) + 's2', r[2]) 31 | super(InlierNet, self).add_module(str(i) + 's3', r[3]) 32 | 33 | # output are 1D sampling weights (log probabilities) 34 | self.p_out = torch.nn.Conv2d(128, 1, 1, 1, 0) 35 | 36 | def forward(self, inputs): 37 | x = inputs 38 | x = F.relu(self.p_in(x)) 39 | 40 | for r in self.res_blocks: 41 | res = x 42 | x = F.relu(r[1](F.instance_norm(r[0](x)))) 43 | x = F.relu(r[3](F.instance_norm(r[2](x)))) 44 | x = x + res 45 | return self.p_out(x) 46 | -------------------------------------------------------------------------------- /kp2d/networks/keypoint_net.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import os 4 | from math import pi 5 | 6 | import torch 7 | import torch.nn.functional as F 8 | from PIL import Image 9 | 10 | from kp2d.utils.image import image_grid 11 | 12 | 13 | class KeypointNet(torch.nn.Module): 14 | """ 15 | Keypoint detection network. 16 | 17 | Parameters 18 | ---------- 19 | use_color : bool 20 | Use color or grayscale images. 21 | do_upsample: bool 22 | Upsample desnse descriptor map. 23 | with_drop : bool 24 | Use dropout. 25 | do_cross: bool 26 | Predict keypoints outside cell borders. 27 | kwargs : dict 28 | Extra parameters 29 | """ 30 | 31 | def __init__(self, use_color=True, do_upsample=True, with_drop=True, do_cross=True, **kwargs): 32 | super().__init__() 33 | 34 | self.training = True 35 | 36 | self.use_color = use_color 37 | self.with_drop = with_drop 38 | self.do_cross = do_cross 39 | self.do_upsample = do_upsample 40 | 41 | if self.use_color: 42 | c0 = 3 43 | else: 44 | c0 = 1 45 | 46 | self.bn_momentum = 0.1 47 | self.cross_ratio = 2.0 48 | 49 | if self.do_cross is False: 50 | self.cross_ratio = 1.0 51 | 52 | c1, c2, c3, c4, c5, d1 = 32, 64, 128, 256, 256, 512 53 | 54 | self.conv1a = torch.nn.Sequential(torch.nn.Conv2d(c0, c1, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c1,momentum=self.bn_momentum)) 55 | self.conv1b = torch.nn.Sequential(torch.nn.Conv2d(c1, c1, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c1,momentum=self.bn_momentum)) 56 | self.conv2a = torch.nn.Sequential(torch.nn.Conv2d(c1, c2, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c2,momentum=self.bn_momentum)) 57 | self.conv2b = torch.nn.Sequential(torch.nn.Conv2d(c2, c2, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c2,momentum=self.bn_momentum)) 58 | self.conv3a = torch.nn.Sequential(torch.nn.Conv2d(c2, c3, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c3,momentum=self.bn_momentum)) 59 | self.conv3b = torch.nn.Sequential(torch.nn.Conv2d(c3, c3, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c3,momentum=self.bn_momentum)) 60 | self.conv4a = torch.nn.Sequential(torch.nn.Conv2d(c3, c4, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c4,momentum=self.bn_momentum)) 61 | self.conv4b = torch.nn.Sequential(torch.nn.Conv2d(c4, c4, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c4,momentum=self.bn_momentum)) 62 | 63 | # Score Head. 64 | self.convDa = torch.nn.Sequential(torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c4,momentum=self.bn_momentum)) 65 | self.convDb = torch.nn.Conv2d(c5, 1, kernel_size=3, stride=1, padding=1) 66 | 67 | # Location Head. 68 | self.convPa = torch.nn.Sequential(torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c4,momentum=self.bn_momentum)) 69 | self.convPb = torch.nn.Conv2d(c5, 2, kernel_size=3, stride=1, padding=1) 70 | 71 | # Desc Head. 72 | self.convFa = torch.nn.Sequential(torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c4,momentum=self.bn_momentum)) 73 | self.convFb = torch.nn.Sequential(torch.nn.Conv2d(c5, d1, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(d1,momentum=self.bn_momentum)) 74 | self.convFaa = torch.nn.Sequential(torch.nn.Conv2d(c4, c5, kernel_size=3, stride=1, padding=1, bias=False), torch.nn.BatchNorm2d(c5,momentum=self.bn_momentum)) 75 | self.convFbb = torch.nn.Conv2d(c5, 256, kernel_size=3, stride=1, padding=1) 76 | 77 | self.relu = torch.nn.LeakyReLU(inplace=True) 78 | if self.with_drop: 79 | self.dropout = torch.nn.Dropout2d(0.2) 80 | else: 81 | self.dropout = None 82 | self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2) 83 | 84 | self.cell = 8 85 | self.upsample = torch.nn.PixelShuffle(upscale_factor=2) 86 | 87 | def forward(self, x): 88 | """ 89 | Processes a batch of images. 90 | 91 | Parameters 92 | ---------- 93 | x : torch.Tensor 94 | Batch of input images (B, 3, H, W) 95 | 96 | Returns 97 | ------- 98 | score : torch.Tensor 99 | Score map (B, 1, H_out, W_out) 100 | coord: torch.Tensor 101 | Keypoint coordinates (B, 2, H_out, W_out) 102 | feat: torch.Tensor 103 | Keypoint descriptors (B, 256, H_out, W_out) 104 | """ 105 | B, _, H, W = x.shape 106 | 107 | x = self.relu(self.conv1a(x)) 108 | x = self.relu(self.conv1b(x)) 109 | if self.dropout: 110 | x = self.dropout(x) 111 | x = self.pool(x) 112 | x = self.relu(self.conv2a(x)) 113 | x = self.relu(self.conv2b(x)) 114 | if self.dropout: 115 | x = self.dropout(x) 116 | x = self.pool(x) 117 | x = self.relu(self.conv3a(x)) 118 | skip = self.relu(self.conv3b(x)) 119 | if self.dropout: 120 | skip = self.dropout(skip) 121 | x = self.pool(skip) 122 | x = self.relu(self.conv4a(x)) 123 | x = self.relu(self.conv4b(x)) 124 | if self.dropout: 125 | x = self.dropout(x) 126 | 127 | B, _, Hc, Wc = x.shape 128 | 129 | score = self.relu(self.convDa(x)) 130 | if self.dropout: 131 | score = self.dropout(score) 132 | score = self.convDb(score).sigmoid() 133 | 134 | border_mask = torch.ones(B, Hc, Wc) 135 | border_mask[:, 0] = 0 136 | border_mask[:, Hc - 1] = 0 137 | border_mask[:, :, 0] = 0 138 | border_mask[:, :, Wc - 1] = 0 139 | border_mask = border_mask.unsqueeze(1) 140 | score = score * border_mask.to(score.device) 141 | 142 | center_shift = self.relu(self.convPa(x)) 143 | if self.dropout: 144 | center_shift = self.dropout(center_shift) 145 | center_shift = self.convPb(center_shift).tanh() 146 | 147 | step = (self.cell-1) / 2. 148 | center_base = image_grid(B, Hc, Wc, 149 | dtype=center_shift.dtype, 150 | device=center_shift.device, 151 | ones=False, normalized=False).mul(self.cell) + step 152 | 153 | coord_un = center_base.add(center_shift.mul(self.cross_ratio * step)) 154 | coord = coord_un.clone() 155 | coord[:, 0] = torch.clamp(coord_un[:, 0], min=0, max=W-1) 156 | coord[:, 1] = torch.clamp(coord_un[:, 1], min=0, max=H-1) 157 | 158 | feat = self.relu(self.convFa(x)) 159 | if self.dropout: 160 | feat = self.dropout(feat) 161 | if self.do_upsample: 162 | feat = self.upsample(self.convFb(feat)) 163 | feat = torch.cat([feat, skip], dim=1) 164 | feat = self.relu(self.convFaa(feat)) 165 | feat = self.convFbb(feat) 166 | 167 | if self.training is False: 168 | coord_norm = coord[:, :2].clone() 169 | coord_norm[:, 0] = (coord_norm[:, 0] / (float(W-1)/2.)) - 1. 170 | coord_norm[:, 1] = (coord_norm[:, 1] / (float(H-1)/2.)) - 1. 171 | coord_norm = coord_norm.permute(0, 2, 3, 1) 172 | 173 | feat = torch.nn.functional.grid_sample(feat, coord_norm, align_corners=True) 174 | 175 | dn = torch.norm(feat, p=2, dim=1) # Compute the norm. 176 | feat = feat.div(torch.unsqueeze(dn, 1)) # Divide by norm to normalize. 177 | return score, coord, feat 178 | -------------------------------------------------------------------------------- /kp2d/networks/keypoint_resnet.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.utils.model_zoo as model_zoo 8 | import torchvision.models as models 9 | from kp2d.utils.image import image_grid 10 | 11 | 12 | def upsample(x): 13 | return F.interpolate(x, scale_factor=2, mode="nearest") 14 | 15 | class conv_bn_elu(nn.Module): 16 | def __init__(self, in_channels, out_channels): 17 | super(conv_bn_elu, self).__init__() 18 | 19 | self.conv = nn.Sequential( 20 | nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False), 21 | nn.BatchNorm2d(out_channels), 22 | nn.LeakyReLU(inplace=True)) 23 | 24 | def forward(self, x): 25 | return self.conv(x) 26 | 27 | class KeypointEncoder(nn.Module): 28 | def __init__(self, pretrained, with_drop): 29 | super(KeypointEncoder, self).__init__() 30 | 31 | self.rn = models.resnet18(pretrained) 32 | self.dropout = nn.Dropout2d(0.2) 33 | self.use_dropout = with_drop 34 | 35 | def forward(self, input_image): 36 | self.features = [] 37 | 38 | x = self.rn.relu(self.rn.bn1(self.rn.conv1(input_image))) 39 | l1 = self.rn.layer1(self.rn.maxpool(x)) if not self.use_dropout else self.dropout(self.rn.layer1(self.rn.maxpool(x))) 40 | l2 = self.rn.layer2(l1) if not self.use_dropout else self.dropout(self.rn.layer2(l1)) 41 | l3 = self.rn.layer3(l2) if not self.use_dropout else self.dropout(self.rn.layer3(l2)) 42 | l4 = self.rn.layer4(l3) if not self.use_dropout else self.dropout(self.rn.layer4(l3)) 43 | 44 | return [x, l1, l2, l3, l4] 45 | 46 | 47 | 48 | class KeypointDecoder(nn.Module): 49 | def __init__(self): 50 | super(KeypointDecoder, self).__init__() 51 | 52 | self.detect_scales = [3] 53 | self.feat_scales = [1] 54 | self.depth_scales = [0] 55 | 56 | self.num_ch_enc = np.array([64, 64, 128, 256, 512]) 57 | self.num_ch_dec = np.array([32, 64, 128, 256, 256]) 58 | 59 | self.sigmoid = nn.Sigmoid() 60 | self.tanh = nn.Tanh() 61 | self.pad = nn.ReflectionPad2d(1) 62 | 63 | # Layer4 64 | self.upconv4_0 = conv_bn_elu(self.num_ch_enc[4], self.num_ch_dec[4]) 65 | self.upconv4_1 = conv_bn_elu(self.num_ch_dec[4] + self.num_ch_enc[3], self.num_ch_dec[4]) 66 | 67 | # Layer3 68 | self.upconv3_0 = conv_bn_elu(self.num_ch_dec[4], self.num_ch_dec[3]) 69 | self.upconv3_1 = conv_bn_elu(self.num_ch_dec[3] + self.num_ch_enc[2], self.num_ch_dec[3]) 70 | 71 | # Layer2 72 | self.upconv2_0 = conv_bn_elu(self.num_ch_dec[3], self.num_ch_dec[2]) 73 | self.upconv2_1 = conv_bn_elu(self.num_ch_dec[2] + self.num_ch_enc[1], self.num_ch_dec[2]) 74 | 75 | # Layer1 76 | self.upconv1_0 = conv_bn_elu(self.num_ch_dec[2], self.num_ch_dec[1]) 77 | self.upconv1_1 = conv_bn_elu(self.num_ch_dec[1] + self.num_ch_enc[0], self.num_ch_dec[1]) 78 | 79 | # Score 80 | self.scoreconv = nn.Sequential( 81 | nn.Conv2d(self.num_ch_dec[3], self.num_ch_dec[3], kernel_size=3, stride=1, padding=1, bias=False), 82 | nn.BatchNorm2d(self.num_ch_dec[3]), 83 | nn.LeakyReLU(inplace=True), 84 | nn.ReflectionPad2d(1), 85 | nn.Conv2d(self.num_ch_dec[3], 1, 3)) 86 | # Detector 87 | self.locconv = nn.Sequential( 88 | nn.Conv2d(self.num_ch_dec[3], self.num_ch_dec[3], kernel_size=3, stride=1, padding=1, bias=False), 89 | nn.BatchNorm2d(self.num_ch_dec[3]), 90 | nn.LeakyReLU(inplace=True), 91 | nn.ReflectionPad2d(1), 92 | nn.Conv2d(self.num_ch_dec[3], 2, 3)) 93 | # Descriptor 94 | self.featconv = nn.Sequential( 95 | nn.Conv2d(self.num_ch_dec[1], self.num_ch_dec[1], kernel_size=3, stride=1, padding=1, bias=False), 96 | nn.BatchNorm2d(self.num_ch_dec[1]), 97 | nn.LeakyReLU(inplace=True), 98 | nn.ReflectionPad2d(1), 99 | nn.Conv2d(self.num_ch_dec[1], self.num_ch_dec[1], 3)) 100 | 101 | def init_weights(self): 102 | for m in self.modules(): 103 | if isinstance(m, torch.nn.Conv2d): 104 | torch.nn.init.xavier_uniform_(m.weight) 105 | if m.bias is not None: 106 | m.bias.data.zero_() 107 | 108 | if isinstance(m, nn.BatchNorm2d): 109 | nn.init.constant_(m.weight, 1) 110 | nn.init.constant_(m.bias, 0) 111 | 112 | 113 | def forward(self, input_features): 114 | self.outputs = {} 115 | 116 | # decoder 117 | x = input_features[4] 118 | # Layer4 119 | x = self.upconv4_0(x) 120 | x = [upsample(x)] 121 | x += [input_features[3]] 122 | x = torch.cat(x, 1) 123 | x = self.upconv4_1(x) 124 | # Layer3 125 | x = self.upconv3_0(x) 126 | x = [upsample(x)] 127 | x += [input_features[2]] 128 | x = torch.cat(x, 1) 129 | x = self.upconv3_1(x) 130 | # Detector and score 131 | self.outputs[("location")] = self.tanh(self.locconv(x)) 132 | self.outputs[("score")] = self.sigmoid(self.scoreconv(x)) 133 | # Layer2 134 | x = self.upconv2_0(x) 135 | x = [upsample(x)] 136 | x += [input_features[1]] 137 | x = torch.cat(x, 1) 138 | x = self.upconv2_1(x) 139 | # Layer1 140 | x = self.upconv1_0(x) 141 | x = [upsample(x)] 142 | x += [input_features[0]] 143 | x = torch.cat(x, 1) 144 | x = self.upconv1_1(x) 145 | # Descriptor features 146 | self.outputs[("feature")] = self.featconv(x) 147 | 148 | return self.outputs 149 | 150 | class KeypointResnet(nn.Module): 151 | def __init__(self, with_drop=True): 152 | super().__init__() 153 | print('Instantiating keypoint resnet') 154 | 155 | pretrained = True 156 | self.encoderK = KeypointEncoder(pretrained=pretrained, with_drop=True) 157 | self.decoderK = KeypointDecoder() 158 | 159 | self.cross_ratio = 2.0 160 | self.cell = 8 161 | 162 | def forward(self, x): 163 | 164 | B, _, H, W = x.shape 165 | 166 | x = self.encoderK(x) 167 | xK = self.decoderK(x) 168 | 169 | score = xK[('score')] 170 | center_shift = xK[('location')] 171 | feat = xK[('feature')] 172 | 173 | _, _, Hc, Wc = score.shape 174 | 175 | ############ Remove border for score ############## 176 | border_mask = torch.ones(B,Hc,Wc) 177 | border_mask[:,0] = 0 178 | border_mask[:,Hc-1] = 0 179 | border_mask[:,:,0] = 0 180 | border_mask[:,:,Wc-1] = 0 181 | border_mask = border_mask.unsqueeze(1) 182 | score = score * border_mask.to(score.device) 183 | 184 | ############ Remap coordinate ############## 185 | step = (self.cell-1) / 2. 186 | center_base = image_grid(B, Hc, Wc, 187 | dtype=center_shift.dtype, 188 | device=center_shift.device, 189 | ones=False, normalized=False).mul(self.cell) + step 190 | 191 | coord_un = center_base.add(center_shift.mul(self.cross_ratio * step)) 192 | coord = coord_un.clone() 193 | coord[:,0] = torch.clamp(coord_un[:,0], min=0, max=W-1) 194 | coord[:,1] = torch.clamp(coord_un[:,1], min=0, max=H-1) 195 | 196 | ############ Sampling feature ############## 197 | if self.training is False: 198 | coord_norm = coord[:,:2].clone() 199 | coord_norm[:,0] = (coord_norm[:,0] / (float(W-1)/2.)) - 1. 200 | coord_norm[:,1] = (coord_norm[:,1] / (float(H-1)/2.)) - 1. 201 | coord_norm = coord_norm.permute(0, 2, 3, 1) 202 | 203 | feat = torch.nn.functional.grid_sample(feat, coord_norm, align_corners=False) 204 | 205 | dn = torch.norm(feat, p=2, dim=1) # Compute the norm. 206 | feat = feat.div(torch.unsqueeze(dn, 1)) # Divide by norm to normalize. 207 | 208 | return score, coord, feat 209 | -------------------------------------------------------------------------------- /kp2d/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/kp2d/utils/__init__.py -------------------------------------------------------------------------------- /kp2d/utils/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import os 4 | 5 | from yacs.config import CfgNode 6 | 7 | from kp2d.configs.base_config import get_cfg_defaults 8 | 9 | 10 | def get_default_config(cfg_default): 11 | """Get default configuration from file""" 12 | config = get_cfg_defaults() 13 | config.merge_from_list(['default', cfg_default]) 14 | return config 15 | 16 | def merge_cfg_file(config, cfg_file=None): 17 | """Merge configuration file""" 18 | if cfg_file is not None: 19 | config.merge_from_file(cfg_file) 20 | config.merge_from_list(['config', cfg_file]) 21 | return config 22 | 23 | def merge_cfgs(original, override): 24 | """ 25 | Updates CfgNode with information from another one 26 | 27 | Parameters 28 | ---------- 29 | original : CfgNode 30 | Original configuration node 31 | override : CfgNode 32 | Another configuration node used for overriding 33 | 34 | Returns 35 | ------- 36 | updated : CfgNode 37 | Updated configuration node 38 | """ 39 | for key, value in original.items(): 40 | if key in override.keys(): 41 | if is_cfg(value): # If it's a configuration node, recursion 42 | original[key] = merge_cfgs(original[key], override[key]) 43 | else: # Otherwise, simply update key 44 | original[key] = override[key] 45 | return original 46 | 47 | def parse_train_config(cfg_default, cfg_file): 48 | """ 49 | Parse model configuration for training 50 | 51 | Parameters 52 | ---------- 53 | cfg_default : str 54 | Default **.py** configuration file 55 | cfg_file : str 56 | Configuration **.yaml** file to override the default parameters 57 | 58 | Returns 59 | ------- 60 | config : CfgNode 61 | Parsed model configuration 62 | """ 63 | # Loads default configuration 64 | config = get_default_config(cfg_default) 65 | # Merge configuration file 66 | config = merge_cfg_file(config, cfg_file) 67 | # Return prepared configuration 68 | return config 69 | 70 | def parse_train_file(file): 71 | """ 72 | Parse file for training 73 | 74 | Parameters 75 | ---------- 76 | file : str 77 | File, can be either a 78 | **.yaml** for a yacs configuration file or a 79 | **.ckpt** for a pre-trained checkpoint file 80 | 81 | Returns 82 | ------- 83 | config : CfgNode 84 | Parsed model configuration 85 | ckpt : str 86 | Parsed checkpoint file 87 | """ 88 | # If it's a .yaml configuration file 89 | if file.endswith('yaml'): 90 | cfg_default = 'configs/default_config' 91 | return parse_train_config(cfg_default, file) 92 | # We have a problem 93 | else: 94 | raise ValueError('You need to provide a .yaml or .ckpt to train') 95 | -------------------------------------------------------------------------------- /kp2d/utils/horovod.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | try: 4 | import horovod.torch as hvd 5 | HAS_HOROVOD = True 6 | except ImportError: 7 | HAS_HOROVOD = False 8 | 9 | 10 | def hvd_init(): 11 | if HAS_HOROVOD: 12 | hvd.init() 13 | return HAS_HOROVOD 14 | 15 | def rank(): 16 | return hvd.rank() if HAS_HOROVOD else 0 17 | 18 | def local_rank(): 19 | return hvd.local_rank() if HAS_HOROVOD else 0 20 | 21 | def world_size(): 22 | return hvd.size() if HAS_HOROVOD else 1 23 | -------------------------------------------------------------------------------- /kp2d/utils/image.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | from functools import lru_cache 4 | 5 | import torch 6 | import torch.nn.functional as F 7 | 8 | 9 | @lru_cache(maxsize=None) 10 | def meshgrid(B, H, W, dtype, device, normalized=False): 11 | """Create mesh-grid given batch size, height and width dimensions. 12 | 13 | Parameters 14 | ---------- 15 | B: int 16 | Batch size 17 | H: int 18 | Grid Height 19 | W: int 20 | Batch size 21 | dtype: torch.dtype 22 | Tensor dtype 23 | device: str 24 | Tensor device 25 | normalized: bool 26 | Normalized image coordinates or integer-grid. 27 | 28 | Returns 29 | ------- 30 | xs: torch.Tensor 31 | Batched mesh-grid x-coordinates (BHW). 32 | ys: torch.Tensor 33 | Batched mesh-grid y-coordinates (BHW). 34 | """ 35 | if normalized: 36 | xs = torch.linspace(-1, 1, W, device=device, dtype=dtype) 37 | ys = torch.linspace(-1, 1, H, device=device, dtype=dtype) 38 | else: 39 | xs = torch.linspace(0, W-1, W, device=device, dtype=dtype) 40 | ys = torch.linspace(0, H-1, H, device=device, dtype=dtype) 41 | ys, xs = torch.meshgrid([ys, xs]) 42 | return xs.repeat([B, 1, 1]), ys.repeat([B, 1, 1]) 43 | 44 | 45 | @lru_cache(maxsize=None) 46 | def image_grid(B, H, W, dtype, device, ones=True, normalized=False): 47 | """Create an image mesh grid with shape B3HW given image shape BHW 48 | 49 | Parameters 50 | ---------- 51 | B: int 52 | Batch size 53 | H: int 54 | Grid Height 55 | W: int 56 | Batch size 57 | dtype: str 58 | Tensor dtype 59 | device: str 60 | Tensor device 61 | ones : bool 62 | Use (x, y, 1) coordinates 63 | normalized: bool 64 | Normalized image coordinates or integer-grid. 65 | 66 | Returns 67 | ------- 68 | grid: torch.Tensor 69 | Mesh-grid for the corresponding image shape (B3HW) 70 | """ 71 | xs, ys = meshgrid(B, H, W, dtype, device, normalized=normalized) 72 | coords = [xs, ys] 73 | if ones: 74 | coords.append(torch.ones_like(xs)) # BHW 75 | grid = torch.stack(coords, dim=1) # B3HW 76 | return grid 77 | 78 | 79 | def to_gray_normalized(images): 80 | """Performs image normalization and converts images to grayscale (preserving dimensions) 81 | 82 | Parameters 83 | ---------- 84 | images: torch.Tensor 85 | Input images. 86 | 87 | Returns 88 | ------- 89 | normalized_images: torch.Tensor 90 | Normalized grayscale images. 91 | """ 92 | assert len(images.shape) == 4 93 | images -= 0.5 94 | images *= 0.225 95 | normalized_images = images.mean(1).unsqueeze(1) 96 | return normalized_images 97 | 98 | 99 | def to_color_normalized(images): 100 | """Performs image normalization and converts images to grayscale (preserving dimensions) 101 | 102 | Parameters 103 | ---------- 104 | images: torch.Tensor 105 | Input images. 106 | 107 | Returns 108 | ------- 109 | normalized_images: torch.Tensor 110 | Normalized grayscale images. 111 | """ 112 | assert len(images.shape) == 4 113 | images -= 0.5 114 | images *= 0.225 115 | return images 116 | -------------------------------------------------------------------------------- /kp2d/utils/keypoints.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | 7 | def warp_keypoints(keypoints, H): 8 | """Warp keypoints given a homography 9 | 10 | Parameters 11 | ---------- 12 | keypoints: numpy.ndarray (N,2) 13 | Keypoint vector. 14 | H: numpy.ndarray (3,3) 15 | Homography. 16 | 17 | Returns 18 | ------- 19 | warped_keypoints: numpy.ndarray (N,2) 20 | Warped keypoints vector. 21 | """ 22 | num_points = keypoints.shape[0] 23 | homogeneous_points = np.concatenate([keypoints, np.ones((num_points, 1))], axis=1) 24 | warped_points = np.dot(homogeneous_points, np.transpose(H)) 25 | return warped_points[:, :2] / warped_points[:, 2:] 26 | 27 | 28 | def draw_keypoints(img_l, top_uvz, color=(255, 0, 0), idx=0): 29 | """Draw keypoints on an image""" 30 | vis_xyd = top_uvz.permute(0, 2, 1)[idx].detach().cpu().clone().numpy() 31 | vis = img_l.copy() 32 | cnt = 0 33 | for pt in vis_xyd[:,:2].astype(np.int32): 34 | x, y = int(pt[0]), int(pt[1]) 35 | cv2.circle(vis, (x,y), 2, color, -1) 36 | return vis 37 | -------------------------------------------------------------------------------- /kp2d/utils/logging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | """Logging utilities for training 4 | """ 5 | import os 6 | 7 | from termcolor import colored 8 | import horovod.torch as hvd 9 | import numpy as np 10 | import torch 11 | 12 | from kp2d.utils.wandb import WandBLogger 13 | 14 | 15 | def printcolor_single(message, color="white"): 16 | """Print a message in a certain color""" 17 | print(colored(message, color)) 18 | 19 | 20 | def printcolor(message, color="white"): 21 | "Print a message in a certain color (only rank 0)" 22 | if hvd.rank() == 0: 23 | print(colored(message, color)) 24 | 25 | 26 | class SummaryWriter: 27 | """Wrapper class for tensorboard and WandB logging""" 28 | def __init__(self, log_path, params, 29 | description=None, 30 | project='monodepth', 31 | entity='tri', 32 | mode='run', 33 | job_type='train', 34 | log_wb=True): 35 | self.log_wb = log_wb 36 | self._global_step = 0 37 | if self.log_wb: 38 | os.environ['WANDB_DIR'] = log_path 39 | self.wb_logger = WandBLogger( 40 | params, description=description, 41 | project=project, entity=entity, mode=mode, job_type=job_type) 42 | 43 | @property 44 | def run_name(self): 45 | return self.wb_logger.run_name 46 | 47 | @property 48 | def run_url(self): 49 | return self.wb_logger.run_url 50 | 51 | @property 52 | def global_step(self): 53 | return self._global_step 54 | 55 | def log_wandb(self, value): 56 | self.log_wb = value 57 | 58 | def add_scalar(self, tag, scalar_value): 59 | if self.log_wb: 60 | self.wb_logger.log_values(tag, scalar_value, now=False) 61 | 62 | def add_image(self, tag, img_tensor): 63 | assert img_tensor.max() <= 1.0 64 | assert (isinstance(img_tensor, torch.Tensor) and img_tensor.device == torch.device( 65 | 'cpu')) or isinstance(img_tensor, np.ndarray) 66 | if self.log_wb: 67 | caption = tag 68 | if isinstance(img_tensor, torch.Tensor): 69 | # shape: (C, H, W) 70 | size = tuple(img_tensor.shape[-2:][::-1]) 71 | assert img_tensor.shape[0] == 1 or img_tensor.shape[0] == 3, \ 72 | 'Expects CHW with C=1 or 3, provided {}'.format(img_tensor.shape) 73 | self.wb_logger.log_tensor_image(img_tensor * 255, tag, caption, size=size, now=False) 74 | else: 75 | # shape: (H, W, C) 76 | size = tuple(img_tensor.shape[:2][::-1]) 77 | assert img_tensor.shape[-1] == 1 or img_tensor.shape[-1] == 3, \ 78 | 'Expects HWC with C=1 or 3, provided {}'.format(img_tensor.shape) 79 | self.wb_logger.log_numpy_image((img_tensor * 255).astype(np.uint8), tag, caption, size=size, now=False) 80 | 81 | def commit_log(self): 82 | if self.log_wb and self._global_step >= 0: 83 | self.wb_logger.commit_log() 84 | self._global_step += 1 -------------------------------------------------------------------------------- /kp2d/utils/wandb.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import numbers 4 | import os 5 | import socket 6 | 7 | import numpy as np 8 | from PIL import Image 9 | 10 | import wandb 11 | 12 | 13 | class WandBLogger: 14 | """Dedicated logger class for WandB. Creates a group name based on the description. 15 | Training/validation sub-runs can be viewed within that group. Timestamp is required 16 | to create unique description name in the UI. 17 | Parameters 18 | ---------- 19 | params: dict 20 | Dictionary containing all configuration parameters. If `groupd_id` 21 | is set in params, then the wandb_logger will associate the run with 22 | that group 23 | description: str 24 | Name for experiment in WandB UI. 25 | unique_id: str 26 | Unique id for the run 27 | project: str, default: wandb-debug 28 | Name of the project in the WandB UI 29 | entity: str, default: tri 30 | Team name or username. CMI uses `tri` 31 | mode: str, default: run 32 | Mode for WandB logging. Set to `dryrun` to not sync to cloud. 33 | job_type: str, default: debug 34 | Used for filtering UI in WandB 35 | """ 36 | 37 | def __init__(self, 38 | params, 39 | description=None, 40 | project='debug_wandb', 41 | entity='tri', 42 | mode='run', 43 | job_type='train'): 44 | 45 | super().__init__() 46 | 47 | # Set up environment variables to work with wandb 48 | os.environ['WANDB_PROJECT'] = project 49 | os.environ['WANDB_ENTITY'] = entity 50 | os.environ['WANDB_MODE'] = mode 51 | 52 | pwd = os.getenv('PWD') 53 | params.update({'run_dir': pwd, 54 | 'job_type': job_type, 55 | 'description': description, 56 | 'name': ''}) 57 | 58 | self._wandb_logger = wandb.init(config=params, allow_val_change=True) 59 | wandb.run.save() 60 | 61 | self.description = description 62 | self.run_id = wandb.run.id 63 | self.run_name = wandb.run.name 64 | 65 | print('-'*50) 66 | print(self._wandb_logger, self.run_id, self.run_name, self.run_url) 67 | 68 | @property 69 | def run_url(self) -> str: 70 | """Returns run URL.""" 71 | return 'https://app.wandb.ai/{}/{}/runs/{}'.format( 72 | wandb.run.entity, wandb.run.project, wandb.run.id) if self._wandb_logger else None 73 | 74 | def log_values(self, key, values, now=True): 75 | """Add metrics to the logging buffer 76 | Parameters 77 | ---------- 78 | key: str 79 | Name of the value to log. 80 | values: number or dictionary 81 | Value or values to be logged. If dictionary and 82 | key is provided, key will be applied as prefix 83 | now: bool, default: True 84 | Flag to log immediately to cloud 85 | """ 86 | temp = self._parse_values_for_logging(key, values) 87 | self._wandb_logger.log(temp) 88 | 89 | def log_summary(self, key, values, now=True): 90 | """Add metrics to the summary statistics 91 | Parameters 92 | ---------- 93 | key: str 94 | Name of the value to log 95 | values: number or dictionary 96 | Value or values to be logged. If dictionary and 97 | key is provided, key will be applied as prefix 98 | now: bool, default: True 99 | Flag to log immediately to cloud 100 | """ 101 | temp = self._parse_values_for_logging(key, values) 102 | self._wandb_logger.log(self._metrics) 103 | 104 | def log_dictionary_subset(self, keys, dictionary, now=True): 105 | """"Log a subset of a dictionary based on keys 106 | Parameters 107 | ---------- 108 | keys: list of str, default: None 109 | List of keys to log 110 | dictionary: dict 111 | Dictionary containing values to log 112 | now: bool, default: True 113 | Flag to log immediately to cloud 114 | """ 115 | log_populated = False 116 | for _k in keys: 117 | if _k in dictionary: 118 | log_populated = True 119 | self.log_values(_k, dictionary[_k]) 120 | 121 | def commit_log(self): 122 | """Send buffer to wandb, and create a new row. 123 | Initialize the next point in history. 124 | """ 125 | self._wandb_logger.history.add() 126 | 127 | def log_tensor_image(self, image, key, caption, size=(400, 400), now=True): 128 | """Log image to wandb 129 | Parameters 130 | ---------- 131 | image: torch.FloatTensor (RGB float32 0-255 range) (C, H, W) 132 | Image to log 133 | key: str 134 | Key associated with image 135 | caption: str 136 | Caption for the image 137 | size: tuple (int, int), default: (400, 400) 138 | Size of image to upload 139 | now: bool, default: True 140 | Flag to log immediately to cloud 141 | """ 142 | image = image.numpy() 143 | image = np.array(image, dtype=np.uint8).transpose(1, 2, 0) # (C, H, W) -> (H, W, C) 144 | self.log_numpy_image(image, key, caption, size=size, now=now) 145 | 146 | def log_numpy_image(self, image, key, caption, size=(400, 400), now=True): 147 | """Log numpy image to WandB 148 | Parameters 149 | ---------- 150 | image: (H, W, C) numpy.ndarray (RGB float32 or uint8 0-255 range) 151 | Image to log 152 | key: str 153 | Key associated with image 154 | caption: str 155 | Caption for the image 156 | size: tuple (int, int), default: (400, 400) 157 | Size of image to upload 158 | now: bool, default: False 159 | Flag to log immediately to cloud 160 | """ 161 | height, width, _n_channels = image.shape 162 | assert _n_channels == 3 163 | image_to_log = Image.fromarray(image) 164 | image_to_log = image_to_log.resize(size, Image.BILINEAR) 165 | self._wandb_logger.log({key: [wandb.Image(image_to_log, caption=caption)]}) 166 | 167 | def _parse_values_for_logging(self, key, values): 168 | """Utility to prep dictionary of values for logging""" 169 | if isinstance(values, dict): 170 | temp = {} 171 | for _k, _v in values.items(): 172 | if key: 173 | temp[key + '_' + _k] = _v 174 | else: 175 | temp[_k] = _v 176 | # WandB has a length limit of 256 on keys so this fixes that error 177 | for _k in temp.copy(): 178 | new_key = _k[:76] + ' ...' if len(_k) > 80 else _k 179 | temp[new_key] = temp.pop(_k) 180 | elif isinstance(values, numbers.Number): 181 | temp = {key: values} 182 | elif values is None: 183 | temp = {} 184 | else: 185 | raise TypeError("{} is of of type {}, cannot log it.".format(values, type(values))) 186 | return temp 187 | 188 | def end_log(self): 189 | """Uses global process to kill subprocesses. Changes coming in future""" 190 | try: 191 | wandb.join() 192 | except TypeError: 193 | pass 194 | -------------------------------------------------------------------------------- /media/gifs/compressed_h2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/compressed_h2.gif -------------------------------------------------------------------------------- /media/gifs/compressed_v2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/compressed_v2.gif -------------------------------------------------------------------------------- /media/gifs/compressed_w2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/compressed_w2.gif -------------------------------------------------------------------------------- /media/gifs/h1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/h1.gif -------------------------------------------------------------------------------- /media/gifs/v1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/v1.gif -------------------------------------------------------------------------------- /media/gifs/w1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/gifs/w1.gif -------------------------------------------------------------------------------- /media/imgs/diagram_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/diagram_architecture.png -------------------------------------------------------------------------------- /media/imgs/l1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/l1.png -------------------------------------------------------------------------------- /media/imgs/l2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/l2.png -------------------------------------------------------------------------------- /media/imgs/p1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/p1.png -------------------------------------------------------------------------------- /media/imgs/p2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/p2.png -------------------------------------------------------------------------------- /media/imgs/r1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/r1.png -------------------------------------------------------------------------------- /media/imgs/r2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TRI-ML/KP2D/77633af57d65c5a0cd7609eaeab7e6cb758ca791/media/imgs/r2.png -------------------------------------------------------------------------------- /scripts/eval_keypoint_net.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | # Example usage: python scripts/eval_keypoint_net.sh --pretrained_model /data/models/kp2d/v4.pth --input_dir /data/datasets/kp2d/HPatches/ 3 | 4 | import argparse 5 | import os 6 | import pickle 7 | import random 8 | import subprocess 9 | 10 | import cv2 11 | import numpy as np 12 | import torch 13 | from PIL import Image 14 | from termcolor import colored 15 | from torch.utils.data import DataLoader, Dataset 16 | from tqdm import tqdm 17 | 18 | from kp2d.datasets.patches_dataset import PatchesDataset 19 | from kp2d.evaluation.evaluate import evaluate_keypoint_net 20 | from kp2d.networks.keypoint_net import KeypointNet 21 | from kp2d.networks.keypoint_resnet import KeypointResnet 22 | 23 | 24 | def main(): 25 | parser = argparse.ArgumentParser( 26 | description='Script for KeyPointNet testing', 27 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 28 | parser.add_argument("--pretrained_model", type=str, help="pretrained model path") 29 | parser.add_argument("--input_dir", required=True, type=str, help="Folder containing input images") 30 | 31 | args = parser.parse_args() 32 | checkpoint = torch.load(args.pretrained_model) 33 | model_args = checkpoint['config']['model']['params'] 34 | 35 | # Check model type 36 | if 'keypoint_net_type' in checkpoint['config']['model']['params']: 37 | net_type = checkpoint['config']['model']['params'] 38 | else: 39 | net_type = KeypointNet # default when no type is specified 40 | 41 | # Create and load keypoint net 42 | if net_type is KeypointNet: 43 | keypoint_net = KeypointNet(use_color=model_args['use_color'], 44 | do_upsample=model_args['do_upsample'], 45 | do_cross=model_args['do_cross']) 46 | else: 47 | keypoint_net = KeypointResnet() 48 | 49 | keypoint_net.load_state_dict(checkpoint['state_dict']) 50 | keypoint_net = keypoint_net.cuda() 51 | keypoint_net.eval() 52 | print('Loaded KeypointNet from {}'.format(args.pretrained_model)) 53 | print('KeypointNet params {}'.format(model_args)) 54 | 55 | eval_params = [{'res': (320, 240), 'top_k': 300, }] if net_type is KeypointNet else [{'res': (320, 256), 'top_k': 300, }] # KeypointResnet needs (320,256) 56 | eval_params += [{'res': (640, 480), 'top_k': 1000, }] 57 | 58 | for params in eval_params: 59 | hp_dataset = PatchesDataset(root_dir=args.input_dir, use_color=True, 60 | output_shape=params['res'], type='a') 61 | data_loader = DataLoader(hp_dataset, 62 | batch_size=1, 63 | pin_memory=False, 64 | shuffle=False, 65 | num_workers=8, 66 | worker_init_fn=None, 67 | sampler=None) 68 | 69 | print(colored('Evaluating for {} -- top_k {}'.format(params['res'], params['top_k']),'green')) 70 | rep, loc, c1, c3, c5, mscore = evaluate_keypoint_net( 71 | data_loader, 72 | keypoint_net, 73 | output_shape=params['res'], 74 | top_k=params['top_k'], 75 | use_color=True) 76 | 77 | print('Repeatability {0:.3f}'.format(rep)) 78 | print('Localization Error {0:.3f}'.format(loc)) 79 | print('Correctness d1 {:.3f}'.format(c1)) 80 | print('Correctness d3 {:.3f}'.format(c3)) 81 | print('Correctness d5 {:.3f}'.format(c5)) 82 | print('MScore {:.3f}'.format(mscore)) 83 | 84 | 85 | if __name__ == '__main__': 86 | main() 87 | -------------------------------------------------------------------------------- /scripts/train_keypoint_net_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Toyota Research Institute. All rights reserved. 2 | 3 | import os 4 | import random 5 | 6 | import numpy as np 7 | import torch 8 | import torch.utils.data.distributed 9 | from torch.utils.data import ConcatDataset, DataLoader 10 | 11 | import horovod.torch as hvd 12 | from kp2d.datasets.augmentations import (ha_augment_sample, resize_sample, 13 | spatial_augment_sample, 14 | to_tensor_sample) 15 | from kp2d.datasets.coco import COCOLoader 16 | from kp2d.utils.horovod import rank, world_size 17 | 18 | 19 | def sample_to_cuda(data): 20 | if isinstance(data, str): 21 | return data 22 | if isinstance(data, dict): 23 | data_cuda = {} 24 | for key in data.keys(): 25 | data_cuda[key] = sample_to_cuda(data[key]) 26 | return data_cuda 27 | elif isinstance(data, list): 28 | data_cuda = [] 29 | for key in data: 30 | data_cuda.append(sample_to_cuda(key)) 31 | return data_cuda 32 | else: 33 | return data.to('cuda') 34 | 35 | 36 | def image_transforms(shape, jittering): 37 | def train_transforms(sample): 38 | sample = resize_sample(sample, image_shape=shape) 39 | sample = spatial_augment_sample(sample) 40 | sample = to_tensor_sample(sample) 41 | sample = ha_augment_sample(sample, jitter_paramters=jittering) 42 | return sample 43 | 44 | return {'train': train_transforms} 45 | 46 | 47 | def _set_seeds(seed=42): 48 | """Set Python random seeding and PyTorch seeds. 49 | Parameters 50 | ---------- 51 | seed: int, default: 42 52 | Random number generator seeds for PyTorch and python 53 | """ 54 | np.random.seed(seed) 55 | random.seed(seed) 56 | torch.manual_seed(seed) 57 | torch.cuda.manual_seed_all(seed) 58 | 59 | 60 | def setup_datasets_and_dataloaders(config): 61 | """Prepare datasets for training, validation and test.""" 62 | def _worker_init_fn(worker_id): 63 | """Worker init fn to fix the seed of the workers""" 64 | _set_seeds(42 + worker_id) 65 | 66 | data_transforms = image_transforms(shape=config.augmentation.image_shape, jittering=config.augmentation.jittering) 67 | train_dataset = COCOLoader(config.train.path, data_transform=data_transforms['train']) 68 | # Concatenate dataset to produce a larger one 69 | if config.train.repeat > 1: 70 | train_dataset = ConcatDataset([train_dataset for _ in range(config.train.repeat)]) 71 | 72 | # Create loaders 73 | if world_size() > 1: 74 | sampler = torch.utils.data.distributed.DistributedSampler( 75 | train_dataset, num_replicas=world_size(), rank=rank()) 76 | else: 77 | sampler = None 78 | 79 | train_loader = DataLoader(train_dataset, 80 | batch_size=config.train.batch_size, 81 | pin_memory=True, 82 | shuffle=not (world_size() > 1), 83 | num_workers=config.train.num_workers, 84 | worker_init_fn=_worker_init_fn, 85 | sampler=sampler) 86 | return train_dataset, train_loader 87 | --------------------------------------------------------------------------------