├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── cfgs ├── default.yaml ├── default_test.yaml └── default_train.yaml ├── install.sh └── pose_diffusion ├── datasets ├── co3d_v2.py ├── re10k.py └── re10k_test_1800.txt ├── demo.py ├── models ├── __init__.py ├── denoiser.py ├── gaussian_diffuser.py ├── image_feature_extractor.py └── pose_diffusion_model.py ├── samples └── apple │ ├── frame000007.jpg │ ├── frame000012.jpg │ ├── frame000017.jpg │ ├── frame000019.jpg │ ├── frame000024.jpg │ ├── frame000025.jpg │ ├── frame000043.jpg │ ├── frame000052.jpg │ ├── frame000070.jpg │ ├── frame000077.jpg │ ├── frame000085.jpg │ ├── frame000096.jpg │ ├── frame000128.jpg │ ├── frame000145.jpg │ ├── frame000160.jpg │ ├── frame000162.jpg │ ├── frame000168.jpg │ ├── frame000172.jpg │ ├── frame000191.jpg │ ├── frame000200.jpg │ └── gt_cameras.npz ├── test.py ├── train.py └── util ├── __init__.py ├── camera_transform.py ├── embedding.py ├── geometry_guided_sampling.py ├── get_fundamental_matrix.py ├── load_img_folder.py ├── match_extraction.py ├── metric.py ├── normalize_cameras.py ├── train_util.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | tmp/ 4 | **/tmp/ 5 | .vscode/ 6 | *.egg-info/ 7 | **/__pycache__/ 8 | **/outputs/ 9 | *-checkpoint.ipynb 10 | **/.ipynb_checkpoints 11 | **/.ipynb_checkpoints/** 12 | test_visualizations/** 13 | visualisations/** 14 | dependency/ 15 | **/data/** 16 | *.log 17 | # Ignore .DS_Store files in any folder 18 | .DS_Store 19 | **/.DS_Store 20 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when there is a 56 | reasonable belief that an individual's behavior may have a negative impact on 57 | the project or its community. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported by contacting the project team at . All 63 | complaints will be reviewed and investigated and will result in a response that 64 | is deemed necessary and appropriate to the circumstances. The project team is 65 | obligated to maintain confidentiality with regard to the reporter of an incident. 66 | Further details of specific enforcement policies may be posted separately. 67 | 68 | Project maintainers who do not follow or enforce the Code of Conduct in good 69 | faith may face temporary or permanent repercussions as determined by other 70 | members of the project's leadership. 71 | 72 | ## Attribution 73 | 74 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 75 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 76 | 77 | [homepage]: https://www.contributor-covenant.org 78 | 79 | For answers to common questions about this code of conduct, see 80 | https://www.contributor-covenant.org/faq 81 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to PoseDiffusion 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Pull Requests 6 | We actively welcome your pull requests. 7 | 8 | 1. Fork the repo and create your branch from `main`. 9 | 2. If you've added code that should be tested, add tests. 10 | 3. If you've changed APIs, update the documentation. 11 | 4. Ensure the test suite passes. 12 | 5. Make sure your code lints. 13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 14 | 15 | ## Contributor License Agreement ("CLA") 16 | In order to accept your pull request, we need you to submit a CLA. You only need 17 | to do this once to work on any of Facebook's open source projects. 18 | 19 | Complete your CLA here: 20 | 21 | ## Issues 22 | We use GitHub issues to track public bugs. Please ensure your description is 23 | clear and has sufficient instructions to be able to reproduce the issue. 24 | 25 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 26 | disclosure of security bugs. In those cases, please go through the process 27 | outlined on that page and do not file a public issue. 28 | 29 | ## License 30 | By contributing to PoseDiffusion, you agree that your contributions will be licensed 31 | under the LICENSE file in the root directory of this source tree. -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More_considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial 4.0 International Public 58 | License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial 4.0 International Public License ("Public 63 | License"). To the extent this Public License may be interpreted as a 64 | contract, You are granted the Licensed Rights in consideration of Your 65 | acceptance of these terms and conditions, and the Licensor grants You 66 | such rights in consideration of benefits the Licensor receives from 67 | making the Licensed Material available under these terms and 68 | conditions. 69 | 70 | Section 1 -- Definitions. 71 | 72 | a. Adapted Material means material subject to Copyright and Similar 73 | Rights that is derived from or based upon the Licensed Material 74 | and in which the Licensed Material is translated, altered, 75 | arranged, transformed, or otherwise modified in a manner requiring 76 | permission under the Copyright and Similar Rights held by the 77 | Licensor. For purposes of this Public License, where the Licensed 78 | Material is a musical work, performance, or sound recording, 79 | Adapted Material is always produced where the Licensed Material is 80 | synched in timed relation with a moving image. 81 | 82 | b. Adapter's License means the license You apply to Your Copyright 83 | and Similar Rights in Your contributions to Adapted Material in 84 | accordance with the terms and conditions of this Public License. 85 | 86 | c. Copyright and Similar Rights means copyright and/or similar rights 87 | closely related to copyright including, without limitation, 88 | performance, broadcast, sound recording, and Sui Generis Database 89 | Rights, without regard to how the rights are labeled or 90 | categorized. For purposes of this Public License, the rights 91 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 92 | Rights. 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. NonCommercial means not primarily intended for or directed towards 116 | commercial advantage or monetary compensation. For purposes of 117 | this Public License, the exchange of the Licensed Material for 118 | other material subject to Copyright and Similar Rights by digital 119 | file-sharing or similar means is NonCommercial provided there is 120 | no payment of monetary compensation in connection with the 121 | exchange. 122 | 123 | j. Share means to provide material to the public by any means or 124 | process that requires permission under the Licensed Rights, such 125 | as reproduction, public display, public performance, distribution, 126 | dissemination, communication, or importation, and to make material 127 | available to the public including in ways that members of the 128 | public may access the material from a place and at a time 129 | individually chosen by them. 130 | 131 | k. Sui Generis Database Rights means rights other than copyright 132 | resulting from Directive 96/9/EC of the European Parliament and of 133 | the Council of 11 March 1996 on the legal protection of databases, 134 | as amended and/or succeeded, as well as other essentially 135 | equivalent rights anywhere in the world. 136 | 137 | l. You means the individual or entity exercising the Licensed Rights 138 | under this Public License. Your has a corresponding meaning. 139 | 140 | Section 2 -- Scope. 141 | 142 | a. License grant. 143 | 144 | 1. Subject to the terms and conditions of this Public License, 145 | the Licensor hereby grants You a worldwide, royalty-free, 146 | non-sublicensable, non-exclusive, irrevocable license to 147 | exercise the Licensed Rights in the Licensed Material to: 148 | 149 | a. reproduce and Share the Licensed Material, in whole or 150 | in part, for NonCommercial purposes only; and 151 | 152 | b. produce, reproduce, and Share Adapted Material for 153 | NonCommercial purposes only. 154 | 155 | 2. Exceptions and Limitations. For the avoidance of doubt, where 156 | Exceptions and Limitations apply to Your use, this Public 157 | License does not apply, and You do not need to comply with 158 | its terms and conditions. 159 | 160 | 3. Term. The term of this Public License is specified in Section 161 | 6(a). 162 | 163 | 4. Media and formats; technical modifications allowed. The 164 | Licensor authorizes You to exercise the Licensed Rights in 165 | all media and formats whether now known or hereafter created, 166 | and to make technical modifications necessary to do so. The 167 | Licensor waives and/or agrees not to assert any right or 168 | authority to forbid You from making technical modifications 169 | necessary to exercise the Licensed Rights, including 170 | technical modifications necessary to circumvent Effective 171 | Technological Measures. For purposes of this Public License, 172 | simply making modifications authorized by this Section 2(a) 173 | (4) never produces Adapted Material. 174 | 175 | 5. Downstream recipients. 176 | 177 | a. Offer from the Licensor -- Licensed Material. Every 178 | recipient of the Licensed Material automatically 179 | receives an offer from the Licensor to exercise the 180 | Licensed Rights under the terms and conditions of this 181 | Public License. 182 | 183 | b. No downstream restrictions. You may not offer or impose 184 | any additional or different terms or conditions on, or 185 | apply any Effective Technological Measures to, the 186 | Licensed Material if doing so restricts exercise of the 187 | Licensed Rights by any recipient of the Licensed 188 | Material. 189 | 190 | 6. No endorsement. Nothing in this Public License constitutes or 191 | may be construed as permission to assert or imply that You 192 | are, or that Your use of the Licensed Material is, connected 193 | with, or sponsored, endorsed, or granted official status by, 194 | the Licensor or others designated to receive attribution as 195 | provided in Section 3(a)(1)(A)(i). 196 | 197 | b. Other rights. 198 | 199 | 1. Moral rights, such as the right of integrity, are not 200 | licensed under this Public License, nor are publicity, 201 | privacy, and/or other similar personality rights; however, to 202 | the extent possible, the Licensor waives and/or agrees not to 203 | assert any such rights held by the Licensor to the limited 204 | extent necessary to allow You to exercise the Licensed 205 | Rights, but not otherwise. 206 | 207 | 2. Patent and trademark rights are not licensed under this 208 | Public License. 209 | 210 | 3. To the extent possible, the Licensor waives any right to 211 | collect royalties from You for the exercise of the Licensed 212 | Rights, whether directly or through a collecting society 213 | under any voluntary or waivable statutory or compulsory 214 | licensing scheme. In all other cases the Licensor expressly 215 | reserves any right to collect such royalties, including when 216 | the Licensed Material is used other than for NonCommercial 217 | purposes. 218 | 219 | Section 3 -- License Conditions. 220 | 221 | Your exercise of the Licensed Rights is expressly made subject to the 222 | following conditions. 223 | 224 | a. Attribution. 225 | 226 | 1. If You Share the Licensed Material (including in modified 227 | form), You must: 228 | 229 | a. retain the following if it is supplied by the Licensor 230 | with the Licensed Material: 231 | 232 | i. identification of the creator(s) of the Licensed 233 | Material and any others designated to receive 234 | attribution, in any reasonable manner requested by 235 | the Licensor (including by pseudonym if 236 | designated); 237 | 238 | ii. a copyright notice; 239 | 240 | iii. a notice that refers to this Public License; 241 | 242 | iv. a notice that refers to the disclaimer of 243 | warranties; 244 | 245 | v. a URI or hyperlink to the Licensed Material to the 246 | extent reasonably practicable; 247 | 248 | b. indicate if You modified the Licensed Material and 249 | retain an indication of any previous modifications; and 250 | 251 | c. indicate the Licensed Material is licensed under this 252 | Public License, and include the text of, or the URI or 253 | hyperlink to, this Public License. 254 | 255 | 2. You may satisfy the conditions in Section 3(a)(1) in any 256 | reasonable manner based on the medium, means, and context in 257 | which You Share the Licensed Material. For example, it may be 258 | reasonable to satisfy the conditions by providing a URI or 259 | hyperlink to a resource that includes the required 260 | information. 261 | 262 | 3. If requested by the Licensor, You must remove any of the 263 | information required by Section 3(a)(1)(A) to the extent 264 | reasonably practicable. 265 | 266 | 4. If You Share Adapted Material You produce, the Adapter's 267 | License You apply must not prevent recipients of the Adapted 268 | Material from complying with this Public License. 269 | 270 | Section 4 -- Sui Generis Database Rights. 271 | 272 | Where the Licensed Rights include Sui Generis Database Rights that 273 | apply to Your use of the Licensed Material: 274 | 275 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 276 | to extract, reuse, reproduce, and Share all or a substantial 277 | portion of the contents of the database for NonCommercial purposes 278 | only; 279 | 280 | b. if You include all or a substantial portion of the database 281 | contents in a database in which You have Sui Generis Database 282 | Rights, then the database in which You have Sui Generis Database 283 | Rights (but not its individual contents) is Adapted Material; and 284 | 285 | c. You must comply with the conditions in Section 3(a) if You Share 286 | all or a substantial portion of the contents of the database. 287 | 288 | For the avoidance of doubt, this Section 4 supplements and does not 289 | replace Your obligations under this Public License where the Licensed 290 | Rights include other Copyright and Similar Rights. 291 | 292 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 293 | 294 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 295 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 296 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 297 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 298 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 299 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 300 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 301 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 302 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 303 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 304 | 305 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 306 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 307 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 308 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 309 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 310 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 311 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 312 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 313 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 314 | 315 | c. The disclaimer of warranties and limitation of liability provided 316 | above shall be interpreted in a manner that, to the extent 317 | possible, most closely approximates an absolute disclaimer and 318 | waiver of all liability. 319 | 320 | Section 6 -- Term and Termination. 321 | 322 | a. This Public License applies for the term of the Copyright and 323 | Similar Rights licensed here. However, if You fail to comply with 324 | this Public License, then Your rights under this Public License 325 | terminate automatically. 326 | 327 | b. Where Your right to use the Licensed Material has terminated under 328 | Section 6(a), it reinstates: 329 | 330 | 1. automatically as of the date the violation is cured, provided 331 | it is cured within 30 days of Your discovery of the 332 | violation; or 333 | 334 | 2. upon express reinstatement by the Licensor. 335 | 336 | For the avoidance of doubt, this Section 6(b) does not affect any 337 | right the Licensor may have to seek remedies for Your violations 338 | of this Public License. 339 | 340 | c. For the avoidance of doubt, the Licensor may also offer the 341 | Licensed Material under separate terms or conditions or stop 342 | distributing the Licensed Material at any time; however, doing so 343 | will not terminate this Public License. 344 | 345 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 346 | License. 347 | 348 | Section 7 -- Other Terms and Conditions. 349 | 350 | a. The Licensor shall not be bound by any additional or different 351 | terms or conditions communicated by You unless expressly agreed. 352 | 353 | b. Any arrangements, understandings, or agreements regarding the 354 | Licensed Material not stated herein are separate from and 355 | independent of the terms and conditions of this Public License. 356 | 357 | Section 8 -- Interpretation. 358 | 359 | a. For the avoidance of doubt, this Public License does not, and 360 | shall not be interpreted to, reduce, limit, restrict, or impose 361 | conditions on any use of the Licensed Material that could lawfully 362 | be made without permission under this Public License. 363 | 364 | b. To the extent possible, if any provision of this Public License is 365 | deemed unenforceable, it shall be automatically reformed to the 366 | minimum extent necessary to make it enforceable. If the provision 367 | cannot be reformed, it shall be severed from this Public License 368 | without affecting the enforceability of the remaining terms and 369 | conditions. 370 | 371 | c. No term or condition of this Public License will be waived and no 372 | failure to comply consented to unless expressly agreed to by the 373 | Licensor. 374 | 375 | d. Nothing in this Public License constitutes or may be interpreted 376 | as a limitation upon, or waiver of, any privileges and immunities 377 | that apply to the Licensor or You, including from the legal 378 | processes of any jurisdiction or authority. 379 | 380 | ======================================================================= 381 | 382 | Creative Commons is not a party to its public 383 | licenses. Notwithstanding, Creative Commons may elect to apply one of 384 | its public licenses to material it publishes and in those instances 385 | will be considered the “Licensor.” The text of the Creative Commons 386 | public licenses is dedicated to the public domain under the CC0 Public 387 | Domain Dedication. Except for the limited purpose of indicating that 388 | material is shared under a Creative Commons public license or as 389 | otherwise permitted by the Creative Commons policies published at 390 | creativecommons.org/policies, Creative Commons does not authorize the 391 | use of the trademark "Creative Commons" or any other trademark or logo 392 | of Creative Commons without its prior written consent including, 393 | without limitation, in connection with any unauthorized modifications 394 | to any of its public licenses or any other arrangements, 395 | understandings, or agreements concerning use of licensed material. For 396 | the avoidance of doubt, this paragraph does not form part of the 397 | public licenses. 398 | 399 | Creative Commons may be contacted at creativecommons.org. 400 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PoseDiffusion: Solving Pose Estimation via Diffusion-aided Bundle Adjustment 2 | 3 | ![Teaser](https://raw.githubusercontent.com/posediffusion/posediffusion.github.io/main/resources/teaser.gif) 4 | 5 |

[Paper] 6 | [Project Page]

7 | 8 | 9 | **Updates:** 10 | 11 | - [Aug 6, 2024] To access the raw data for Figures 5, 7, and 10, please download our submission from arXiv. You can do this by clicking the [Download source] button at https://arxiv.org/format/2306.15667. The submission includes the following files: `plots_co3dv2.tex`, `plots_re10k.tex`, and `plots_co3dv2_suppl.tex`. These files use data from `csvs.tex` and `csvs_suppl.tex`, which are also included in the LaTeX submission source, to generate the figures. The split we used for Re10k can be found at [re10k_test_1800.txt](https://github.com/facebookresearch/PoseDiffusion/blob/main/pose_diffusion/datasets/re10k_test_1800.txt). 12 | 13 | - [Apr 24, 2024] You may also have an interest in [VGGSfM](https://github.com/facebookresearch/vggsfm), where a model similar to PoseDiffusion is used as the camera predictor. It also supports to optimize camera parameters through bundle adjustment. 14 | 15 | - [Apr 24, 2024] Updated the checkpoint for RealEstate10K dataset. 16 | 17 | 18 | 19 | ## Installation 20 | We provide a simple installation script that, by default, sets up a conda environment with Python 3.9, PyTorch 1.13, and CUDA 11.6. 21 | 22 | ```.bash 23 | source install.sh 24 | ``` 25 | 26 | ## Quick Start 27 | 28 | ### 1. Download Checkpoint 29 | 30 | You can download the model checkpoint trained on [Co3D](https://drive.google.com/file/d/14Waj4RCrdezx_jG9Am9WlvLfvhyuCQ3k/view?usp=sharing) or [RealEstate10K](https://drive.google.com/file/d/18kyF8XLKsTGtKEFb0NAF2tceS2eoIV-a/view?usp=sharing). Please note that the checkpoint for RealEstate10K was re-trained on an image size of 336 (you need to change the `image_size` in the coorresponding config). The predicted camera poses and focal lengths are defined in [NDC coordinate](https://pytorch3d.org/docs/cameras). 31 | 32 | 33 | 34 | 35 | 36 | ### 2. Run the Demo 37 | 38 | ```.bash 39 | python demo.py image_folder="samples/apple" ckpt="/PATH/TO/DOWNLOADED/CKPT" 40 | ``` 41 | 42 | You can experiment with your own data by specifying a different `image_folder`. 43 | 44 | 45 | On a Quadro GP100 GPU, the inference time for a 20-frame sequence is approximately 0.8 seconds without GGS and around 80 seconds with GGS (including 20 seconds for matching extraction). 46 | 47 | You can choose to enable or disable GGS (or other settings) in `./cfgs/default.yaml`. 48 | 49 | We use [Visdom](https://github.com/fossasia/visdom) by default for visualization. Ensure your Visdom settings are correctly configured to visualize the results accurately. However, Visdom is not necessary for running the model. 50 | 51 | ## Training 52 | 53 | ### 1. Preprocess Annotations 54 | 55 | Start by following the instructions [here](https://github.com/amyxlase/relpose-plus-plus#pre-processing-co3d) to preprocess the annotations of the Co3D V2 dataset. This will significantly reduce data processing time during training. 56 | 57 | ### 2. Specify Paths 58 | 59 | Next, specify the paths for `CO3D_DIR` and `CO3D_ANNOTATION_DIR` in `./cfgs/default_train.yaml`. `CO3D_DIR` should be set to the path where your downloaded Co3D dataset is located, while `CO3D_ANNOTATION_DIR` should point to the location of the annotation files generated after completing the preprocessing in step 1. 60 | 61 | ### 3. Start Training 62 | 63 | - For 1-GPU Training: 64 | ```bash 65 | python train.py 66 | ``` 67 | 68 | - For multi-GPU training, launch the training script using [accelerate](https://huggingface.co/docs/accelerate/basic_tutorials/launch), e.g., training on 8 GPUs (processes) in 1 node (machines): 69 | ```bash 70 | accelerate launch --num_processes=8 --multi_gpu --num_machines=1 train.py 71 | ``` 72 | 73 | All configurations are specified inside `./cfgs/default_train.yaml`. Please notice that we use Visdom to record logs. 74 | 75 | For each iteration, the training should take around 1~3 seconds depending on difference devices. You can check it by looking at the `sec/it` of the log. The whole training should take around 2-3 days on 8 A100 GPUs. 76 | 77 | **NOTE**: In some clusters we found the publicly released training code can be super slow when using multiple GPUs. This looks because that `accelerate` does not work well under some settings and hence the data loading is very slow (if not hangs out). The simplest solution is to remove `accelerate (accelerator)` from the code, and use pytorch's own distributed trainer or [pytorch-lighting](https://github.com/Lightning-AI/pytorch-lightning) to launch the training. This problem does not affect single GPU training. Please submit an issue if you observed a higher number or report your case [here](https://github.com/facebookresearch/PoseDiffusion/issues/33) (this should be related to the data loading of `accelerate`, a simple solution is to use pytorch's own distributed training). 78 | 79 | 80 | ## Testing 81 | 82 | ### 1. Specify Paths 83 | 84 | Please specify the paths `CO3D_DIR`, `CO3D_ANNOTATION_DIR`, and `resume_ckpt` in `./cfgs/default_test.yaml`. The flag `resume_ckpt` refers to your downloaded model checkpoint. 85 | 86 | ### 2. Run Testing 87 | 88 | ```bash 89 | python test.py 90 | ``` 91 | 92 | You can check different testing settings by adjusting `num_frames`, `GGS.enable`, and others in `./cfgs/default_test.yaml`. 93 | 94 | 95 | ## Acknowledgement 96 | 97 | Thanks for the great implementation of [denoising-diffusion-pytorch](https://github.com/lucidrains/denoising-diffusion-pytorch), [guided-diffusion](https://github.com/openai/guided-diffusion), [hloc](https://github.com/cvg/Hierarchical-Localization), [relpose](https://github.com/jasonyzhang/relpose). 98 | 99 | 100 | ## License 101 | See the [LICENSE](./LICENSE) file for details about the license under which this code is made available. 102 | -------------------------------------------------------------------------------- /cfgs/default.yaml: -------------------------------------------------------------------------------- 1 | image_folder: samples/apple 2 | image_size: 224 3 | ckpt: co3d_model1.pth 4 | seed: 0 5 | 6 | GGS: 7 | enable: True 8 | start_step: 10 9 | learning_rate: 0.01 10 | iter_num: 100 11 | sampson_max: 10 12 | min_matches: 10 13 | alpha: 0.0001 14 | 15 | 16 | MODEL: 17 | _target_: models.PoseDiffusionModel 18 | 19 | pose_encoding_type: absT_quaR_logFL 20 | 21 | IMAGE_FEATURE_EXTRACTOR: 22 | _target_: models.MultiScaleImageFeatureExtractor 23 | freeze: False 24 | 25 | DENOISER: 26 | _target_: models.Denoiser 27 | TRANSFORMER: 28 | _target_: models.TransformerEncoderWrapper 29 | d_model: 512 30 | nhead: 4 31 | dim_feedforward: 1024 32 | num_encoder_layers: 8 33 | dropout: 0.1 34 | batch_first: True 35 | norm_first: True 36 | 37 | 38 | DIFFUSER: 39 | _target_: models.GaussianDiffusion 40 | beta_schedule: custom 41 | -------------------------------------------------------------------------------- /cfgs/default_test.yaml: -------------------------------------------------------------------------------- 1 | seed: 0 2 | exp_name: exp001_test 3 | 4 | 5 | 6 | 7 | test: 8 | # Please Specify Your Own Path 9 | CO3D_DIR: "/datasets01/co3dv2/080422/" 10 | CO3D_ANNOTATION_DIR: "/fsx-repligen/jianyuan/datasets/co3d_relpose/" 11 | resume_ckpt: "/data/home/jianyuan/src/PoseDiffusion/pose_diffusion/tmp/co3d_model_Apr16.pth" 12 | 13 | random_order: True 14 | num_frames: 10 15 | 16 | ########################## 17 | 18 | img_size: 224 19 | category: seen 20 | 21 | normalize_cameras: True 22 | persistent_workers: True 23 | 24 | preload_image: False 25 | cudnnbenchmark: False 26 | first_camera_transform: True 27 | min_num_images: 50 28 | compute_optical: True 29 | 30 | GGS: 31 | enable: True 32 | start_step: 10 33 | learning_rate: 0.01 34 | iter_num: 100 35 | sampson_max: 10 36 | min_matches: 10 37 | alpha: 0.0001 38 | 39 | 40 | 41 | debug: False 42 | 43 | 44 | MODEL: 45 | _target_: models.PoseDiffusionModel 46 | 47 | pose_encoding_type: absT_quaR_logFL 48 | 49 | IMAGE_FEATURE_EXTRACTOR: 50 | _target_: models.MultiScaleImageFeatureExtractor 51 | modelname: "dino_vits16" 52 | freeze: False 53 | 54 | DENOISER: 55 | _target_: models.Denoiser 56 | TRANSFORMER: 57 | _target_: models.TransformerEncoderWrapper 58 | d_model: 512 59 | nhead: 4 60 | dim_feedforward: 1024 61 | num_encoder_layers: 8 62 | dropout: 0.1 63 | batch_first: True 64 | norm_first: True 65 | 66 | DIFFUSER: 67 | _target_: models.GaussianDiffusion 68 | beta_schedule: custom 69 | -------------------------------------------------------------------------------- /cfgs/default_train.yaml: -------------------------------------------------------------------------------- 1 | seed: 0 2 | exp_name: exp001_train 3 | exp_dir: "name_your_own" 4 | 5 | train: 6 | # Please Specify Your Own Path 7 | CO3D_DIR: "/datasets01/co3dv2/080422/" 8 | CO3D_ANNOTATION_DIR: "/fsx-repligen/jianyuan/datasets/co3d_relpose/" 9 | 10 | 11 | img_size: 224 12 | category: seen 13 | restart_num: 50 14 | lr: 0.0001 15 | resume_ckpt: False 16 | epochs: 100 17 | ckpt_interval: 5 18 | num_workers: 8 19 | 20 | eval_interval: 5 21 | 22 | print_interval: 10 23 | 24 | len_train: 16384 25 | len_eval: 256 26 | 27 | max_images: 512 28 | normalize_cameras: True 29 | persistent_workers: True 30 | 31 | pin_memory: False 32 | clip_grad: 1.0 33 | preload_image: False 34 | cudnnbenchmark: False 35 | first_camera_transform: True 36 | min_num_images: 50 37 | images_per_seq: [3, 51] 38 | compute_optical: True 39 | color_aug: True 40 | erase_aug: False 41 | batch_repeat: 90 42 | 43 | debug: False 44 | 45 | 46 | MODEL: 47 | _target_: models.PoseDiffusionModel 48 | 49 | pose_encoding_type: absT_quaR_logFL 50 | 51 | IMAGE_FEATURE_EXTRACTOR: 52 | _target_: models.MultiScaleImageFeatureExtractor 53 | modelname: "dino_vits16" 54 | freeze: False 55 | 56 | DENOISER: 57 | _target_: models.Denoiser 58 | TRANSFORMER: 59 | _target_: models.TransformerEncoderWrapper 60 | d_model: 512 61 | nhead: 4 62 | dim_feedforward: 1024 63 | num_encoder_layers: 8 64 | dropout: 0.1 65 | batch_first: True 66 | norm_first: True 67 | 68 | DIFFUSER: 69 | _target_: models.GaussianDiffusion 70 | beta_schedule: custom 71 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | # This Script Assumes Python 3.9, CUDA 11.6 8 | 9 | conda deactivate 10 | 11 | # Set environment variables 12 | export ENV_NAME=posediffusion 13 | export PYTHON_VERSION=3.9 14 | export PYTORCH_VERSION=1.13.0 15 | export CUDA_VERSION=11.6 16 | 17 | # Create a new conda environment and activate it 18 | conda create -n $ENV_NAME python=$PYTHON_VERSION 19 | conda activate $ENV_NAME 20 | 21 | # Install PyTorch, torchvision, and PyTorch3D using conda 22 | conda install pytorch=$PYTORCH_VERSION torchvision pytorch-cuda=$CUDA_VERSION -c pytorch -c nvidia 23 | conda install -c fvcore -c iopath -c conda-forge fvcore iopath 24 | conda install pytorch3d -c pytorch3d 25 | 26 | # Install pip packages 27 | pip install hydra-core --upgrade 28 | pip install omegaconf opencv-python einops visdom 29 | pip install accelerate==0.24.0 30 | 31 | # Install HLoc for extracting 2D matches (optional if GGS is not needed) 32 | git clone --recursive https://github.com/cvg/Hierarchical-Localization.git dependency/hloc 33 | 34 | cd dependency/hloc 35 | python -m pip install -e . 36 | cd ../../ 37 | 38 | # Ensure the version of pycolmap is not 0.5.0 39 | pip install --upgrade "pycolmap>=0.3.0,<=0.4.0" 40 | 41 | -------------------------------------------------------------------------------- /pose_diffusion/datasets/co3d_v2.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | """ 8 | Adapted from code originally written by Jason Zhang. 9 | """ 10 | 11 | import gzip 12 | import json 13 | import os.path as osp 14 | import random 15 | 16 | import numpy as np 17 | import torch 18 | from PIL import Image, ImageFile 19 | from pytorch3d.renderer import PerspectiveCameras 20 | from torch.utils.data import Dataset 21 | from torchvision import transforms 22 | 23 | from util.normalize_cameras import normalize_cameras 24 | 25 | from multiprocessing import Pool 26 | import tqdm 27 | from util.camera_transform import adjust_camera_to_bbox_crop_, adjust_camera_to_image_scale_, bbox_xyxy_to_xywh 28 | 29 | Image.MAX_IMAGE_PIXELS = None 30 | ImageFile.LOAD_TRUNCATED_IMAGES = True 31 | 32 | 33 | class Co3dDataset(Dataset): 34 | def __init__( 35 | self, 36 | category=("all",), 37 | split="train", 38 | transform=None, 39 | debug=False, 40 | random_aug=True, 41 | jitter_scale=[0.8, 1.2], 42 | jitter_trans=[-0.07, 0.07], 43 | min_num_images=50, 44 | img_size=224, 45 | eval_time=False, 46 | normalize_cameras=False, 47 | first_camera_transform=True, 48 | mask_images=False, 49 | CO3D_DIR=None, 50 | CO3D_ANNOTATION_DIR=None, 51 | foreground_crop=True, 52 | center_box=True, 53 | sort_by_filename=False, 54 | compute_optical=False, 55 | color_aug=True, 56 | erase_aug=False, 57 | ): 58 | """ 59 | Args: 60 | category (iterable): List of categories to use. If "all" is in the list, 61 | all training categories are used. 62 | num_images (int): Default number of images in each batch. 63 | normalize_cameras (bool): If True, normalizes cameras so that the 64 | intersection of the optical axes is placed at the origin and the norm 65 | of the first camera translation is 1. 66 | first_camera_transform (bool): If True, tranforms the cameras such that 67 | camera 1 has extrinsics [I | 0]. 68 | mask_images (bool): If True, masks out the background of the images. 69 | """ 70 | if "seen" in category: 71 | category = TRAINING_CATEGORIES 72 | 73 | if "unseen" in category: 74 | category = TEST_CATEGORIES 75 | 76 | if "all" in category: 77 | category = TRAINING_CATEGORIES + TEST_CATEGORIES 78 | 79 | category = sorted(category) 80 | self.category = category 81 | 82 | if split == "train": 83 | split_name = "train" 84 | elif split == "test": 85 | split_name = "test" 86 | 87 | self.low_quality_translations = [] 88 | self.rotations = {} 89 | self.category_map = {} 90 | 91 | if CO3D_DIR == None: 92 | raise ValueError("CO3D_DIR is not specified") 93 | 94 | print(f"CO3D_DIR is {CO3D_DIR}") 95 | 96 | self.CO3D_DIR = CO3D_DIR 97 | self.CO3D_ANNOTATION_DIR = CO3D_ANNOTATION_DIR 98 | self.center_box = center_box 99 | self.split_name = split_name 100 | self.min_num_images = min_num_images 101 | self.foreground_crop = foreground_crop 102 | 103 | for c in category: 104 | annotation_file = osp.join(self.CO3D_ANNOTATION_DIR, f"{c}_{split_name}.jgz") 105 | with gzip.open(annotation_file, "r") as fin: 106 | annotation = json.loads(fin.read()) 107 | 108 | counter = 0 109 | for seq_name, seq_data in annotation.items(): 110 | counter += 1 111 | if len(seq_data) < min_num_images: 112 | continue 113 | 114 | filtered_data = [] 115 | self.category_map[seq_name] = c 116 | bad_seq = False 117 | for data in seq_data: 118 | # Make sure translations are not ridiculous 119 | if data["T"][0] + data["T"][1] + data["T"][2] > 1e5: 120 | bad_seq = True 121 | self.low_quality_translations.append(seq_name) 122 | break 123 | 124 | # Ignore all unnecessary information. 125 | filtered_data.append( 126 | { 127 | "filepath": data["filepath"], 128 | "bbox": data["bbox"], 129 | "R": data["R"], 130 | "T": data["T"], 131 | "focal_length": data["focal_length"], 132 | "principal_point": data["principal_point"], 133 | } 134 | ) 135 | 136 | if not bad_seq: 137 | self.rotations[seq_name] = filtered_data 138 | 139 | print(annotation_file) 140 | print(counter) 141 | 142 | self.sequence_list = list(self.rotations.keys()) 143 | 144 | self.split = split 145 | self.debug = debug 146 | self.sort_by_filename = sort_by_filename 147 | 148 | if transform is None: 149 | self.transform = transforms.Compose([transforms.ToTensor(), transforms.Resize(img_size, antialias=True)]) 150 | else: 151 | self.transform = transform 152 | 153 | if random_aug and not eval_time: 154 | self.jitter_scale = jitter_scale 155 | self.jitter_trans = jitter_trans 156 | else: 157 | self.jitter_scale = [1, 1] 158 | self.jitter_trans = [0, 0] 159 | 160 | self.img_size = img_size 161 | self.eval_time = eval_time 162 | self.normalize_cameras = normalize_cameras 163 | self.first_camera_transform = first_camera_transform 164 | self.mask_images = mask_images 165 | self.compute_optical = compute_optical 166 | self.color_aug = color_aug 167 | self.erase_aug = erase_aug 168 | 169 | if self.color_aug: 170 | self.color_jitter = transforms.Compose( 171 | [ 172 | transforms.RandomApply( 173 | [transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1)], p=0.65 174 | ), 175 | transforms.RandomGrayscale(p=0.15), 176 | ] 177 | ) 178 | if self.erase_aug: 179 | self.rand_erase = transforms.RandomErasing( 180 | p=0.1, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False 181 | ) 182 | 183 | print(f"Low quality translation sequences, not used: {self.low_quality_translations}") 184 | print(f"Data size: {len(self)}") 185 | 186 | def __len__(self): 187 | return len(self.sequence_list) 188 | 189 | def _jitter_bbox(self, bbox): 190 | # Random aug to bounding box shape 191 | 192 | bbox = square_bbox(bbox.astype(np.float32)) 193 | s = np.random.uniform(self.jitter_scale[0], self.jitter_scale[1]) 194 | tx, ty = np.random.uniform(self.jitter_trans[0], self.jitter_trans[1], size=2) 195 | 196 | side_length = bbox[2] - bbox[0] 197 | center = (bbox[:2] + bbox[2:]) / 2 + np.array([tx, ty]) * side_length 198 | extent = side_length / 2 * s 199 | 200 | # Final coordinates need to be integer for cropping. 201 | ul = (center - extent).round().astype(int) 202 | lr = ul + np.round(2 * extent).astype(int) 203 | return np.concatenate((ul, lr)) 204 | 205 | def _crop_image(self, image, bbox, white_bg=False): 206 | if white_bg: 207 | # Only support PIL Images 208 | image_crop = Image.new("RGB", (bbox[2] - bbox[0], bbox[3] - bbox[1]), (255, 255, 255)) 209 | image_crop.paste(image, (-bbox[0], -bbox[1])) 210 | else: 211 | image_crop = transforms.functional.crop( 212 | image, top=bbox[1], left=bbox[0], height=bbox[3] - bbox[1], width=bbox[2] - bbox[0] 213 | ) 214 | return image_crop 215 | 216 | def __getitem__(self, idx_N): 217 | """Fetch item by index and a dynamic variable n_per_seq.""" 218 | 219 | # Different from most pytorch datasets, 220 | # here we not only get index, but also a dynamic variable n_per_seq 221 | # supported by DynamicBatchSampler 222 | 223 | index, n_per_seq = idx_N 224 | sequence_name = self.sequence_list[index] 225 | metadata = self.rotations[sequence_name] 226 | ids = np.random.choice(len(metadata), n_per_seq, replace=False) 227 | return self.get_data(index=index, ids=ids) 228 | 229 | def get_data(self, index=None, sequence_name=None, ids=(0, 1), no_images=False, return_path = False): 230 | if sequence_name is None: 231 | sequence_name = self.sequence_list[index] 232 | metadata = self.rotations[sequence_name] 233 | category = self.category_map[sequence_name] 234 | 235 | annos = [metadata[i] for i in ids] 236 | 237 | if self.sort_by_filename: 238 | annos = sorted(annos, key=lambda x: x["filepath"]) 239 | 240 | images = [] 241 | rotations = [] 242 | translations = [] 243 | focal_lengths = [] 244 | principal_points = [] 245 | image_paths = [] 246 | 247 | for anno in annos: 248 | filepath = anno["filepath"] 249 | image_path = osp.join(self.CO3D_DIR, filepath) 250 | image = Image.open(image_path).convert("RGB") 251 | 252 | if self.mask_images: 253 | white_image = Image.new("RGB", image.size, (255, 255, 255)) 254 | mask_name = osp.basename(filepath.replace(".jpg", ".png")) 255 | 256 | mask_path = osp.join(self.CO3D_DIR, category, sequence_name, "masks", mask_name) 257 | mask = Image.open(mask_path).convert("L") 258 | 259 | if mask.size != image.size: 260 | mask = mask.resize(image.size) 261 | mask = Image.fromarray(np.array(mask) > 125) 262 | image = Image.composite(image, white_image, mask) 263 | 264 | images.append(image) 265 | rotations.append(torch.tensor(anno["R"])) 266 | translations.append(torch.tensor(anno["T"])) 267 | focal_lengths.append(torch.tensor(anno["focal_length"])) 268 | principal_points.append(torch.tensor(anno["principal_point"])) 269 | image_paths.append(image_path) 270 | 271 | crop_parameters = [] 272 | images_transformed = [] 273 | 274 | new_fls = [] 275 | new_pps = [] 276 | 277 | for i, (anno, image) in enumerate(zip(annos, images)): 278 | w, h = image.width, image.height 279 | 280 | if self.center_box: 281 | min_dim = min(h, w) 282 | top = (h - min_dim) // 2 283 | left = (w - min_dim) // 2 284 | bbox = np.array([left, top, left + min_dim, top + min_dim]) 285 | else: 286 | bbox = np.array(anno["bbox"]) 287 | 288 | if not self.eval_time: 289 | bbox_jitter = self._jitter_bbox(bbox) 290 | else: 291 | bbox_jitter = bbox 292 | 293 | bbox_xywh = torch.FloatTensor(bbox_xyxy_to_xywh(bbox_jitter)) 294 | (focal_length_cropped, principal_point_cropped) = adjust_camera_to_bbox_crop_( 295 | focal_lengths[i], principal_points[i], torch.FloatTensor(image.size), bbox_xywh 296 | ) 297 | 298 | image = self._crop_image(image, bbox_jitter, white_bg=self.mask_images) 299 | 300 | (new_focal_length, new_principal_point) = adjust_camera_to_image_scale_( 301 | focal_length_cropped, 302 | principal_point_cropped, 303 | torch.FloatTensor(image.size), 304 | torch.FloatTensor([self.img_size, self.img_size]), 305 | ) 306 | 307 | new_fls.append(new_focal_length) 308 | new_pps.append(new_principal_point) 309 | 310 | images_transformed.append(self.transform(image)) 311 | crop_center = (bbox_jitter[:2] + bbox_jitter[2:]) / 2 312 | cc = (2 * crop_center / min(h, w)) - 1 313 | crop_width = 2 * (bbox_jitter[2] - bbox_jitter[0]) / min(h, w) 314 | 315 | crop_parameters.append(torch.tensor([-cc[0], -cc[1], crop_width]).float()) 316 | 317 | images = images_transformed 318 | 319 | batch = {"seq_id": sequence_name, "category": category, "n": len(metadata), "ind": torch.tensor(ids)} 320 | 321 | new_fls = torch.stack(new_fls) 322 | new_pps = torch.stack(new_pps) 323 | 324 | if self.normalize_cameras: 325 | cameras = PerspectiveCameras( 326 | focal_length=new_fls.numpy(), 327 | principal_point=new_pps.numpy(), 328 | R=[data["R"] for data in annos], 329 | T=[data["T"] for data in annos], 330 | ) 331 | 332 | normalized_cameras = normalize_cameras( 333 | cameras, compute_optical=self.compute_optical, first_camera=self.first_camera_transform 334 | ) 335 | 336 | if normalized_cameras == -1: 337 | print("Error in normalizing cameras: camera scale was 0") 338 | raise RuntimeError 339 | 340 | batch["R"] = normalized_cameras.R 341 | batch["T"] = normalized_cameras.T 342 | batch["crop_params"] = torch.stack(crop_parameters) 343 | batch["R_original"] = torch.stack([torch.tensor(anno["R"]) for anno in annos]) 344 | batch["T_original"] = torch.stack([torch.tensor(anno["T"]) for anno in annos]) 345 | 346 | batch["fl"] = normalized_cameras.focal_length 347 | batch["pp"] = normalized_cameras.principal_point 348 | 349 | if torch.any(torch.isnan(batch["T"])): 350 | print(ids) 351 | print(category) 352 | print(sequence_name) 353 | raise RuntimeError 354 | 355 | else: 356 | batch["R"] = torch.stack(rotations) 357 | batch["T"] = torch.stack(translations) 358 | batch["crop_params"] = torch.stack(crop_parameters) 359 | batch["fl"] = new_fls 360 | batch["pp"] = new_pps 361 | 362 | if self.transform is not None: 363 | images = torch.stack(images) 364 | 365 | if self.color_aug and (not self.eval_time): 366 | images = self.color_jitter(images) 367 | if self.erase_aug: 368 | images = self.rand_erase(images) 369 | 370 | batch["image"] = images 371 | 372 | if return_path: 373 | return batch, image_paths 374 | 375 | return batch 376 | 377 | 378 | def square_bbox(bbox, padding=0.0, astype=None): 379 | """ 380 | Computes a square bounding box, with optional padding parameters. 381 | 382 | Args: 383 | bbox: Bounding box in xyxy format (4,). 384 | 385 | Returns: 386 | square_bbox in xyxy format (4,). 387 | """ 388 | if astype is None: 389 | astype = type(bbox[0]) 390 | bbox = np.array(bbox) 391 | center = (bbox[:2] + bbox[2:]) / 2 392 | extents = (bbox[2:] - bbox[:2]) / 2 393 | s = max(extents) * (1 + padding) 394 | square_bbox = np.array([center[0] - s, center[1] - s, center[0] + s, center[1] + s], dtype=astype) 395 | return square_bbox 396 | 397 | 398 | TRAINING_CATEGORIES = [ 399 | "apple", 400 | "backpack", 401 | "banana", 402 | "baseballbat", 403 | "baseballglove", 404 | "bench", 405 | "bicycle", 406 | "bottle", 407 | "bowl", 408 | "broccoli", 409 | "cake", 410 | "car", 411 | "carrot", 412 | "cellphone", 413 | "chair", 414 | "cup", 415 | "donut", 416 | "hairdryer", 417 | "handbag", 418 | "hydrant", 419 | "keyboard", 420 | "laptop", 421 | "microwave", 422 | "motorcycle", 423 | "mouse", 424 | "orange", 425 | "parkingmeter", 426 | "pizza", 427 | "plant", 428 | "stopsign", 429 | "teddybear", 430 | "toaster", 431 | "toilet", 432 | "toybus", 433 | "toyplane", 434 | "toytrain", 435 | "toytruck", 436 | "tv", 437 | "umbrella", 438 | "vase", 439 | "wineglass", 440 | ] 441 | 442 | TEST_CATEGORIES = ["ball", "book", "couch", "frisbee", "hotdog", "kite", "remote", "sandwich", "skateboard", "suitcase"] 443 | 444 | DEBUG_CATEGORIES = ["apple", "teddybear"] 445 | -------------------------------------------------------------------------------- /pose_diffusion/datasets/re10k.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | import gzip 9 | import json 10 | import os.path as osp 11 | import random 12 | import os 13 | 14 | import numpy as np 15 | import torch 16 | from PIL import Image, ImageFile 17 | from pytorch3d.renderer import PerspectiveCameras 18 | from pytorch3d.renderer.cameras import get_ndc_to_screen_transform 19 | from torch.utils.data import Dataset 20 | from torchvision import transforms 21 | import pickle 22 | 23 | from util.normalize_cameras import normalize_cameras 24 | 25 | import h5py 26 | from io import BytesIO 27 | 28 | from multiprocessing import Pool 29 | import tqdm 30 | from util.camera_transform import adjust_camera_to_bbox_crop_, adjust_camera_to_image_scale_, bbox_xyxy_to_xywh 31 | 32 | import matplotlib.pyplot as plt 33 | 34 | 35 | Image.MAX_IMAGE_PIXELS = None 36 | ImageFile.LOAD_TRUNCATED_IMAGES = True 37 | 38 | 39 | 40 | class Re10KDataset(Dataset): 41 | def __init__( 42 | self, 43 | split="train", 44 | transform=None, 45 | debug=False, 46 | random_aug=True, 47 | jitter_scale=[0.8, 1.0], 48 | jitter_trans=[-0.07, 0.07], 49 | min_num_images=50, 50 | img_size=224, 51 | eval_time=False, 52 | normalize_cameras=False, 53 | first_camera_transform=True, 54 | first_camera_rotation_only=False, 55 | mask_images=False, 56 | Re10K_DIR=None, 57 | Re10K_ANNOTATION_DIR = None, 58 | center_box=True, 59 | crop_longest=False, 60 | sort_by_filename=False, 61 | compute_optical=False, 62 | color_aug=True, 63 | erase_aug=False, 64 | ): 65 | 66 | self.Re10K_DIR = Re10K_DIR 67 | if Re10K_DIR == None: 68 | raise NotImplementedError 69 | 70 | if split == "train": 71 | self.train_dir = os.path.join(Re10K_DIR, "frames/train") 72 | video_loc = os.path.join(Re10K_DIR, "frames/train/video_loc.txt") 73 | scenes = np.loadtxt(video_loc, dtype=np.str_) 74 | self.scene_info_dir = os.path.join(Re10K_ANNOTATION_DIR, "train") 75 | self.scenes = scenes 76 | else: 77 | raise ValueError("only implemneted training at this stage") 78 | 79 | 80 | print(f"Re10K_DIR is {Re10K_DIR}") 81 | 82 | 83 | self.center_box = center_box 84 | self.crop_longest = crop_longest 85 | self.min_num_images = min_num_images 86 | 87 | self.build_dataset() 88 | 89 | self.sequence_list = sorted(list(self.wholedata.keys())) 90 | self.sequence_list_len = len(self.sequence_list) 91 | 92 | self.debug = debug 93 | self.sort_by_filename = sort_by_filename 94 | 95 | if transform is None: 96 | self.transform = transforms.Compose( 97 | [ 98 | transforms.ToTensor(), 99 | transforms.Resize(img_size, antialias=True), 100 | ] 101 | ) 102 | else: 103 | self.transform = transform 104 | 105 | if random_aug and not eval_time: 106 | self.jitter_scale = jitter_scale 107 | self.jitter_trans = jitter_trans 108 | else: 109 | self.jitter_scale = [1, 1] 110 | self.jitter_trans = [0, 0] 111 | 112 | self.img_size = img_size 113 | self.eval_time = eval_time 114 | self.normalize_cameras = normalize_cameras 115 | self.first_camera_transform = first_camera_transform 116 | self.mask_images = mask_images 117 | self.compute_optical = compute_optical 118 | self.color_aug = color_aug 119 | self.erase_aug = erase_aug 120 | 121 | 122 | if self.color_aug: 123 | self.color_jitter = transforms.Compose( 124 | [ 125 | transforms.RandomApply([transforms.ColorJitter(brightness=0.3, contrast=0.4, saturation=0.2, hue=0.1)], p=0.75), 126 | transforms.RandomGrayscale(p=0.05), 127 | transforms.RandomApply([transforms.GaussianBlur(5, sigma=(0.1, 1.0))], p=0.05), 128 | ] 129 | ) 130 | 131 | if self.erase_aug: 132 | self.rand_erase = transforms.RandomErasing(p=0.1, scale=(0.02, 0.05), ratio=(0.3, 3.3), value=0, inplace=False) 133 | 134 | 135 | print(f"Data size: {len(self)}") 136 | 137 | def __len__(self): 138 | return len(self.sequence_list) 139 | 140 | def build_dataset(self): 141 | self.wholedata = {} 142 | 143 | cached_pkl = os.path.join(os.path.dirname(os.path.dirname(self.scene_info_dir)), "processed.pkl") 144 | 145 | if os.path.exists(cached_pkl): 146 | # if you have processed annos, load it 147 | with open(cached_pkl, 'rb') as file: 148 | self.wholedata = pickle.load(file) 149 | else: 150 | for scene in self.scenes: 151 | print(scene) 152 | scene_name = "re10k" + scene 153 | 154 | scene_info_name = os.path.join(self.scene_info_dir, os.path.basename(scene)+".txt") 155 | scene_info = np.loadtxt(scene_info_name, delimiter=' ', dtype=np.float64, skiprows=1) 156 | 157 | filtered_data = [] 158 | 159 | for frame_idx in range(len(scene_info)): 160 | # using try here because some images may be missing 161 | try: 162 | raw_line = scene_info[frame_idx] 163 | timestamp = raw_line[0] 164 | intrinsics = raw_line[1:7] 165 | extrinsics = raw_line[7:] 166 | 167 | imgpath = os.path.join(self.train_dir, scene, '%s'%int(timestamp)+'.png') 168 | image_size = Image.open(imgpath).size 169 | posemat = extrinsics.reshape(3, 4).astype('float64') 170 | focal_length = intrinsics[:2]* image_size 171 | principal_point = intrinsics[2:4]* image_size 172 | 173 | data = { 174 | "filepath": imgpath, 175 | "R": posemat[:3,:3], 176 | "T": posemat[:3,-1], 177 | "focal_length": focal_length, 178 | "principal_point":principal_point, 179 | } 180 | 181 | filtered_data.append(data) 182 | except: 183 | print("this image is missing") 184 | 185 | if len(filtered_data) > self.min_num_images: 186 | self.wholedata[scene_name] = filtered_data 187 | else: 188 | print(f"scene {scene_name} does not have enough image nums") 189 | 190 | print("finished") 191 | 192 | 193 | def _jitter_bbox(self, bbox): 194 | # Random aug to cropping box shape 195 | 196 | bbox = square_bbox(bbox.astype(np.float32)) 197 | s = np.random.uniform(self.jitter_scale[0], self.jitter_scale[1]) 198 | tx, ty = np.random.uniform(self.jitter_trans[0], self.jitter_trans[1], size=2) 199 | 200 | side_length = bbox[2] - bbox[0] 201 | center = (bbox[:2] + bbox[2:]) / 2 + np.array([tx, ty]) * side_length 202 | extent = side_length / 2 * s 203 | 204 | # Final coordinates need to be integer for cropping. 205 | ul = (center - extent).round().astype(int) 206 | lr = ul + np.round(2 * extent).astype(int) 207 | return np.concatenate((ul, lr)) 208 | 209 | def _crop_image(self, image, bbox, white_bg=False): 210 | if white_bg: 211 | # Only support PIL Images 212 | image_crop = Image.new("RGB", (bbox[2] - bbox[0], bbox[3] - bbox[1]), (255, 255, 255)) 213 | image_crop.paste(image, (-bbox[0], -bbox[1])) 214 | else: 215 | image_crop = transforms.functional.crop(image, top=bbox[1], left=bbox[0], height=bbox[3] - bbox[1], width=bbox[2] - bbox[0]) 216 | 217 | return image_crop 218 | 219 | def __getitem__(self, idx_N): 220 | """Fetch item by index and a dynamic variable n_per_seq.""" 221 | 222 | # Different from most pytorch datasets, 223 | # here we not only get index, but also a dynamic variable n_per_seq 224 | # supported by DynamicBatchSampler 225 | 226 | index, n_per_seq = idx_N 227 | sequence_name = self.sequence_list[index] 228 | metadata = self.wholedata[sequence_name] 229 | ids = np.random.choice(len(metadata), n_per_seq, replace=False) 230 | return self.get_data(index=index, ids=ids) 231 | 232 | def get_data(self, index=None, sequence_name=None, ids=(0, 1), no_images=False, return_path=False): 233 | if sequence_name is None: 234 | sequence_name = self.sequence_list[index] 235 | 236 | metadata = self.wholedata[sequence_name] 237 | 238 | assert len(np.unique(ids)) == len(ids) 239 | 240 | annos = [metadata[i] for i in ids] 241 | 242 | if self.sort_by_filename: 243 | annos = sorted(annos, key=lambda x: x["filepath"]) 244 | 245 | images = [] 246 | rotations = [] 247 | translations = [] 248 | focal_lengths = [] 249 | principal_points = [] 250 | image_paths = [] 251 | 252 | for anno in annos: 253 | filepath = anno["filepath"] 254 | image_path = osp.join(self.Re10K_DIR, filepath) 255 | image = Image.open(image_path).convert("RGB") 256 | 257 | images.append(image) 258 | rotations.append(torch.tensor(anno["R"])) 259 | translations.append(torch.tensor(anno["T"])) 260 | image_paths.append(image_path) 261 | 262 | ######## to make the convention of pytorch 3D happy 263 | # Raw FL PP. If you want to use them, uncomment here 264 | # focal_lengths_raw.append(torch.tensor(anno["focal_length"])) 265 | # principal_points_raw.append(torch.tensor(anno["principal_point"])) 266 | 267 | # PT3D FL PP 268 | original_size_wh = np.array(image.size) 269 | scale = min(original_size_wh) / 2 270 | c0 = original_size_wh / 2.0 271 | focal_pytorch3d = anno["focal_length"] / scale 272 | # mirrored principal point 273 | p0_pytorch3d = -(anno["principal_point"] - c0) / scale 274 | focal_lengths.append(torch.tensor(focal_pytorch3d)) 275 | principal_points.append(torch.tensor(p0_pytorch3d)) 276 | ######## 277 | 278 | 279 | crop_parameters = [] 280 | images_transformed = [] 281 | 282 | new_fls = [] 283 | new_pps = [] 284 | 285 | 286 | for i, (anno, image) in enumerate(zip(annos, images)): 287 | w, h = image.width, image.height 288 | 289 | if self.crop_longest: 290 | crop_dim = max(h, w) 291 | top = (h - crop_dim) // 2 292 | left = (w - crop_dim) // 2 293 | bbox = np.array([left, top, left + crop_dim, top + crop_dim]) 294 | elif self.center_box: 295 | crop_dim = min(h, w) 296 | top = (h - crop_dim) // 2 297 | left = (w - crop_dim) // 2 298 | bbox = np.array([left, top, left + crop_dim, top + crop_dim]) 299 | else: 300 | bbox = np.array(anno["bbox"]) 301 | 302 | if self.eval_time: 303 | bbox_jitter = bbox 304 | else: 305 | bbox_jitter = self._jitter_bbox(bbox) 306 | 307 | bbox_xywh = torch.FloatTensor(bbox_xyxy_to_xywh(bbox_jitter)) 308 | 309 | ### cropping images 310 | focal_length_cropped, principal_point_cropped = adjust_camera_to_bbox_crop_( 311 | focal_lengths[i], principal_points[i], torch.FloatTensor(image.size), bbox_xywh 312 | ) 313 | 314 | image = self._crop_image(image, bbox_jitter, white_bg=self.mask_images) 315 | 316 | ### resizing images 317 | new_focal_length, new_principal_point = adjust_camera_to_image_scale_( 318 | focal_length_cropped, 319 | principal_point_cropped, 320 | torch.FloatTensor(image.size), 321 | torch.FloatTensor([self.img_size, self.img_size]) 322 | ) 323 | 324 | images_transformed.append(self.transform(image)) 325 | new_fls.append(new_focal_length) 326 | new_pps.append(new_principal_point) 327 | 328 | crop_center = (bbox_jitter[:2] + bbox_jitter[2:]) / 2 329 | cc = (2 * crop_center / min(h, w)) - 1 330 | crop_width = 2 * (bbox_jitter[2] - bbox_jitter[0]) / min(h, w) 331 | 332 | crop_parameters.append(torch.tensor([-cc[0], -cc[1], crop_width]).float()) 333 | 334 | ################################################################ 335 | images = images_transformed 336 | 337 | new_fls = torch.stack(new_fls) 338 | new_pps = torch.stack(new_pps) 339 | 340 | batchR = torch.cat([torch.tensor(data["R"][None]) for data in annos]) 341 | batchT = torch.cat([torch.tensor(data["T"][None]) for data in annos]) 342 | 343 | # From COLMAP to PT3D 344 | batchR = batchR.clone().permute(0, 2, 1) 345 | batchR[:, :, :2] *= -1 346 | batchT[:, :2] *= -1 347 | 348 | cameras = PerspectiveCameras(focal_length=new_fls.float(), 349 | principal_point=new_pps.float(), 350 | R=batchR.float(), 351 | T=batchT.float()) 352 | 353 | if self.normalize_cameras: 354 | ################################################################################################################ 355 | norm_cameras = normalize_cameras( 356 | cameras, 357 | compute_optical=self.compute_optical, 358 | first_camera=self.first_camera_transform, 359 | normalize_T=True, 360 | ) 361 | if norm_cameras == -1: 362 | print("Error in normalizing cameras: camera scale was 0") 363 | raise RuntimeError 364 | else: 365 | raise NotImplementedError("please normalize cameras") 366 | 367 | 368 | crop_params = torch.stack(crop_parameters) 369 | 370 | 371 | batch = { 372 | "seq_name": sequence_name, 373 | "frame_num": len(metadata), 374 | } 375 | 376 | # Add images 377 | if self.transform is not None: 378 | images = torch.stack(images) 379 | 380 | if self.color_aug and (not self.eval_time): 381 | for augidx in range(len(images)): 382 | if self.erase_aug: 383 | if random.random() < 0.15: 384 | ex, ey, eh, ew, ev = self.rand_erase.get_params(images[augidx], scale=self.rand_erase.scale,ratio=self.rand_erase.ratio, value=[self.rand_erase.value]) 385 | images[augidx] = transforms.functional.erase(images[augidx], ex, ey, eh, ew, ev, self.rand_erase.inplace) 386 | 387 | images[augidx] = self.color_jitter(images[augidx]) 388 | 389 | batch["image"] = images.clamp(0,1) 390 | 391 | batch["R"] = norm_cameras.R 392 | batch["T"] = norm_cameras.T 393 | 394 | batch["fl"] = norm_cameras.focal_length 395 | batch["pp"] = norm_cameras.principal_point 396 | batch["crop_params"] = torch.stack(crop_parameters) 397 | 398 | 399 | if return_path: 400 | return batch, image_paths 401 | 402 | return batch 403 | 404 | 405 | 406 | 407 | def square_bbox(bbox, padding=0.0, astype=None): 408 | """ 409 | Computes a square bounding box, with optional padding parameters. 410 | 411 | Args: 412 | bbox: Bounding box in xyxy format (4,). 413 | 414 | Returns: 415 | square_bbox in xyxy format (4,). 416 | """ 417 | if astype is None: 418 | astype = type(bbox[0]) 419 | bbox = np.array(bbox) 420 | center = (bbox[:2] + bbox[2:]) / 2 421 | extents = (bbox[2:] - bbox[:2]) / 2 422 | s = max(extents) * (1 + padding) 423 | square_bbox = np.array([center[0] - s, center[1] - s, center[0] + s, center[1] + s], dtype=astype) 424 | return square_bbox 425 | 426 | -------------------------------------------------------------------------------- /pose_diffusion/datasets/re10k_test_1800.txt: -------------------------------------------------------------------------------- 1 | 1839244b04a05e5a 2 | 2da38ca64192354f 3 | 4308efab35deb3ec 4 | 06db5bb2465ae58e 5 | 3f265c5edb13f00d 6 | 37d4e43b2b029a80 7 | 30127e00a789ed7c 8 | 1db274e904e3fb07 9 | 1d46e25b06eef337 10 | 0f12b97e0e4c7e21 11 | 007876f71baf453f 12 | 0f59c103684c0437 13 | 133ee6e537353604 14 | 2dd67f5e68c8d72b 15 | 4175cb4c71c984ec 16 | 0ede2c8fbe52c1d3 17 | 13211c9e31fa4b14 18 | 21a23c81331b0027 19 | 2dc3af70d25d3043 20 | 095fb57435b7d890 21 | 07ba5489b56b7d62 22 | 2b4c1f50687b2bcf 23 | 0954f5a326941fb1 24 | 02b59cd60efb924e 25 | 1840ae9e2494443e 26 | 3b3880eb01373479 27 | 16ef89980e2ceef7 28 | 31b8eb8bbebed9b8 29 | 08d5cde674e47324 30 | 2cc8ef9e5319d5d7 31 | 46de062e5ff787c5 32 | 0f5bb0704084e290 33 | 3176f1532a468cbd 34 | 03906f66d3bca71a 35 | 18502a6651367e71 36 | 31262e902165f348 37 | 19694d2dc528d75d 38 | 3824335ebd7a4097 39 | 0807e84457d5ef58 40 | 138bb7b0b25e4669 41 | 4091c41c6909da3b 42 | 068140e09ae5ae8f 43 | 4675ea4e00c2544d 44 | 10fbe4690dec6258 45 | 1da33647873725fd 46 | 44c16554a21aa6af 47 | 2e554e99d045b484 48 | 4b86587ecd3325f4 49 | 4757ffa4f1da98cb 50 | 365fc12b4f33ada3 51 | 36977643258aa392 52 | 1593596b99e2dde9 53 | 2c16104a0ed6c8aa 54 | 1526707312c94a92 55 | 23beb4b246e236fb 56 | 4b41d03353967b40 57 | 318cc6a39c0acc71 58 | 2ab72e30a616dd21 59 | 10c4c9600bfaa4d8 60 | 2347b1e3e70842ac 61 | 1513c8f030f4cbe2 62 | 17a489f3cec39fea 63 | 4cb669bd62a4ffb8 64 | 03b3f603a1001de0 65 | 04fe4ec70781a0e8 66 | 4a0368a338dbde67 67 | 1b881261742799f7 68 | 0e5b9dcdb891b82b 69 | 0a45b99f42fb0ecb 70 | 444d7d8445cd444e 71 | 0ab14ffa7e541b0b 72 | 3c31d7b9f2792ed9 73 | 3776e900791c1553 74 | 0e48b9ee438238f1 75 | 03e756bff92d49dd 76 | 2cf32e2408107ea7 77 | 20e7a3651ec30386 78 | 41aa58e688a04336 79 | 1b88fb5063cd8916 80 | 4ce58504b055463e 81 | 0e3951bf1db22064 82 | 16cd4f1cb2a467c1 83 | 455964aa4ead1e2f 84 | 3d6ed8b43655929b 85 | 1fb651cd12893f99 86 | 4a736d7c30ae9280 87 | 33a93f85d5713a71 88 | 09265a9e57075e7c 89 | 45e81a557d2dd78b 90 | 15ffcb1f98d41218 91 | 3c84329b60bfa7cb 92 | 05ce34e3cd48c449 93 | 1c840c855f0c8421 94 | 3a10eb9788bcdfa1 95 | 403951b5d632b5ab 96 | 0a78dcb828c506f1 97 | 36e94b0ad3a62c7d 98 | 25ad11c04b852de5 99 | 4636beec02aa8dce 100 | 2e64a2d17f9a76f7 101 | 2af206730de6f439 102 | 0cd63c88350eef60 103 | 4c502551adddea8b 104 | 0d5a4dcdf8ec9d36 105 | 3115672ced7c5694 106 | 1bc87f52eba89cc4 107 | 1564900dd040c718 108 | 3b1f9cedfc40b06c 109 | 30bd5b88e47f6d0e 110 | 4b3644bfc6083588 111 | 1efbd8f8949b15e8 112 | 2a3c7ba09ed503d5 113 | 31a642fe4cd1e232 114 | 040a26b288e7bda4 115 | 30b1d229ad4c6353 116 | 4260ca20e2430c67 117 | 145708c0216a06a7 118 | 07559b44fa10672c 119 | 3a0328ed13dd8c8b 120 | 1f4279d98e283206 121 | 3edff71624eac3ee 122 | 428fc13fba69054c 123 | 0c6b149da098b121 124 | 0b211a1457076450 125 | 4043989a4ae95a01 126 | 2d16da80d7e3b64b 127 | 454fc1e32db7cc41 128 | 0286d2ed56e8f107 129 | 3f6c97f1ac96dada 130 | 33a5a85f06fcc77b 131 | 0387ef3895b1393c 132 | 08291107fc9e9849 133 | 1fd5f9af785e6e5c 134 | 37d09bda74c92a93 135 | 330b925cef643b3f 136 | 0c4c5d5f751aabf5 137 | 00ae21ab50209282 138 | 2c6e76b362eed8e4 139 | 45108618c40e26a7 140 | 082087c82daa295d 141 | 407eefe8017f6070 142 | 487e2f9d93c162c9 143 | 460455f96fa1a1d6 144 | 2ac7eecd3cd0252f 145 | 47c88dcfb1134255 146 | 146ef4db9655fd67 147 | 333c649f75c3c7bc 148 | 17a39d87a22ac1ec 149 | 0712476a67734dee 150 | 2e715d4c5bf6c45e 151 | 240e89ef33ff15b6 152 | 3d8d29bf0d9f24a4 153 | 37d0f351f07ee925 154 | 1225476a1221ce08 155 | 2beffa088960f673 156 | 16b667d681f8cf25 157 | 05dec89f80cabf23 158 | 232a0b3133326242 159 | 4c69bf407b142b93 160 | 1fcf851e236dec35 161 | 2a1fed061b29b25b 162 | 40c517d28a412a5a 163 | 29f52c76f269ae48 164 | 06954737f53b8688 165 | 24fea6c2c7caa434 166 | 0c1012a308ee2788 167 | 3dfaa97cd48a0332 168 | 1837b8eebc9c2457 169 | 0d01d4d6c5d5297e 170 | 37a6b3200493fac0 171 | 2dd86d1f9e2d3474 172 | 3e1af0b953407ef7 173 | 452fff658953aab1 174 | 0a5e107e1961d01d 175 | 41c600ed9f88871b 176 | 36f4df3e0a1ade5f 177 | 1be5a5c98a51b1cc 178 | 08c87b4b6b23895f 179 | 289ce0f2b82dcd0e 180 | 196069a792ebbf31 181 | 1e969786d2a8c7a2 182 | 047c29e9138af233 183 | 464d63c227f26d09 184 | 08138c1a3ba1ce8d 185 | 1f084607245e4462 186 | 29927d9ef0b472d7 187 | 0a6680fe6e8e09d7 188 | 2bd7cee1fa9c8996 189 | 154072be49bb3c1d 190 | 043c48135c5e8cc2 191 | 452e9c4e4729ddff 192 | 0c8438d86bb28f7d 193 | 2e4cac06a4f92261 194 | 1fe394077e7c3de0 195 | 46200541f9943d16 196 | 3faac9603907b329 197 | 0e728af85650dcb3 198 | 3c44f7a30e0ad967 199 | 0be9a0dcbfe032f1 200 | 0eadbf8806794990 201 | 07225d96742d2a6c 202 | 47573da5cb0e5e44 203 | 4c4fa41951e37e78 204 | 42cab73e14195475 205 | 2e261d7661282e40 206 | 02b2358ff02d3ce8 207 | 2e86767798c005df 208 | 4aa594c0ad661f28 209 | 3bad929f21fc4336 210 | 10ea3faaa29f4a88 211 | 01497290d8b93a9b 212 | 395f5ae56e94e344 213 | 4a582ee23dd05a8d 214 | 49d5b942442449b4 215 | 3e02762a89de4c7d 216 | 4c791225522d45ba 217 | 252a24d25a1ea81d 218 | 07479835711d6f8d 219 | 0043978734eec081 220 | 374fa34fe701a30d 221 | 4a1d79baac733df7 222 | 0e2653d00e3fc05a 223 | 0c356641df7c72b8 224 | 0b564b685315d8ff 225 | 04e4c841b349bf5c 226 | 3dcdffe3b9c6235b 227 | 2ed398e8368e0c6f 228 | 0a4cf8d9b81b4c6e 229 | 0ff193c92d415b18 230 | 4526c5ac1bfebcfb 231 | 018f7907401f2fef 232 | 46502a6038bd288f 233 | 210fc445b9e254f5 234 | 11ee8f1e1bdf1c2c 235 | 4a7e531e1a35d424 236 | 16a75920e79c3710 237 | 391bc7d21641283e 238 | 362be988d0b68a4c 239 | 223c9627b978d127 240 | 1f925fcf391591df 241 | 47cae76cd53c752a 242 | 3e5f747d06bc84a3 243 | 0bc9fd5c8e50d0ee 244 | 16b3eea75ad753ef 245 | 01e18dbbf22ff263 246 | 00a54225c5cb1913 247 | 365cc620c2fcbb05 248 | 44be029ec85609c5 249 | 2cc330488326fd4f 250 | 06058474f164c53d 251 | 2bff9ec89ca982c9 252 | 0277b87a9c943ed5 253 | 1ca02bf1c0b65675 254 | 44d2532c5b5296a1 255 | 18a86c01aaaaaa8b 256 | 29a09527214b3dc5 257 | 1c36b2b8144d29ec 258 | 1c742e548e8698c9 259 | 430d7b5b77861810 260 | 2e04dfa4a1671292 261 | 3d2fdcb64b0352ff 262 | 4aa1973c40d2eb93 263 | 0b1b293ffb0e2f51 264 | 02b618a34bc12ff9 265 | 086e9118bec887be 266 | 1c29b1d8fd1dab3f 267 | 2ca8e34592e0c415 268 | 1623f47d7e74e848 269 | 4c447e587919525c 270 | 2a3baeaa72b86812 271 | 2b41e71d509a8b8f 272 | 462f64859631a099 273 | 2c9b5e69fe7f0338 274 | 4c144eb40a09a0dd 275 | 0ece034988793847 276 | 2bec33eeeab0bb9d 277 | 0f4f779411b45b6f 278 | 0404d32e97ec1cdb 279 | 4a7f0556fb58a5cc 280 | 39f444469cf39006 281 | 1fcfd8a36e171639 282 | 1f079ee70c21002d 283 | 24895a02057db66e 284 | 2228cda919976437 285 | 14f0b962fabfaae2 286 | 2b625e92f2cf9de4 287 | 3094afb27266ee6a 288 | 3087828bc27bc4c7 289 | 3e4057a188e15ac3 290 | 17519e763b34fe14 291 | 220a9caffae81adc 292 | 0efeb5654da456c6 293 | 3953f37661087a95 294 | 46067fb6d992860e 295 | 0de78cb98105f8c2 296 | 0e00a382b62667c0 297 | 39c662fa32a3b5c6 298 | 4319ddbd5e8f20e1 299 | 0555b07fe6239b4a 300 | 20fba1d53c349851 301 | 236f9dc6456cf32c 302 | 283334520a3f8a43 303 | 477a0a9f77c00480 304 | 0869b66f912b845d 305 | 293e02c7c1fa31a8 306 | 1778e784d47e035e 307 | 2a39e3b6061dd887 308 | 2ebedb0f027df101 309 | 3ced9d0b56769bb7 310 | 33e23b97daf5d9f0 311 | 2cecdd7df86ff8d3 312 | 3fb3327a177a0175 313 | 08f82d3899d6b726 314 | 35d5a242ba40f31f 315 | 498688760312447b 316 | 373ab0a1009e0316 317 | 2c5e21f9f91e2e09 318 | 2e06abf6286040e2 319 | 0bd819cb30a432c6 320 | 12d4ca1236a2cf26 321 | 474afb2d4641a228 322 | 4bae8b3980bf32d0 323 | 0ecf489d873b7f52 324 | 2b4e1061f6415a4a 325 | 00beb03ef95dc637 326 | 0ef68550315f57c7 327 | 3e8363be673dafa2 328 | 36707b9a2d7c344a 329 | 32615afea87b52dd 330 | 46df36a031f50a04 331 | 379884ce61c4daa7 332 | 1e1e13de4ebea05a 333 | 2341162bce213f2e 334 | 0c824455996db331 335 | 2e7ffcba51990c93 336 | 1ebb496a04a1bd76 337 | 0425df3e42ba0de3 338 | 179ff8424ec7ad13 339 | 464e3851f923f8d0 340 | 0f64c5e4fead6cf2 341 | 13c4059da4e56a8c 342 | 283059a56e7f3e75 343 | 0a3b5fb184936a83 344 | 31607cc68ada0108 345 | 332294213fa15c56 346 | 0445459f7afb0f48 347 | 126067199873816a 348 | 11e62395c85c250d 349 | 472e2674ece00632 350 | 2be655d4137e6e29 351 | 00cf0a94235771bb 352 | 28190e57702bcfbc 353 | 15b93cbe9fc5220d 354 | 2a41503583d146ed 355 | 460e2066b64b2a40 356 | 3fd084afa49b6499 357 | 2b1a013698fea3a1 358 | 025192166c704a39 359 | 10e931268a81f228 360 | 08868143749f321b 361 | 0c8c4363e0dca250 362 | 01fa6190cd47d125 363 | 05c57211be152630 364 | 0e2f2538b26a179c 365 | 1d2cab92bdcc1453 366 | 41fafa6144b58c39 367 | 2ff40df261e17697 368 | 457a1ed78b1ddb01 369 | 0b81411e1b5ec798 370 | 15f8a54e4822f355 371 | 1203cc23b881ab8d 372 | 1648b3e7471e3766 373 | 3fc2c221557a205c 374 | 31ee8cabc96b9a62 375 | 14417ac810f2024f 376 | 0ba1cadcb191dc0a 377 | 0972074fece891f2 378 | 07842cc567e9beec 379 | 173a82eeea56aed3 380 | 1f73124222492f1c 381 | 1eab4db6941be725 382 | 166818269e4e2568 383 | 0d5112a7eb22d61a 384 | 3a52947c66de5920 385 | 0baa633d2094d2c1 386 | 000db54a47bd43fe 387 | 24668d960406587f 388 | 04f5153fc5255516 389 | 05a0ad1e2aa632e7 390 | 0b4d5beb7d3bd867 391 | 00a5a2af678f37d5 392 | 0708389923510354 393 | 14bfd05497764243 394 | 453c980210335f26 395 | 193c3bd339eb0a75 396 | 1d704b9365e9c86b 397 | 0f540553fc30f16d 398 | 0ff896ed26db5da3 399 | 0e060f89ae0a469a 400 | 3c9c37132583a3d2 401 | 02ee66b3efbf3b0a 402 | 3fcf6c1b81b14af5 403 | 008cd8c450342e49 404 | 3af43fe8d514f7c5 405 | 24f06a46ea08c03c 406 | 132bbf5a9e9626ce 407 | 30b1dffa5f783ecb 408 | 02f801e372d67cfd 409 | 2a89b2a52cee9f5f 410 | 3f33ed2971149ea0 411 | 45ac5168bda9d3e2 412 | 27f3ccfb3199499e 413 | 12497730f691d00c 414 | 18a3593609eb3269 415 | 25aa5f50072ce7f4 416 | 0ebaed7e3d044bc2 417 | 3c85dbda51f7e9b3 418 | 219825f542e6ee4b 419 | 4560f57598efe5ab 420 | 227c21a8dd87a153 421 | 33a3fc21efdc8547 422 | 1fcc400e42725a95 423 | 29c8267c1d10b23e 424 | 004dd4b46a06e5be 425 | 32db375ab51d77a4 426 | 32a2c04fd8321bbb 427 | 1db5a4df1ab8b8e7 428 | 3d4645318868a4f3 429 | 44f9aac9faaac569 430 | 371c9182ffd46ce3 431 | 13fcc228c40a0e67 432 | 3f11a54c6d0703a0 433 | 1c144e2404c5da89 434 | 2482c4388b32f225 435 | 1c9772d765e0679e 436 | 218ba0fa826d3eea 437 | 3cf461bc6d626ed0 438 | 2e109379f53bb221 439 | 43c329f7c0b40258 440 | 1beb8e6662d36ee6 441 | 388cb2f0ff1a6cad 442 | 4a2d6753676df096 443 | 22d6e3fefb1ee7fe 444 | 46fb9c990b6f8114 445 | 2cd1705407546b72 446 | 05c48ff6535fbf55 447 | 095441304a817fe9 448 | 10ac0ae67d317d11 449 | 329aba411f341398 450 | 072a60bc7e0b0dfd 451 | 31a087ee5b1976da 452 | 446f557155994097 453 | 2f3ec1f2335489d2 454 | 1cd3638cceebed08 455 | 3c5163ede747b187 456 | 06f4bfa5f9d5fe0e 457 | 3b8167415736169c 458 | 30029fac7c5621de 459 | 2a3bd0a2ac422822 460 | 0f61837b9749da34 461 | 406bdec5b68b1a71 462 | 3a86a812a1eaa20e 463 | 41be40880094d8d1 464 | 2b1e0225f0952a09 465 | 06e499374ddafbff 466 | 0c11dbe781b1c11c 467 | 232aee1c62a1cd8b 468 | 414e2bf42ee45cc4 469 | 3260a42ccfba8973 470 | 24ac24abf3057732 471 | 4938177a0e6e2fbe 472 | 45e6fa48ddd00e87 473 | 38615248b52e2834 474 | 17535adda2aa3b90 475 | 0e714d042fa59506 476 | 0ecde93bfa1f08d9 477 | 41649e3e8f9a4be0 478 | 49e48d66787ecb8f 479 | 3b9e04113b202116 480 | 1edc6b95e84127b6 481 | 378cb83947bf2d23 482 | 3b84ac07fd85bb3f 483 | 171b3a4c2f95f981 484 | 316035dd285c5e27 485 | 38f7ba7fd9a83069 486 | 46eccb4820f5a4eb 487 | 3efda95897eb23d1 488 | 233d7cca6c4c628b 489 | 114d9c301b847239 490 | 05330a153a103386 491 | 0c52996355b23d76 492 | 225c5f2cdcd2753c 493 | 1047cc04fa16e0d7 494 | 1f1a76ed6db1dae5 495 | 497364635884d8aa 496 | 2fe5274c4baf665f 497 | 0cda9adbbedd7948 498 | 1be80ff36848e758 499 | 3de41ace235a3a13 500 | 03a78406de1d0993 501 | 2b43428fa1cf1a7e 502 | 3d5114f5d7496cdd 503 | 15d4a976e4e7d3dd 504 | 3232f7457b27dbbc 505 | 375a7fbf80d09c92 506 | 05b77cb7c0f79f0f 507 | 375f9c448cf31ccb 508 | 43759ced44693671 509 | 41f3291d82fc4d93 510 | 00620c2b77518524 511 | 2d524e9324228d6e 512 | 17428a1f23edf411 513 | 2f4bcd593fe37158 514 | 107b78daf075d371 515 | 39b3e7b2a8bf30a6 516 | 16a2c55f96e6aa18 517 | 2cc5f95fbe24ffe5 518 | 382a5736d9134153 519 | 0f18fb6736efb1c2 520 | 4596160a24b1af1f 521 | 14e540cf0ff7ff91 522 | 01eca393f86d37c5 523 | 3f2bf7371b72e40f 524 | 0c4a239e265ae1c3 525 | 3f1c5b36d217d345 526 | 2b3e3c5d30c17bd9 527 | 107d3d674fdebd31 528 | 027c8c3fc3e7d056 529 | 11a680776863b321 530 | 0a6c499522efa0d5 531 | 4b85062505816744 532 | 2f98ee24d3fc43a2 533 | 390ddb7ee9b716ca 534 | 45dce690caec2917 535 | 29f0d7c051d80035 536 | 459a954b63f98d8a 537 | 377769942e6a748d 538 | 18ce480be0ececbd 539 | 08b8b63abbec8780 540 | 4c0ef61c55467706 541 | 4161944d7d592071 542 | 04b580ea1f4df0a5 543 | 424f597efdad3067 544 | 2c55c5a96b50ab36 545 | 45166f266dd609a3 546 | 3ce8f87fcfc988a8 547 | 32d28b7513f873be 548 | 01aaf4ebb084dc16 549 | 3a88be7c404596ad 550 | 1ec6fa3de6fa1bf0 551 | 36d85599a9cbb6b3 552 | 004334c94bbc8bd5 553 | 0d08611c8b251e15 554 | 2b4a934049f932d0 555 | 01866b81c3b90f2c 556 | 24fe21f0a899701f 557 | 39b58270c2e99310 558 | 21005252fe2383ba 559 | 03482c3bd66de195 560 | 0dd9e020b6d9d687 561 | 021575237abe0684 562 | 17a75f0b036c9cf8 563 | 3d394fbabe0e733d 564 | 0cbbc98eec80360a 565 | 0871e5f582cd933c 566 | 2e96b2142fd337dd 567 | 431e6542fde13130 568 | 3cfb6cb5052ce744 569 | 4393f3c42606c573 570 | 1227d00562c106e2 571 | 35b994780d720894 572 | 2b0b2259a7216762 573 | 3ab70559ec30a57c 574 | 36afe96c11a8211c 575 | 0223924f43297881 576 | 145da324f69d1c6b 577 | 04db26572a791881 578 | 02e6fb86b0172f0b 579 | 058c67085c217b96 580 | 48638883e537ccda 581 | 3a9fa6535917a07a 582 | 3af70052616f7fa1 583 | 0bcde26e5a802638 584 | 2c2cfc0ac780a3aa 585 | 19ec130ecea98d5e 586 | 1c7b9f93752085f4 587 | 1fe9f9bc178a1778 588 | 1cca8650a292e7b0 589 | 20d1b02740ef1124 590 | 3b1c57027302837f 591 | 4c943ac66f6c277e 592 | 0553c19e8933374a 593 | 2e35fc35559543f2 594 | 484cfd0b6334c43a 595 | 4b341307a872487c 596 | 1f0e06e4388dd600 597 | 4bdfa30358809038 598 | 42565e9d863220ae 599 | 203c8a4d66c74338 600 | 23df266716914368 601 | 10f9d6f46e438d36 602 | 2fffc623e6a34e23 603 | 23808c0cfcc72e72 604 | 0d0a99d7f22aab71 605 | 10eebcbb9021f437 606 | 01842c6b21e1d679 607 | 11d5f4e7b0b17565 608 | 247bc2e47eb7f6fa 609 | 46e653208e529783 610 | 0b194567eff966f5 611 | 1de1b73fe4d6aa77 612 | 45a00d135c5388fc 613 | 3ed3ffd0ae9c3224 614 | 2060cce4dad6f988 615 | 040a7af97273204f 616 | 4bc1c3a888a8bfba 617 | 1bf668db0194cf83 618 | 1bf4fe9301893904 619 | 2d3e1349898addb6 620 | 22085848f943c2c6 621 | 2c5249093fc26fde 622 | 3858fc4475d10c78 623 | 388b75bd17c5332e 624 | 203b7543bb3387ce 625 | 21b9d476c5a49c63 626 | 4227369e7d0e735a 627 | 39074bca3524418b 628 | 07e8ffa32746c7ce 629 | 20b38e0a985506ba 630 | 12b7562944c06836 631 | 4c169a41e66b6599 632 | 3c44d53659dbe4fe 633 | 471abe46b812be64 634 | 195ca8350ff27f6f 635 | 0db2394602b8b81c 636 | 44138776bdbfe28c 637 | 4ba7caa04cea37b7 638 | 372f324a1f4d6898 639 | 361d722ef5009e09 640 | 41c2ed4944dec77f 641 | 12a70416c92a9483 642 | 1b87f55dec310243 643 | 15d8b7e256ffd066 644 | 241aa9bcdbdc7ac6 645 | 036f135766f38f78 646 | 17dba5fa8138ed92 647 | 21f88a42bf424000 648 | 0a8f10a9a68236f7 649 | 2d60837ef2e52abd 650 | 33c37aeeda88c3bf 651 | 0a0027a48d9ff2ae 652 | 21e97eb0cfbff775 653 | 32eba3e4cfb61f93 654 | 118f563fe2ed4998 655 | 0095ddd83beb3b8d 656 | 49b8f80c849dc341 657 | 088e115752ce9e56 658 | 43db35d743e6be54 659 | 3d2486ac8822da47 660 | 000c3ab189999a83 661 | 037e8191b3985142 662 | 41d8b4350913ca64 663 | 0368abd976e8d82e 664 | 08718fb99eaafea7 665 | 04422a07336e32da 666 | 049a98f70ce9f471 667 | 2f7f2369486cc959 668 | 128bf83073de3ba1 669 | 0326e5c562bdf1b5 670 | 104c9a27980f9bb7 671 | 24548ce6c15bc2cf 672 | 039b153af4fbfba7 673 | 20384754a6e5d1b0 674 | 430c6d1f8676fabc 675 | 1e80fd6e7507e3be 676 | 49d4a7288f6b5dac 677 | 0d7f00ff38b135f7 678 | 024908906fadb408 679 | 25468a86d9cd851e 680 | 45a4515834848010 681 | 0afdc571e4667a44 682 | 11eb4e9eec5048f2 683 | 308681a294d1417b 684 | 27daccf898b206de 685 | 3fe382b2ae6c9361 686 | 368fa2dd830843c7 687 | 23e428c0dc43f046 688 | 483c2b4c67e32c19 689 | 47191aa41a979900 690 | 15ca8e1fce488c19 691 | 0718f733a326d65f 692 | 38d44ebf460ac132 693 | 024152256b6bcac7 694 | 4634124b21b763f6 695 | 0b970b3417969c89 696 | 436a235ed74c3d89 697 | 3f0d9e856d93b8b3 698 | 004e9db3337e8206 699 | 397bbed49e1ee8dc 700 | 4c9030a5917a1328 701 | 40f6d540b9b16531 702 | 3628ec0337eae7be 703 | 24d97ac8e96e7a5e 704 | 12f45658983d380d 705 | 08e076c11a67b54b 706 | 4881a65d7476d6dd 707 | 2ad50852a84faf51 708 | 30abedc6c413510e 709 | 432a9cfcf53ef717 710 | 3d16e4b4719bc256 711 | 31838e9542a906be 712 | 18c6473be3bd827a 713 | 1ef9f5dfa615fafe 714 | 396065ee739ea046 715 | 2b38cc883c900d33 716 | 11337164b772b7c9 717 | 0362399a61c18ad5 718 | 3a488ff3afa463e2 719 | 11357e0934f26aed 720 | 2deacec5c281fbac 721 | 29832cbdb4144601 722 | 05ddd2fee689399c 723 | 4b461c1ec52a3076 724 | 1424acd0007d40b5 725 | 16b8ab24bd231f9a 726 | 2313ea0fb17cbed6 727 | 31c79c843555c2c6 728 | 45f5a75e63afd4a2 729 | 30efdcef9f38568b 730 | 32dc25ef78b564a1 731 | 02c0c9192fb9a6a1 732 | 3b676f25b54dcc1c 733 | 1d8017cad8dc1d56 734 | 420bdc53a6928b32 735 | 2c80f9eb0d3b2bb4 736 | 214bae1626cba843 737 | 304dd8f38dbeef0f 738 | 0af60a9ffd747a1c 739 | 3f89e23583c36441 740 | 06a8196a66e125af 741 | 0aa8646901d156e4 742 | 0a72a3fd46a88ef6 743 | 12e6ba92e82c7ca4 744 | 397037082e6eb839 745 | 23341c3a0b420e54 746 | 08c9e7365f0707a4 747 | 13ac6a6a3a4f5e5d 748 | 3eb718c3170fcd8e 749 | 2d0bd035f7df86b0 750 | 186f18684ed4b516 751 | 0bcef9ed1c18f74d 752 | 30140756550dc38e 753 | 0b3674ffb90b641a 754 | 0e512d350465a63c 755 | 3970859f54703c88 756 | 3e3d858083d20eab 757 | 3d0a0fecfbdada35 758 | 2ea3133861ebde3b 759 | 3f4f553239e96d90 760 | 1150003196de2529 761 | 493bb055f33cb256 762 | 1072aae07584e091 763 | 066ac822934d52d6 764 | 3317c40fd3e0a7b7 765 | 36fb4d41b00581c4 766 | 2eb515bff528d3c3 767 | 2d7d7fb53d960909 768 | 4a1b9fb940541809 769 | 0de8a88480533be6 770 | 38b7a1d23745fbf3 771 | 0871e2540b0a6804 772 | 06d8995be6aa4db6 773 | 3b434b5302dea908 774 | 0f97153fcaafc80d 775 | 28f59b68509ce59d 776 | 03c61595d13e121c 777 | 3185302b2275b009 778 | 3e94e6706fcdccfa 779 | 2d6f9fa00dcee664 780 | 214c597029aebf9f 781 | 2b23144adffe2e49 782 | 4bd922d1e75cc936 783 | 38a9a0f5e76103d2 784 | 0fbe6d76015f75d4 785 | 0190fe72a727c853 786 | 33e5bb3820c171a5 787 | 2cb589c80f31524f 788 | 36bd5d3f3ee292cc 789 | 2fbb94b8cf388ba7 790 | 03cf40616d79cb6a 791 | 227d63ed9a678fb1 792 | 0d4de33c6888a754 793 | 439762b81d4b908c 794 | 1eb03f0e3088edf0 795 | 164e33fbbcb5d223 796 | 0d06be83296cf911 797 | 168e0f4071d2fe58 798 | 06c7c747b4542273 799 | 0588138dfec165a1 800 | 242fc4972c7bb385 801 | 2552fd444d04ef21 802 | 226646771975e6db 803 | 09754b77eeea6dae 804 | 03bcb03930ff1ace 805 | 213286ef58a8c73d 806 | 171403db6cb88926 807 | 4393f3a15ed6fb9a 808 | 29791bf60e718c6b 809 | 31d903660e1647c5 810 | 4bc203e17758f3a0 811 | 1bf49251fdd23cc0 812 | 39c41be5e76c79c1 813 | 44ab295bc3092c28 814 | 3a6dc09185951ae3 815 | 1859766466c069b1 816 | 11dc36548bbd85da 817 | 2075a2388413899d 818 | 4989f6cc2b43d2c6 819 | 02b406d1e5e31d5c 820 | 4c2383e60aaf26cc 821 | 1f0f5e82e9d0f9ee 822 | 4118895a33890c5a 823 | 33288d55dde83e72 824 | 05ef56b2656c9318 825 | 055e4612c1ea70f9 826 | 07b667b34838336d 827 | 308de3d523189c72 828 | 3e8dd5a6930ecb92 829 | 20d86cff490c0c42 830 | 37c99410741fbca5 831 | 2e7b5f8836ab642c 832 | 3fc266558ec5c07b 833 | 002ae53df0e0afe2 834 | 2236eab1c5c86fc4 835 | 36c5e00f55c4f217 836 | 1402fce28722610b 837 | 14cdf4aa7a2de14b 838 | 0beae06611ead92b 839 | 2e2ad99d45033d6a 840 | 0da6a36b24eaf5db 841 | 139055b26734436f 842 | 43fee307c6339b5e 843 | 424980541ccdb10d 844 | 28b23ec38ac5c0b1 845 | 2c04a38ae16197b2 846 | 44d12349e0609ba3 847 | 3115ce06e0160828 848 | 098263de57257005 849 | 202a627de66ad397 850 | 28b06f7087798198 851 | 07d1d5769e8d797c 852 | 220d718317f7a025 853 | 0d1aa0f47c9d2f6d 854 | 3105d330651c2726 855 | 379470c7d22c498a 856 | 47a66fa042406908 857 | 12691b0622a823ba 858 | 4879ebbcfd888f5b 859 | 1e847ebd7cd1174e 860 | 1f56ccfabbfce568 861 | 0e41af1514f92887 862 | 28742766eb882cd2 863 | 1d9254a5cb93d4ff 864 | 17fc81293f337cc3 865 | 246579087204ba8f 866 | 0c2b3463c27c5ac3 867 | 15ac594106229c62 868 | 0d0f4080d36dfc68 869 | 06eba57d1c333a3c 870 | 069597e1fe899530 871 | 401e10a4352fba1c 872 | 020a41f988981396 873 | 16c4484a4093e2f6 874 | 0ab163a1b88f1128 875 | 23a6c9168abdb38e 876 | 4766f2062abaaf74 877 | 15aba05919bae167 878 | 063b857e6470addb 879 | 23099812f662b3ec 880 | 00cfc0ecd345deb4 881 | 03e141d7afac53e4 882 | 0c9c387ae23d090a 883 | 1d1cafa3e27da040 884 | 1ec7e7dce1175aee 885 | 36cb04da872d3bce 886 | 4bdb70500b99c91f 887 | 2b8f367d01df3601 888 | 20e9bf845c4bd9e7 889 | 12dce44829d88985 890 | 3e577a3be646152b 891 | 37ff2186f55b3fd8 892 | 2f5af4b429b2c992 893 | 2ffe00ad70fe9c00 894 | 0e1a7abc82b1afb2 895 | 35eeb3ce1b3dd01d 896 | 3289ad7a811d2348 897 | 22b86e38854ad186 898 | 41a55418bee59b11 899 | 0c0f298ace7c875b 900 | 0ed4c9cac4a615fd 901 | 200ad247448c5577 902 | 195074aeac8bbd76 903 | 443e5a7e679e3e94 904 | 2d6d5e82bda0611c 905 | 485f996ecf360da7 906 | 0d20062086f6d05c 907 | 443b1691d94c1b3a 908 | 2ce75a0f430e2387 909 | 2c596a5abfd67267 910 | 3f9b08ed34ec795a 911 | 4213b6b3b673f9b5 912 | 0992802044bf665d 913 | 450cf402f042bfd2 914 | 00969c45a093d43c 915 | 1d4d81f8629b119e 916 | 413062cf685711e7 917 | 206d9828d717139e 918 | 19a28e5c25feb31f 919 | 00d83c48cb78ec83 920 | 156b422215789c18 921 | 1dd869f66c3c9497 922 | 2e470f6e2c83566a 923 | 37dbb9846f2fdc01 924 | 3f7b6f511421e395 925 | 4be8bf31940bd475 926 | 129b5e9b80cc4b4b 927 | 37a6ae1e1c6eff66 928 | 3d986ec2fd6d210d 929 | 0b79ada01eb45be9 930 | 4200282fe9b4015a 931 | 385f9444d20eb160 932 | 4a653d46e89d4202 933 | 20171db88f887218 934 | 49be5de41d619cb1 935 | 072b4bb46d80484e 936 | 2868a1b43e9eceff 937 | 322d03d487fc0f01 938 | 2afb8a0a98e15155 939 | 32294ad73efca3db 940 | 4cc4c8a8cfa8e944 941 | 4773f5327489d57a 942 | 14f477e7d5af5b91 943 | 233dda1ab6796b0a 944 | 2c02607ae436a9fb 945 | 12e3d03a933c2eb4 946 | 1ce68f950e7cdf8c 947 | 45536907ffef7585 948 | 2a695d52faf1949e 949 | 3cc40f129447cb31 950 | 296c87d370b03f17 951 | 45f2d7abb5fafaa8 952 | 296241182c2df900 953 | 1025622b7f308760 954 | 0b07051f912592d7 955 | 2111cd087a82344b 956 | 4a6a057fc644624e 957 | 3dd211f3865fc234 958 | 389f65e97bd902b2 959 | 3069cc190d78e55e 960 | 33ac471b97ddf5c5 961 | 4803cf5deca2b38a 962 | 2837dd5c75e026f5 963 | 1e2a2be2df033527 964 | 44b13221c50914d9 965 | 1e0f97ec8f5aa374 966 | 12a5cf6bbe330edb 967 | 179a1357a581ad51 968 | 3a79b9aefafb0b8d 969 | 06dcbfe7cd79bb66 970 | 01be77405b16df11 971 | 2d0e6766c725becc 972 | 071e8c0978097efd 973 | 29e0bfbad00f0d5e 974 | 1c5514d49d61bafc 975 | 33a3f65849195eba 976 | 0d76da0fcac26af8 977 | 1ed5e32330ec25e8 978 | 28f5ebf3c3e2fe54 979 | 0b1e61c69c98026b 980 | 00ca5123d8ff6f83 981 | 1d748783383ad977 982 | 0bb99505a71035cc 983 | 103777494841b376 984 | 291bc22350620114 985 | 440b5d1587251680 986 | 146581180e89666d 987 | 22e6c736f2f7227b 988 | 4b6e9a02975ef9a4 989 | 24bcb936908f3a31 990 | 3cb1489b614e5f39 991 | 3eac186b3e7badb2 992 | 36bc6918e9fc837c 993 | 0ac6adb37a92f549 994 | 3b6d8db52c54b174 995 | 2c48ab563b92a1d5 996 | 03f551fc4abedc08 997 | 3349f3089ea84d6b 998 | 4a1920283e3087de 999 | 298c394f21c62ae6 1000 | 30fceb3b40ec062f 1001 | 395caf9235fde098 1002 | 3bb70a92a0d384e1 1003 | 09b92d14a157d130 1004 | 0463d74358aca878 1005 | 47b5d62899ea4869 1006 | 32991f419c96ea0e 1007 | 46ea97f6f3757209 1008 | 41a3a167ea5d9e88 1009 | 4befac16ffdf8489 1010 | 09d860b12f6604cb 1011 | 37697c41773d597a 1012 | 01a2277ee817b310 1013 | 4b7071b34e8cc67d 1014 | 2a7387e017c241a4 1015 | 3ac32347c3ff7d38 1016 | 433742b23712bf06 1017 | 12293b264f68673d 1018 | 24b4e8ff5a9a6439 1019 | 1695a74d194b65c0 1020 | 1bce163cad1e1d20 1021 | 154365ab5a4e067b 1022 | 2a1769dddc1dbf8d 1023 | 0f68374b76390082 1024 | 3e746126204810a4 1025 | 35b15e97674edec3 1026 | 20de87f0b3f2d136 1027 | 297b57a9296052ce 1028 | 3021337b3fbdb2f3 1029 | 1eca36ec55b88fe4 1030 | 1b74274269c75c8d 1031 | 11eb02d24a3241a9 1032 | 0915a60e1ae6a826 1033 | 10c8e54590f715f7 1034 | 0deb1b80eb8481c6 1035 | 0c609c435b1f7114 1036 | 0fb6678e63316201 1037 | 0c25287b812367cd 1038 | 0896b4819e39caf2 1039 | 19267b6a68d2701b 1040 | 10a3511b61f40243 1041 | 3f6e7ee98174056b 1042 | 4c2ed13774ae4613 1043 | 411c4dd047c49cde 1044 | 0e8a52a174610350 1045 | 3e6d44a66c0d7a0a 1046 | 1c7b3ccd5482f834 1047 | 0f7061acbeed50dd 1048 | 32dfef9109202812 1049 | 17d9303ee77c3a3d 1050 | 466150f780ad7b80 1051 | 2b0cbc443e6c3c6d 1052 | 4130aefaca885090 1053 | 0498c9066256055c 1054 | 3d7a1ebc77f683b4 1055 | 389df03f3c2d7291 1056 | 43db8c6515021c01 1057 | 432fb354aa710e62 1058 | 1e9b1dc1c096d68d 1059 | 180542b70f713d5b 1060 | 2d4efa3897a4ba2d 1061 | 0894f0072a8c5fd0 1062 | 21e794f71e31becb 1063 | 1247b2ac5986205b 1064 | 282ad05cb0113543 1065 | 2807e5ac66c140cc 1066 | 0c9cc2f6d62336f1 1067 | 3c5bb5694853c36a 1068 | 3a9c883b11e86530 1069 | 3e034bde9426ae9f 1070 | 30fc5fc78c5a716e 1071 | 11bf4b38f88bfe9b 1072 | 04145b4b73b2d313 1073 | 416f82fdbad68e21 1074 | 12627e75d51b372e 1075 | 0720ee62bf014834 1076 | 15d324ad8ff2dd83 1077 | 1deec169175eb15b 1078 | 4953a5140e2f439b 1079 | 1214f2a11a9fc1ed 1080 | 09e4d5e8eef7b9c1 1081 | 171fdcc554d303d8 1082 | 3172ad0d099430da 1083 | 3f79dc32d575bcdc 1084 | 1defdda324307269 1085 | 418ad7b9e78208cb 1086 | 1fa073504b4facaf 1087 | 31afa3dcf3b737fa 1088 | 0c884aee4b01366f 1089 | 2e15ad61e7078fdb 1090 | 3bceca99e87d64c5 1091 | 422d976591ab629e 1092 | 196ffec2c68cafd5 1093 | 111356766833a7df 1094 | 2cb9869cb05a9a01 1095 | 0122933cf8ab3317 1096 | 3bb1007fcf0e03ff 1097 | 0516a5d959b58cb3 1098 | 01fe225e2f261d1a 1099 | 42b086af2a1e5d98 1100 | 0c788e368d993870 1101 | 33a29a351c1d9800 1102 | 061f829d3dd2e46e 1103 | 2473b5003a95628c 1104 | 0485a8528fa72698 1105 | 03f1781c4cc126e6 1106 | 3b7443b24830d388 1107 | 2defb4625a3ccb54 1108 | 4b7e7de9132f4149 1109 | 249fd0890d439aa9 1110 | 2c6fb46edb748fe5 1111 | 19310eb8261d4bb2 1112 | 4acd145b0c133dca 1113 | 01cf55ae3e378faf 1114 | 3dcef43736468b29 1115 | 0c5ed899789e60ad 1116 | 3172feb32990cf09 1117 | 4500d9faefff3a41 1118 | 48f9cd996f80c34d 1119 | 1e1742072c0b2d6b 1120 | 16ca2db2f920c1b4 1121 | 39987911e4cf003c 1122 | 04ec725465dc5329 1123 | 10002e18d04c3d93 1124 | 2c805f56d92a2e22 1125 | 22b16c2f5af0f3ef 1126 | 4cda491521679291 1127 | 394037c064421c3e 1128 | 2e619c31122ee40f 1129 | 33f242465f51563c 1130 | 18d9631f5eb45b87 1131 | 224e9686747afbb5 1132 | 3916390b35258215 1133 | 4cc48509585e4157 1134 | 00e12e215c028984 1135 | 0951fdaf9d399411 1136 | 3290731e5f908b92 1137 | 3b273cb40c55db95 1138 | 1cdfd3abfcc3a64e 1139 | 094fd37f09dc318c 1140 | 44b2ab5292c06a7e 1141 | 33f4eeb64d0c9c1a 1142 | 322c4bf5043ffd95 1143 | 10c551ef9644ea03 1144 | 392c21ee30b21459 1145 | 298276fb3c0330e5 1146 | 19b0cd79a126e8bd 1147 | 12f6faddcc88bcc5 1148 | 431795f999dc215f 1149 | 39e5f256790c3343 1150 | 15e0783c6b9683be 1151 | 3ea8d9787998f70a 1152 | 2c3a96fb820e1ab7 1153 | 131773f46d989860 1154 | 39f31c4461ede05f 1155 | 3c90ad3bb72adcf8 1156 | 37ce88a12d77382b 1157 | 0134d6a876481ed8 1158 | 3c19657356e9e229 1159 | 2516b6023683fc3e 1160 | 2fdfa70413053b84 1161 | 16da9b4fdfe4883b 1162 | 4bc7ca44cc62b8b1 1163 | 15b902774d67a394 1164 | 20100b779d28b6d5 1165 | 314c584ff3842715 1166 | 1cddaac7be8ecfa5 1167 | 1ca4db19711258a0 1168 | 01a628e2c509b823 1169 | 4422b38e60e3bc2f 1170 | 40f92f1e65a5e1dd 1171 | 02241c9f162966e3 1172 | 312c8e1f1f9f3594 1173 | 0e6f8d0eb4103baf 1174 | 0e5e7fbe8914352c 1175 | 221205ddf59c5156 1176 | 3f5454f2f53e2103 1177 | 446626a2bd617d24 1178 | 16a23081fd821e92 1179 | 374b2f4abac6dbb9 1180 | 188e6f96fa74ebe7 1181 | 4ac044dcaa428723 1182 | 404043fe2f398440 1183 | 2ef881551a7fda22 1184 | 372bb866143b6e35 1185 | 015d8a2a2834d38c 1186 | 3c35b868a8ec3433 1187 | 3d5125567924e37b 1188 | 23d7f14af4b7ba08 1189 | 172c99489b18e0e6 1190 | 3c64a373bc1c53bd 1191 | 1cc1ff58dc89d230 1192 | 4317016b336431be 1193 | 3928ea8b8c134846 1194 | 234271629f7099df 1195 | 154813fc1d6820dc 1196 | 449ee1308d0710b5 1197 | 0e4f56edbc3d8cd7 1198 | 286393e1e797cdad 1199 | 2b4f6fdcabf53d59 1200 | 3af4c4e5a8ced21e 1201 | 36e69606a7599644 1202 | 0196dedebec3dad2 1203 | 214df1c2863d2959 1204 | 05ac37966de4e7fb 1205 | 3c91fa87850fd1a8 1206 | 42b218cc2f794026 1207 | 3eac742acbd69adf 1208 | 2bcb26e95f5d152b 1209 | 0e2c96cd97e73a38 1210 | 2aca85b3bdf90a09 1211 | 059058768c222bd6 1212 | 15a138312ad94718 1213 | 12dc074fab6ada73 1214 | 46f840365cee9c44 1215 | 0302fcf06bfba582 1216 | 2aa1e311e4bc039b 1217 | 3b0b55657925fb34 1218 | 2864dc6c129cf3cc 1219 | 20f1c3ade1608b4d 1220 | 15d4131d721f1b5e 1221 | 3d2f4958db5aefbf 1222 | 0b55abc1ca2fe909 1223 | 22274e48b847c860 1224 | 0c3d3b45ff4a4326 1225 | 1fef543188ace6e5 1226 | 42f761f7e655bdce 1227 | 39f1b33acc70ad7b 1228 | 05b1462991e38e4d 1229 | 2a4d835a6e023621 1230 | 419b773d04a986b7 1231 | 3015a3eab4b6d042 1232 | 01b08e2f20321127 1233 | 4b0fdb10ae15684b 1234 | 33afdeba3cd5af05 1235 | 369f3639d9605255 1236 | 17c234c2eec050a0 1237 | 42d8b53a15001cd5 1238 | 4ae4456267802484 1239 | 0bd7e6e9f0185aa3 1240 | 125c92c36a04a68a 1241 | 167c2e0c6e9ffa5d 1242 | 4949361d0831c838 1243 | 01a5cc3805e94c21 1244 | 04c441c7ce273dcc 1245 | 47d9493675e58f3b 1246 | 497f507a5901bf4b 1247 | 2276982dbc5a23e2 1248 | 2a8ef9e44f580d13 1249 | 0ed7ecf45f945ead 1250 | 321911ae4a16f038 1251 | 24ad46fd2b26b208 1252 | 4c7df9d3840b2d63 1253 | 2ddca94aefd55b8f 1254 | 0181b66a65650830 1255 | 22a0db80d91128e4 1256 | 2a2d971fd44ae258 1257 | 084fe29cd9d008db 1258 | 339c95e2e709d044 1259 | 0c8b534612a0a776 1260 | 1f2153b5fb50d41a 1261 | 4c8fd5318ae8d467 1262 | 33baf3e18e5d7256 1263 | 24d1c4f7497f8e77 1264 | 36fc018c7b62b997 1265 | 234c14a79d4da1ff 1266 | 039cc34e9cdbcf8f 1267 | 177d39d72e983b69 1268 | 3fdaa028b8baad4a 1269 | 2e9d6a76c40b707f 1270 | 3d8d753f0851bf3b 1271 | 27f772c12c97b594 1272 | 3ebaecd85db14943 1273 | 1a04733e4ee45c90 1274 | 0e01be9445403642 1275 | 4ade6d5fe4b32738 1276 | 3cc4c306db84c6fe 1277 | 1b747b8eba6f7b45 1278 | 052430ff6e2c07c4 1279 | 053e78d3134437a5 1280 | 20b541350492e3ad 1281 | 1fd615fea825fe87 1282 | 0d82dba8f137e3da 1283 | 49c9324758b5e867 1284 | 05bacb6d6a4741b0 1285 | 007b4ae7c05f2ea2 1286 | 3cc97c3d778975f2 1287 | 0d4b941f4678267b 1288 | 06ca8f480c91e9eb 1289 | 42f700b22cb0be39 1290 | 4571dd58b16ba385 1291 | 14a5b002ce46d4d3 1292 | 09c1b7a0876c08df 1293 | 2b1da1fbe7f18f7e 1294 | 3784777516b00247 1295 | 2ceec371086f5d82 1296 | 090c672e7e394397 1297 | 285aef90afaaf565 1298 | 37a960afb176c485 1299 | 2e715b2e0162f768 1300 | 0720bead0cc7cbbf 1301 | 1d36cd02a549e244 1302 | 4b5a6dd314bebe88 1303 | 0a5eeb4466dd19bb 1304 | 1512ea7a9754ac34 1305 | 22666111b2180af9 1306 | 09b505bb829c1d12 1307 | 442ad5ba8e834889 1308 | 325ff82707386438 1309 | 0a10d55239d83d99 1310 | 4bf9ef4705f35e8f 1311 | 038137c9569c60eb 1312 | 41210bec1c0c87e8 1313 | 21d9134faec148f2 1314 | 18089956e2be2289 1315 | 0cb83cef3177a006 1316 | 204fc9ff2c7ff92c 1317 | 489f9441d513634e 1318 | 2d39f39fb8254c27 1319 | 45e5fcd5c8978342 1320 | 32cccd10c84b4529 1321 | 117335f5d67368ca 1322 | 46e2ddf094d0c3a1 1323 | 2a8cd9f87b3c9a2b 1324 | 0a9f2831a3e73de8 1325 | 3d584707e2f3ccf3 1326 | 32d2163aa65c0e8a 1327 | 075278a4d0af74f7 1328 | 0145c694b53b120d 1329 | 17ca3b8ad5815b35 1330 | 0f25241e37e16f56 1331 | 48bb743178166598 1332 | 0181d3b41c2cf87c 1333 | 005dd9a58df1ba3c 1334 | 42c2c85060ab5233 1335 | 49cc37f7f96be5a9 1336 | 1e5548d951e91a40 1337 | 3a4d7cbcf0c84668 1338 | 2990bbc008d9fa82 1339 | 3ad4793daf6adc19 1340 | 4828fb60e4a871ee 1341 | 47a76ca10546fe8f 1342 | 477df7ad0c2e7fdb 1343 | 0d6a534d75f20921 1344 | 427c035484d45682 1345 | 49d21e0a4c3eace5 1346 | 3be029de36008afc 1347 | 17f552ef56d85c55 1348 | 49235d402cbb8895 1349 | 3f8d1edf59e70df3 1350 | 0cf444aef3ba16bd 1351 | 02c485bd207116d4 1352 | 02cb3a4fd80ee0cc 1353 | 139a615209ee09ac 1354 | 0f1f245fa1c181ae 1355 | 1be3758972b35151 1356 | 03aa0437e5d62d58 1357 | 2d4e81e66ce80039 1358 | 4221bc1d4aea1a02 1359 | 13686755488a9d51 1360 | 04ef6a410f034514 1361 | 46a4d49d61a86d37 1362 | 03ef5f13e0a30864 1363 | 28e8300e004ab30b 1364 | 3059f523501dfd97 1365 | 0d46043105cf3185 1366 | 482c3e92080f18c5 1367 | 41e428b3c7a16695 1368 | 18e659699338f835 1369 | 0aacb1732fee7a3c 1370 | 237ef12cd69e2aa0 1371 | 31bf989cb15492d4 1372 | 2f8e1946600c65d4 1373 | 494f87170e713843 1374 | 4c2d32a7f2b62657 1375 | 4c96e034f8af77d6 1376 | 306e2b7785657539 1377 | 2ad4ea800caafe09 1378 | 43361dbc0c5a2808 1379 | 487d83675a8d1574 1380 | 2bd43375196ce1a7 1381 | 1b65111d34f57bb0 1382 | 0278b3d8abd9654d 1383 | 3714123f055e06a1 1384 | 4cf74ffa5bfd5904 1385 | 0598fec76ecc7bd6 1386 | 2c52d9d606a3ece2 1387 | 180bf845cc8cada3 1388 | 2245fcd7f76c2ecd 1389 | 41abd737e0228c1a 1390 | 2d62a5d66d6e4931 1391 | 05a6149f1fcee38d 1392 | 3a126bd9702ee8f7 1393 | 291db63458af0613 1394 | 478e22bb4c242aa9 1395 | 1c11709814a1a2f2 1396 | 2b1303680b081ccb 1397 | 32ce9b303717a29d 1398 | 04cb1526cf3c43cc 1399 | 3eb185c04280412d 1400 | 0c209edeb7637dff 1401 | 1c58aa75858147f1 1402 | 4833b3d2a8184313 1403 | 0f7e8bf1137abcac 1404 | 40904cd4b9e0579d 1405 | 3aae131a319acd17 1406 | 4ca952ede2af6578 1407 | 489254d4b26a04c6 1408 | 02679535c5f06a19 1409 | 3a48dfbd2f0977f9 1410 | 04ed1812719e05f0 1411 | 2fe4e7ab61b23c85 1412 | 1798c9640d8875e6 1413 | 329abf340d23b0b9 1414 | 48d4444b94c2a2c0 1415 | 4cbb82a6bab25a0f 1416 | 48614bc62c3acbf8 1417 | 0f7267e7e369b7d6 1418 | 33dbdc4396938ae1 1419 | 398c4688209874c9 1420 | 1433f61e9591ea9b 1421 | 040de715f9303ba5 1422 | 3dba1838ed366ab5 1423 | 0de79d4f3d7a9171 1424 | 33f1be3a9ccf4e4b 1425 | 25b3854e6efb747d 1426 | 182054e13eaf58fc 1427 | 310cbf1c65c52fa7 1428 | 037bdac76bdcd7c6 1429 | 18be7c1b9895691f 1430 | 03fe94a439456692 1431 | 2b121397791014af 1432 | 44095e87bee5475e 1433 | 39424be692a88364 1434 | 1df20a29cdec61a4 1435 | 49c758aa3c35ed86 1436 | 04ca03945611febb 1437 | 134d7e5a74497a82 1438 | 2b81fbc1af01f0ad 1439 | 2e4c69143b09033c 1440 | 0ffd1083a70c6968 1441 | 1930a64d9a119b13 1442 | 14e3fec07ba502d7 1443 | 07d0229847bd7408 1444 | 37b0b70a1a0c25d3 1445 | 18cde229723f22df 1446 | 24d95746999b7f7e 1447 | 47d11d4bee6608e0 1448 | 0f47577ab3441480 1449 | 401a94bd9d84f501 1450 | 259f6f1f002d2d94 1451 | 3ce90c0ea2537c48 1452 | 1ffd3b706d708774 1453 | 1f62165dfec00c3e 1454 | 066c35b1abc706be 1455 | 06c71ce295284689 1456 | 0ecdc87c3391ce98 1457 | 034677cf3d80162d 1458 | 0e7b68884ac4d959 1459 | 05ec2d3e4c027220 1460 | 23dcee801bca67bf 1461 | 0131c9aed0fb3940 1462 | 22552c9a2a2a2ce7 1463 | 4263257ec6099434 1464 | 1c8d34a791deaaf1 1465 | 21b548b570c0b415 1466 | 11491a312c6b8f58 1467 | 482ce5c63038e5b4 1468 | 17d841670d2da942 1469 | 0ef054fbdabce0cd 1470 | 45823117f0acb627 1471 | 0c89e266974e8b90 1472 | 115fa3a1923b7c9f 1473 | 21840c44aed0ae43 1474 | 430d8fece8e0f7e5 1475 | 335794d48a9b168a 1476 | 0757b4bc82bf26b1 1477 | 172bd46c6ddda95d 1478 | 3d6e04af63ebfea4 1479 | 3cfb4c69b14a1970 1480 | 41016527728cae5b 1481 | 195fbef2c08715e9 1482 | 078b8bdc29565cac 1483 | 40284c1baec06ac9 1484 | 3613c77d8c234008 1485 | 3662b5f2916b470c 1486 | 48aff7218b00d843 1487 | 368fae6f3bc0b0f7 1488 | 29460ac4580ea232 1489 | 0068e97c1c1f61aa 1490 | 33f7565ccb685cb7 1491 | 00e8df74b6805da7 1492 | 316ed1b489ff8f40 1493 | 22c8b35c589276c4 1494 | 06ffaa9ffc2eea95 1495 | 3d410da4d7fd9f64 1496 | 3e07add8413f8157 1497 | 2a9d8ba86290db0e 1498 | 375cff10cab07955 1499 | 2da082dfb7a66b4d 1500 | 17a3e731c4fe0aaf 1501 | 4811a66f87b0dd6a 1502 | 1c375830155fe6dc 1503 | 15729869d1862b7f 1504 | 130cbf8f12764687 1505 | 015631b21f792a12 1506 | 12e3b1ad12a752f6 1507 | 040895d45bf4e580 1508 | 0e16d64d961fe855 1509 | 2d5e1c16ba1f89c2 1510 | 10351dc7a37a44c1 1511 | 2cf1a544b179b1a7 1512 | 3bae42d603be2266 1513 | 156f4c7dca878ff2 1514 | 2d3f982ada31489c 1515 | 3a642c6d0e43510b 1516 | 03de2844d3c8314e 1517 | 08072d6cc8e8711d 1518 | 39f03d5fb1807102 1519 | 21cfc9b1266a6bd0 1520 | 311d8515bd115aba 1521 | 0c9d930d226d6bd6 1522 | 0dad7f2ef3496f13 1523 | 12bed927641025a0 1524 | 15bc9eb752c6dcbd 1525 | 2177ca3a775a9ee9 1526 | 289bcb973ed702a1 1527 | 0f2197967bb7fa43 1528 | 33842f4b169e4145 1529 | 397b050e345d73fd 1530 | 0542630de1d734de 1531 | 0752baf20fbc2285 1532 | 3ee30754edfbdb3f 1533 | 0c061512de79b744 1534 | 3eef492bf5120757 1535 | 45a0fe252a89e008 1536 | 36056faee50a621f 1537 | 46b2a13f6ab0be05 1538 | 311db764bdc7f537 1539 | 074653ff3928b9fe 1540 | 2d815b3e5e9bb237 1541 | 21b26eda16f7cb88 1542 | 12cb4aa3f5b59ad6 1543 | 21f3cc00e0cfe8bc 1544 | 088b93f15ca8745d 1545 | 15762acaba295de1 1546 | 3c33566bacd602f2 1547 | 27d163d7046d36b5 1548 | 284efc2041b1d1d8 1549 | 2ad09b7837010330 1550 | 40954e72e02dc771 1551 | 03b440db4696d8e7 1552 | 0ce3839aa5b66e3f 1553 | 47a07f51fd3fef77 1554 | 35e8c6c2168dd087 1555 | 11fcbcdb1dfefb38 1556 | 178cef169356d4a5 1557 | 122cb7d5ea4a99df 1558 | 42742db2633d2eb5 1559 | 077d42bb51ee2793 1560 | 3dc0058dce3828d9 1561 | 44adc8d00568380f 1562 | 0ccb28128213f19d 1563 | 3862411e9bf455cd 1564 | 3eb795302924e912 1565 | 2445756494ef6e3d 1566 | 4c5fd496905b91ce 1567 | 46e0654ccb5d88cf 1568 | 16b48792e910cf49 1569 | 2e4013ea92d04301 1570 | 2ac712ac8d2fd488 1571 | 4ce642bc93f1bb5a 1572 | 3052da93ceeda447 1573 | 2b973e6f676eb243 1574 | 46889bb1803c5cd7 1575 | 4a8f9d6889992fd9 1576 | 424397db4b1cf634 1577 | 0fe2286088ece98e 1578 | 04433dcf217ad9a0 1579 | 37bbec9e46c8c1a9 1580 | 12e985eaa4b79298 1581 | 367446f773123e02 1582 | 397da8e32c2edd65 1583 | 069a4c442912c405 1584 | 1bc87c160d1dc982 1585 | 2c45f54d9432d25c 1586 | 388ed39170b69946 1587 | 1058fe0400a873e0 1588 | 3e1236935a5f70ae 1589 | 2e4c0705bde13d32 1590 | 449c34eaea295942 1591 | 306d435307fef477 1592 | 0d68a05801d48984 1593 | 0c916bcc9351521e 1594 | 35f89d3ac607bd5a 1595 | 1dad44855584a4ea 1596 | 18297e1f8e25d3ee 1597 | 0aa49c0b75e51ba4 1598 | 10f4acad3ed87288 1599 | 0ebb04534d7f2ba7 1600 | 0b04644621e97d30 1601 | 4246a11f0971a231 1602 | 127884736471b631 1603 | 367345ad0b29f8a3 1604 | 46fdfa2a16c7c811 1605 | 01cefee9563f691a 1606 | 1250e369ad1e2fbc 1607 | 1f5df6019b0bb73c 1608 | 22598e2596e6bae7 1609 | 07d449efdb66c20d 1610 | 2326e9820982ad81 1611 | 2e30ae101611cca8 1612 | 40ca76de44a6e1a9 1613 | 281452e730c39fd0 1614 | 33c1bb87a88e59f1 1615 | 45064c8142f3a360 1616 | 2b8778726c1f2fe4 1617 | 44accffce93c7e87 1618 | 1735d8d1b4015669 1619 | 2d99ba7951695f79 1620 | 3cc1fcf538c81442 1621 | 3018aa8ad3eb5dca 1622 | 33be1ba5aec86c96 1623 | 2fca5797ae48529b 1624 | 4054d32655ba5eda 1625 | 1ce8503fd200fed2 1626 | 4a763e1b87e495a7 1627 | 1f7770ac5cbb41eb 1628 | 3b59c7d97b900724 1629 | 2b5b5b4f4fc526ba 1630 | 3e68931874661724 1631 | 3fe783b9c7c8f492 1632 | 0722819bdf5b2737 1633 | 20f8c6738e22e764 1634 | 48e49bdd1aa706e1 1635 | 3b650a9e2ebdfde2 1636 | 0667d5bedfdbc555 1637 | 33517e9838fe5f20 1638 | 227e06087cbffb2b 1639 | 0f5c5385dbcd96df 1640 | 0bf152ef84195293 1641 | 296ec0d98f4d4151 1642 | 293c7c1ccaa6861a 1643 | 31b5667e16de1d94 1644 | 1d04977fb85a8b3c 1645 | 01570ac1c73e9ca2 1646 | 28760a14d0a5ff3a 1647 | 15c9a45c9c3d73f2 1648 | 288dd40199dee268 1649 | 0e101fbd21daf79c 1650 | 2f878176347bcf9e 1651 | 0282160b901229a7 1652 | 1c919c7e4ec601de 1653 | 3a3bc11b9ebb7d44 1654 | 19391676f0fc7982 1655 | 322261824c4a3003 1656 | 2064e46352532375 1657 | 44a85c75cf4a6da8 1658 | 47a1f1f01e2b7be6 1659 | 49f6f14c580b71b9 1660 | 05c423623c9f6f56 1661 | 4214dda1f9a61b1b 1662 | 1e01b910ceba4573 1663 | 3783162ee796a21c 1664 | 0c9b371cc6225682 1665 | 1910e79a60d57aa7 1666 | 2422f760ea77551d 1667 | 2e6876c6c1e40652 1668 | 3f68a1e365e94eb4 1669 | 1bdf9dd7628ddb0b 1670 | 13c510a7403f8231 1671 | 42b88f7ee71a7ba9 1672 | 399cfd9cfacc0499 1673 | 4615277ffb68ca9d 1674 | 2da2eeb966bc0ef8 1675 | 1c1b2e56952040cc 1676 | 20bab82d0268c877 1677 | 0cca84503a86574c 1678 | 45cb862034851efe 1679 | 16930ebf3f0f6b84 1680 | 2a058bafbecaccf9 1681 | 28ed97894371982f 1682 | 4955f54f807ef5aa 1683 | 0fd536fc3c8fdf19 1684 | 20764a96cc70fe46 1685 | 0ed25f15cbccd939 1686 | 37de8da2580d0c1d 1687 | 0932a5bf82eb2f5c 1688 | 0e8995dcbdd22f48 1689 | 0d8fd962cbfc81b7 1690 | 3e00b129b656fbce 1691 | 13adf913ea857ddf 1692 | 22da7610855d6b9d 1693 | 2aab03e1aec0222c 1694 | 0d22ced53b1db7d3 1695 | 3b122e1becb5fcb7 1696 | 3f45b8234504020c 1697 | 15bc7fa1ed5567cc 1698 | 11e9cb1ccb9abe9f 1699 | 150b45a39c57623d 1700 | 309bed43e4406d72 1701 | 14cf1f92ca13d605 1702 | 3984d005557cbd6f 1703 | 0c24590c68af865f 1704 | 144b95c0c3fbe3b0 1705 | 2945a940639798ce 1706 | 1515d37824dd6b22 1707 | 495f50f0997e986e 1708 | 48b1808c546c7e87 1709 | 075654f497170f90 1710 | 2ccd2d98696c87e0 1711 | 144c2c2c52734f15 1712 | 15c28f4ada02cf91 1713 | 3164f4c30188d403 1714 | 389851bf0ac38227 1715 | 4c76898a3d535741 1716 | 0f620bfafa25fcf5 1717 | 28c9d20b865f5d56 1718 | 069a9416dc6a373b 1719 | 28a5318660ab60ba 1720 | 10ad4fc499c48b38 1721 | 3b67613d97aac1df 1722 | 118ffef2ad3950f5 1723 | 3c83e9817c9e022d 1724 | 3c054be9bdb304ee 1725 | 1dcd8aee9a39a61a 1726 | 2f18f5579583e648 1727 | 0539bcecbc483dfe 1728 | 3814a3a8046c8af3 1729 | 1fa7929f55b1fdd5 1730 | 0f6206df8a8e440a 1731 | 0017ce4c6a39d122 1732 | 16a7b5a41f31feb8 1733 | 0de6bc7da518fcae 1734 | 1d125b16063c96c4 1735 | 2c7e35b25a2e4b8b 1736 | 329177dabfe2951d 1737 | 45122648522d4180 1738 | 2d8f1ccdb70c156a 1739 | 1c73def8a62301a9 1740 | 2cd27189549897bf 1741 | 49ec7608e51f7ee2 1742 | 486970d685c0b746 1743 | 0583c7a746238f79 1744 | 06a2911e9add96c6 1745 | 3b9420585a1e66fc 1746 | 1692fab166811028 1747 | 3aaed2e6422d7d57 1748 | 20422596003ac855 1749 | 0ed8b86b87a30d38 1750 | 075f0d808a621ae2 1751 | 48a3049cabb54c0d 1752 | 0c9ea3bf67254e95 1753 | 022a21a897f2a904 1754 | 3d60041ab79f46fc 1755 | 21a6081709444ebf 1756 | 24598987691df957 1757 | 41936ce6152fee64 1758 | 4a177be7db12edcd 1759 | 225ae1d37a7fa519 1760 | 4b4c0c27204604a3 1761 | 4b5619958277861f 1762 | 16ed45d1ce9017df 1763 | 061e49ba3a5386c7 1764 | 19c6e67783781a43 1765 | 3b58206d99feb4b7 1766 | 31812ed5877b73ab 1767 | 0b530eea368f626e 1768 | 37ff932a6a608c24 1769 | 2a30c5309018e00f 1770 | 42c75d578535b0fe 1771 | 177ff3969577b8de 1772 | 0761e0a3a4e25c53 1773 | 473e6ec61583d90f 1774 | 42cc82972397863b 1775 | 4908fab97c9bcec7 1776 | 04e2be0415136fa9 1777 | 0cdfa29561cb24e0 1778 | 23cfdadab7cc51a1 1779 | 474d403238a41315 1780 | 20d0e788abca4aa9 1781 | 0bb7da710cbf4bb9 1782 | 4ace59951acbae4e 1783 | 2d29ff162920db5e 1784 | 2f25826f0d0ef09a 1785 | 0b429a4733089487 1786 | 45592a7f307bccd0 1787 | 280443260e3dced9 1788 | 4905bc8817511dd2 1789 | 338ff9f6c02a6a40 1790 | 44b78f9fcb5cd8d8 1791 | 4bc47dc7f8781812 1792 | 2217c43ddaa29027 1793 | 0aa284f8166e19e4 1794 | 1d7af31482baf61c 1795 | 01cf2d900cb03afb 1796 | 46c9e2d86e7d4c41 1797 | 4a566b7e6eeaf9b4 1798 | 166bb958d7f4798a 1799 | 01f7915dce639515 1800 | 1259726fc1f8e966 1801 | 314c56aad151508d 1802 | 17d35e133dc3ce90 1803 | 223b5e20753d4fa3 1804 | 1cbab4f69b2d48ce 1805 | 464d97e527dd5f8a 1806 | 093b1fe6bc1fd024 1807 | 2007d4829b187feb 1808 | 165696025b477097 1809 | 2412e9f45282fd15 1810 | 0ef15055b44649e3 1811 | 4089ef1b1bdb1d36 1812 | 0898d467d34ff7b2 1813 | 4a046d13e389b505 1814 | 497d2450ed65a678 1815 | 0090cc64d7b7bb24 1816 | 1961bb85524de229 1817 | 2c9018ef57c6b061 1818 | 3dba9cb74bfb79b2 1819 | 168c85ce00de0c6b 1820 | 0c72eaf6bbb7c681 1821 | 384fc7a71b9faca7 1822 | 0a7c052273895bb3 1823 | 4b457a008376cd73 1824 | 021d7121906d6cab 1825 | 228ed4b87c8a6ea7 1826 | 0e241f40ce0cd802 1827 | 3db49ddb3f470436 1828 | 30006eb23f62aa57 1829 | 454197dc5b50b45f 1830 | 020991bdfbdbe504 1831 | 4242fb49c775710c 1832 | 41f438dd19aae981 1833 | -------------------------------------------------------------------------------- /pose_diffusion/demo.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import argparse 8 | import datetime 9 | import glob 10 | import os 11 | import re 12 | import time 13 | from pathlib import Path 14 | import numpy as np 15 | import torch 16 | from typing import Dict, List, Optional, Union 17 | from omegaconf import OmegaConf, DictConfig 18 | import hydra 19 | from hydra.utils import instantiate, get_original_cwd 20 | import models 21 | import time 22 | from functools import partial 23 | from pytorch3d.renderer.cameras import PerspectiveCameras 24 | from pytorch3d.ops import corresponding_cameras_alignment 25 | from pytorch3d.implicitron.tools import model_io, vis_utils 26 | from pytorch3d.vis.plotly_vis import plot_scene 27 | 28 | from util.utils import seed_all_random_engines 29 | from util.match_extraction import extract_match 30 | from util.load_img_folder import load_and_preprocess_images 31 | from util.geometry_guided_sampling import geometry_guided_sampling 32 | from util.metric import compute_ARE 33 | from visdom import Visdom 34 | 35 | 36 | @hydra.main(config_path="../cfgs/", config_name="default") 37 | def demo(cfg: DictConfig) -> None: 38 | OmegaConf.set_struct(cfg, False) 39 | print("Model Config:") 40 | print(OmegaConf.to_yaml(cfg)) 41 | 42 | # Check for GPU availability and set the device 43 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 44 | 45 | # Instantiate the model 46 | model = instantiate(cfg.MODEL, _recursive_=False) 47 | 48 | # Load and preprocess images 49 | original_cwd = get_original_cwd() # Get original working directory 50 | folder_path = os.path.join(original_cwd, cfg.image_folder) 51 | images, image_info = load_and_preprocess_images(folder_path, cfg.image_size) 52 | 53 | # Load checkpoint 54 | ckpt_path = os.path.join(original_cwd, cfg.ckpt) 55 | if os.path.isfile(ckpt_path): 56 | checkpoint = torch.load(ckpt_path, map_location=device) 57 | model.load_state_dict(checkpoint, strict=True) 58 | print(f"Loaded checkpoint from: {ckpt_path}") 59 | else: 60 | raise ValueError(f"No checkpoint found at: {ckpt_path}") 61 | 62 | # Move model and images to the GPU 63 | model = model.to(device) 64 | images = images.to(device) 65 | 66 | # Evaluation Mode 67 | model.eval() 68 | 69 | # Seed random engines 70 | seed_all_random_engines(cfg.seed) 71 | 72 | # Start the timer 73 | start_time = time.time() 74 | 75 | # Perform match extraction 76 | if cfg.GGS.enable: 77 | # Optional TODO: remove the keypoints outside the cropped region? 78 | 79 | kp1, kp2, i12 = extract_match(image_folder_path=folder_path, image_info=image_info) 80 | 81 | if kp1 is not None: 82 | keys = ["kp1", "kp2", "i12", "img_shape"] 83 | values = [kp1, kp2, i12, images.shape] 84 | matches_dict = dict(zip(keys, values)) 85 | 86 | cfg.GGS.pose_encoding_type = cfg.MODEL.pose_encoding_type 87 | GGS_cfg = OmegaConf.to_container(cfg.GGS) 88 | 89 | cond_fn = partial(geometry_guided_sampling, matches_dict=matches_dict, GGS_cfg=GGS_cfg) 90 | print("=====> Sampling with GGS <=====") 91 | else: 92 | cond_fn = None 93 | else: 94 | cond_fn = None 95 | print("=====> Sampling without GGS <=====") 96 | 97 | images = images.unsqueeze(0) 98 | 99 | # Forward 100 | with torch.no_grad(): 101 | # Obtain predicted camera parameters 102 | # pred_cameras is a PerspectiveCameras object with attributes 103 | # pred_cameras.R, pred_cameras.T, pred_cameras.focal_length 104 | 105 | # The poses and focal length are defined as 106 | # NDC coordinate system in 107 | # https://github.com/facebookresearch/pytorch3d/blob/main/docs/notes/cameras.md 108 | predictions = model(image=images, cond_fn=cond_fn, cond_start_step=cfg.GGS.start_step, training=False) 109 | 110 | pred_cameras = predictions["pred_cameras"] 111 | 112 | # Stop the timer and calculate elapsed time 113 | end_time = time.time() 114 | elapsed_time = end_time - start_time 115 | print("Time taken: {:.4f} seconds".format(elapsed_time)) 116 | 117 | # Compute metrics if gt is available 118 | 119 | # Load gt poses 120 | if os.path.exists(os.path.join(folder_path, "gt_cameras.npz")): 121 | gt_cameras_dict = np.load(os.path.join(folder_path, "gt_cameras.npz")) 122 | gt_cameras = PerspectiveCameras( 123 | focal_length=gt_cameras_dict["gtFL"], R=gt_cameras_dict["gtR"], T=gt_cameras_dict["gtT"], device=device 124 | ) 125 | 126 | # 7dof alignment, using Umeyama's algorithm 127 | pred_cameras_aligned = corresponding_cameras_alignment( 128 | cameras_src=pred_cameras, cameras_tgt=gt_cameras, estimate_scale=True, mode="extrinsics", eps=1e-9 129 | ) 130 | 131 | # Compute the absolute rotation error 132 | ARE = compute_ARE(pred_cameras_aligned.R, gt_cameras.R).mean() 133 | print(f"For {folder_path}: the absolute rotation error is {ARE:.6f} degrees.") 134 | else: 135 | print(f"No GT provided. No evaluation conducted.") 136 | 137 | 138 | # Visualization 139 | try: 140 | viz = Visdom() 141 | 142 | cams_show = {"ours_pred": pred_cameras, "ours_pred_aligned": pred_cameras_aligned, "gt_cameras": gt_cameras} 143 | 144 | fig = plot_scene({f"{folder_path}": cams_show}) 145 | 146 | viz.plotlyplot(fig, env="visual", win="cams") 147 | except: 148 | print("Please check your visdom connection") 149 | 150 | 151 | 152 | if __name__ == "__main__": 153 | demo() 154 | -------------------------------------------------------------------------------- /pose_diffusion/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .pose_diffusion_model import PoseDiffusionModel 2 | 3 | 4 | from .denoiser import Denoiser, TransformerEncoderWrapper 5 | from .gaussian_diffuser import GaussianDiffusion 6 | from .image_feature_extractor import MultiScaleImageFeatureExtractor 7 | -------------------------------------------------------------------------------- /pose_diffusion/models/denoiser.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import logging 8 | from collections import defaultdict 9 | from dataclasses import field, dataclass 10 | from typing import Any, Dict, List, Optional, Tuple, Union, Callable 11 | from util.embedding import TimeStepEmbedding, PoseEmbedding 12 | 13 | import torch 14 | import torch.nn as nn 15 | 16 | from hydra.utils import instantiate 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | class Denoiser(nn.Module): 23 | def __init__( 24 | self, 25 | TRANSFORMER: Dict, 26 | target_dim: int = 9, # TODO: reduce fl dim from 2 to 1 27 | pivot_cam_onehot: bool = True, 28 | z_dim: int = 384, 29 | mlp_hidden_dim: bool = 128, 30 | ): 31 | super().__init__() 32 | 33 | self.pivot_cam_onehot = pivot_cam_onehot 34 | self.target_dim = target_dim 35 | 36 | self.time_embed = TimeStepEmbedding() 37 | self.pose_embed = PoseEmbedding(target_dim=self.target_dim) 38 | 39 | first_dim = self.time_embed.out_dim + self.pose_embed.out_dim + z_dim + int(self.pivot_cam_onehot) 40 | 41 | d_model = TRANSFORMER.d_model 42 | self._first = nn.Linear(first_dim, d_model) 43 | 44 | # slightly different from the paper that 45 | # we use 2 encoder layers and 6 decoder layers 46 | # here we use a transformer with 8 encoder layers 47 | # call TransformerEncoderWrapper() to build a encoder-only transformer 48 | self._trunk = instantiate(TRANSFORMER, _recursive_=False) 49 | 50 | # TODO: change the implementation of MLP to a more mature one 51 | self._last = MLP(d_model, [mlp_hidden_dim, self.target_dim], norm_layer=nn.LayerNorm) 52 | 53 | def forward(self, x: torch.Tensor, t: torch.Tensor, z: torch.Tensor): # B x N x dim # B # B x N x dim_z 54 | B, N, _ = x.shape 55 | 56 | t_emb = self.time_embed(t) 57 | # expand t from B x C to B x N x C 58 | t_emb = t_emb.view(B, 1, t_emb.shape[-1]).expand(-1, N, -1) 59 | 60 | x_emb = self.pose_embed(x) 61 | 62 | if self.pivot_cam_onehot: 63 | # add the one hot vector identifying the first camera as pivot 64 | cam_pivot_id = torch.zeros_like(z[..., :1]) 65 | cam_pivot_id[:, 0, ...] = 1.0 66 | z = torch.cat([z, cam_pivot_id], dim=-1) 67 | 68 | feed_feats = torch.cat([x_emb, t_emb, z], dim=-1) 69 | 70 | input_ = self._first(feed_feats) 71 | 72 | feats_ = self._trunk(input_) 73 | 74 | output = self._last(feats_) 75 | 76 | return output 77 | 78 | 79 | def TransformerEncoderWrapper( 80 | d_model: int, 81 | nhead: int, 82 | num_encoder_layers: int, 83 | dim_feedforward: int = 2048, 84 | dropout: float = 0.1, 85 | norm_first: bool = True, 86 | batch_first: bool = True, 87 | ): 88 | encoder_layer = torch.nn.TransformerEncoderLayer( 89 | d_model=d_model, 90 | nhead=nhead, 91 | dim_feedforward=dim_feedforward, 92 | dropout=dropout, 93 | batch_first=batch_first, 94 | norm_first=norm_first, 95 | ) 96 | 97 | _trunk = torch.nn.TransformerEncoder(encoder_layer, num_encoder_layers) 98 | return _trunk 99 | 100 | 101 | class MLP(torch.nn.Sequential): 102 | """This block implements the multi-layer perceptron (MLP) module. 103 | 104 | Args: 105 | in_channels (int): Number of channels of the input 106 | hidden_channels (List[int]): List of the hidden channel dimensions 107 | norm_layer (Callable[..., torch.nn.Module], optional): 108 | Norm layer that will be stacked on top of the convolution layer. 109 | If ``None`` this layer wont be used. Default: ``None`` 110 | activation_layer (Callable[..., torch.nn.Module], optional): 111 | Activation function which will be stacked on top of the 112 | normalization layer (if not None), otherwise on top of the 113 | conv layer. If ``None`` this layer wont be used. 114 | Default: ``torch.nn.ReLU`` 115 | inplace (bool): Parameter for the activation layer, which can 116 | optionally do the operation in-place. Default ``True`` 117 | bias (bool): Whether to use bias in the linear layer. Default ``True`` 118 | dropout (float): The probability for the dropout layer. Default: 0.0 119 | """ 120 | 121 | def __init__( 122 | self, 123 | in_channels: int, 124 | hidden_channels: List[int], 125 | norm_layer: Optional[Callable[..., torch.nn.Module]] = None, 126 | activation_layer: Optional[Callable[..., torch.nn.Module]] = torch.nn.ReLU, 127 | inplace: Optional[bool] = True, 128 | bias: bool = True, 129 | norm_first: bool = False, 130 | dropout: float = 0.0, 131 | ): 132 | # The addition of `norm_layer` is inspired from 133 | # the implementation of TorchMultimodal: 134 | # https://github.com/facebookresearch/multimodal/blob/5dec8a/torchmultimodal/modules/layers/mlp.py 135 | params = {} if inplace is None else {"inplace": inplace} 136 | 137 | layers = [] 138 | in_dim = in_channels 139 | 140 | for hidden_dim in hidden_channels[:-1]: 141 | if norm_first and norm_layer is not None: 142 | layers.append(norm_layer(in_dim)) 143 | 144 | layers.append(torch.nn.Linear(in_dim, hidden_dim, bias=bias)) 145 | 146 | if not norm_first and norm_layer is not None: 147 | layers.append(norm_layer(hidden_dim)) 148 | 149 | layers.append(activation_layer(**params)) 150 | 151 | if dropout > 0: 152 | layers.append(torch.nn.Dropout(dropout, **params)) 153 | 154 | in_dim = hidden_dim 155 | 156 | if norm_first and norm_layer is not None: 157 | layers.append(norm_layer(in_dim)) 158 | 159 | layers.append(torch.nn.Linear(in_dim, hidden_channels[-1], bias=bias)) 160 | if dropout > 0: 161 | layers.append(torch.nn.Dropout(dropout, **params)) 162 | 163 | super().__init__(*layers) 164 | -------------------------------------------------------------------------------- /pose_diffusion/models/gaussian_diffuser.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | # Adapted from https://github.com/lucidrains/denoising-diffusion-pytorch/blob/beb2f2d8dd9b4f2bd5be4719f37082fe061ee450/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py 9 | 10 | import math 11 | import copy 12 | from pathlib import Path 13 | from random import random 14 | from functools import partial 15 | from collections import namedtuple 16 | from multiprocessing import cpu_count 17 | 18 | import torch 19 | from torch import nn, einsum 20 | import torch.nn.functional as F 21 | from torch.utils.data import Dataset, DataLoader 22 | 23 | from torch.optim import Adam 24 | from torchvision import transforms as T, utils 25 | 26 | from einops import rearrange, reduce 27 | from einops.layers.torch import Rearrange 28 | 29 | from tqdm.auto import tqdm 30 | from typing import Any, Dict, List, Optional, Tuple, Union 31 | 32 | # constants 33 | 34 | ModelPrediction = namedtuple("ModelPrediction", ["pred_noise", "pred_x_start"]) 35 | 36 | # helpers functions 37 | 38 | 39 | def exists(x): 40 | return x is not None 41 | 42 | 43 | def default(val, d): 44 | if exists(val): 45 | return val 46 | return d() if callable(d) else d 47 | 48 | 49 | def extract(a, t, x_shape): 50 | b, *_ = t.shape 51 | out = a.gather(-1, t) 52 | return out.reshape(b, *((1,) * (len(x_shape) - 1))) 53 | 54 | 55 | def linear_beta_schedule(timesteps): 56 | scale = 1000 / timesteps 57 | beta_start = scale * 0.0001 58 | beta_end = scale * 0.02 59 | return torch.linspace(beta_start, beta_end, timesteps, dtype=torch.float64) 60 | 61 | 62 | def cosine_beta_schedule(timesteps, s=0.008): 63 | """ 64 | cosine schedule 65 | as proposed in https://openreview.net/forum?id=-NEXDKk8gZ 66 | """ 67 | steps = timesteps + 1 68 | x = torch.linspace(0, timesteps, steps, dtype=torch.float64) 69 | alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * math.pi * 0.5) ** 2 70 | alphas_cumprod = alphas_cumprod / alphas_cumprod[0] 71 | betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1]) 72 | return torch.clip(betas, 0, 0.999) 73 | 74 | 75 | class GaussianDiffusion(nn.Module): 76 | def __init__( 77 | self, 78 | timesteps=100, 79 | sampling_timesteps=None, 80 | beta_1=0.0001, 81 | beta_T=0.1, 82 | loss_type="l1", 83 | objective="pred_noise", 84 | beta_schedule="custom", 85 | p2_loss_weight_gamma=0.0, 86 | p2_loss_weight_k=1, 87 | ): 88 | super().__init__() 89 | 90 | self.objective = objective 91 | 92 | assert objective in { 93 | "pred_noise", 94 | "pred_x0", 95 | }, "objective must be either pred_noise (predict noise) \ 96 | or pred_x0 (predict image start)" 97 | 98 | self.timesteps = timesteps 99 | self.sampling_timesteps = sampling_timesteps 100 | self.beta_1 = beta_1 101 | self.beta_T = beta_T 102 | self.loss_type = loss_type 103 | self.objective = objective 104 | self.beta_schedule = beta_schedule 105 | self.p2_loss_weight_gamma = p2_loss_weight_gamma 106 | self.p2_loss_weight_k = p2_loss_weight_k 107 | 108 | self.init_diff_hyper( 109 | self.timesteps, 110 | self.sampling_timesteps, 111 | self.beta_1, 112 | self.beta_T, 113 | self.loss_type, 114 | self.objective, 115 | self.beta_schedule, 116 | self.p2_loss_weight_gamma, 117 | self.p2_loss_weight_k, 118 | ) 119 | 120 | def init_diff_hyper( 121 | self, 122 | timesteps, 123 | sampling_timesteps, 124 | beta_1, 125 | beta_T, 126 | loss_type, 127 | objective, 128 | beta_schedule, 129 | p2_loss_weight_gamma, 130 | p2_loss_weight_k, 131 | ): 132 | if beta_schedule == "linear": 133 | betas = linear_beta_schedule(timesteps) 134 | elif beta_schedule == "cosine": 135 | betas = cosine_beta_schedule(timesteps) 136 | elif beta_schedule == "custom": 137 | betas = torch.linspace(beta_1, beta_T, timesteps, dtype=torch.float64) 138 | else: 139 | raise ValueError(f"unknown beta schedule {beta_schedule}") 140 | 141 | alphas = 1.0 - betas 142 | alphas_cumprod = torch.cumprod(alphas, axis=0) 143 | alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0) 144 | 145 | (timesteps,) = betas.shape 146 | self.num_timesteps = int(timesteps) 147 | self.loss_type = loss_type 148 | 149 | # sampling related parameters 150 | self.sampling_timesteps = default( 151 | sampling_timesteps, timesteps 152 | ) # default num sampling timesteps to number of timesteps at training 153 | 154 | assert self.sampling_timesteps <= timesteps 155 | 156 | # helper function to register buffer from float64 to float32 157 | register_buffer = lambda name, val: self.register_buffer(name, val.to(torch.float32)) 158 | 159 | register_buffer("betas", betas) 160 | register_buffer("alphas_cumprod", alphas_cumprod) 161 | register_buffer("alphas_cumprod_prev", alphas_cumprod_prev) 162 | 163 | # calculations for diffusion q(x_t | x_{t-1}) and others 164 | register_buffer("sqrt_alphas_cumprod", torch.sqrt(alphas_cumprod)) 165 | register_buffer("sqrt_one_minus_alphas_cumprod", torch.sqrt(1.0 - alphas_cumprod)) 166 | register_buffer("log_one_minus_alphas_cumprod", torch.log(1.0 - alphas_cumprod)) 167 | register_buffer("sqrt_recip_alphas_cumprod", torch.sqrt(1.0 / alphas_cumprod)) 168 | register_buffer("sqrt_recipm1_alphas_cumprod", torch.sqrt(1.0 / alphas_cumprod - 1)) 169 | 170 | # calculations for posterior q(x_{t-1} | x_t, x_0) 171 | posterior_variance = betas * (1.0 - alphas_cumprod_prev) / (1.0 - alphas_cumprod) 172 | 173 | # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) 174 | register_buffer("posterior_variance", posterior_variance) 175 | 176 | # below: log calculation clipped because the posterior variance is 0 177 | # at the beginning of the diffusion chain 178 | register_buffer("posterior_log_variance_clipped", torch.log(posterior_variance.clamp(min=1e-20))) 179 | register_buffer("posterior_mean_coef1", betas * torch.sqrt(alphas_cumprod_prev) / (1.0 - alphas_cumprod)) 180 | register_buffer( 181 | "posterior_mean_coef2", (1.0 - alphas_cumprod_prev) * torch.sqrt(alphas) / (1.0 - alphas_cumprod) 182 | ) 183 | 184 | # calculate p2 reweighting 185 | register_buffer( 186 | "p2_loss_weight", (p2_loss_weight_k + alphas_cumprod / (1 - alphas_cumprod)) ** -p2_loss_weight_gamma 187 | ) 188 | 189 | # helper functions 190 | def predict_start_from_noise(self, x_t, t, noise): 191 | return ( 192 | extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t 193 | - extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise 194 | ) 195 | 196 | def predict_noise_from_start(self, x_t, t, x0): 197 | return (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / extract( 198 | self.sqrt_recipm1_alphas_cumprod, t, x_t.shape 199 | ) 200 | 201 | def q_posterior(self, x_start, x_t, t): 202 | posterior_mean = ( 203 | extract(self.posterior_mean_coef1, t, x_t.shape) * x_start 204 | + extract(self.posterior_mean_coef2, t, x_t.shape) * x_t 205 | ) 206 | 207 | posterior_variance = extract(self.posterior_variance, t, x_t.shape) 208 | posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape) 209 | return (posterior_mean, posterior_variance, posterior_log_variance_clipped) 210 | 211 | def q_sample(self, x_start, t, noise=None): 212 | noise = default(noise, lambda: torch.randn_like(x_start)) 213 | return ( 214 | extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start 215 | + extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise 216 | ) 217 | 218 | def model_predictions(self, x, t, z, x_self_cond=None): 219 | model_output = self.model(x, t, z) 220 | 221 | if self.objective == "pred_noise": 222 | pred_noise = model_output 223 | x_start = self.predict_start_from_noise(x, t, model_output) 224 | 225 | elif self.objective == "pred_x0": 226 | pred_noise = self.predict_noise_from_start(x, t, model_output) 227 | x_start = model_output 228 | 229 | return ModelPrediction(pred_noise, x_start) 230 | 231 | def p_mean_variance( 232 | self, x: torch.Tensor, t: int, z: torch.Tensor, x_self_cond=None, clip_denoised=False # B x N_x x dim 233 | ): 234 | preds = self.model_predictions(x, t, z) 235 | 236 | x_start = preds.pred_x_start 237 | 238 | if clip_denoised: 239 | raise NotImplementedError( 240 | "We don't clip the output because \ 241 | pose does not have a clear bound." 242 | ) 243 | 244 | (model_mean, posterior_variance, posterior_log_variance) = self.q_posterior(x_start=x_start, x_t=x, t=t) 245 | 246 | return model_mean, posterior_variance, posterior_log_variance, x_start 247 | 248 | @torch.no_grad() 249 | def p_sample( 250 | self, 251 | x: torch.Tensor, # B x N_x x dim 252 | t: int, 253 | z: torch.Tensor, 254 | x_self_cond=None, 255 | clip_denoised=False, 256 | cond_fn=None, 257 | cond_start_step=0, 258 | ): 259 | ################################################################################ 260 | # from util.utils import seed_all_random_engines 261 | # seed_all_random_engines(0) 262 | ################################################################################ 263 | 264 | b, *_, device = *x.shape, x.device 265 | batched_times = torch.full((x.shape[0],), t, device=x.device, dtype=torch.long) 266 | model_mean, _, model_log_variance, x_start = self.p_mean_variance( 267 | x=x, t=batched_times, z=z, x_self_cond=x_self_cond, clip_denoised=clip_denoised 268 | ) 269 | 270 | if cond_fn is not None and t < cond_start_step: 271 | # print(model_mean[...,3:7].norm(dim=-1, keepdim=True)) 272 | # tmp = model_mean.clone() 273 | model_mean = cond_fn(model_mean, t) 274 | # diff_norm = torch.norm(tmp-model_mean) 275 | # print(f"the diff norm is {diff_norm}") 276 | noise = 0.0 277 | else: 278 | noise = torch.randn_like(x) if t > 0 else 0.0 # no noise if t == 0 279 | 280 | pred = model_mean + (0.5 * model_log_variance).exp() * noise 281 | 282 | return pred, x_start 283 | 284 | @torch.no_grad() 285 | def p_sample_loop(self, shape, z: torch.Tensor, cond_fn=None, cond_start_step=0): 286 | batch, device = shape[0], self.betas.device 287 | 288 | # Init here 289 | pose = torch.randn(shape, device=device) 290 | 291 | x_start = None 292 | 293 | pose_process = [] 294 | pose_process.append(pose.unsqueeze(0)) 295 | 296 | for t in reversed(range(0, self.num_timesteps)): 297 | pose, _ = self.p_sample(x=pose, t=t, z=z, cond_fn=cond_fn, cond_start_step=cond_start_step) 298 | pose_process.append(pose.unsqueeze(0)) 299 | 300 | return pose, torch.cat(pose_process) 301 | 302 | @torch.no_grad() 303 | def sample(self, shape, z, cond_fn=None, cond_start_step=0): 304 | # TODO: add more variants 305 | sample_fn = self.p_sample_loop 306 | return sample_fn(shape, z=z, cond_fn=cond_fn, cond_start_step=cond_start_step) 307 | 308 | def p_losses(self, x_start, t, z=None, noise=None): 309 | noise = default(noise, lambda: torch.randn_like(x_start)) 310 | # noise sample 311 | x = self.q_sample(x_start=x_start, t=t, noise=noise) 312 | model_out = self.model(x, t, z) 313 | 314 | if self.objective == "pred_noise": 315 | target = noise 316 | x_0_pred = self.predict_start_from_noise(x, t, model_out) 317 | elif self.objective == "pred_x0": 318 | target = x_start 319 | x_0_pred = model_out 320 | else: 321 | raise ValueError(f"unknown objective {self.objective}") 322 | 323 | loss = self.loss_fn(model_out, target, reduction="none") 324 | # loss = reduce(loss, "b ... -> b (...)", "mean") 325 | # loss = loss * extract(self.p2_loss_weight, t, loss.shape) 326 | 327 | return {"loss": loss, "noise": noise, "x_0_pred": x_0_pred, "x_t": x, "t": t} 328 | 329 | def forward(self, pose, z=None, *args, **kwargs): 330 | b = len(pose) 331 | t = torch.randint(0, self.num_timesteps, (b,), device=pose.device).long() 332 | return self.p_losses(pose, t, z=z, *args, **kwargs) 333 | 334 | @property 335 | def loss_fn(self): 336 | if self.loss_type == "l1": 337 | return F.l1_loss 338 | elif self.loss_type == "l2": 339 | return F.mse_loss 340 | else: 341 | raise ValueError(f"invalid loss type {self.loss_type}") 342 | -------------------------------------------------------------------------------- /pose_diffusion/models/image_feature_extractor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import logging 8 | import math 9 | import warnings 10 | from collections import defaultdict 11 | from dataclasses import field, dataclass 12 | from typing import Any, Dict, List, Optional, Tuple, Union, Callable 13 | 14 | 15 | import torch 16 | import torch.nn as nn 17 | import torchvision 18 | 19 | import io 20 | import numpy as np 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | _RESNET_MEAN = [0.485, 0.456, 0.406] 25 | _RESNET_STD = [0.229, 0.224, 0.225] 26 | 27 | 28 | class MultiScaleImageFeatureExtractor(nn.Module): 29 | def __init__(self, modelname: str = "dino_vits16", freeze: bool = False, scale_factors: list = [1, 1 / 2, 1 / 3]): 30 | super().__init__() 31 | self.freeze = freeze 32 | self.scale_factors = scale_factors 33 | 34 | if "res" in modelname: 35 | self._net = getattr(torchvision.models, modelname)(pretrained=True) 36 | self._output_dim = self._net.fc.weight.shape[1] 37 | self._net.fc = nn.Identity() 38 | elif "dinov2" in modelname: 39 | self._net = torch.hub.load("facebookresearch/dinov2", modelname) 40 | self._output_dim = self._net.norm.weight.shape[0] 41 | elif "dino" in modelname: 42 | self._net = torch.hub.load("facebookresearch/dino:main", modelname) 43 | self._output_dim = self._net.norm.weight.shape[0] 44 | else: 45 | raise ValueError(f"Unknown model name {modelname}") 46 | 47 | for name, value in (("_resnet_mean", _RESNET_MEAN), ("_resnet_std", _RESNET_STD)): 48 | self.register_buffer(name, torch.FloatTensor(value).view(1, 3, 1, 1), persistent=False) 49 | 50 | if self.freeze: 51 | for param in self.parameters(): 52 | param.requires_grad = False 53 | 54 | def get_output_dim(self): 55 | return self._output_dim 56 | 57 | def forward(self, image_rgb: torch.Tensor) -> torch.Tensor: 58 | img_normed = self._resnet_normalize_image(image_rgb) 59 | features = self._compute_multiscale_features(img_normed) 60 | return features 61 | 62 | def _resnet_normalize_image(self, img: torch.Tensor) -> torch.Tensor: 63 | return (img - self._resnet_mean) / self._resnet_std 64 | 65 | def _compute_multiscale_features(self, img_normed: torch.Tensor) -> torch.Tensor: 66 | multiscale_features = None 67 | 68 | if len(self.scale_factors) <= 0: 69 | raise ValueError(f"Wrong format of self.scale_factors: {self.scale_factors}") 70 | 71 | for scale_factor in self.scale_factors: 72 | if scale_factor == 1: 73 | inp = img_normed 74 | else: 75 | inp = self._resize_image(img_normed, scale_factor) 76 | 77 | if multiscale_features is None: 78 | multiscale_features = self._net(inp) 79 | else: 80 | multiscale_features += self._net(inp) 81 | 82 | averaged_features = multiscale_features / len(self.scale_factors) 83 | return averaged_features 84 | 85 | @staticmethod 86 | def _resize_image(image: torch.Tensor, scale_factor: float) -> torch.Tensor: 87 | return nn.functional.interpolate(image, scale_factor=scale_factor, mode="bilinear", align_corners=False) 88 | -------------------------------------------------------------------------------- /pose_diffusion/models/pose_diffusion_model.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # This source code is licensed under the BSD-style license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # Standard library imports 7 | import base64 8 | import io 9 | import logging 10 | import math 11 | import pickle 12 | import warnings 13 | from collections import defaultdict 14 | from dataclasses import field, dataclass 15 | from typing import Any, Dict, List, Optional, Tuple, Union 16 | 17 | # Third-party library imports 18 | import cv2 19 | import numpy as np 20 | import torch 21 | import torch.nn as nn 22 | from PIL import Image 23 | 24 | from pytorch3d.renderer.cameras import CamerasBase 25 | from pytorch3d.transforms import se3_exp_map, se3_log_map, Transform3d, so3_relative_angle 26 | from util.camera_transform import pose_encoding_to_camera, camera_to_pose_encoding 27 | 28 | import models 29 | from hydra.utils import instantiate 30 | from pytorch3d.renderer.cameras import PerspectiveCameras 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | class PoseDiffusionModel(nn.Module): 36 | def __init__(self, pose_encoding_type: str, IMAGE_FEATURE_EXTRACTOR: Dict, DIFFUSER: Dict, DENOISER: Dict): 37 | """Initializes a PoseDiffusion model. 38 | 39 | Args: 40 | pose_encoding_type (str): 41 | Defines the encoding type for extrinsics and intrinsics 42 | Currently, only `"absT_quaR_logFL"` is supported - 43 | a concatenation of the translation vector, 44 | rotation quaternion, and logarithm of focal length. 45 | image_feature_extractor_cfg (Dict): 46 | Configuration for the image feature extractor. 47 | diffuser_cfg (Dict): 48 | Configuration for the diffuser. 49 | denoiser_cfg (Dict): 50 | Configuration for the denoiser. 51 | """ 52 | 53 | super().__init__() 54 | 55 | self.pose_encoding_type = pose_encoding_type 56 | 57 | self.image_feature_extractor = instantiate(IMAGE_FEATURE_EXTRACTOR, _recursive_=False) 58 | self.diffuser = instantiate(DIFFUSER, _recursive_=False) 59 | 60 | denoiser = instantiate(DENOISER, _recursive_=False) 61 | self.diffuser.model = denoiser 62 | 63 | self.target_dim = denoiser.target_dim 64 | 65 | self.apply(self._init_weights) 66 | 67 | def _init_weights(self, m): 68 | if isinstance(m, nn.Linear): 69 | nn.init.trunc_normal_(m.weight, std=0.02) 70 | if isinstance(m, nn.Linear) and m.bias is not None: 71 | nn.init.constant_(m.bias, 0) 72 | elif isinstance(m, nn.LayerNorm): 73 | nn.init.constant_(m.bias, 0) 74 | nn.init.constant_(m.weight, 1.0) 75 | 76 | def forward( 77 | self, 78 | image: torch.Tensor, 79 | gt_cameras: Optional[CamerasBase] = None, 80 | sequence_name: Optional[List[str]] = None, 81 | cond_fn=None, 82 | cond_start_step=0, 83 | training=True, 84 | batch_repeat=-1, 85 | ): 86 | """ 87 | Forward pass of the PoseDiffusionModel. 88 | 89 | Args: 90 | image (torch.Tensor): 91 | Input image tensor, Bx3xHxW. 92 | gt_cameras (Optional[CamerasBase], optional): 93 | Camera object. Defaults to None. 94 | sequence_name (Optional[List[str]], optional): 95 | List of sequence names. Defaults to None. 96 | cond_fn ([type], optional): 97 | Conditional function. Wrapper for GGS or other functions. 98 | cond_start_step (int, optional): 99 | The sampling step to start using conditional function. 100 | 101 | Returns: 102 | PerspectiveCameras: PyTorch3D camera object. 103 | """ 104 | 105 | shapelist = list(image.shape) 106 | batch_num = shapelist[0] 107 | frame_num = shapelist[1] 108 | 109 | reshaped_image = image.reshape(batch_num * frame_num, *shapelist[2:]) 110 | z = self.image_feature_extractor(reshaped_image).reshape(batch_num, frame_num, -1) 111 | if training: 112 | pose_encoding = camera_to_pose_encoding(gt_cameras, pose_encoding_type=self.pose_encoding_type) 113 | 114 | if batch_repeat > 0: 115 | pose_encoding = pose_encoding.reshape(batch_num * batch_repeat, -1, self.target_dim) 116 | z = z.repeat(batch_repeat, 1, 1) 117 | else: 118 | pose_encoding = pose_encoding.reshape(batch_num, -1, self.target_dim) 119 | 120 | diffusion_results = self.diffuser(pose_encoding, z=z) 121 | 122 | diffusion_results["pred_cameras"] = pose_encoding_to_camera( 123 | diffusion_results["x_0_pred"], pose_encoding_type=self.pose_encoding_type 124 | ) 125 | 126 | return diffusion_results 127 | else: 128 | B, N, _ = z.shape 129 | 130 | target_shape = [B, N, self.target_dim] 131 | 132 | # sampling 133 | (pose_encoding, pose_encoding_diffusion_samples) = self.diffuser.sample( 134 | shape=target_shape, z=z, cond_fn=cond_fn, cond_start_step=cond_start_step 135 | ) 136 | 137 | # convert the encoded representation to PyTorch3D cameras 138 | pred_cameras = pose_encoding_to_camera(pose_encoding, pose_encoding_type=self.pose_encoding_type) 139 | 140 | diffusion_results = {"pred_cameras": pred_cameras, "z": z} 141 | 142 | return diffusion_results 143 | -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000007.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000012.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000012.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000017.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000017.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000019.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000019.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000024.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000024.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000025.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000025.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000043.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000043.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000052.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000052.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000070.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000070.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000077.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000077.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000085.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000085.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000096.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000096.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000128.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000128.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000145.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000145.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000160.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000160.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000162.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000162.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000168.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000168.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000172.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000172.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000191.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000191.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/frame000200.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/frame000200.jpg -------------------------------------------------------------------------------- /pose_diffusion/samples/apple/gt_cameras.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/samples/apple/gt_cameras.npz -------------------------------------------------------------------------------- /pose_diffusion/test.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import time 9 | from collections import OrderedDict 10 | from functools import partial 11 | from typing import Dict, List, Optional, Union 12 | from multiprocessing import Pool 13 | 14 | 15 | import hydra 16 | import torch 17 | import numpy as np 18 | from hydra.utils import instantiate, get_original_cwd 19 | from accelerate import Accelerator 20 | from omegaconf import DictConfig, OmegaConf 21 | from pytorch3d.ops import corresponding_cameras_alignment 22 | from pytorch3d.renderer.cameras import PerspectiveCameras 23 | from pytorch3d.vis.plotly_vis import plot_scene 24 | 25 | from datasets.co3d_v2 import TRAINING_CATEGORIES, TEST_CATEGORIES, DEBUG_CATEGORIES 26 | from util.match_extraction import extract_match 27 | from util.geometry_guided_sampling import geometry_guided_sampling 28 | from util.metric import camera_to_rel_deg, calculate_auc_np 29 | from util.load_img_folder import load_and_preprocess_images 30 | from util.train_util import ( 31 | get_co3d_dataset_test, 32 | set_seed_and_print, 33 | ) 34 | 35 | 36 | 37 | 38 | @hydra.main(config_path="../cfgs/", config_name="default_test") 39 | def test_fn(cfg: DictConfig): 40 | OmegaConf.set_struct(cfg, False) 41 | accelerator = Accelerator(even_batches=False, device_placement=False) 42 | 43 | # Print configuration and accelerator state 44 | accelerator.print("Model Config:", OmegaConf.to_yaml(cfg), accelerator.state) 45 | 46 | torch.backends.cudnn.benchmark = cfg.test.cudnnbenchmark if not cfg.debug else False 47 | if cfg.debug: 48 | accelerator.print("********DEBUG MODE********") 49 | torch.backends.cudnn.deterministic = True 50 | 51 | set_seed_and_print(cfg.seed) 52 | 53 | # Model instantiation 54 | model = instantiate(cfg.MODEL, _recursive_=False) 55 | model = model.to(accelerator.device) 56 | 57 | # Accelerator setup 58 | model = accelerator.prepare(model) 59 | 60 | if cfg.test.resume_ckpt: 61 | checkpoint = torch.load(cfg.test.resume_ckpt) 62 | try: 63 | model.load_state_dict(prefix_with_module(checkpoint), strict=True) 64 | except: 65 | model.load_state_dict(checkpoint, strict=True) 66 | 67 | accelerator.print(f"Successfully resumed from {cfg.test.resume_ckpt}") 68 | 69 | 70 | categories = cfg.test.category 71 | 72 | if "seen" in categories: 73 | categories = TRAINING_CATEGORIES 74 | 75 | if "unseen" in categories: 76 | categories = TEST_CATEGORIES 77 | 78 | if "debug" in categories: 79 | categories = DEBUG_CATEGORIES 80 | 81 | if "all" in categories: 82 | categories = TRAINING_CATEGORIES + TEST_CATEGORIES 83 | 84 | categories = sorted(categories) 85 | 86 | print("-"*100) 87 | print(f"Testing on {categories}") 88 | print("-"*100) 89 | 90 | category_dict = {} 91 | metric_name = ["Auc_30", "Racc_5", "Racc_15", "Racc_30", "Tacc_5", "Tacc_15", "Tacc_30"] 92 | 93 | for m_name in metric_name: 94 | category_dict[m_name] = {} 95 | 96 | 97 | for category in categories: 98 | print("-"*100) 99 | print(f"Category {category} Start") 100 | 101 | error_dict = _test_one_category( 102 | model = model, 103 | category = category, 104 | cfg = cfg, 105 | num_frames = cfg.test.num_frames, 106 | random_order = cfg.test.random_order, 107 | accelerator = accelerator, 108 | ) 109 | 110 | rError = np.array(error_dict['rError']) 111 | tError = np.array(error_dict['tError']) 112 | 113 | category_dict["Racc_5"][category] = np.mean(rError < 5) * 100 114 | category_dict["Racc_15"][category] = np.mean(rError < 15) * 100 115 | category_dict["Racc_30"][category] = np.mean(rError < 30) * 100 116 | 117 | category_dict["Tacc_5"][category] = np.mean(tError < 5) * 100 118 | category_dict["Tacc_15"][category] = np.mean(tError < 15) * 100 119 | category_dict["Tacc_30"][category] = np.mean(tError < 30) * 100 120 | 121 | Auc_30 = calculate_auc_np(rError, tError, max_threshold=30) 122 | category_dict["Auc_30"][category] = Auc_30 * 100 123 | 124 | print("-"*100) 125 | print(f"Category {category} Done") 126 | 127 | for m_name in metric_name: 128 | category_dict[m_name]["mean"] = np.mean(list(category_dict[m_name].values())) 129 | 130 | for c_name in (categories + ["mean"]): 131 | print_str = f"{c_name.ljust(20)}: " 132 | for m_name in metric_name: 133 | print_num = np.mean(category_dict[m_name][c_name]) 134 | print_str = print_str + f"{m_name} is {print_num:.3f} | " 135 | 136 | if c_name == "mean": 137 | print("-"*100) 138 | print(print_str) 139 | 140 | 141 | return True 142 | 143 | def _test_one_category(model, category, cfg, num_frames, random_order, accelerator): 144 | model.eval() 145 | 146 | print(f"******************************** test on {category} ********************************") 147 | 148 | # Data loading 149 | test_dataset = get_co3d_dataset_test(cfg, category = category) 150 | 151 | category_error_dict = {"rError":[], "tError":[]} 152 | 153 | for seq_name in test_dataset.sequence_list: 154 | print(f"Testing the sequence {seq_name.ljust(15, ' ')} of category {category.ljust(15, ' ')}") 155 | metadata = test_dataset.rotations[seq_name] 156 | 157 | if len(metadata) Sampling with GGS <=====") 188 | else: 189 | cond_fn = None 190 | else: 191 | cond_fn = None 192 | print("=====> Sampling without GGS <=====") 193 | 194 | 195 | translation = batch["T"].to(accelerator.device) 196 | rotation = batch["R"].to(accelerator.device) 197 | fl = batch["fl"].to(accelerator.device) 198 | pp = batch["pp"].to(accelerator.device) 199 | 200 | gt_cameras = PerspectiveCameras( 201 | focal_length=fl.reshape(-1, 2), 202 | R=rotation.reshape(-1, 3, 3), 203 | T=translation.reshape(-1, 3), 204 | device=accelerator.device, 205 | ) 206 | 207 | # expand to 1 x N x 3 x H x W 208 | images = images.unsqueeze(0) 209 | batch_size = len(images) 210 | 211 | with torch.no_grad(): 212 | predictions = model(images, cond_fn=cond_fn, cond_start_step=cfg.GGS.start_step, training=False) 213 | 214 | pred_cameras = predictions["pred_cameras"] 215 | 216 | # compute metrics 217 | rel_rangle_deg, rel_tangle_deg = camera_to_rel_deg(pred_cameras, gt_cameras, accelerator.device, batch_size) 218 | 219 | print(f" -- Pair Rot Error (Deg): {rel_rangle_deg.mean():10.2f}") 220 | print(f" -- Pair Trans Error (Deg): {rel_tangle_deg.mean():10.2f}") 221 | 222 | category_error_dict["rError"].extend(rel_rangle_deg.cpu().numpy()) 223 | category_error_dict["tError"].extend(rel_tangle_deg.cpu().numpy()) 224 | 225 | return category_error_dict 226 | 227 | 228 | def prefix_with_module(checkpoint): 229 | prefixed_checkpoint = OrderedDict() 230 | for key, value in checkpoint.items(): 231 | prefixed_key = "module." + key 232 | prefixed_checkpoint[prefixed_key] = value 233 | return prefixed_checkpoint 234 | 235 | 236 | if __name__ == "__main__": 237 | test_fn() 238 | -------------------------------------------------------------------------------- /pose_diffusion/train.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import time 9 | from collections import OrderedDict 10 | from functools import partial 11 | from typing import Dict, List, Optional, Union 12 | from multiprocessing import Pool 13 | 14 | import hydra 15 | import torch 16 | from hydra.utils import instantiate, get_original_cwd 17 | from accelerate import Accelerator 18 | from omegaconf import DictConfig, OmegaConf 19 | from pytorch3d.renderer.cameras import PerspectiveCameras 20 | from pytorch3d.vis.plotly_vis import plot_scene 21 | from util.metric import camera_to_rel_deg, calculate_auc 22 | from util.train_util import ( 23 | DynamicBatchSampler, 24 | VizStats, 25 | WarmupCosineRestarts, 26 | get_co3d_dataset, 27 | plotly_scene_visualization, 28 | set_seed_and_print, 29 | view_color_coded_images_for_visdom, 30 | ) 31 | 32 | 33 | @hydra.main(config_path="../cfgs/", config_name="default_train") 34 | def train_fn(cfg: DictConfig): 35 | OmegaConf.set_struct(cfg, False) 36 | accelerator = Accelerator(even_batches=False, device_placement=False) 37 | 38 | # Print configuration and accelerator state 39 | accelerator.print("Model Config:", OmegaConf.to_yaml(cfg), accelerator.state) 40 | 41 | torch.backends.cudnn.benchmark = cfg.train.cudnnbenchmark if not cfg.debug else False 42 | if cfg.debug: 43 | accelerator.print("********DEBUG MODE********") 44 | torch.backends.cudnn.deterministic = True 45 | 46 | set_seed_and_print(cfg.seed) 47 | 48 | # Visualization setup 49 | if accelerator.is_main_process: 50 | try: 51 | from visdom import Visdom 52 | 53 | viz = Visdom() 54 | # cams_show = {"ours_pred": pred_cameras, "ours_pred_aligned": pred_cameras_aligned, "gt_cameras": gt_cameras} 55 | # fig = plot_scene({f"{folder_path}": cams_show}) 56 | # viz.plotlyplot(fig, env="visual", win="cams") 57 | except: 58 | print("Warning: please check your visdom connection for visualization") 59 | 60 | # Data loading 61 | dataset, eval_dataset = get_co3d_dataset(cfg) 62 | dataloader = get_dataloader(cfg, dataset) 63 | eval_dataloader = get_dataloader(cfg, eval_dataset, is_eval=True) 64 | 65 | accelerator.print("length of train dataloader is: ", len(dataloader)) 66 | accelerator.print("length of eval dataloader is: ", len(eval_dataloader)) 67 | 68 | # Model instantiation 69 | model = instantiate(cfg.MODEL, _recursive_=False) 70 | model = model.to(accelerator.device) 71 | 72 | # Optimizer and Scheduler 73 | optimizer = torch.optim.AdamW(params=model.parameters(), lr=cfg.train.lr) 74 | 75 | lr_scheduler = WarmupCosineRestarts( 76 | optimizer=optimizer, T_0=cfg.train.restart_num, iters_per_epoch=len(dataloader), warmup_ratio=0.1 77 | ) 78 | # torch.optim.lr_scheduler.OneCycleLR() can achieve similar performance 79 | 80 | # Accelerator setup 81 | model, dataloader, optimizer, lr_scheduler = accelerator.prepare(model, dataloader, optimizer, lr_scheduler) 82 | 83 | start_epoch = 0 84 | if cfg.train.resume_ckpt: 85 | checkpoint = torch.load(cfg.train.resume_ckpt) 86 | try: 87 | model.load_state_dict(prefix_with_module(checkpoint), strict=True) 88 | except: 89 | model.load_state_dict(checkpoint, strict=True) 90 | 91 | accelerator.print(f"Successfully resumed from {cfg.train.resume_ckpt}") 92 | 93 | # metrics to record 94 | stats = VizStats(("loss", "lr", "sec/it", "Auc_30", "Racc_5", "Racc_15", "Racc_30", "Tacc_5", "Tacc_15", "Tacc_30")) 95 | num_epochs = cfg.train.epochs 96 | 97 | for epoch in range(start_epoch, num_epochs): 98 | stats.new_epoch() 99 | 100 | set_seed_and_print(cfg.seed + epoch) 101 | 102 | # Evaluation 103 | if (epoch != 0) and (epoch % cfg.train.eval_interval == 0): 104 | # if (epoch%cfg.train.eval_interval ==0): 105 | accelerator.print(f"----------Start to eval at epoch {epoch}----------") 106 | _train_or_eval_fn( 107 | model, 108 | eval_dataloader, 109 | cfg, 110 | optimizer, 111 | stats, 112 | accelerator, 113 | lr_scheduler, 114 | training=False, 115 | visualize=False, 116 | ) 117 | accelerator.print(f"----------Finish the eval at epoch {epoch}----------") 118 | else: 119 | accelerator.print(f"----------Skip the eval at epoch {epoch}----------") 120 | 121 | # Training 122 | accelerator.print(f"----------Start to train at epoch {epoch}----------") 123 | _train_or_eval_fn( 124 | model, dataloader, cfg, optimizer, stats, accelerator, lr_scheduler, training=True, visualize=False 125 | ) 126 | accelerator.print(f"----------Finish the train at epoch {epoch}----------") 127 | 128 | if accelerator.is_main_process: 129 | lr = lr_scheduler.get_last_lr()[0] 130 | accelerator.print(f"----------LR is {lr}----------") 131 | accelerator.print(f"----------Saving stats to {cfg.exp_name}----------") 132 | stats.update({"lr": lr}, stat_set="train") 133 | stats.plot_stats(viz=viz, visdom_env=cfg.exp_name) 134 | accelerator.print(f"----------Done----------") 135 | 136 | if epoch % cfg.train.ckpt_interval == 0: 137 | accelerator.wait_for_everyone() 138 | ckpt_path = os.path.join(cfg.exp_dir, f"ckpt_{epoch:06}") 139 | accelerator.print(f"----------Saving the ckpt at epoch {epoch} to {ckpt_path}----------") 140 | accelerator.save_state(output_dir=ckpt_path, safe_serialization=False) 141 | 142 | if accelerator.is_main_process: 143 | stats.save(cfg.exp_dir + "stats") 144 | 145 | accelerator.wait_for_everyone() 146 | accelerator.save_state(output_dir=os.path.join(cfg.exp_dir, f"ckpt_{epoch:06}"), safe_serialization=False) 147 | 148 | return True 149 | 150 | 151 | def _train_or_eval_fn( 152 | model, dataloader, cfg, optimizer, stats, accelerator, lr_scheduler, training=True, visualize=False 153 | ): 154 | if training: 155 | model.train() 156 | else: 157 | model.eval() 158 | 159 | time_start = time.time() 160 | max_it = len(dataloader) 161 | 162 | stat_set = "train" if training else "eval" 163 | 164 | for step, batch in enumerate(dataloader): 165 | # data preparation 166 | images = batch["image"].to(accelerator.device) 167 | translation = batch["T"].to(accelerator.device) 168 | rotation = batch["R"].to(accelerator.device) 169 | fl = batch["fl"].to(accelerator.device) 170 | pp = batch["pp"].to(accelerator.device) 171 | 172 | if training and cfg.train.batch_repeat > 0: 173 | # repeat samples by several times 174 | # to accelerate training 175 | br = cfg.train.batch_repeat 176 | gt_cameras = PerspectiveCameras( 177 | focal_length=fl.reshape(-1, 2).repeat(br, 1), 178 | R=rotation.reshape(-1, 3, 3).repeat(br, 1, 1), 179 | T=translation.reshape(-1, 3).repeat(br, 1), 180 | device=accelerator.device, 181 | ) 182 | batch_size = len(images) * br 183 | else: 184 | gt_cameras = PerspectiveCameras( 185 | focal_length=fl.reshape(-1, 2), 186 | R=rotation.reshape(-1, 3, 3), 187 | T=translation.reshape(-1, 3), 188 | device=accelerator.device, 189 | ) 190 | batch_size = len(images) 191 | 192 | if training: 193 | predictions = model(images, gt_cameras=gt_cameras, training=True, batch_repeat=cfg.train.batch_repeat) 194 | predictions["loss"] = predictions["loss"].mean() 195 | loss = predictions["loss"] 196 | else: 197 | with torch.no_grad(): 198 | predictions = model(images, training=False) 199 | 200 | pred_cameras = predictions["pred_cameras"] 201 | 202 | # compute metrics 203 | rel_rangle_deg, rel_tangle_deg = camera_to_rel_deg(pred_cameras, gt_cameras, accelerator.device, batch_size) 204 | 205 | # metrics to report 206 | Racc_5 = (rel_rangle_deg < 5).float().mean() 207 | Racc_15 = (rel_rangle_deg < 15).float().mean() 208 | Racc_30 = (rel_rangle_deg < 30).float().mean() 209 | 210 | Tacc_5 = (rel_tangle_deg < 5).float().mean() 211 | Tacc_15 = (rel_tangle_deg < 15).float().mean() 212 | Tacc_30 = (rel_tangle_deg < 30).float().mean() 213 | 214 | # also called mAA in some literature 215 | Auc_30 = calculate_auc(rel_rangle_deg, rel_tangle_deg, max_threshold=30) 216 | 217 | predictions["Racc_5"] = Racc_5 218 | predictions["Racc_15"] = Racc_15 219 | predictions["Racc_30"] = Racc_30 220 | predictions["Tacc_5"] = Tacc_5 221 | predictions["Tacc_15"] = Tacc_15 222 | predictions["Tacc_30"] = Tacc_30 223 | predictions["Auc_30"] = Auc_30 224 | 225 | if visualize: 226 | # an example if trying to conduct visualization by visdom 227 | frame_num = images.shape[1] 228 | 229 | camera_dict = {"pred_cameras": {}, "gt_cameras": {}} 230 | 231 | for visidx in range(frame_num): 232 | camera_dict["pred_cameras"][visidx] = pred_cameras[visidx] 233 | camera_dict["gt_cameras"][visidx] = gt_cameras[visidx] 234 | 235 | fig = plotly_scene_visualization(camera_dict, frame_num) 236 | viz.plotlyplot(fig, env=cfg.exp_name, win="cams") 237 | 238 | show_img = view_color_coded_images_for_visdom(images[0]) 239 | viz.images(show_img, env=cfg.exp_name, win="imgs") 240 | 241 | stats.update(predictions, time_start=time_start, stat_set=stat_set) 242 | if step % cfg.train.print_interval == 0: 243 | accelerator.print(stats.get_status_string(stat_set=stat_set, max_it=max_it)) 244 | 245 | if training: 246 | optimizer.zero_grad() 247 | accelerator.backward(loss) 248 | if cfg.train.clip_grad > 0 and accelerator.sync_gradients: 249 | accelerator.clip_grad_norm_(model.parameters(), cfg.train.clip_grad) 250 | optimizer.step() 251 | lr_scheduler.step() 252 | 253 | return True 254 | 255 | 256 | def get_dataloader(cfg, dataset, is_eval=False): 257 | """Utility function to get DataLoader.""" 258 | prefix = "eval" if is_eval else "train" 259 | batch_sampler = DynamicBatchSampler( 260 | len(dataset), 261 | dataset_len=getattr(cfg.train, f"len_{prefix}"), 262 | max_images=cfg.train.max_images // (2 if is_eval else 1), 263 | images_per_seq=cfg.train.images_per_seq, 264 | ) 265 | dataloader = torch.utils.data.DataLoader( 266 | dataset, 267 | batch_sampler=batch_sampler, 268 | num_workers=cfg.train.num_workers, 269 | pin_memory=cfg.train.pin_memory, 270 | persistent_workers=cfg.train.persistent_workers, 271 | ) 272 | dataloader.batch_sampler.drop_last = True 273 | dataloader.batch_sampler.sampler = dataloader.batch_sampler 274 | return dataloader 275 | 276 | 277 | def prefix_with_module(checkpoint): 278 | prefixed_checkpoint = OrderedDict() 279 | for key, value in checkpoint.items(): 280 | prefixed_key = "module." + key 281 | prefixed_checkpoint[prefixed_key] = value 282 | return prefixed_checkpoint 283 | 284 | 285 | if __name__ == "__main__": 286 | train_fn() 287 | -------------------------------------------------------------------------------- /pose_diffusion/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/PoseDiffusion/b138198e2891a0f1a1c3435614b9490adb7fd4d6/pose_diffusion/util/__init__.py -------------------------------------------------------------------------------- /pose_diffusion/util/camera_transform.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | 8 | import torch 9 | from pytorch3d.transforms.rotation_conversions import matrix_to_quaternion, quaternion_to_matrix 10 | from pytorch3d.renderer.cameras import CamerasBase, PerspectiveCameras 11 | import numpy as np 12 | 13 | 14 | def bbox_xyxy_to_xywh(xyxy): 15 | wh = xyxy[2:] - xyxy[:2] 16 | xywh = np.concatenate([xyxy[:2], wh]) 17 | return xywh 18 | 19 | 20 | def adjust_camera_to_bbox_crop_(fl, pp, image_size_wh: torch.Tensor, clamp_bbox_xywh: torch.Tensor): 21 | focal_length_px, principal_point_px = _convert_ndc_to_pixels(fl, pp, image_size_wh) 22 | principal_point_px_cropped = principal_point_px - clamp_bbox_xywh[:2] 23 | 24 | focal_length, principal_point_cropped = _convert_pixels_to_ndc( 25 | focal_length_px, principal_point_px_cropped, clamp_bbox_xywh[2:] 26 | ) 27 | 28 | return focal_length, principal_point_cropped 29 | 30 | 31 | def adjust_camera_to_image_scale_(fl, pp, original_size_wh: torch.Tensor, new_size_wh: torch.LongTensor): 32 | focal_length_px, principal_point_px = _convert_ndc_to_pixels(fl, pp, original_size_wh) 33 | 34 | # now scale and convert from pixels to NDC 35 | image_size_wh_output = new_size_wh.float() 36 | scale = (image_size_wh_output / original_size_wh).min(dim=-1, keepdim=True).values 37 | focal_length_px_scaled = focal_length_px * scale 38 | principal_point_px_scaled = principal_point_px * scale 39 | 40 | focal_length_scaled, principal_point_scaled = _convert_pixels_to_ndc( 41 | focal_length_px_scaled, principal_point_px_scaled, image_size_wh_output 42 | ) 43 | return focal_length_scaled, principal_point_scaled 44 | 45 | 46 | def _convert_ndc_to_pixels(focal_length: torch.Tensor, principal_point: torch.Tensor, image_size_wh: torch.Tensor): 47 | half_image_size = image_size_wh / 2 48 | rescale = half_image_size.min() 49 | principal_point_px = half_image_size - principal_point * rescale 50 | focal_length_px = focal_length * rescale 51 | return focal_length_px, principal_point_px 52 | 53 | 54 | def _convert_pixels_to_ndc( 55 | focal_length_px: torch.Tensor, principal_point_px: torch.Tensor, image_size_wh: torch.Tensor 56 | ): 57 | half_image_size = image_size_wh / 2 58 | rescale = half_image_size.min() 59 | principal_point = (half_image_size - principal_point_px) / rescale 60 | focal_length = focal_length_px / rescale 61 | return focal_length, principal_point 62 | 63 | 64 | def pose_encoding_to_camera( 65 | pose_encoding, 66 | pose_encoding_type="absT_quaR_logFL", 67 | log_focal_length_bias=1.8, 68 | min_focal_length=0.1, 69 | max_focal_length=20, 70 | return_dict=False, 71 | ): 72 | """ 73 | Args: 74 | pose_encoding: A tensor of shape `BxNxC`, containing a batch of 75 | `BxN` `C`-dimensional pose encodings. 76 | pose_encoding_type: The type of pose encoding, 77 | only "absT_quaR_logFL" is supported. 78 | """ 79 | 80 | pose_encoding_reshaped = pose_encoding.reshape(-1, pose_encoding.shape[-1]) # Reshape to BNxC 81 | 82 | if pose_encoding_type == "absT_quaR_logFL": 83 | # forced that 3 for absT, 4 for quaR, 2 logFL 84 | # TODO: converted to 1 dim for logFL, consistent with our paper 85 | abs_T = pose_encoding_reshaped[:, :3] 86 | quaternion_R = pose_encoding_reshaped[:, 3:7] 87 | R = quaternion_to_matrix(quaternion_R) 88 | 89 | log_focal_length = pose_encoding_reshaped[:, 7:9] 90 | 91 | # log_focal_length_bias was the hyperparameter 92 | # to ensure the mean of logFL close to 0 during training 93 | # Now converted back 94 | focal_length = (log_focal_length + log_focal_length_bias).exp() 95 | 96 | # clamp to avoid weird fl values 97 | focal_length = torch.clamp(focal_length, min=min_focal_length, max=max_focal_length) 98 | else: 99 | raise ValueError(f"Unknown pose encoding {pose_encoding_type}") 100 | 101 | if return_dict: 102 | return {"focal_length": focal_length, "R": R, "T": abs_T} 103 | 104 | pred_cameras = PerspectiveCameras(focal_length=focal_length, R=R, T=abs_T, device=R.device) 105 | return pred_cameras 106 | 107 | 108 | def camera_to_pose_encoding( 109 | camera, pose_encoding_type="absT_quaR_logFL", log_focal_length_bias=1.8, min_focal_length=0.1, max_focal_length=20 110 | ): 111 | """ """ 112 | 113 | if pose_encoding_type == "absT_quaR_logFL": 114 | # Convert rotation matrix to quaternion 115 | quaternion_R = matrix_to_quaternion(camera.R) 116 | 117 | # Calculate log_focal_length 118 | log_focal_length = ( 119 | torch.log(torch.clamp(camera.focal_length, min=min_focal_length, max=max_focal_length)) 120 | - log_focal_length_bias 121 | ) 122 | 123 | # Concatenate to form pose_encoding 124 | pose_encoding = torch.cat([camera.T, quaternion_R, log_focal_length], dim=-1) 125 | 126 | else: 127 | raise ValueError(f"Unknown pose encoding {pose_encoding_type}") 128 | 129 | return pose_encoding 130 | -------------------------------------------------------------------------------- /pose_diffusion/util/embedding.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | import torch.nn as nn 9 | import math 10 | from pytorch3d.renderer import HarmonicEmbedding 11 | 12 | 13 | class TimeStepEmbedding(nn.Module): 14 | # learned from https://github.com/openai/guided-diffusion/blob/main/guided_diffusion/nn.py 15 | def __init__(self, dim=256, max_period=10000): 16 | super().__init__() 17 | self.dim = dim 18 | self.max_period = max_period 19 | 20 | self.linear = nn.Sequential(nn.Linear(dim, dim // 2), nn.SiLU(), nn.Linear(dim // 2, dim // 2)) 21 | 22 | self.out_dim = dim // 2 23 | 24 | def _compute_freqs(self, half): 25 | freqs = torch.exp(-math.log(self.max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half) 26 | return freqs 27 | 28 | def forward(self, timesteps): 29 | half = self.dim // 2 30 | freqs = self._compute_freqs(half).to(device=timesteps.device) 31 | args = timesteps[:, None].float() * freqs[None] 32 | embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) 33 | if self.dim % 2: 34 | embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) 35 | 36 | output = self.linear(embedding) 37 | return output 38 | 39 | 40 | class PoseEmbedding(nn.Module): 41 | def __init__(self, target_dim, n_harmonic_functions=10, append_input=True): 42 | super().__init__() 43 | 44 | self._emb_pose = HarmonicEmbedding(n_harmonic_functions=n_harmonic_functions, append_input=append_input) 45 | 46 | self.out_dim = self._emb_pose.get_output_dim(target_dim) 47 | 48 | def forward(self, pose_encoding): 49 | e_pose_encoding = self._emb_pose(pose_encoding) 50 | return e_pose_encoding 51 | -------------------------------------------------------------------------------- /pose_diffusion/util/geometry_guided_sampling.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | from typing import Dict, List, Optional, Union 9 | from util.camera_transform import pose_encoding_to_camera 10 | from util.get_fundamental_matrix import get_fundamental_matrices 11 | from pytorch3d.renderer.cameras import CamerasBase, PerspectiveCameras 12 | 13 | 14 | def geometry_guided_sampling(model_mean: torch.Tensor, t: int, matches_dict: Dict, GGS_cfg: Dict): 15 | # pre-process matches 16 | b, c, h, w = matches_dict["img_shape"] 17 | device = model_mean.device 18 | 19 | def _to_device(tensor): 20 | return torch.from_numpy(tensor).to(device) 21 | 22 | kp1 = _to_device(matches_dict["kp1"]) 23 | kp2 = _to_device(matches_dict["kp2"]) 24 | i12 = _to_device(matches_dict["i12"]) 25 | 26 | pair_idx = i12[:, 0] * b + i12[:, 1] 27 | pair_idx = pair_idx.long() 28 | 29 | def _to_homogeneous(tensor): 30 | return torch.nn.functional.pad(tensor, [0, 1], value=1) 31 | 32 | kp1_homo = _to_homogeneous(kp1) 33 | kp2_homo = _to_homogeneous(kp2) 34 | 35 | i1, i2 = [i.reshape(-1) for i in torch.meshgrid(torch.arange(b), torch.arange(b))] 36 | 37 | processed_matches = { 38 | "kp1_homo": kp1_homo, 39 | "kp2_homo": kp2_homo, 40 | "i1": i1, 41 | "i2": i2, 42 | "h": h, 43 | "w": w, 44 | "pair_idx": pair_idx, 45 | } 46 | 47 | # conduct GGS 48 | model_mean = GGS_optimize(model_mean, t, processed_matches, **GGS_cfg) 49 | 50 | # Optimize FL, R, and T separately 51 | model_mean = GGS_optimize( 52 | model_mean, t, processed_matches, update_T=False, update_R=False, update_FL=True, **GGS_cfg 53 | ) # only optimize FL 54 | 55 | model_mean = GGS_optimize( 56 | model_mean, t, processed_matches, update_T=False, update_R=True, update_FL=False, **GGS_cfg 57 | ) # only optimize R 58 | 59 | model_mean = GGS_optimize( 60 | model_mean, t, processed_matches, update_T=True, update_R=False, update_FL=False, **GGS_cfg 61 | ) # only optimize T 62 | 63 | model_mean = GGS_optimize(model_mean, t, processed_matches, **GGS_cfg) 64 | return model_mean 65 | 66 | 67 | def GGS_optimize( 68 | model_mean: torch.Tensor, 69 | t: int, 70 | processed_matches: Dict, 71 | update_R: bool = True, 72 | update_T: bool = True, 73 | update_FL: bool = True, 74 | # the args below come from **GGS_cfg 75 | alpha: float = 0.0001, 76 | learning_rate: float = 1e-2, 77 | iter_num: int = 100, 78 | sampson_max: int = 10, 79 | min_matches: int = 10, 80 | pose_encoding_type: str = "absT_quaR_logFL", 81 | **kwargs, 82 | ): 83 | with torch.enable_grad(): 84 | model_mean.requires_grad_(True) 85 | 86 | if update_R and update_T and update_FL: 87 | iter_num = iter_num * 2 88 | 89 | optimizer = torch.optim.SGD([model_mean], lr=learning_rate, momentum=0.9) 90 | batch_size = model_mean.shape[1] 91 | 92 | for _ in range(iter_num): 93 | valid_sampson, sampson_to_print = compute_sampson_distance( 94 | model_mean, 95 | t, 96 | processed_matches, 97 | update_R=update_R, 98 | update_T=update_T, 99 | update_FL=update_FL, 100 | pose_encoding_type=pose_encoding_type, 101 | sampson_max=sampson_max, 102 | ) 103 | 104 | if min_matches > 0: 105 | valid_match_per_frame = len(valid_sampson) / batch_size 106 | if valid_match_per_frame < min_matches: 107 | print("Drop this pair because of insufficient valid matches") 108 | break 109 | 110 | loss = valid_sampson.mean() 111 | optimizer.zero_grad() 112 | loss.backward() 113 | 114 | grads = model_mean.grad 115 | grad_norm = grads.norm() 116 | grad_mask = (grads.abs() > 0).detach() 117 | model_mean_norm = (model_mean * grad_mask).norm() 118 | 119 | max_norm = alpha * model_mean_norm / learning_rate 120 | 121 | total_norm = torch.nn.utils.clip_grad_norm_(model_mean, max_norm) 122 | optimizer.step() 123 | 124 | print(f"t={t:02d} | sampson={sampson_to_print:05f}") 125 | model_mean = model_mean.detach() 126 | return model_mean 127 | 128 | 129 | def compute_sampson_distance( 130 | model_mean: torch.Tensor, 131 | t: int, 132 | processed_matches: Dict, 133 | update_R=True, 134 | update_T=True, 135 | update_FL=True, 136 | pose_encoding_type: str = "absT_quaR_logFL", 137 | sampson_max: int = 10, 138 | ): 139 | camera = pose_encoding_to_camera(model_mean, pose_encoding_type) 140 | 141 | # pick the mean of the predicted focal length 142 | camera.focal_length = camera.focal_length.mean(dim=0).repeat(len(camera.focal_length), 1) 143 | 144 | if not update_R: 145 | camera.R = camera.R.detach() 146 | 147 | if not update_T: 148 | camera.T = camera.T.detach() 149 | 150 | if not update_FL: 151 | camera.focal_length = camera.focal_length.detach() 152 | 153 | kp1_homo, kp2_homo, i1, i2, he, wi, pair_idx = processed_matches.values() 154 | F_2_to_1 = get_fundamental_matrices(camera, he, wi, i1, i2, l2_normalize_F=False) 155 | F = F_2_to_1.permute(0, 2, 1) # y1^T F y2 = 0 156 | 157 | def _sampson_distance(F, kp1_homo, kp2_homo, pair_idx): 158 | left = torch.bmm(kp1_homo[:, None], F[pair_idx]) 159 | right = torch.bmm(F[pair_idx], kp2_homo[..., None]) 160 | 161 | bottom = left[:, :, 0].square() + left[:, :, 1].square() + right[:, 0, :].square() + right[:, 1, :].square() 162 | top = torch.bmm(left, kp2_homo[..., None]).square() 163 | 164 | sampson = top[:, 0] / bottom 165 | return sampson 166 | 167 | sampson = _sampson_distance(F, kp1_homo.float(), kp2_homo.float(), pair_idx) 168 | 169 | sampson_to_print = sampson.detach().clone().clamp(max=sampson_max).mean() 170 | sampson = sampson[sampson < sampson_max] 171 | 172 | return sampson, sampson_to_print 173 | -------------------------------------------------------------------------------- /pose_diffusion/util/get_fundamental_matrix.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import torch 8 | import pytorch3d 9 | from pytorch3d.utils import opencv_from_cameras_projection 10 | from pytorch3d.transforms.so3 import hat 11 | from pytorch3d.renderer.cameras import CamerasBase, PerspectiveCameras 12 | 13 | 14 | def get_fundamental_matrices( 15 | camera: CamerasBase, 16 | height: int, 17 | width: int, 18 | index1: torch.LongTensor, 19 | index2: torch.LongTensor, 20 | l2_normalize_F=False, 21 | ): 22 | """Compute fundamental matrices for given camera parameters.""" 23 | batch_size = camera.R.shape[0] 24 | 25 | # Convert to opencv / colmap / Hartley&Zisserman convention 26 | image_size_t = torch.LongTensor([height, width])[None].repeat(batch_size, 1).to(camera.device) 27 | R, t, K = opencv_from_cameras_projection(camera, image_size=image_size_t) 28 | 29 | F, E = get_fundamental_matrix(K[index1], R[index1], t[index1], K[index2], R[index2], t[index2]) 30 | 31 | if l2_normalize_F: 32 | F_scale = torch.norm(F, dim=(1, 2)) 33 | F_scale = F_scale.clamp(min=0.0001) 34 | F = F / F_scale[:, None, None] 35 | 36 | return F 37 | 38 | 39 | def get_fundamental_matrix(K1, R1, t1, K2, R2, t2): 40 | E = get_essential_matrix(R1, t1, R2, t2) 41 | F = K2.inverse().permute(0, 2, 1).matmul(E).matmul(K1.inverse()) 42 | return F, E # p2^T F p1 = 0 43 | 44 | 45 | def get_essential_matrix(R1, t1, R2, t2): 46 | R12 = R2.matmul(R1.permute(0, 2, 1)) 47 | t12 = t2 - R12.matmul(t1[..., None])[..., 0] 48 | E_R = R12 49 | E_t = -E_R.permute(0, 2, 1).matmul(t12[..., None])[..., 0] 50 | E = E_R.matmul(hat(E_t)) 51 | return E 52 | -------------------------------------------------------------------------------- /pose_diffusion/util/load_img_folder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import numpy as np 9 | from PIL import Image 10 | import torch 11 | import torch.nn.functional as F 12 | from typing import Any, ClassVar, Dict, Iterable, List, Optional, Sequence, Tuple, Type, TYPE_CHECKING, Union 13 | 14 | 15 | def load_and_preprocess_images(folder_path = None, image_size: int = 224, image_paths = None, mode: str = "bilinear") -> torch.Tensor: 16 | if image_paths is None: 17 | image_paths = [ 18 | os.path.join(folder_path, file) 19 | for file in os.listdir(folder_path) 20 | if file.lower().endswith((".png", ".jpg", ".jpeg")) 21 | ] 22 | image_paths.sort() 23 | 24 | images = [] 25 | bboxes_xyxy = [] 26 | scales = [] 27 | 28 | for path in image_paths: 29 | image = _load_image(path) 30 | image, bbox_xyxy, min_hw = _center_crop_square(image) 31 | minscale = image_size / min_hw 32 | 33 | imre = F.interpolate( 34 | torch.from_numpy(image)[None], 35 | size=(image_size, image_size), 36 | mode=mode, 37 | align_corners=False if mode == "bilinear" else None, 38 | )[0] 39 | 40 | images.append(imre.numpy()) 41 | bboxes_xyxy.append(bbox_xyxy.numpy()) 42 | scales.append(minscale) 43 | 44 | images_tensor = torch.from_numpy(np.stack(images)) 45 | 46 | # assume all the images have the same shape for GGS 47 | image_info = {"size": (min_hw, min_hw), "bboxes_xyxy": np.stack(bboxes_xyxy), "resized_scales": np.stack(scales)} 48 | return images_tensor, image_info 49 | 50 | 51 | # helper functions 52 | 53 | 54 | def _load_image(path) -> np.ndarray: 55 | with Image.open(path) as pil_im: 56 | im = np.array(pil_im.convert("RGB")) 57 | im = im.transpose((2, 0, 1)) 58 | im = im.astype(np.float32) / 255.0 59 | return im 60 | 61 | 62 | def _center_crop_square(image: np.ndarray) -> np.ndarray: 63 | h, w = image.shape[1:] 64 | min_dim = min(h, w) 65 | top = (h - min_dim) // 2 66 | left = (w - min_dim) // 2 67 | cropped_image = image[:, top : top + min_dim, left : left + min_dim] 68 | 69 | # bbox_xywh: the cropped region 70 | bbox_xywh = torch.tensor([left, top, min_dim, min_dim]) 71 | 72 | # the format from xywh to xyxy 73 | bbox_xyxy = _clamp_box_to_image_bounds_and_round( 74 | _get_clamp_bbox(bbox_xywh, box_crop_context=0.0), image_size_hw=(h, w) 75 | ) 76 | return cropped_image, bbox_xyxy, min_dim 77 | 78 | 79 | def _get_clamp_bbox(bbox: torch.Tensor, box_crop_context: float = 0.0) -> torch.Tensor: 80 | # box_crop_context: rate of expansion for bbox 81 | # returns possibly expanded bbox xyxy as float 82 | 83 | bbox = bbox.clone() # do not edit bbox in place 84 | 85 | # increase box size 86 | if box_crop_context > 0.0: 87 | c = box_crop_context 88 | bbox = bbox.float() 89 | bbox[0] -= bbox[2] * c / 2 90 | bbox[1] -= bbox[3] * c / 2 91 | bbox[2] += bbox[2] * c 92 | bbox[3] += bbox[3] * c 93 | 94 | if (bbox[2:] <= 1.0).any(): 95 | raise ValueError(f"squashed image!! The bounding box contains no pixels.") 96 | 97 | bbox[2:] = torch.clamp(bbox[2:], 2) # set min height, width to 2 along both axes 98 | bbox_xyxy = _bbox_xywh_to_xyxy(bbox, clamp_size=2) 99 | 100 | return bbox_xyxy 101 | 102 | 103 | def _bbox_xywh_to_xyxy(xywh: torch.Tensor, clamp_size: Optional[int] = None) -> torch.Tensor: 104 | xyxy = xywh.clone() 105 | if clamp_size is not None: 106 | xyxy[2:] = torch.clamp(xyxy[2:], clamp_size) 107 | xyxy[2:] += xyxy[:2] 108 | return xyxy 109 | 110 | 111 | def _clamp_box_to_image_bounds_and_round(bbox_xyxy: torch.Tensor, image_size_hw: Tuple[int, int]) -> torch.LongTensor: 112 | bbox_xyxy = bbox_xyxy.clone() 113 | bbox_xyxy[[0, 2]] = torch.clamp(bbox_xyxy[[0, 2]], 0, image_size_hw[-1]) 114 | bbox_xyxy[[1, 3]] = torch.clamp(bbox_xyxy[[1, 3]], 0, image_size_hw[-2]) 115 | if not isinstance(bbox_xyxy, torch.LongTensor): 116 | bbox_xyxy = bbox_xyxy.round().long() 117 | return bbox_xyxy # pyre-ignore [7] 118 | 119 | 120 | if __name__ == "__main__": 121 | # Example usage: 122 | folder_path = "path/to/your/folder" 123 | image_size = 224 124 | images_tensor = load_and_preprocess_images(folder_path, image_size) 125 | print(images_tensor.shape) 126 | -------------------------------------------------------------------------------- /pose_diffusion/util/match_extraction.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import os 8 | import shutil 9 | import tempfile 10 | from pathlib import Path 11 | 12 | import numpy as np 13 | import pycolmap 14 | from typing import Optional, List, Dict, Any 15 | from hloc import extract_features, logger, match_features, pairs_from_exhaustive 16 | from hloc.triangulation import ( 17 | import_features, 18 | import_matches, 19 | estimation_and_geometric_verification, 20 | parse_option_args, 21 | OutputCapture, 22 | ) 23 | from hloc.utils.database import COLMAPDatabase, image_ids_to_pair_id, pair_id_to_image_ids 24 | from hloc.reconstruction import create_empty_db, import_images, get_image_ids 25 | 26 | 27 | def extract_match(image_paths = None, image_folder_path = None, image_info = None): 28 | # Now only supports SPSG 29 | 30 | with tempfile.TemporaryDirectory() as tmpdir: 31 | tmp_mapping = os.path.join(tmpdir, "mapping") 32 | os.makedirs(tmp_mapping) 33 | 34 | if image_paths is None: 35 | for filename in os.listdir(image_folder_path): 36 | if filename.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff")): 37 | shutil.copy(os.path.join(image_folder_path, filename), os.path.join(tmp_mapping, filename)) 38 | else: 39 | for filename in image_paths: 40 | if filename.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff")): 41 | shutil.copy(filename, os.path.join(tmp_mapping, os.path.basename(filename))) 42 | matches, keypoints = run_hloc(tmpdir) 43 | 44 | # From the format of colmap to PyTorch3D 45 | kp1, kp2, i12 = colmap_keypoint_to_pytorch3d(matches, keypoints, image_info) 46 | 47 | return kp1, kp2, i12 48 | 49 | 50 | def colmap_keypoint_to_pytorch3d(matches, keypoints, image_info): 51 | kp1, kp2, i12 = [], [], [] 52 | bbox_xyxy, scale = image_info["bboxes_xyxy"], image_info["resized_scales"] 53 | 54 | for idx in keypoints: 55 | # coordinate change from COLMAP to OpenCV 56 | cur_keypoint = keypoints[idx] - 0.5 57 | 58 | # go to the coordiante after cropping 59 | # use idx - 1 here because the COLMAP format starts from 1 instead of 0 60 | cur_keypoint = cur_keypoint - [bbox_xyxy[idx - 1][0], bbox_xyxy[idx - 1][1]] 61 | cur_keypoint = cur_keypoint * scale[idx - 1] 62 | keypoints[idx] = cur_keypoint 63 | 64 | for (r_idx, q_idx), pair_match in matches.items(): 65 | if pair_match is not None: 66 | kp1.append(keypoints[r_idx][pair_match[:, 0]]) 67 | kp2.append(keypoints[q_idx][pair_match[:, 1]]) 68 | 69 | i12_pair = np.array([[r_idx - 1, q_idx - 1]]) 70 | i12.append(np.repeat(i12_pair, len(pair_match), axis=0)) 71 | 72 | if kp1: 73 | kp1, kp2, i12 = map(np.concatenate, (kp1, kp2, i12), (0, 0, 0)) 74 | else: 75 | kp1 = kp2 = i12 = None 76 | 77 | return kp1, kp2, i12 78 | 79 | 80 | def run_hloc(output_dir: str): 81 | # learned from 82 | # https://github.com/cvg/Hierarchical-Localization/blob/master/pipeline_SfM.ipynb 83 | 84 | images = Path(output_dir) 85 | outputs = Path(os.path.join(output_dir, "output")) 86 | sfm_pairs = outputs / "pairs-sfm.txt" 87 | sfm_dir = outputs / "sfm" 88 | features = outputs / "features.h5" 89 | matches = outputs / "matches.h5" 90 | 91 | feature_conf = extract_features.confs["superpoint_inloc"] # or superpoint_max 92 | matcher_conf = match_features.confs["superglue"] 93 | 94 | references = [p.relative_to(images).as_posix() for p in (images / "mapping/").iterdir()] 95 | 96 | extract_features.main(feature_conf, images, image_list=references, feature_path=features) 97 | pairs_from_exhaustive.main(sfm_pairs, image_list=references) 98 | match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches) 99 | 100 | matches, keypoints = compute_matches_and_keypoints( 101 | sfm_dir, images, sfm_pairs, features, matches, image_list=references 102 | ) 103 | 104 | return matches, keypoints 105 | 106 | 107 | def compute_matches_and_keypoints( 108 | sfm_dir: Path, 109 | image_dir: Path, 110 | pairs: Path, 111 | features: Path, 112 | matches: Path, 113 | camera_mode: pycolmap.CameraMode = pycolmap.CameraMode.AUTO, 114 | verbose: bool = False, 115 | min_match_score: Optional[float] = None, 116 | image_list: Optional[List[str]] = None, 117 | image_options: Optional[Dict[str, Any]] = None, 118 | ) -> pycolmap.Reconstruction: 119 | # learned from 120 | # https://github.com/cvg/Hierarchical-Localization/blob/master/hloc/reconstruction.py 121 | 122 | sfm_dir.mkdir(parents=True, exist_ok=True) 123 | database = sfm_dir / "database.db" 124 | 125 | create_empty_db(database) 126 | import_images(image_dir, database, camera_mode, image_list, image_options) 127 | image_ids = get_image_ids(database) 128 | import_features(image_ids, database, features) 129 | import_matches(image_ids, database, pairs, matches, min_match_score) 130 | estimation_and_geometric_verification(database, pairs, verbose) 131 | 132 | db = COLMAPDatabase.connect(database) 133 | 134 | matches = dict( 135 | (pair_id_to_image_ids(pair_id), _blob_to_array_safe(data, np.uint32, (-1, 2))) 136 | for pair_id, data in db.execute("SELECT pair_id, data FROM matches") 137 | ) 138 | 139 | keypoints = dict( 140 | (image_id, _blob_to_array_safe(data, np.float32, (-1, 2))) 141 | for image_id, data in db.execute("SELECT image_id, data FROM keypoints") 142 | ) 143 | 144 | db.close() 145 | 146 | return matches, keypoints 147 | 148 | 149 | def _blob_to_array_safe(blob, dtype, shape=(-1,)): 150 | if blob is not None: 151 | return np.fromstring(blob, dtype=dtype).reshape(*shape) 152 | else: 153 | return blob 154 | -------------------------------------------------------------------------------- /pose_diffusion/util/metric.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import random 8 | import numpy as np 9 | import torch 10 | 11 | from pytorch3d.transforms import so3_relative_angle 12 | 13 | 14 | def camera_to_rel_deg(pred_cameras, gt_cameras, device, batch_size): 15 | """ 16 | Calculate relative rotation and translation angles between predicted and ground truth cameras. 17 | 18 | Args: 19 | - pred_cameras: Predicted camera. 20 | - gt_cameras: Ground truth camera. 21 | - accelerator: The device for moving tensors to GPU or others. 22 | - batch_size: Number of data samples in one batch. 23 | 24 | Returns: 25 | - rel_rotation_angle_deg, rel_translation_angle_deg: Relative rotation and translation angles in degrees. 26 | """ 27 | 28 | with torch.no_grad(): 29 | # Convert cameras to 4x4 SE3 transformation matrices 30 | gt_se3 = gt_cameras.get_world_to_view_transform().get_matrix() 31 | pred_se3 = pred_cameras.get_world_to_view_transform().get_matrix() 32 | 33 | # Generate pairwise indices to compute relative poses 34 | pair_idx_i1, pair_idx_i2 = batched_all_pairs(batch_size, gt_se3.shape[0] // batch_size) 35 | pair_idx_i1 = pair_idx_i1.to(device) 36 | 37 | # Compute relative camera poses between pairs 38 | # We use closed_form_inverse to avoid potential numerical loss by torch.inverse() 39 | # This is possible because of SE3 40 | relative_pose_gt = closed_form_inverse(gt_se3[pair_idx_i1]).bmm(gt_se3[pair_idx_i2]) 41 | relative_pose_pred = closed_form_inverse(pred_se3[pair_idx_i1]).bmm(pred_se3[pair_idx_i2]) 42 | 43 | # Compute the difference in rotation and translation 44 | # between the ground truth and predicted relative camera poses 45 | rel_rangle_deg = rotation_angle(relative_pose_gt[:, :3, :3], relative_pose_pred[:, :3, :3]) 46 | rel_tangle_deg = translation_angle(relative_pose_gt[:, 3, :3], relative_pose_pred[:, 3, :3]) 47 | 48 | return rel_rangle_deg, rel_tangle_deg 49 | 50 | 51 | def calculate_auc_np(r_error, t_error, max_threshold=30): 52 | """ 53 | Calculate the Area Under the Curve (AUC) for the given error arrays. 54 | 55 | :param r_error: numpy array representing R error values (Degree). 56 | :param t_error: numpy array representing T error values (Degree). 57 | :param max_threshold: maximum threshold value for binning the histogram. 58 | :return: cumulative sum of normalized histogram of maximum error values. 59 | """ 60 | 61 | # Concatenate the error arrays along a new axis 62 | error_matrix = np.concatenate((r_error[:, None], t_error[:, None]), axis=1) 63 | 64 | # Compute the maximum error value for each pair 65 | max_errors = np.max(error_matrix, axis=1) 66 | 67 | # Define histogram bins 68 | bins = np.arange(max_threshold + 1) 69 | 70 | # Calculate histogram of maximum error values 71 | histogram, _ = np.histogram(max_errors, bins=bins) 72 | 73 | # Normalize the histogram 74 | num_pairs = float(len(max_errors)) 75 | normalized_histogram = histogram.astype(float) / num_pairs 76 | 77 | # Compute and return the cumulative sum of the normalized histogram 78 | return np.mean(np.cumsum(normalized_histogram)) 79 | 80 | 81 | def calculate_auc(r_error, t_error, max_threshold=30): 82 | """ 83 | Calculate the Area Under the Curve (AUC) for the given error arrays using PyTorch. 84 | 85 | :param r_error: torch.Tensor representing R error values (Degree). 86 | :param t_error: torch.Tensor representing T error values (Degree). 87 | :param max_threshold: maximum threshold value for binning the histogram. 88 | :return: cumulative sum of normalized histogram of maximum error values. 89 | """ 90 | 91 | # Concatenate the error tensors along a new axis 92 | error_matrix = torch.stack((r_error, t_error), dim=1) 93 | 94 | # Compute the maximum error value for each pair 95 | max_errors, _ = torch.max(error_matrix, dim=1) 96 | 97 | # Define histogram bins 98 | bins = torch.arange(max_threshold + 1) 99 | 100 | # Calculate histogram of maximum error values 101 | histogram = torch.histc(max_errors, bins=max_threshold + 1, min=0, max=max_threshold) 102 | 103 | # Normalize the histogram 104 | num_pairs = float(max_errors.size(0)) 105 | normalized_histogram = histogram / num_pairs 106 | 107 | # Compute and return the cumulative sum of the normalized histogram 108 | return torch.cumsum(normalized_histogram, dim=0).mean() 109 | 110 | 111 | def batched_all_pairs(B, N): 112 | # B, N = se3.shape[:2] 113 | i1_, i2_ = torch.combinations(torch.arange(N), 2, with_replacement=False).unbind(-1) 114 | i1, i2 = [(i[None] + torch.arange(B)[:, None] * N).reshape(-1) for i in [i1_, i2_]] 115 | 116 | return i1, i2 117 | 118 | 119 | def closed_form_inverse(se3): 120 | """ 121 | Computes the inverse of each 4x4 SE3 matrix in the batch. 122 | 123 | Args: 124 | - se3 (Tensor): Nx4x4 tensor of SE3 matrices. 125 | 126 | Returns: 127 | - Tensor: Nx4x4 tensor of inverted SE3 matrices. 128 | """ 129 | R = se3[:, :3, :3] 130 | T = se3[:, 3:, :3] 131 | 132 | # Compute the transpose of the rotation 133 | R_transposed = R.transpose(1, 2) 134 | 135 | # Compute the left part of the inverse transformation 136 | left_bottom = -T.bmm(R_transposed) 137 | left_combined = torch.cat((R_transposed, left_bottom), dim=1) 138 | 139 | # Keep the right-most column as it is 140 | right_col = se3[:, :, 3:].detach().clone() 141 | inverted_matrix = torch.cat((left_combined, right_col), dim=-1) 142 | 143 | return inverted_matrix 144 | 145 | 146 | def rotation_angle(rot_gt, rot_pred, batch_size=None): 147 | # rot_gt, rot_pred (B, 3, 3) 148 | rel_angle_cos = so3_relative_angle(rot_gt, rot_pred, eps=1e-4) 149 | rel_rangle_deg = rel_angle_cos * 180 / np.pi 150 | 151 | if batch_size is not None: 152 | rel_rangle_deg = rel_rangle_deg.reshape(batch_size, -1) 153 | 154 | return rel_rangle_deg 155 | 156 | 157 | def translation_angle(tvec_gt, tvec_pred, batch_size=None): 158 | # tvec_gt, tvec_pred (B, 3,) 159 | rel_tangle_deg = compare_translation_by_angle(tvec_gt, tvec_pred) 160 | rel_tangle_deg = rel_tangle_deg * 180.0 / np.pi 161 | 162 | if batch_size is not None: 163 | rel_tangle_deg = rel_tangle_deg.reshape(batch_size, -1) 164 | 165 | return rel_tangle_deg 166 | 167 | 168 | def compare_translation_by_angle(t_gt, t, eps=1e-15, default_err=1e6): 169 | """Normalize the translation vectors and compute the angle between them.""" 170 | t_norm = torch.norm(t, dim=1, keepdim=True) 171 | t = t / (t_norm + eps) 172 | 173 | t_gt_norm = torch.norm(t_gt, dim=1, keepdim=True) 174 | t_gt = t_gt / (t_gt_norm + eps) 175 | 176 | loss_t = torch.clamp_min(1.0 - torch.sum(t * t_gt, dim=1) ** 2, eps) 177 | err_t = torch.acos(torch.sqrt(1 - loss_t)) 178 | 179 | err_t[torch.isnan(err_t) | torch.isinf(err_t)] = default_err 180 | return err_t 181 | 182 | def compute_ARE(rotation1, rotation2): 183 | if isinstance(rotation1, torch.Tensor): 184 | rotation1 = rotation1.cpu().detach().numpy() 185 | if isinstance(rotation2, torch.Tensor): 186 | rotation2 = rotation2.cpu().detach().numpy() 187 | 188 | R_rel = np.einsum("Bij,Bjk ->Bik", rotation1.transpose(0, 2, 1), rotation2) 189 | t = (np.trace(R_rel, axis1=1, axis2=2) - 1) / 2 190 | theta = np.arccos(np.clip(t, -1, 1)) 191 | error = theta * 180 / np.pi 192 | return np.minimum(error, np.abs(180 - error)) -------------------------------------------------------------------------------- /pose_diffusion/util/normalize_cameras.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | """ 8 | Adapted from code originally written by David Novotny. 9 | """ 10 | import torch 11 | from pytorch3d.transforms import Rotate, Translate 12 | import math 13 | 14 | 15 | def intersect_skew_line_groups(p, r, mask): 16 | # p, r both of shape (B, N, n_intersected_lines, 3) 17 | # mask of shape (B, N, n_intersected_lines) 18 | p_intersect, r = intersect_skew_lines_high_dim(p, r, mask=mask) 19 | _, p_line_intersect = _point_line_distance(p, r, p_intersect[..., None, :].expand_as(p)) 20 | intersect_dist_squared = ((p_line_intersect - p_intersect[..., None, :]) ** 2).sum(dim=-1) 21 | return p_intersect, p_line_intersect, intersect_dist_squared, r 22 | 23 | 24 | def intersect_skew_lines_high_dim(p, r, mask=None): 25 | # Implements https://en.wikipedia.org/wiki/Skew_lines In more than two dimensions 26 | dim = p.shape[-1] 27 | # make sure the heading vectors are l2-normed 28 | if mask is None: 29 | mask = torch.ones_like(p[..., 0]) 30 | r = torch.nn.functional.normalize(r, dim=-1) 31 | 32 | eye = torch.eye(dim, device=p.device, dtype=p.dtype)[None, None] 33 | I_min_cov = (eye - (r[..., None] * r[..., None, :])) * mask[..., None, None] 34 | sum_proj = I_min_cov.matmul(p[..., None]).sum(dim=-3) 35 | p_intersect = torch.linalg.lstsq(I_min_cov.sum(dim=-3), sum_proj).solution[..., 0] 36 | 37 | if torch.any(torch.isnan(p_intersect)): 38 | print(p_intersect) 39 | raise ValueError(f"p_intersect is NaN") 40 | 41 | return p_intersect, r 42 | 43 | 44 | def _point_line_distance(p1, r1, p2): 45 | df = p2 - p1 46 | proj_vector = df - ((df * r1).sum(dim=-1, keepdim=True) * r1) 47 | line_pt_nearest = p2 - proj_vector 48 | d = (proj_vector).norm(dim=-1) 49 | return d, line_pt_nearest 50 | 51 | 52 | def compute_optical_axis_intersection(cameras): 53 | centers = cameras.get_camera_center() 54 | principal_points = cameras.principal_point 55 | 56 | one_vec = torch.ones((len(cameras), 1)) 57 | optical_axis = torch.cat((principal_points, one_vec), -1) 58 | 59 | pp = cameras.unproject_points(optical_axis, from_ndc=True, world_coordinates=True) 60 | 61 | pp2 = pp[torch.arange(pp.shape[0]), torch.arange(pp.shape[0])] 62 | 63 | directions = pp2 - centers 64 | centers = centers.unsqueeze(0).unsqueeze(0) 65 | directions = directions.unsqueeze(0).unsqueeze(0) 66 | 67 | p_intersect, p_line_intersect, _, r = intersect_skew_line_groups(p=centers, r=directions, mask=None) 68 | 69 | p_intersect = p_intersect.squeeze().unsqueeze(0) 70 | dist = (p_intersect - centers).norm(dim=-1) 71 | 72 | return p_intersect, dist, p_line_intersect, pp2, r 73 | 74 | 75 | def normalize_cameras(cameras, compute_optical=True, first_camera=True, scale=1.0, normalize_T = False): 76 | """ 77 | Normalizes cameras such that the optical axes point to the origin and the average 78 | distance to the origin is 1. 79 | 80 | Args: 81 | cameras (List[camera]). 82 | """ 83 | # Let distance from first camera to origin be unit 84 | new_cameras = cameras.clone() 85 | 86 | if compute_optical: 87 | new_transform = new_cameras.get_world_to_view_transform() 88 | 89 | (p_intersect, dist, p_line_intersect, pp, r) = compute_optical_axis_intersection(cameras) 90 | t = Translate(p_intersect) 91 | 92 | scale = dist.squeeze()[0] 93 | 94 | # Degenerate case 95 | if scale == 0: 96 | scale = torch.norm(new_cameras.T, dim=(0, 1)) 97 | scale = torch.sqrt(scale) 98 | new_cameras.T = new_cameras.T / scale 99 | else: 100 | new_matrix = t.compose(new_transform).get_matrix() 101 | new_cameras.R = new_matrix[:, :3, :3] 102 | new_cameras.T = new_matrix[:, 3, :3] / scale 103 | else: 104 | scale = torch.norm(new_cameras.T, dim=(0, 1)) 105 | scale = torch.sqrt(scale) 106 | new_cameras.T = new_cameras.T / scale 107 | 108 | if first_camera: 109 | new_cameras = first_camera_transform(new_cameras) 110 | 111 | if normalize_T: 112 | new_cameras = normalize_Trans(new_cameras) 113 | 114 | return new_cameras 115 | 116 | 117 | 118 | def normalize_Trans(new_cameras): 119 | 120 | t_gt = new_cameras.T.clone() 121 | t_gt = t_gt[1:,:] 122 | t_gt_scale = torch.norm(t_gt, dim=(0, 1)) 123 | t_gt_scale = t_gt_scale / math.sqrt(len(t_gt)) 124 | t_gt_scale = t_gt_scale / 2 125 | t_gt_scale = t_gt_scale.clamp(min=0.01, max = 100) 126 | new_cameras.T = new_cameras.T / t_gt_scale 127 | 128 | return new_cameras 129 | 130 | 131 | 132 | def first_camera_transform(cameras, rotation_only=False): 133 | # Let distance from first camera to origin be unit 134 | new_cameras = cameras.clone() 135 | new_transform = new_cameras.get_world_to_view_transform() 136 | tR = Rotate(new_cameras.R[0].unsqueeze(0)) 137 | if rotation_only: 138 | t = tR.inverse() 139 | else: 140 | tT = Translate(new_cameras.T[0].unsqueeze(0)) 141 | t = tR.compose(tT).inverse() 142 | 143 | new_matrix = t.compose(new_transform).get_matrix() 144 | 145 | new_cameras.R = new_matrix[:, :3, :3] 146 | new_cameras.T = new_matrix[:, 3, :3] 147 | 148 | return new_cameras 149 | -------------------------------------------------------------------------------- /pose_diffusion/util/train_util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import logging 8 | import math 9 | import numpy as np 10 | from itertools import cycle 11 | import matplotlib 12 | import matplotlib.pyplot as plt 13 | from matplotlib import colors as mcolors, cm 14 | import torch.optim 15 | from torch.utils.data import BatchSampler 16 | from accelerate import Accelerator 17 | from accelerate.utils import set_seed as accelerate_set_seed 18 | from pytorch3d.implicitron.tools.stats import Stats 19 | from pytorch3d.vis.plotly_vis import plot_scene 20 | from pytorch3d.implicitron.tools.vis_utils import get_visdom_connection 21 | from datasets.co3d_v2 import Co3dDataset 22 | 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | class DynamicBatchSampler(BatchSampler): 28 | def __init__(self, num_sequences, dataset_len=1024, max_images=128, images_per_seq=(3, 20)): 29 | # Batch sampler with a dynamic number of sequences 30 | # max_images >= number_of_sequences * images_per_sequence 31 | 32 | self.max_images = max_images 33 | self.images_per_seq = list(range(images_per_seq[0], images_per_seq[1])) 34 | self.num_sequences = num_sequences 35 | self.dataset_len = dataset_len 36 | 37 | def __iter__(self): 38 | for _ in range(self.dataset_len): 39 | # number per sequence 40 | n_per_seq = np.random.choice(self.images_per_seq) 41 | # number of sequences 42 | n_seqs = self.max_images // n_per_seq 43 | 44 | # randomly select sequences 45 | chosen_seq = self._capped_random_choice(self.num_sequences, n_seqs) 46 | 47 | # get item 48 | batches = [(bidx, n_per_seq) for bidx in chosen_seq] 49 | yield batches 50 | 51 | def _capped_random_choice(self, x, size, replace: bool = True): 52 | len_x = x if isinstance(x, int) else len(x) 53 | if replace: 54 | return np.random.choice(x, size=size, replace=len_x < size) 55 | else: 56 | return np.random.choice(x, size=min(size, len_x), replace=False) 57 | 58 | def __len__(self): 59 | return self.dataset_len 60 | 61 | 62 | class WarmupCosineRestarts(torch.optim.lr_scheduler._LRScheduler): 63 | def __init__( 64 | self, optimizer, T_0, iters_per_epoch, T_mult=1, eta_min=0, warmup_ratio=0.1, warmup_lr_init=1e-7, last_epoch=-1 65 | ): 66 | # Similar to torch.optim.lr_scheduler.OneCycleLR() 67 | # But allow multiple cycles and a warmup 68 | self.T_0 = T_0 * iters_per_epoch 69 | self.T_mult = T_mult 70 | self.eta_min = eta_min 71 | self.warmup_iters = int(T_0 * warmup_ratio * iters_per_epoch) 72 | self.warmup_lr_init = warmup_lr_init 73 | super(WarmupCosineRestarts, self).__init__(optimizer, last_epoch) 74 | 75 | def get_lr(self): 76 | if self.T_mult == 1: 77 | i_restart = self.last_epoch // self.T_0 78 | T_cur = self.last_epoch - i_restart * self.T_0 79 | else: 80 | n = int(math.log((self.last_epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult)) 81 | T_cur = self.last_epoch - self.T_0 * (self.T_mult**n - 1) // (self.T_mult - 1) 82 | 83 | if T_cur < self.warmup_iters: 84 | warmup_ratio = T_cur / self.warmup_iters 85 | return [self.warmup_lr_init + (base_lr - self.warmup_lr_init) * warmup_ratio for base_lr in self.base_lrs] 86 | else: 87 | T_cur_adjusted = T_cur - self.warmup_iters 88 | T_i = self.T_0 - self.warmup_iters 89 | return [ 90 | self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * T_cur_adjusted / T_i)) / 2 91 | for base_lr in self.base_lrs 92 | ] 93 | 94 | 95 | def get_co3d_dataset(cfg): 96 | # Common dataset parameters 97 | common_params = { 98 | "category": (cfg.train.category,), 99 | "debug": False, 100 | "mask_images": False, 101 | "img_size": cfg.train.img_size, 102 | "normalize_cameras": cfg.train.normalize_cameras, 103 | "min_num_images": cfg.train.min_num_images, 104 | "CO3D_DIR": cfg.train.CO3D_DIR, 105 | "CO3D_ANNOTATION_DIR": cfg.train.CO3D_ANNOTATION_DIR, 106 | "first_camera_transform": cfg.train.first_camera_transform, 107 | "compute_optical": cfg.train.compute_optical, 108 | "color_aug": cfg.train.color_aug, 109 | "erase_aug": cfg.train.erase_aug, 110 | } 111 | 112 | # Create the train dataset 113 | dataset = Co3dDataset(**common_params, split="train") 114 | 115 | # Create the eval dataset 116 | eval_dataset = Co3dDataset(**common_params, split="test", eval_time=True) 117 | 118 | return dataset, eval_dataset 119 | 120 | 121 | def get_co3d_dataset_test(cfg, category = None): 122 | # Common dataset parameters 123 | if category is None: 124 | category = cfg.test.category 125 | 126 | common_params = { 127 | "category": (category,), 128 | "debug": False, 129 | "mask_images": False, 130 | "img_size": cfg.test.img_size, 131 | "normalize_cameras": cfg.test.normalize_cameras, 132 | "min_num_images": cfg.test.min_num_images, 133 | "CO3D_DIR": cfg.test.CO3D_DIR, 134 | "CO3D_ANNOTATION_DIR": cfg.test.CO3D_ANNOTATION_DIR, 135 | "first_camera_transform": cfg.test.first_camera_transform, 136 | "compute_optical": cfg.test.compute_optical, 137 | "sort_by_filename": True, # to ensure images are aligned with extracted matches 138 | } 139 | 140 | # Create the test dataset 141 | test_dataset = Co3dDataset(**common_params, split="test", eval_time=True) 142 | 143 | return test_dataset 144 | 145 | 146 | def set_seed_and_print(seed): 147 | accelerate_set_seed(seed, device_specific=True) 148 | print(f"----------Seed is set to {np.random.get_state()[1][0]} now----------") 149 | 150 | 151 | class VizStats(Stats): 152 | def plot_stats(self, viz=None, visdom_env=None, plot_file=None, visdom_server=None, visdom_port=None): 153 | # use the cached visdom env if none supplied 154 | if visdom_env is None: 155 | visdom_env = self.visdom_env 156 | if visdom_server is None: 157 | visdom_server = self.visdom_server 158 | if visdom_port is None: 159 | visdom_port = self.visdom_port 160 | if plot_file is None: 161 | plot_file = self.plot_file 162 | 163 | stat_sets = list(self.stats.keys()) 164 | 165 | logger.debug(f"printing charts to visdom env '{visdom_env}' ({visdom_server}:{visdom_port})") 166 | 167 | novisdom = False 168 | 169 | if viz is None: 170 | viz = get_visdom_connection(server=visdom_server, port=visdom_port) 171 | 172 | if viz is None or not viz.check_connection(): 173 | logger.info("no visdom server! -> skipping visdom plots") 174 | novisdom = True 175 | 176 | lines = [] 177 | 178 | # plot metrics 179 | if not novisdom: 180 | viz.close(env=visdom_env, win=None) 181 | 182 | for stat in self.log_vars: 183 | vals = [] 184 | stat_sets_now = [] 185 | for stat_set in stat_sets: 186 | val = self.stats[stat_set][stat].get_epoch_averages() 187 | if val is None: 188 | continue 189 | else: 190 | val = np.array(val).reshape(-1) 191 | stat_sets_now.append(stat_set) 192 | vals.append(val) 193 | 194 | if len(vals) == 0: 195 | continue 196 | 197 | lines.append((stat_sets_now, stat, vals)) 198 | 199 | if not novisdom: 200 | for tmodes, stat, vals in lines: 201 | title = "%s" % stat 202 | opts = {"title": title, "legend": list(tmodes)} 203 | for i, (tmode, val) in enumerate(zip(tmodes, vals)): 204 | update = "append" if i > 0 else None 205 | valid = np.where(np.isfinite(val))[0] 206 | if len(valid) == 0: 207 | continue 208 | x = np.arange(len(val)) 209 | viz.line( 210 | Y=val[valid], 211 | X=x[valid], 212 | env=visdom_env, 213 | opts=opts, 214 | win=f"stat_plot_{title}", 215 | name=tmode, 216 | update=update, 217 | ) 218 | 219 | if plot_file: 220 | logger.info(f"plotting stats to {plot_file}") 221 | ncol = 3 222 | nrow = int(np.ceil(float(len(lines)) / ncol)) 223 | matplotlib.rcParams.update({"font.size": 5}) 224 | color = cycle(plt.cm.tab10(np.linspace(0, 1, 10))) 225 | fig = plt.figure(1) 226 | plt.clf() 227 | for idx, (tmodes, stat, vals) in enumerate(lines): 228 | c = next(color) 229 | plt.subplot(nrow, ncol, idx + 1) 230 | plt.gca() 231 | for vali, vals_ in enumerate(vals): 232 | c_ = c * (1.0 - float(vali) * 0.3) 233 | valid = np.where(np.isfinite(vals_))[0] 234 | if len(valid) == 0: 235 | continue 236 | x = np.arange(len(vals_)) 237 | plt.plot(x[valid], vals_[valid], c=c_, linewidth=1) 238 | plt.ylabel(stat) 239 | plt.xlabel("epoch") 240 | plt.gca().yaxis.label.set_color(c[0:3] * 0.75) 241 | plt.legend(tmodes) 242 | gcolor = np.array(mcolors.to_rgba("lightgray")) 243 | grid_params = {"visible": True, "color": gcolor} 244 | plt.grid(**grid_params, which="major", linestyle="-", linewidth=0.4) 245 | plt.grid(**grid_params, which="minor", linestyle="--", linewidth=0.2) 246 | plt.minorticks_on() 247 | 248 | plt.tight_layout() 249 | plt.show() 250 | try: 251 | fig.savefig(plot_file) 252 | except PermissionError: 253 | warnings.warn("Cant dump stats due to insufficient permissions!") 254 | 255 | 256 | def view_color_coded_images_for_visdom(images): 257 | num_frames, _, height, width = images.shape 258 | cmap = cm.get_cmap("hsv") 259 | bordered_images = [] 260 | 261 | for i in range(num_frames): 262 | img = images[i] 263 | color = torch.tensor(np.array(cmap(i / num_frames))[:3], dtype=img.dtype, device=img.device) 264 | # Create colored borders 265 | thickness = 5 # Border thickness 266 | # Left border 267 | img[:, :, :thickness] = color[:, None, None] 268 | 269 | # Right border 270 | img[:, :, -thickness:] = color[:, None, None] 271 | 272 | # Top border 273 | img[:, :thickness, :] = color[:, None, None] 274 | 275 | # Bottom border 276 | img[:, -thickness:, :] = color[:, None, None] 277 | 278 | bordered_images.append(img) 279 | 280 | return torch.stack(bordered_images) 281 | 282 | 283 | def plotly_scene_visualization(camera_dict, batch_size): 284 | fig = plot_scene(camera_dict, camera_scale=0.03, ncols=2) 285 | fig.update_scenes(aspectmode="data") 286 | 287 | cmap = plt.get_cmap("hsv") 288 | 289 | for i in range(batch_size): 290 | fig.data[i].line.color = matplotlib.colors.to_hex(cmap(i / (batch_size))) 291 | fig.data[i + batch_size].line.color = matplotlib.colors.to_hex(cmap(i / (batch_size))) 292 | 293 | return fig 294 | -------------------------------------------------------------------------------- /pose_diffusion/util/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | 7 | import random 8 | 9 | import numpy as np 10 | import torch 11 | import tempfile 12 | 13 | 14 | def seed_all_random_engines(seed: int) -> None: 15 | np.random.seed(seed) 16 | torch.manual_seed(seed) 17 | random.seed(seed) 18 | --------------------------------------------------------------------------------