├── .gitignore
├── LICENSE
├── README.md
├── demo
├── TIPS_demo.ipynb
├── test_df2df.py
├── test_df2rw.py
└── tips
│ ├── __init__.py
│ ├── models
│ ├── __init__.py
│ ├── pose2pose.py
│ ├── refinenet.py
│ └── text2pose.py
│ ├── tips.py
│ └── visualization.py
├── docs
├── index.html
└── static
│ ├── colab_enjoyer.svg
│ ├── favicon.ico
│ ├── network_architecture.svg
│ ├── poster.pdf
│ ├── results.svg
│ ├── social_preview.png
│ └── teaser.svg
├── notebooks
└── TIPS_demo.ipynb
├── pose2pose
└── README.md
├── refinenet
├── dataloader.py
├── refinenet.py
├── test.py
├── train.py
└── visualization.py
├── requirements.txt
└── text2pose
├── data
├── __init__.py
└── dataloader.py
├── generate_heatmaps.py
├── models
├── __init__.py
├── base_model.py
├── netD.py
└── netG.py
├── text2pose_model.py
├── train.py
└── utils
├── __init__.py
├── heatmap.py
└── visualization.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 |
131 | # Project exlusions
132 | datasets/
133 | output/
134 | demo/checkpoints/
135 | demo/data/
136 | demo/output/
137 | notebooks/checkpoints/
138 | notebooks/data/
139 | notebooks/output/
140 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ### Official code for TIPS: Text-Induced Pose Synthesis.
2 |
3 | *Accepted in the European Conference on Computer Vision (ECCV) 2022.*
4 |
5 | [](https://pytorch.org/)
6 | [](https://colab.research.google.com/github/prasunroy/tips/blob/main/notebooks/TIPS_demo.ipynb)
7 |
8 | 
9 |
10 | ### Abstract
11 |
12 | In computer vision, human pose synthesis and transfer deal with probabilistic image generation of a person in a previously unseen pose from an already available observation of that person. Though researchers have recently proposed several methods to achieve this task, most of these techniques derive the target pose directly from the desired target image on a specific dataset, making the underlying process challenging to apply in real-world scenarios as the generation of the target image is the actual aim. In this paper, we first present the shortcomings of current pose transfer algorithms and then propose a novel text-based pose transfer technique to address those issues. We divide the problem into three independent stages: (a) text to pose representation, (b) pose refinement, and (c) pose rendering. To the best of our knowledge, this is one of the first attempts to develop a text-based pose transfer framework where we also introduce a new dataset DF-PASS, by adding descriptive pose annotations for the images of the DeepFashion dataset. The proposed method generates promising results with significant qualitative and quantitative scores in our experiments.
13 |
14 |
15 |
16 |
17 | ### Network Architecture
18 | 
19 |
20 | The pipeline is divided into three stages. In stage 1, we estimate the target pose keypoints from the corresponding text description embedding. In stage 2, we regressively refine the initial estimation of the facial keypoints and obtain the refined target pose keypoints. Finally, in stage 3, we render the target image by conditioning the pose transfer on the source image.
21 |
22 |
23 |
24 |
25 | ### Generation Results
26 | 
27 |
28 | Keypoints-guided methods tend to produce structurally inaccurate results when the physical appearance of the target pose reference significantly differs from the condition image. This observation is more frequent for the out of distribution target poses than the within distribution target poses. On the other hand, the existing text-guided method occasionally misinterprets the target pose due to a limited set of basic poses used for pose representation. The proposed text-guided technique successfully addresses these issues while retaining the ability to generate visually decent results close to the keypoints-guided baseline.
29 |
30 |
31 |
32 |
33 | ### Try the TIPS inference pipeline demo in Colab
34 | [](https://colab.research.google.com/github/prasunroy/tips/blob/main/notebooks/TIPS_demo.ipynb)
35 |
36 | [](https://colab.research.google.com/github/prasunroy/tips/blob/main/notebooks/TIPS_demo.ipynb)
37 |
38 |
39 |
40 | ### :zap: Getting Started
41 | * Clone the project repository and install dependencies.
42 | ```bash
43 | git clone https://github.com/prasunroy/tips.git
44 | cd tips
45 | mkdir datasets
46 | pip install -r requirements.txt
47 | ```
48 | * Download the DF-PASS dataset from [Google Drive](https://drive.google.com/drive/folders/17cvo22Eh_Z_S6fb-J-c6qw97WH6UeIHo) and extract into `datasets/DF-PASS` directory.
49 | ```
50 | tips
51 | ├───datasets
52 | │ └───DF-PASS
53 | │ ├───gaussian_heatmaps
54 | │ ├───descriptions.csv
55 | │ ├───encodings.csv
56 | │ ├───test_img_keypoints.csv
57 | │ ├───test_img_list.csv
58 | │ ├───test_img_pairs.csv
59 | │ ├───train_img_keypoints.csv
60 | │ ├───train_img_list.csv
61 | │ └───train_img_pairs.csv
62 | └─── ...
63 | ```
64 |
65 |
66 |
67 | ### :rocket: Running the demo locally
68 | * Download the pretrained checkpoints and test data from [Google Drive](https://drive.google.com/uc?export=download&id=1zTG9M06ckW0z4MvJks3-JSC8sJguiZJH) and extract into `tips/demo` directory.
69 | ```
70 | tips
71 | ├───demo
72 | │ ├───checkpoints
73 | │ │ ├───pose2pose_260500.pth
74 | │ │ ├───refinenet_100.pth
75 | │ │ └───text2pose_75000.pth
76 | │ ├───data
77 | │ │ ├───images
78 | │ │ ├───descriptions.csv
79 | │ │ ├───encodings.csv
80 | │ │ ├───img_pairs_df2df.csv
81 | │ │ ├───img_pairs_df2rw.csv
82 | │ │ ├───keypoints.csv
83 | │ │ └───FreeMono.ttf
84 | │ └─── ...
85 | └─── ...
86 | ```
87 | * Run the demo notebook from `tips/demo` directory.
88 | ```bash
89 | cd demo
90 | jupyter notebook TIPS_demo.ipynb
91 | ```
92 |
93 |
94 |
95 | ### External Links
96 |
103 |
104 |
105 |
106 | ### Citation
107 | ```
108 | @inproceedings{roy2022tips,
109 | title = {TIPS: Text-Induced Pose Synthesis},
110 | author = {Roy, Prasun and Ghosh, Subhankar and Bhattacharya, Saumik and Pal, Umapada and Blumenstein, Michael},
111 | booktitle = {The European Conference on Computer Vision (ECCV)},
112 | month = {October},
113 | year = {2022}
114 | }
115 | ```
116 |
117 |
118 |
119 | ### Related Publications
120 |
121 | [1] [Multi-scale Attention Guided Pose Transfer](https://arxiv.org/abs/2202.06777) (PR 2023).
122 |
123 | [2] [Scene Aware Person Image Generation through Global Contextual Conditioning](https://arxiv.org/abs/2206.02717) (ICPR 2022).
124 |
125 | [3] [Text Guided Person Image Synthesis](https://openaccess.thecvf.com/content_CVPR_2019/html/Zhou_Text_Guided_Person_Image_Synthesis_CVPR_2019_paper.html) (CVPR 2019).
126 |
127 | [4] [Progressive Pose Attention Transfer for Person Image Generation](https://openaccess.thecvf.com/content_CVPR_2019/html/Zhu_Progressive_Pose_Attention_Transfer_for_Person_Image_Generation_CVPR_2019_paper.html) (CVPR 2019).
128 |
129 | [5] [DeepFashion: Powering Robust Clothes Recognition and Retrieval With Rich Annotations](https://openaccess.thecvf.com/content_cvpr_2016/html/Liu_DeepFashion_Powering_Robust_CVPR_2016_paper.html) (CVPR 2016).
130 |
131 |
132 |
133 | ### License
134 | ```
135 | Copyright 2022 by the authors
136 |
137 | Licensed under the Apache License, Version 2.0 (the "License");
138 | you may not use this file except in compliance with the License.
139 | You may obtain a copy of the License at
140 |
141 | http://www.apache.org/licenses/LICENSE-2.0
142 |
143 | Unless required by applicable law or agreed to in writing, software
144 | distributed under the License is distributed on an "AS IS" BASIS,
145 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
146 | See the License for the specific language governing permissions and
147 | limitations under the License.
148 | ```
149 |
150 | >The DF-PASS dataset and the pretrained models are released under Creative Commons Attribution 4.0 International ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)) license.
151 |
152 |
153 |
154 | ##### Made with :heart: and :pizza: on Earth.
155 |
--------------------------------------------------------------------------------
/demo/TIPS_demo.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "EIRGdFExTMaA"
7 | },
8 | "source": [
9 | "# **TIPS: Text-Induced Pose Synthesis**\n",
10 | "\n",
11 | "This notebook demonstrates the inference pipeline of TIPS.\n",
12 | "\n",
13 | "*Accepted in The European Conference on Computer Vision (ECCV) 2022.*\n",
14 | "\n",
15 | "https://prasunroy.github.io/tips\n"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {
21 | "id": "llQwjAlyVUGg"
22 | },
23 | "source": [
24 | "## Getting started"
25 | ]
26 | },
27 | {
28 | "cell_type": "markdown",
29 | "metadata": {
30 | "id": "oIecdeBgX1OT"
31 | },
32 | "source": [
33 | "Import dependencies"
34 | ]
35 | },
36 | {
37 | "cell_type": "code",
38 | "execution_count": null,
39 | "metadata": {
40 | "id": "-xi6AyqVYPiY"
41 | },
42 | "outputs": [],
43 | "source": [
44 | "import datetime\n",
45 | "import numpy as np\n",
46 | "import os\n",
47 | "import pandas as pd\n",
48 | "from PIL import Image"
49 | ]
50 | },
51 | {
52 | "cell_type": "code",
53 | "execution_count": null,
54 | "metadata": {
55 | "id": "R67U4fbTYzMx"
56 | },
57 | "outputs": [],
58 | "source": [
59 | "from tips import TIPS\n",
60 | "from tips import visualize_skeletons, visualize"
61 | ]
62 | },
63 | {
64 | "cell_type": "markdown",
65 | "metadata": {
66 | "id": "GF5ZPPY_Y4GD"
67 | },
68 | "source": [
69 | "Configure environment"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": null,
75 | "metadata": {
76 | "id": "mZ8GBlzhZIa2"
77 | },
78 | "outputs": [],
79 | "source": [
80 | "prng = np.random.default_rng(1)\n",
81 | "\n",
82 | "ckpt_text2pose = './checkpoints/text2pose_75000.pth'\n",
83 | "ckpt_refinenet = './checkpoints/refinenet_100.pth'\n",
84 | "ckpt_pose2pose = './checkpoints/pose2pose_260500.pth'\n",
85 | "\n",
86 | "timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')\n",
87 | "\n",
88 | "data_root = './data'\n",
89 | "save_root_df2df = f'./output/{timestamp}/df2df'\n",
90 | "save_root_df2rw = f'./output/{timestamp}/df2rw'\n",
91 | "\n",
92 | "keypoints = pd.read_csv('./data/keypoints.csv', index_col='file_id')\n",
93 | "encodings = pd.read_csv('./data/encodings.csv', index_col='file_id')\n",
94 | "img_descs = pd.read_csv('./data/descriptions.csv', index_col='file_id')\n",
95 | "img_pairs_df2df = pd.read_csv('./data/img_pairs_df2df.csv')\n",
96 | "img_pairs_df2rw = pd.read_csv('./data/img_pairs_df2rw.csv')\n",
97 | "\n",
98 | "font = './data/FreeMono.ttf'\n",
99 | "bbox = (40, 0, 216, 256)\n",
100 | "\n",
101 | "file_id = lambda path: os.path.splitext(os.path.basename(path))[0]\n",
102 | "\n",
103 | "if not os.path.isdir(save_root_df2df): os.makedirs(save_root_df2df)\n",
104 | "if not os.path.isdir(save_root_df2rw): os.makedirs(save_root_df2rw)\n",
105 | "\n",
106 | "# Sample a random noise vector from a standard normal distribution\n",
107 | "z = prng.normal(size=128).astype(np.float32)"
108 | ]
109 | },
110 | {
111 | "cell_type": "markdown",
112 | "metadata": {
113 | "id": "Pod3SkPOaFGk"
114 | },
115 | "source": [
116 | "## Initialize TIPS"
117 | ]
118 | },
119 | {
120 | "cell_type": "code",
121 | "execution_count": null,
122 | "metadata": {
123 | "id": "OdMxzLtGaN2w"
124 | },
125 | "outputs": [],
126 | "source": [
127 | "tips = TIPS(ckpt_text2pose, ckpt_refinenet, ckpt_pose2pose)"
128 | ]
129 | },
130 | {
131 | "cell_type": "markdown",
132 | "metadata": {
133 | "id": "v-AOEIS94Qok"
134 | },
135 | "source": [
136 | "## Generation with DeepFashion targets (*within distribution*)"
137 | ]
138 | },
139 | {
140 | "cell_type": "markdown",
141 | "metadata": {
142 | "id": "5lI4ExM9uXUf"
143 | },
144 | "source": [
145 | "#### Load a random test sample"
146 | ]
147 | },
148 | {
149 | "cell_type": "code",
150 | "execution_count": null,
151 | "metadata": {
152 | "id": "xAj3GmX2vMCl"
153 | },
154 | "outputs": [],
155 | "source": [
156 | "index = np.random.randint(0, len(img_pairs_df2df))\n",
157 | "\n",
158 | "fpA = img_pairs_df2df.iloc[index].imgA\n",
159 | "fpB = img_pairs_df2df.iloc[index].imgB\n",
160 | "\n",
161 | "source_image = Image.open(f'{data_root}/{fpA}')\n",
162 | "target_image = Image.open(f'{data_root}/{fpB}')\n",
163 | "\n",
164 | "source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
165 | "target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
166 | "\n",
167 | "source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
168 | "target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
169 | "\n",
170 | "source_text_description = img_descs.loc[file_id(fpA)].description\n",
171 | "target_text_description = img_descs.loc[file_id(fpB)].description"
172 | ]
173 | },
174 | {
175 | "cell_type": "markdown",
176 | "metadata": {
177 | "id": "ske1_3M47slz"
178 | },
179 | "source": [
180 | "#### Keypoints guided benchmark"
181 | ]
182 | },
183 | {
184 | "cell_type": "code",
185 | "execution_count": null,
186 | "metadata": {
187 | "id": "rOEyuU4Q70Pj"
188 | },
189 | "outputs": [],
190 | "source": [
191 | "generated_image = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
192 | "\n",
193 | "images_dict = {\n",
194 | " 'iA': source_image.crop(bbox),\n",
195 | " 'iB': target_image.crop(bbox),\n",
196 | " 'iB_k': generated_image.crop(bbox),\n",
197 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
198 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox)\n",
199 | "}\n",
200 | "\n",
201 | "layout = [['iA', 'kA', 'iB', 'kB', 'iB_k']]\n",
202 | "\n",
203 | "grid = visualize(images_dict, layout, True, font)\n",
204 | "\n",
205 | "display(grid)"
206 | ]
207 | },
208 | {
209 | "cell_type": "markdown",
210 | "metadata": {
211 | "id": "rC3wDS3H2WXe"
212 | },
213 | "source": [
214 | "#### Partially text guided pipeline"
215 | ]
216 | },
217 | {
218 | "cell_type": "code",
219 | "execution_count": null,
220 | "metadata": {
221 | "id": "PdGCFL0y2rnQ"
222 | },
223 | "outputs": [],
224 | "source": [
225 | "out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
226 | "\n",
227 | "images_dict = {\n",
228 | " 'iA': source_image.crop(bbox),\n",
229 | " 'iB': target_image.crop(bbox),\n",
230 | " 'iB_c': out1['iB_c'].crop(bbox),\n",
231 | " 'iB_f': out1['iB_f'].crop(bbox),\n",
232 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
233 | " 'kB_c': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
234 | " 'kB_f': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
235 | "}\n",
236 | "\n",
237 | "layout = [['iA', 'kA', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA', 'iB', 'kB_f', 'iB_f']]\n",
238 | "\n",
239 | "grid = visualize(images_dict, layout, True, font)\n",
240 | "\n",
241 | "display(grid)\n",
242 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
243 | ]
244 | },
245 | {
246 | "cell_type": "markdown",
247 | "metadata": {
248 | "id": "GAEc3Pwu68en"
249 | },
250 | "source": [
251 | "#### Fully text guided pipeline"
252 | ]
253 | },
254 | {
255 | "cell_type": "code",
256 | "execution_count": null,
257 | "metadata": {
258 | "id": "A68vTs717Dta"
259 | },
260 | "outputs": [],
261 | "source": [
262 | "out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
263 | "\n",
264 | "images_dict = {\n",
265 | " 'iA': source_image.crop(bbox),\n",
266 | " 'iB': target_image.crop(bbox),\n",
267 | " 'iB_c': out2['iB_c'].crop(bbox),\n",
268 | " 'iB_f': out2['iB_f'].crop(bbox),\n",
269 | " 'kA_c': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
270 | " 'kA_f': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
271 | " 'kB_c': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
272 | " 'kB_f': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
273 | "}\n",
274 | "\n",
275 | "layout = [['iA', 'kA_c', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA_f', 'iB', 'kB_f', 'iB_f']]\n",
276 | "\n",
277 | "grid = visualize(images_dict, layout, True, font)\n",
278 | "\n",
279 | "display(grid)\n",
280 | "print('\\nSource description:\\n\\n' + source_text_description.replace('. ', '.\\n'))\n",
281 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
282 | ]
283 | },
284 | {
285 | "cell_type": "markdown",
286 | "metadata": {
287 | "id": "AhC_maow-hbD"
288 | },
289 | "source": [
290 | "## Generation with Real World targets (*out of distribution*)"
291 | ]
292 | },
293 | {
294 | "cell_type": "markdown",
295 | "metadata": {
296 | "id": "LPdVn2xr-hbY"
297 | },
298 | "source": [
299 | "#### Load a random test sample"
300 | ]
301 | },
302 | {
303 | "cell_type": "code",
304 | "execution_count": null,
305 | "metadata": {
306 | "id": "kwDWzqpu-hbZ"
307 | },
308 | "outputs": [],
309 | "source": [
310 | "index = np.random.randint(0, len(img_pairs_df2rw))\n",
311 | "\n",
312 | "fpA = img_pairs_df2rw.iloc[index].imgA\n",
313 | "fpB = img_pairs_df2rw.iloc[index].imgB\n",
314 | "\n",
315 | "source_image = Image.open(f'{data_root}/{fpA}')\n",
316 | "target_image = Image.open(f'{data_root}/{fpB}')\n",
317 | "\n",
318 | "source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
319 | "target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
320 | "\n",
321 | "source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
322 | "target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
323 | "\n",
324 | "source_text_description = img_descs.loc[file_id(fpA)].description\n",
325 | "target_text_description = img_descs.loc[file_id(fpB)].description"
326 | ]
327 | },
328 | {
329 | "cell_type": "markdown",
330 | "metadata": {
331 | "id": "VpeZ-tg9-hbb"
332 | },
333 | "source": [
334 | "#### Keypoints guided benchmark"
335 | ]
336 | },
337 | {
338 | "cell_type": "code",
339 | "execution_count": null,
340 | "metadata": {
341 | "id": "Dz_19CRq-hbc"
342 | },
343 | "outputs": [],
344 | "source": [
345 | "generated_image = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
346 | "\n",
347 | "images_dict = {\n",
348 | " 'iA': source_image.crop(bbox),\n",
349 | " 'iB': target_image.crop(bbox),\n",
350 | " 'iB_k': generated_image.crop(bbox),\n",
351 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
352 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox)\n",
353 | "}\n",
354 | "\n",
355 | "layout = [['iA', 'kA', 'iB', 'kB', 'iB_k']]\n",
356 | "\n",
357 | "grid = visualize(images_dict, layout, True, font)\n",
358 | "\n",
359 | "display(grid)"
360 | ]
361 | },
362 | {
363 | "cell_type": "markdown",
364 | "metadata": {
365 | "id": "TgYMrH9d-hbd"
366 | },
367 | "source": [
368 | "#### Partially text guided pipeline"
369 | ]
370 | },
371 | {
372 | "cell_type": "code",
373 | "execution_count": null,
374 | "metadata": {
375 | "id": "hE1skWtc-hbd"
376 | },
377 | "outputs": [],
378 | "source": [
379 | "out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
380 | "\n",
381 | "images_dict = {\n",
382 | " 'iA': source_image.crop(bbox),\n",
383 | " 'iB': target_image.crop(bbox),\n",
384 | " 'iB_c': out1['iB_c'].crop(bbox),\n",
385 | " 'iB_f': out1['iB_f'].crop(bbox),\n",
386 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
387 | " 'kB_c': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
388 | " 'kB_f': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
389 | "}\n",
390 | "\n",
391 | "layout = [['iA', 'kA', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA', 'iB', 'kB_f', 'iB_f']]\n",
392 | "\n",
393 | "grid = visualize(images_dict, layout, True, font)\n",
394 | "\n",
395 | "display(grid)\n",
396 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
397 | ]
398 | },
399 | {
400 | "cell_type": "markdown",
401 | "metadata": {
402 | "id": "DJ0iHUeq-hbe"
403 | },
404 | "source": [
405 | "#### Fully text guided pipeline"
406 | ]
407 | },
408 | {
409 | "cell_type": "code",
410 | "execution_count": null,
411 | "metadata": {
412 | "id": "o-CHrbRo-hbf"
413 | },
414 | "outputs": [],
415 | "source": [
416 | "out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
417 | "\n",
418 | "images_dict = {\n",
419 | " 'iA': source_image.crop(bbox),\n",
420 | " 'iB': target_image.crop(bbox),\n",
421 | " 'iB_c': out2['iB_c'].crop(bbox),\n",
422 | " 'iB_f': out2['iB_f'].crop(bbox),\n",
423 | " 'kA_c': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
424 | " 'kA_f': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
425 | " 'kB_c': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
426 | " 'kB_f': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
427 | "}\n",
428 | "\n",
429 | "layout = [['iA', 'kA_c', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA_f', 'iB', 'kB_f', 'iB_f']]\n",
430 | "\n",
431 | "grid = visualize(images_dict, layout, True, font)\n",
432 | "\n",
433 | "display(grid)\n",
434 | "print('\\nSource description:\\n\\n' + source_text_description.replace('. ', '.\\n'))\n",
435 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
436 | ]
437 | },
438 | {
439 | "cell_type": "markdown",
440 | "metadata": {
441 | "id": "fvHwRacqEGBO"
442 | },
443 | "source": [
444 | "## Generate all *within distribution* samples\n",
445 | "\n",
446 | "This will generate all *within distribution* test samples for reproducibility.\n"
447 | ]
448 | },
449 | {
450 | "cell_type": "code",
451 | "execution_count": null,
452 | "metadata": {
453 | "id": "ie7X3hK9FPY5"
454 | },
455 | "outputs": [],
456 | "source": [
457 | "layout = [\n",
458 | " ['iA', 'kA', 'iB', 'kB', 'iB_k0'],\n",
459 | " ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],\n",
460 | " ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],\n",
461 | " ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],\n",
462 | " ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']\n",
463 | "]\n",
464 | "\n",
465 | "for i in range(len(img_pairs_df2df)):\n",
466 | " fpA = img_pairs_df2df.iloc[i].imgA\n",
467 | " fpB = img_pairs_df2df.iloc[i].imgB\n",
468 | " \n",
469 | " source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
470 | " target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
471 | " \n",
472 | " source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
473 | " target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
474 | " \n",
475 | " source_image = Image.open(f'{data_root}/{fpA}')\n",
476 | " target_image = Image.open(f'{data_root}/{fpB}')\n",
477 | " \n",
478 | " iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
479 | " out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
480 | " out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
481 | " \n",
482 | " images_dict = {\n",
483 | " 'iA': source_image.crop(bbox),\n",
484 | " 'iB': target_image.crop(bbox),\n",
485 | " 'iB_k0': iB_k.crop(bbox),\n",
486 | " 'iB_c1': out1['iB_c'].crop(bbox),\n",
487 | " 'iB_f1': out1['iB_f'].crop(bbox),\n",
488 | " 'iB_c2': out2['iB_c'].crop(bbox),\n",
489 | " 'iB_f2': out2['iB_f'].crop(bbox),\n",
490 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
491 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
492 | " 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
493 | " 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
494 | " 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
495 | " 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
496 | " 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
497 | " 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
498 | " }\n",
499 | " \n",
500 | " grid = visualize(images_dict, layout, True, font)\n",
501 | " grid.save(f'{save_root_df2df}/{file_id(fpA)}____{file_id(fpB)}.png')\n",
502 | " print(f'\\r[DF2DF] Testing TIPS inference pipeline... {i+1}/{len(img_pairs_df2df)}', end='')\n",
503 | "\n",
504 | "print('')"
505 | ]
506 | },
507 | {
508 | "cell_type": "markdown",
509 | "metadata": {
510 | "id": "WcQ16H0BJNvn"
511 | },
512 | "source": [
513 | "## Generate all *out of distribution* samples\n",
514 | "\n",
515 | "This will generate all *out of distribution* test samples for reproducibility.\n"
516 | ]
517 | },
518 | {
519 | "cell_type": "code",
520 | "execution_count": null,
521 | "metadata": {
522 | "id": "7oiaNr0FJNv9"
523 | },
524 | "outputs": [],
525 | "source": [
526 | "layout = [\n",
527 | " ['iA', 'kA', 'iB', 'kB', 'iB_k0'],\n",
528 | " ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],\n",
529 | " ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],\n",
530 | " ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],\n",
531 | " ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']\n",
532 | "]\n",
533 | "\n",
534 | "for i in range(len(img_pairs_df2rw)):\n",
535 | " fpA = img_pairs_df2rw.iloc[i].imgA\n",
536 | " fpB = img_pairs_df2rw.iloc[i].imgB\n",
537 | " \n",
538 | " source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
539 | " target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
540 | " \n",
541 | " source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
542 | " target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
543 | " \n",
544 | " source_image = Image.open(f'{data_root}/{fpA}')\n",
545 | " target_image = Image.open(f'{data_root}/{fpB}')\n",
546 | " \n",
547 | " iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
548 | " out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
549 | " out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
550 | " \n",
551 | " images_dict = {\n",
552 | " 'iA': source_image.crop(bbox),\n",
553 | " 'iB': target_image.crop(bbox),\n",
554 | " 'iB_k0': iB_k.crop(bbox),\n",
555 | " 'iB_c1': out1['iB_c'].crop(bbox),\n",
556 | " 'iB_f1': out1['iB_f'].crop(bbox),\n",
557 | " 'iB_c2': out2['iB_c'].crop(bbox),\n",
558 | " 'iB_f2': out2['iB_f'].crop(bbox),\n",
559 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
560 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
561 | " 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
562 | " 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
563 | " 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
564 | " 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
565 | " 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
566 | " 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
567 | " }\n",
568 | " \n",
569 | " grid = visualize(images_dict, layout, True, font)\n",
570 | " grid.save(f'{save_root_df2rw}/{file_id(fpA)}____{file_id(fpB)}.png')\n",
571 | " print(f'\\r[DF2RW] Testing TIPS inference pipeline... {i+1}/{len(img_pairs_df2rw)}', end='')\n",
572 | "\n",
573 | "print('')"
574 | ]
575 | },
576 | {
577 | "cell_type": "markdown",
578 | "metadata": {
579 | "id": "IH66_yKYR6v6"
580 | },
581 | "source": [
582 | "# ***Thank you for checking out TIPS!***\n"
583 | ]
584 | }
585 | ],
586 | "metadata": {
587 | "accelerator": "GPU",
588 | "colab": {
589 | "collapsed_sections": [],
590 | "name": "TIPS_demo.ipynb",
591 | "provenance": [],
592 | "toc_visible": true
593 | },
594 | "kernelspec": {
595 | "display_name": "Python 3 (ipykernel)",
596 | "language": "python",
597 | "name": "python3"
598 | },
599 | "language_info": {
600 | "codemirror_mode": {
601 | "name": "ipython",
602 | "version": 3
603 | },
604 | "file_extension": ".py",
605 | "mimetype": "text/x-python",
606 | "name": "python",
607 | "nbconvert_exporter": "python",
608 | "pygments_lexer": "ipython3",
609 | "version": "3.11.3"
610 | }
611 | },
612 | "nbformat": 4,
613 | "nbformat_minor": 4
614 | }
615 |
--------------------------------------------------------------------------------
/demo/test_df2df.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Test TIPS inference pipeline
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import datetime
12 | import numpy as np
13 | import os
14 | import pandas as pd
15 | from PIL import Image
16 | from tips import TIPS
17 | from tips import visualize_skeletons, visualize
18 |
19 |
20 | # -----------------------------------------------------------------------------
21 | prng = np.random.default_rng(1)
22 |
23 | ckpt_text2pose = './checkpoints/text2pose_75000.pth'
24 | ckpt_refinenet = './checkpoints/refinenet_100.pth'
25 | ckpt_pose2pose = './checkpoints/pose2pose_260500.pth'
26 |
27 | data_root = './data'
28 | save_root = f'./output/df2df_{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}'
29 |
30 | keypoints = pd.read_csv('./data/keypoints.csv', index_col='file_id')
31 | encodings = pd.read_csv('./data/encodings.csv', index_col='file_id')
32 | img_pairs = pd.read_csv('./data/img_pairs_df2df.csv')
33 |
34 | font = './data/FreeMono.ttf'
35 | bbox = (40, 0, 216, 256)
36 | # -----------------------------------------------------------------------------
37 |
38 |
39 | def file_id(path):
40 | return os.path.splitext(os.path.basename(path))[0]
41 |
42 |
43 | if not os.path.isdir(save_root):
44 | os.makedirs(save_root)
45 |
46 |
47 | tips = TIPS(ckpt_text2pose, ckpt_refinenet, ckpt_pose2pose)
48 |
49 |
50 | z = prng.normal(size=128).astype(np.float32)
51 |
52 |
53 | layout = [
54 | ['iA', 'kA', 'iB', 'kB', 'iB_k0'],
55 | ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],
56 | ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],
57 | ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],
58 | ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']
59 | ]
60 |
61 |
62 | for i in range(len(img_pairs)):
63 | fpA = img_pairs.iloc[i].imgA
64 | fpB = img_pairs.iloc[i].imgB
65 |
66 | source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)
67 | target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)
68 |
69 | source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)
70 | target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)
71 |
72 | source_image = Image.open(f'{data_root}/{fpA}')
73 | target_image = Image.open(f'{data_root}/{fpB}')
74 |
75 | iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)
76 | out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)
77 | out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)
78 |
79 | images_dict = {
80 | 'iA': source_image.crop(bbox),
81 | 'iB': target_image.crop(bbox),
82 | 'iB_k0': iB_k.crop(bbox),
83 | 'iB_c1': out1['iB_c'].crop(bbox),
84 | 'iB_f1': out1['iB_f'].crop(bbox),
85 | 'iB_c2': out2['iB_c'].crop(bbox),
86 | 'iB_f2': out2['iB_f'].crop(bbox),
87 | 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),
88 | 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),
89 | 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),
90 | 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),
91 | 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),
92 | 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),
93 | 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),
94 | 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),
95 | }
96 |
97 | grid = visualize(images_dict, layout, True, font)
98 | grid.save(f'{save_root}/{file_id(fpA)}____{file_id(fpB)}.png')
99 | print(f'\r[TIPS] Testing inference pipeline... {i+1}/{len(img_pairs)}', end='')
100 |
101 | print('')
102 |
--------------------------------------------------------------------------------
/demo/test_df2rw.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Test TIPS inference pipeline
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import datetime
12 | import numpy as np
13 | import os
14 | import pandas as pd
15 | from PIL import Image
16 | from tips import TIPS
17 | from tips import visualize_skeletons, visualize
18 |
19 |
20 | # -----------------------------------------------------------------------------
21 | prng = np.random.default_rng(1)
22 |
23 | ckpt_text2pose = './checkpoints/text2pose_75000.pth'
24 | ckpt_refinenet = './checkpoints/refinenet_100.pth'
25 | ckpt_pose2pose = './checkpoints/pose2pose_260500.pth'
26 |
27 | data_root = './data'
28 | save_root = f'./output/df2rw_{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}'
29 |
30 | keypoints = pd.read_csv('./data/keypoints.csv', index_col='file_id')
31 | encodings = pd.read_csv('./data/encodings.csv', index_col='file_id')
32 | img_pairs = pd.read_csv('./data/img_pairs_df2rw.csv')
33 |
34 | font = './data/FreeMono.ttf'
35 | bbox = (40, 0, 216, 256)
36 | # -----------------------------------------------------------------------------
37 |
38 |
39 | def file_id(path):
40 | return os.path.splitext(os.path.basename(path))[0]
41 |
42 |
43 | if not os.path.isdir(save_root):
44 | os.makedirs(save_root)
45 |
46 |
47 | tips = TIPS(ckpt_text2pose, ckpt_refinenet, ckpt_pose2pose)
48 |
49 |
50 | z = prng.normal(size=128).astype(np.float32)
51 |
52 |
53 | layout = [
54 | ['iA', 'kA', 'iB', 'kB', 'iB_k0'],
55 | ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],
56 | ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],
57 | ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],
58 | ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']
59 | ]
60 |
61 |
62 | for i in range(len(img_pairs)):
63 | fpA = img_pairs.iloc[i].imgA
64 | fpB = img_pairs.iloc[i].imgB
65 |
66 | source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)
67 | target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)
68 |
69 | source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)
70 | target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)
71 |
72 | source_image = Image.open(f'{data_root}/{fpA}')
73 | target_image = Image.open(f'{data_root}/{fpB}')
74 |
75 | iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)
76 | out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)
77 | out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)
78 |
79 | images_dict = {
80 | 'iA': source_image.crop(bbox),
81 | 'iB': target_image.crop(bbox),
82 | 'iB_k0': iB_k.crop(bbox),
83 | 'iB_c1': out1['iB_c'].crop(bbox),
84 | 'iB_f1': out1['iB_f'].crop(bbox),
85 | 'iB_c2': out2['iB_c'].crop(bbox),
86 | 'iB_f2': out2['iB_f'].crop(bbox),
87 | 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),
88 | 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),
89 | 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),
90 | 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),
91 | 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),
92 | 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),
93 | 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),
94 | 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),
95 | }
96 |
97 | grid = visualize(images_dict, layout, True, font)
98 | grid.save(f'{save_root}/{file_id(fpA)}____{file_id(fpB)}.png')
99 | print(f'\r[TIPS] Testing inference pipeline... {i+1}/{len(img_pairs)}', end='')
100 |
101 | print('')
102 |
--------------------------------------------------------------------------------
/demo/tips/__init__.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Package initialization
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | __version__ = '1.0.0'
12 |
13 | from .tips import TIPS
14 | from .visualization import visualize_skeletons, visualize
15 |
--------------------------------------------------------------------------------
/demo/tips/models/__init__.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Package initialization
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | __version__ = '1.0.0'
12 |
--------------------------------------------------------------------------------
/demo/tips/models/pose2pose.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Stage-3 network: Pose2Pose generator
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import torch
12 | import torch.nn as nn
13 |
14 |
15 | def conv1x1(in_channels, out_channels):
16 | return nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False)
17 |
18 |
19 | def conv3x3(in_channels, out_channels):
20 | return nn.Conv2d(in_channels, out_channels, 3, 1, 1, bias=False)
21 |
22 |
23 | def downconv2x(in_channels, out_channels):
24 | return nn.Conv2d(in_channels, out_channels, 4, 2, 1, bias=False)
25 |
26 |
27 | def upconv2x(in_channels, out_channels):
28 | return nn.ConvTranspose2d(in_channels, out_channels, 4, 2, 1, bias=False)
29 |
30 |
31 | class ResidualBlock(nn.Module):
32 |
33 | def __init__(self, num_channels):
34 | super(ResidualBlock, self).__init__()
35 | layers = [
36 | conv3x3(num_channels, num_channels),
37 | nn.BatchNorm2d(num_channels),
38 | nn.ReLU(inplace=True),
39 | conv3x3(num_channels, num_channels),
40 | nn.BatchNorm2d(num_channels)
41 | ]
42 | self.layers = nn.Sequential(*layers)
43 |
44 | def forward(self, x):
45 | y = self.layers(x) + x
46 | return y
47 |
48 |
49 | class NetG(nn.Module):
50 |
51 | def __init__(self, in1_channels, in2_channels, out_channels, ngf=64):
52 | super(NetG, self).__init__()
53 |
54 | self.in1_conv1 = self.inconv(in1_channels, ngf)
55 | self.in1_down1 = self.down2x(ngf, ngf*2)
56 | self.in1_down2 = self.down2x(ngf*2, ngf*4)
57 | self.in1_down3 = self.down2x(ngf*4, ngf*8)
58 | self.in1_down4 = self.down2x(ngf*8, ngf*16)
59 |
60 | self.in2_conv1 = self.inconv(in2_channels, ngf)
61 | self.in2_down1 = self.down2x(ngf, ngf*2)
62 | self.in2_down2 = self.down2x(ngf*2, ngf*4)
63 | self.in2_down3 = self.down2x(ngf*4, ngf*8)
64 | self.in2_down4 = self.down2x(ngf*8, ngf*16)
65 |
66 | self.out_up1 = self.up2x(ngf*16, ngf*8)
67 | self.out_up2 = self.up2x(ngf*8, ngf*4)
68 | self.out_up3 = self.up2x(ngf*4, ngf*2)
69 | self.out_up4 = self.up2x(ngf*2, ngf)
70 |
71 | self.out_conv1 = self.outconv(ngf, out_channels)
72 |
73 | def inconv(self, in_channels, out_channels):
74 | return nn.Sequential(
75 | conv3x3(in_channels, out_channels),
76 | nn.BatchNorm2d(out_channels),
77 | nn.ReLU(inplace=True)
78 | )
79 |
80 | def outconv(self, in_channels, out_channels):
81 | return nn.Sequential(
82 | ResidualBlock(in_channels),
83 | ResidualBlock(in_channels),
84 | ResidualBlock(in_channels),
85 | ResidualBlock(in_channels),
86 | conv1x1(in_channels, out_channels),
87 | nn.Tanh()
88 | )
89 |
90 | def down2x(self, in_channels, out_channels):
91 | return nn.Sequential(
92 | downconv2x(in_channels, out_channels),
93 | nn.BatchNorm2d(out_channels),
94 | nn.ReLU(inplace=True),
95 | ResidualBlock(out_channels)
96 | )
97 |
98 | def up2x(self, in_channels, out_channels):
99 | return nn.Sequential(
100 | upconv2x(in_channels, out_channels),
101 | nn.BatchNorm2d(out_channels),
102 | nn.ReLU(inplace=True),
103 | ResidualBlock(out_channels)
104 | )
105 |
106 | def forward(self, x1, x2):
107 | x1_c1 = self.in1_conv1(x1)
108 | x1_d1 = self.in1_down1(x1_c1)
109 | x1_d2 = self.in1_down2(x1_d1)
110 | x1_d3 = self.in1_down3(x1_d2)
111 | x1_d4 = self.in1_down4(x1_d3)
112 |
113 | x2_c1 = self.in2_conv1(x2)
114 | x2_d1 = self.in2_down1(x2_c1)
115 | x2_d2 = self.in2_down2(x2_d1)
116 | x2_d3 = self.in2_down3(x2_d2)
117 | x2_d4 = self.in2_down4(x2_d3)
118 |
119 | y = x1_d4 * torch.sigmoid(x2_d4)
120 | y = self.out_up1(y)
121 | y = y * torch.sigmoid(x2_d3)
122 | y = self.out_up2(y)
123 | y = y * torch.sigmoid(x2_d2)
124 | y = self.out_up3(y)
125 | y = y * torch.sigmoid(x2_d1)
126 | y = self.out_up4(y)
127 | y = self.out_conv1(y)
128 |
129 | return y
130 |
--------------------------------------------------------------------------------
/demo/tips/models/refinenet.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Stage-2 network: RefineNet regressor
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import torch
12 | import torch.nn as nn
13 |
14 |
15 | class RefineNet(nn.Module):
16 |
17 | def __init__(self, in_features, out_features, bias=True):
18 | super(RefineNet, self).__init__()
19 | self.linear1 = nn.Linear(in_features, 128, bias=bias)
20 | self.linear2 = nn.Linear(128, 128, bias=bias)
21 | self.linear3 = nn.Linear(128, 128, bias=bias)
22 | self.linear4 = nn.Linear(128, out_features, bias=bias)
23 |
24 | def forward(self, x):
25 | y = torch.relu(self.linear1(x))
26 | y = torch.relu(self.linear2(y))
27 | y = torch.relu(self.linear3(y))
28 | y = torch.tanh(self.linear4(y))
29 | return y
30 |
--------------------------------------------------------------------------------
/demo/tips/models/text2pose.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Stage-1 network: Text2Pose generator
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import torch
12 | import torch.nn as nn
13 |
14 |
15 | def linear(in_features, out_features, bias=True):
16 | return nn.Sequential(
17 | nn.Linear(in_features, out_features, bias=bias),
18 | nn.LeakyReLU(inplace=True)
19 | )
20 |
21 |
22 | def upconv4x(in_channels, out_channels, bias=False):
23 | return nn.Sequential(
24 | nn.ConvTranspose2d(in_channels, out_channels, 4, 4, 0, bias=bias),
25 | nn.BatchNorm2d(out_channels),
26 | nn.ReLU(inplace=True)
27 | )
28 |
29 |
30 | def upconv2x_hidden(in_channels, out_channels, bias=False):
31 | return nn.Sequential(
32 | nn.ConvTranspose2d(in_channels, out_channels, 4, 2, 1, bias=bias),
33 | nn.BatchNorm2d(out_channels),
34 | nn.ReLU(inplace=True)
35 | )
36 |
37 |
38 | def upconv2x_output(in_channels, out_channels, bias=False):
39 | return nn.Sequential(
40 | nn.ConvTranspose2d(in_channels, out_channels, 4, 2, 1, bias=bias),
41 | nn.Tanh()
42 | )
43 |
44 |
45 | class NetG(nn.Module):
46 |
47 | def __init__(self, noise_dim, embed_dim, heatmap_channels, ngf=32):
48 | super(NetG, self).__init__()
49 | self.embed_linear = linear(embed_dim, noise_dim)
50 | self.combined_up1 = upconv4x(noise_dim*2, ngf*8)
51 | self.combined_up2 = upconv2x_hidden(ngf*8, ngf*4)
52 | self.combined_up3 = upconv2x_hidden(ngf*4, ngf*2)
53 | self.combined_up4 = upconv2x_hidden(ngf*2, ngf)
54 | self.combined_up5 = upconv2x_output(ngf, heatmap_channels)
55 |
56 | def forward(self, x1, x2):
57 | x1_noise = x1.view(x1.size(0), -1, 1, 1)
58 |
59 | x2_embed = x2.view(x2.size(0), 1, -1)
60 | x2_embed = self.embed_linear(x2_embed)
61 | x2_embed = x2_embed.view(x2_embed.size(0), -1, 1, 1)
62 |
63 | combined = torch.cat((x1_noise, x2_embed), dim=1)
64 |
65 | y = self.combined_up1(combined)
66 | y = self.combined_up2(y)
67 | y = self.combined_up3(y)
68 | y = self.combined_up4(y)
69 | y = self.combined_up5(y)
70 |
71 | return y
72 |
--------------------------------------------------------------------------------
/demo/tips/tips.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Inference pipeline
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import numpy as np
12 | import torch
13 | import torchvision
14 | from PIL import Image
15 | from .models.text2pose import NetG as Stage1
16 | from .models.refinenet import RefineNet as Stage2
17 | from .models.pose2pose import NetG as Stage3
18 |
19 |
20 | class TIPS(object):
21 |
22 | def __init__(self, ckpt_text2pose, ckpt_refinenet, ckpt_pose2pose):
23 | self.stage1 = Stage1(128, 84, 18, 32).eval()
24 | self.stage2 = Stage2(10, 10, True).eval()
25 | self.stage3 = Stage3(3, 36, 3, 64).eval()
26 |
27 | self.stage1.load_state_dict(torch.load(ckpt_text2pose))
28 | self.stage2.load_state_dict(torch.load(ckpt_refinenet))
29 | self.stage3.load_state_dict(torch.load(ckpt_pose2pose))
30 |
31 | if torch.cuda.is_available():
32 | self.stage1.cuda()
33 | self.stage2.cuda()
34 | self.stage3.cuda()
35 |
36 | self.transforms1 = torchvision.transforms.ToTensor()
37 | self.transforms2 = torchvision.transforms.Compose([
38 | torchvision.transforms.ToTensor(),
39 | torchvision.transforms.Normalize((0.5,), (0.5,))
40 | ])
41 |
42 | def heatmaps2keypoints(self, heatmaps, confidence):
43 | keypoints = []
44 | for k in range(heatmaps.shape[2]):
45 | heatmap_k = heatmaps[:, :, k]
46 | proba_max = np.max(heatmap_k)
47 | if proba_max > confidence:
48 | y, x = np.where(heatmap_k == proba_max)
49 | y, x = y[0], x[0]
50 | else:
51 | y, x = -1, -1
52 | keypoints.append((x, y))
53 | return np.int32(keypoints).reshape(-1)
54 |
55 | def keypoints2heatmaps(self, keypoints, size):
56 | keypoints = keypoints.reshape(-1, 2).astype(np.int32)
57 | heatmaps = np.zeros(size + (keypoints.shape[0],), dtype=np.float32)
58 | for k in range(keypoints.shape[0]):
59 | x, y = keypoints[k]
60 | if x < 0 or y < 0:
61 | continue
62 | heatmaps[y, x, k] = 1
63 | return heatmaps
64 |
65 | def stage1_inference(self, z, text_encoding):
66 | if torch.cuda.is_available():
67 | z = z.cuda()
68 | text_encoding = text_encoding.cuda()
69 | with torch.no_grad():
70 | heatmaps = self.stage1(z, text_encoding)
71 | return (heatmaps.detach().cpu().squeeze().permute(1, 2, 0).numpy() + 1.0) / 2.0
72 |
73 | def stage2_inference(self, keypoints):
74 | head_keypoints = keypoints.reshape(-1, 2)[[0, 14, 15, 16, 17], :].astype(np.int32)
75 | if np.allclose(head_keypoints[0], [-1, -1]):
76 | return keypoints
77 | x = np.where(head_keypoints == [-1, -1], 0, head_keypoints - head_keypoints[0])
78 | x = torch.tensor(x.reshape(1, -1).astype(np.float32)) / 50
79 | if torch.cuda.is_available():
80 | x = x.cuda()
81 | with torch.no_grad():
82 | p = self.stage2(x)
83 | p = (p.detach().cpu().squeeze().numpy() * 50).astype(np.int32).reshape(-1, 2)
84 | p = np.where(head_keypoints == [-1, -1], -1, p + head_keypoints[0])
85 | refined_keypoints = keypoints.reshape(-1, 2).astype(np.int32)
86 | refined_keypoints[[0, 14, 15, 16, 17], :] = p
87 | return refined_keypoints.reshape(keypoints.shape)
88 |
89 | def stage3_inference(self, source_image, source_heatmaps, target_heatmaps):
90 | x1 = source_image.unsqueeze(0)
91 | x2 = torch.cat((source_heatmaps, target_heatmaps), dim=0).unsqueeze(0)
92 | if torch.cuda.is_available():
93 | x1 = x1.cuda()
94 | x2 = x2.cuda()
95 | with torch.no_grad():
96 | p = self.stage3(x1, x2)
97 | p = (p.detach().cpu().squeeze().permute(1, 2, 0).numpy() + 1.0) / 2.0
98 | return np.clip(p * 255, 0, 255).astype(np.uint8)
99 |
100 | def benchmark(self, source_image, source_keypoints, target_keypoints):
101 | iA = self.transforms2(source_image)
102 | pA = self.transforms1(self.keypoints2heatmaps(source_keypoints, (256, 256)).astype(np.float32))
103 | pB = self.transforms1(self.keypoints2heatmaps(target_keypoints, (256, 256)).astype(np.float32))
104 | iB = self.stage3_inference(iA, pA, pB)
105 | return Image.fromarray(iB)
106 |
107 | def pipeline(self, source_image, source_keypoints, target_text_encoding, z):
108 | z = torch.tensor(z.reshape(1, -1).astype(np.float32))
109 | tB = (torch.tensor(target_text_encoding.reshape(1, -1).astype(np.float32)) - 0.5) / 0.5
110 | hB = self.stage1_inference(z, tB)
111 | kB = self.heatmaps2keypoints(hB, 0.2)
112 | kB = np.where(kB < 0, -1, kB * 4)
113 | pB = self.transforms1(self.keypoints2heatmaps(kB, (256, 256)).astype(np.float32))
114 | kB_f = self.stage2_inference(kB)
115 | pB_f = self.transforms1(self.keypoints2heatmaps(kB_f, (256, 256)).astype(np.float32))
116 | kA = source_keypoints.reshape(-1)
117 | pA = self.transforms1(self.keypoints2heatmaps(kA, (256, 256)).astype(np.float32))
118 | iA = self.transforms2(source_image)
119 | iB = self.stage3_inference(iA, pA, pB)
120 | iB_f = self.stage3_inference(iA, pA, pB_f)
121 | return {
122 | 'kB_c': kB,
123 | 'kB_f': kB_f,
124 | 'iB_c': Image.fromarray(iB),
125 | 'iB_f': Image.fromarray(iB_f)
126 | }
127 |
128 | def pipeline_full(self, source_image, source_text_encoding, target_text_encoding, z):
129 | z = torch.tensor(z.reshape(1, -1).astype(np.float32))
130 | tB = (torch.tensor(target_text_encoding.reshape(1, -1).astype(np.float32)) - 0.5) / 0.5
131 | hB = self.stage1_inference(z, tB)
132 | kB = self.heatmaps2keypoints(hB, 0.2)
133 | kB = np.where(kB < 0, -1, kB * 4)
134 | pB = self.transforms1(self.keypoints2heatmaps(kB, (256, 256)).astype(np.float32))
135 | kB_f = self.stage2_inference(kB)
136 | pB_f = self.transforms1(self.keypoints2heatmaps(kB_f, (256, 256)).astype(np.float32))
137 | tA = (torch.tensor(source_text_encoding.reshape(1, -1).astype(np.float32)) - 0.5) / 0.5
138 | hA = self.stage1_inference(z, tA)
139 | kA = self.heatmaps2keypoints(hA, 0.2)
140 | kA = np.where(kA < 0, -1, kA * 4)
141 | pA = self.transforms1(self.keypoints2heatmaps(kA, (256, 256)).astype(np.float32))
142 | kA_f = self.stage2_inference(kA)
143 | pA_f = self.transforms1(self.keypoints2heatmaps(kA_f, (256, 256)).astype(np.float32))
144 | iA = self.transforms2(source_image)
145 | iB = self.stage3_inference(iA, pA, pB)
146 | iB_f = self.stage3_inference(iA, pA_f, pB_f)
147 | return {
148 | 'kA_c': kA,
149 | 'kA_f': kA_f,
150 | 'kB_c': kB,
151 | 'kB_f': kB_f,
152 | 'iB_c': Image.fromarray(iB),
153 | 'iB_f': Image.fromarray(iB_f)
154 | }
155 |
--------------------------------------------------------------------------------
/demo/tips/visualization.py:
--------------------------------------------------------------------------------
1 | """TIPS: Text-Induced Pose Synthesis
2 |
3 | Visualization utilities
4 | Created on Thu Nov 18 10:00:00 2021
5 | Author: Prasun Roy | https://prasunroy.github.io
6 | GitHub: https://github.com/prasunroy/tips
7 |
8 | """
9 |
10 |
11 | import cv2
12 | import numpy as np
13 | from PIL import Image, ImageDraw, ImageFont
14 |
15 |
16 | def _draw_circle(image, point, color, radius=1):
17 | x, y = point
18 | if x >= 0 and y >= 0:
19 | cv2.circle(image, (int(x), int(y)), radius, color, -1, cv2.LINE_AA)
20 | return image
21 |
22 |
23 | def _draw_line(image, point1, point2, color, thickness=1):
24 | x1, y1 = point1
25 | x2, y2 = point2
26 | if x1 >= 0 and y1 >= 0 and x2 >= 0 and y2 >= 0:
27 | cv2.line(image, (int(x1), int(y1)), (int(x2), int(y2)), color, thickness, cv2.LINE_AA)
28 | return image
29 |
30 |
31 | def draw_keypoints(image, keypoints, radius=1, head_color=(128, 128, 128), alpha=1.0):
32 | overlay = image.copy()
33 | for kp in keypoints:
34 | for i, (x, y) in enumerate(kp.reshape(-1, 2)):
35 | if i in [0, 14, 15, 16, 17]:
36 | overlay = _draw_circle(overlay, (x, y), head_color, radius)
37 | else:
38 | overlay = _draw_circle(overlay, (x, y), (128, 128, 128), radius)
39 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
40 |
41 |
42 | def draw_connections(image, keypoints, thickness=1, head_color=(128, 128, 128), alpha=1.0):
43 | overlay = image.copy()
44 | conns_h = [(0, 14), (0, 15), (14, 16), (15, 17)]
45 | conns_b = [(0, 1), (1, 2), (1, 5), (2, 8), (5, 11), (8, 11)]
46 | conns_l = [(5, 6), (6, 7), (11, 12), (12, 13)]
47 | conns_r = [(2, 3), (3, 4), (8, 9), (9, 10)]
48 | for kp in keypoints:
49 | kp = kp.reshape(-1, 2)
50 | for i, j in conns_h:
51 | overlay = _draw_line(overlay, kp[i], kp[j], head_color, thickness)
52 | for i, j in conns_b:
53 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
54 | for i, j in conns_l:
55 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
56 | for i, j in conns_r:
57 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
58 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
59 |
60 |
61 | def visualize_skeletons(keypoints, keypoint_radius=3, connection_thickness=1,
62 | head_color=(128, 128, 128), grid_size=(256, 256), alpha=1.0):
63 | image = np.zeros((grid_size[1], grid_size[0], 3), dtype=np.uint8) + 255
64 | image = draw_connections(image, keypoints, connection_thickness, head_color, alpha)
65 | image = draw_keypoints(image, keypoints, keypoint_radius, head_color, alpha)
66 | return image
67 |
68 |
69 | def visualize(images_dict, layout, labels=False, font_file=None, font_size=20, font_color=(0, 0, 0)):
70 | w, h = np.int32([image.size for image in images_dict.values()]).max(axis=0)
71 | r, c = np.array(layout).shape
72 | grid = Image.new('RGB', (w*c, h*r), (255, 255, 255))
73 | for i in range(r):
74 | for j in range(c):
75 | key = layout[i][j]
76 | if key not in images_dict.keys():
77 | continue
78 | image = images_dict[key].copy()
79 | if labels:
80 | font = ImageFont.truetype(font_file, font_size)
81 | draw = ImageDraw.Draw(image)
82 | draw.text((4, 4), key, font_color, font)
83 | grid.paste(image, (j*w, i*h))
84 | return grid
85 |
--------------------------------------------------------------------------------
/docs/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 | TIPS: Text-Induced Pose Synthesis
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
105 |
106 |
107 |
108 |
109 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
146 |
147 | In computer vision, human pose synthesis and transfer deal with probabilistic image generation of a person in a previously unseen pose from an already available observation of that person. Though researchers have recently proposed several methods to achieve this task, most of these techniques derive the target pose directly from the desired target image on a specific dataset, making the underlying process challenging to apply in real-world scenarios as the generation of the target image is the actual aim. In this paper, we first present the shortcomings of current pose transfer algorithms and then propose a novel text-based pose transfer technique to address those issues. We divide the problem into three independent stages: (a) text to pose representation, (b) pose refinement, and (c) pose rendering. To the best of our knowledge, this is one of the first attempts to develop a text-based pose transfer framework where we also introduce a new dataset DF-PASS, by adding descriptive pose annotations for the images of the DeepFashion dataset. The proposed method generates promising results with significant qualitative and quantitative scores in our experiments.
148 |
149 |
150 |
151 |
152 |
153 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 | Click on the diagram for a zoomed view of the network architecture (Opens a new tab).
166 |
167 |
168 | The workflow is divided into three stages. In stage 1, we estimate a spatial representation \(K^*_B\) for the target pose \(P_B\) from the corresponding text description embedding \(v_B\). In stage 2, we regressively refine the initial estimation of the facial keypoints to obtain the refined target keypoints \(\tilde{K}^*_B\). Finally, in stage 3, we render the target image \(\tilde{I}_B\) by conditioning the pose transfer on the source image \(I_A\) having the keypoints \(K_A\) corresponding to the source pose \(P_A\).
169 |
170 |
171 |
172 |
173 |
174 |
175 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 | Keypoints-guided methods tend to produce structurally inaccurate results when the physical appearance of the target pose reference significantly differs from the condition image. This observation is more frequent for the out of distribution target poses than the within distribution target poses. On the other hand, the existing text-guided method occasionally misinterprets the target pose due to a limited set of basic poses used for pose representation. The proposed text-guided technique successfully addresses these issues while retaining the ability to generate visually decent results close to the keypoints-guided baseline.
186 |
187 |
188 |
189 |
190 |
191 |
192 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
220 |
227 |
228 |
229 |
230 | Code @ GitHub
231 |
232 |
233 |
234 |
241 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
257 |
258 | @inproceedings{roy2022tips,
259 | title = {TIPS: Text-Induced Pose Synthesis},
260 | author = {Roy, Prasun and Ghosh, Subhankar and Bhattacharya, Saumik and Pal, Umapada and Blumenstein, Michael},
261 | booktitle = {The European Conference on Computer Vision (ECCV)},
262 | month = {October},
263 | year = {2022}
264 | }
265 |
266 |
267 |
268 |
269 |
270 |
273 |
274 | VIDEO
275 |
276 |
277 |
285 |
286 |
287 |
315 |
316 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
--------------------------------------------------------------------------------
/docs/static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/docs/static/favicon.ico
--------------------------------------------------------------------------------
/docs/static/poster.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/docs/static/poster.pdf
--------------------------------------------------------------------------------
/docs/static/social_preview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/docs/static/social_preview.png
--------------------------------------------------------------------------------
/notebooks/TIPS_demo.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "TIPS_demo.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "toc_visible": true
10 | },
11 | "kernelspec": {
12 | "name": "python3",
13 | "display_name": "Python 3"
14 | },
15 | "language_info": {
16 | "name": "python"
17 | },
18 | "accelerator": "GPU"
19 | },
20 | "cells": [
21 | {
22 | "cell_type": "markdown",
23 | "metadata": {
24 | "id": "EIRGdFExTMaA"
25 | },
26 | "source": [
27 | "# **TIPS: Text-Induced Pose Synthesis**\n",
28 | "\n",
29 | "This notebook demonstrates the inference pipeline of TIPS.\n",
30 | "\n",
31 | "*Accepted in The European Conference on Computer Vision (ECCV) 2022.*\n",
32 | "\n",
33 | "https://prasunroy.github.io/tips\n"
34 | ]
35 | },
36 | {
37 | "cell_type": "markdown",
38 | "metadata": {
39 | "id": "llQwjAlyVUGg"
40 | },
41 | "source": [
42 | "## Getting started"
43 | ]
44 | },
45 | {
46 | "cell_type": "markdown",
47 | "metadata": {
48 | "id": "WGRaCQXz2K6A"
49 | },
50 | "source": [
51 | "Download and extract the required resources"
52 | ]
53 | },
54 | {
55 | "cell_type": "code",
56 | "metadata": {
57 | "id": "Ks3itvIuVD9z"
58 | },
59 | "source": [
60 | "!gdown 1zTG9M06ckW0z4MvJks3-JSC8sJguiZJH -O tips.zip && unzip -oq tips.zip && rm tips.zip && rm -r sample_data"
61 | ],
62 | "execution_count": null,
63 | "outputs": []
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {
68 | "id": "oIecdeBgX1OT"
69 | },
70 | "source": [
71 | "Import dependencies"
72 | ]
73 | },
74 | {
75 | "cell_type": "code",
76 | "metadata": {
77 | "id": "-xi6AyqVYPiY"
78 | },
79 | "source": [
80 | "import datetime\n",
81 | "import numpy as np\n",
82 | "import os\n",
83 | "import pandas as pd\n",
84 | "from PIL import Image"
85 | ],
86 | "execution_count": null,
87 | "outputs": []
88 | },
89 | {
90 | "cell_type": "code",
91 | "metadata": {
92 | "id": "R67U4fbTYzMx"
93 | },
94 | "source": [
95 | "from tips import TIPS\n",
96 | "from tips import visualize_skeletons, visualize"
97 | ],
98 | "execution_count": null,
99 | "outputs": []
100 | },
101 | {
102 | "cell_type": "code",
103 | "metadata": {
104 | "id": "hzBoq8b2IPw4"
105 | },
106 | "source": [
107 | "from google.colab import files"
108 | ],
109 | "execution_count": null,
110 | "outputs": []
111 | },
112 | {
113 | "cell_type": "markdown",
114 | "metadata": {
115 | "id": "GF5ZPPY_Y4GD"
116 | },
117 | "source": [
118 | "Configure environment"
119 | ]
120 | },
121 | {
122 | "cell_type": "code",
123 | "metadata": {
124 | "id": "mZ8GBlzhZIa2"
125 | },
126 | "source": [
127 | "prng = np.random.default_rng(1)\n",
128 | "\n",
129 | "ckpt_text2pose = './checkpoints/text2pose_75000.pth'\n",
130 | "ckpt_refinenet = './checkpoints/refinenet_100.pth'\n",
131 | "ckpt_pose2pose = './checkpoints/pose2pose_260500.pth'\n",
132 | "\n",
133 | "timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')\n",
134 | "\n",
135 | "data_root = './data'\n",
136 | "save_root_df2df = f'./output/{timestamp}/df2df'\n",
137 | "save_root_df2rw = f'./output/{timestamp}/df2rw'\n",
138 | "\n",
139 | "keypoints = pd.read_csv('./data/keypoints.csv', index_col='file_id')\n",
140 | "encodings = pd.read_csv('./data/encodings.csv', index_col='file_id')\n",
141 | "img_descs = pd.read_csv('./data/descriptions.csv', index_col='file_id')\n",
142 | "img_pairs_df2df = pd.read_csv('./data/img_pairs_df2df.csv')\n",
143 | "img_pairs_df2rw = pd.read_csv('./data/img_pairs_df2rw.csv')\n",
144 | "\n",
145 | "font = './data/FreeMono.ttf'\n",
146 | "bbox = (40, 0, 216, 256)\n",
147 | "\n",
148 | "file_id = lambda path: os.path.splitext(os.path.basename(path))[0]\n",
149 | "\n",
150 | "if not os.path.isdir(save_root_df2df): os.makedirs(save_root_df2df)\n",
151 | "if not os.path.isdir(save_root_df2rw): os.makedirs(save_root_df2rw)\n",
152 | "\n",
153 | "# Sample a random noise vector from a standard normal distribution\n",
154 | "z = prng.normal(size=128).astype(np.float32)"
155 | ],
156 | "execution_count": null,
157 | "outputs": []
158 | },
159 | {
160 | "cell_type": "markdown",
161 | "metadata": {
162 | "id": "Pod3SkPOaFGk"
163 | },
164 | "source": [
165 | "## Initialize TIPS"
166 | ]
167 | },
168 | {
169 | "cell_type": "code",
170 | "metadata": {
171 | "id": "OdMxzLtGaN2w"
172 | },
173 | "source": [
174 | "tips = TIPS(ckpt_text2pose, ckpt_refinenet, ckpt_pose2pose)"
175 | ],
176 | "execution_count": null,
177 | "outputs": []
178 | },
179 | {
180 | "cell_type": "markdown",
181 | "metadata": {
182 | "id": "v-AOEIS94Qok"
183 | },
184 | "source": [
185 | "## Generation with DeepFashion targets (*within distribution*)"
186 | ]
187 | },
188 | {
189 | "cell_type": "markdown",
190 | "metadata": {
191 | "id": "5lI4ExM9uXUf"
192 | },
193 | "source": [
194 | "#### Load a random test sample"
195 | ]
196 | },
197 | {
198 | "cell_type": "code",
199 | "metadata": {
200 | "id": "xAj3GmX2vMCl"
201 | },
202 | "source": [
203 | "index = np.random.randint(0, len(img_pairs_df2df))\n",
204 | "\n",
205 | "fpA = img_pairs_df2df.iloc[index].imgA\n",
206 | "fpB = img_pairs_df2df.iloc[index].imgB\n",
207 | "\n",
208 | "source_image = Image.open(f'{data_root}/{fpA}')\n",
209 | "target_image = Image.open(f'{data_root}/{fpB}')\n",
210 | "\n",
211 | "source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
212 | "target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
213 | "\n",
214 | "source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
215 | "target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
216 | "\n",
217 | "source_text_description = img_descs.loc[file_id(fpA)].description\n",
218 | "target_text_description = img_descs.loc[file_id(fpB)].description"
219 | ],
220 | "execution_count": null,
221 | "outputs": []
222 | },
223 | {
224 | "cell_type": "markdown",
225 | "metadata": {
226 | "id": "ske1_3M47slz"
227 | },
228 | "source": [
229 | "#### Keypoints guided benchmark"
230 | ]
231 | },
232 | {
233 | "cell_type": "code",
234 | "metadata": {
235 | "id": "rOEyuU4Q70Pj"
236 | },
237 | "source": [
238 | "generated_image = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
239 | "\n",
240 | "images_dict = {\n",
241 | " 'iA': source_image.crop(bbox),\n",
242 | " 'iB': target_image.crop(bbox),\n",
243 | " 'iB_k': generated_image.crop(bbox),\n",
244 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
245 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox)\n",
246 | "}\n",
247 | "\n",
248 | "layout = [['iA', 'kA', 'iB', 'kB', 'iB_k']]\n",
249 | "\n",
250 | "grid = visualize(images_dict, layout, True, font)\n",
251 | "\n",
252 | "display(grid)"
253 | ],
254 | "execution_count": null,
255 | "outputs": []
256 | },
257 | {
258 | "cell_type": "markdown",
259 | "metadata": {
260 | "id": "rC3wDS3H2WXe"
261 | },
262 | "source": [
263 | "#### Partially text guided pipeline"
264 | ]
265 | },
266 | {
267 | "cell_type": "code",
268 | "metadata": {
269 | "id": "PdGCFL0y2rnQ"
270 | },
271 | "source": [
272 | "out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
273 | "\n",
274 | "images_dict = {\n",
275 | " 'iA': source_image.crop(bbox),\n",
276 | " 'iB': target_image.crop(bbox),\n",
277 | " 'iB_c': out1['iB_c'].crop(bbox),\n",
278 | " 'iB_f': out1['iB_f'].crop(bbox),\n",
279 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
280 | " 'kB_c': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
281 | " 'kB_f': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
282 | "}\n",
283 | "\n",
284 | "layout = [['iA', 'kA', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA', 'iB', 'kB_f', 'iB_f']]\n",
285 | "\n",
286 | "grid = visualize(images_dict, layout, True, font)\n",
287 | "\n",
288 | "display(grid)\n",
289 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
290 | ],
291 | "execution_count": null,
292 | "outputs": []
293 | },
294 | {
295 | "cell_type": "markdown",
296 | "metadata": {
297 | "id": "GAEc3Pwu68en"
298 | },
299 | "source": [
300 | "#### Fully text guided pipeline"
301 | ]
302 | },
303 | {
304 | "cell_type": "code",
305 | "metadata": {
306 | "id": "A68vTs717Dta"
307 | },
308 | "source": [
309 | "out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
310 | "\n",
311 | "images_dict = {\n",
312 | " 'iA': source_image.crop(bbox),\n",
313 | " 'iB': target_image.crop(bbox),\n",
314 | " 'iB_c': out2['iB_c'].crop(bbox),\n",
315 | " 'iB_f': out2['iB_f'].crop(bbox),\n",
316 | " 'kA_c': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
317 | " 'kA_f': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
318 | " 'kB_c': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
319 | " 'kB_f': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
320 | "}\n",
321 | "\n",
322 | "layout = [['iA', 'kA_c', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA_f', 'iB', 'kB_f', 'iB_f']]\n",
323 | "\n",
324 | "grid = visualize(images_dict, layout, True, font)\n",
325 | "\n",
326 | "display(grid)\n",
327 | "print('\\nSource description:\\n\\n' + source_text_description.replace('. ', '.\\n'))\n",
328 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
329 | ],
330 | "execution_count": null,
331 | "outputs": []
332 | },
333 | {
334 | "cell_type": "markdown",
335 | "metadata": {
336 | "id": "AhC_maow-hbD"
337 | },
338 | "source": [
339 | "## Generation with Real World targets (*out of distribution*)"
340 | ]
341 | },
342 | {
343 | "cell_type": "markdown",
344 | "metadata": {
345 | "id": "LPdVn2xr-hbY"
346 | },
347 | "source": [
348 | "#### Load a random test sample"
349 | ]
350 | },
351 | {
352 | "cell_type": "code",
353 | "metadata": {
354 | "id": "kwDWzqpu-hbZ"
355 | },
356 | "source": [
357 | "index = np.random.randint(0, len(img_pairs_df2rw))\n",
358 | "\n",
359 | "fpA = img_pairs_df2rw.iloc[index].imgA\n",
360 | "fpB = img_pairs_df2rw.iloc[index].imgB\n",
361 | "\n",
362 | "source_image = Image.open(f'{data_root}/{fpA}')\n",
363 | "target_image = Image.open(f'{data_root}/{fpB}')\n",
364 | "\n",
365 | "source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
366 | "target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
367 | "\n",
368 | "source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
369 | "target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
370 | "\n",
371 | "source_text_description = img_descs.loc[file_id(fpA)].description\n",
372 | "target_text_description = img_descs.loc[file_id(fpB)].description"
373 | ],
374 | "execution_count": null,
375 | "outputs": []
376 | },
377 | {
378 | "cell_type": "markdown",
379 | "metadata": {
380 | "id": "VpeZ-tg9-hbb"
381 | },
382 | "source": [
383 | "#### Keypoints guided benchmark"
384 | ]
385 | },
386 | {
387 | "cell_type": "code",
388 | "metadata": {
389 | "id": "Dz_19CRq-hbc"
390 | },
391 | "source": [
392 | "generated_image = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
393 | "\n",
394 | "images_dict = {\n",
395 | " 'iA': source_image.crop(bbox),\n",
396 | " 'iB': target_image.crop(bbox),\n",
397 | " 'iB_k': generated_image.crop(bbox),\n",
398 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
399 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox)\n",
400 | "}\n",
401 | "\n",
402 | "layout = [['iA', 'kA', 'iB', 'kB', 'iB_k']]\n",
403 | "\n",
404 | "grid = visualize(images_dict, layout, True, font)\n",
405 | "\n",
406 | "display(grid)"
407 | ],
408 | "execution_count": null,
409 | "outputs": []
410 | },
411 | {
412 | "cell_type": "markdown",
413 | "metadata": {
414 | "id": "TgYMrH9d-hbd"
415 | },
416 | "source": [
417 | "#### Partially text guided pipeline"
418 | ]
419 | },
420 | {
421 | "cell_type": "code",
422 | "metadata": {
423 | "id": "hE1skWtc-hbd"
424 | },
425 | "source": [
426 | "out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
427 | "\n",
428 | "images_dict = {\n",
429 | " 'iA': source_image.crop(bbox),\n",
430 | " 'iB': target_image.crop(bbox),\n",
431 | " 'iB_c': out1['iB_c'].crop(bbox),\n",
432 | " 'iB_f': out1['iB_f'].crop(bbox),\n",
433 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
434 | " 'kB_c': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
435 | " 'kB_f': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
436 | "}\n",
437 | "\n",
438 | "layout = [['iA', 'kA', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA', 'iB', 'kB_f', 'iB_f']]\n",
439 | "\n",
440 | "grid = visualize(images_dict, layout, True, font)\n",
441 | "\n",
442 | "display(grid)\n",
443 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
444 | ],
445 | "execution_count": null,
446 | "outputs": []
447 | },
448 | {
449 | "cell_type": "markdown",
450 | "metadata": {
451 | "id": "DJ0iHUeq-hbe"
452 | },
453 | "source": [
454 | "#### Fully text guided pipeline"
455 | ]
456 | },
457 | {
458 | "cell_type": "code",
459 | "metadata": {
460 | "id": "o-CHrbRo-hbf"
461 | },
462 | "source": [
463 | "out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
464 | "\n",
465 | "images_dict = {\n",
466 | " 'iA': source_image.crop(bbox),\n",
467 | " 'iB': target_image.crop(bbox),\n",
468 | " 'iB_c': out2['iB_c'].crop(bbox),\n",
469 | " 'iB_f': out2['iB_f'].crop(bbox),\n",
470 | " 'kA_c': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
471 | " 'kA_f': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
472 | " 'kB_c': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
473 | " 'kB_f': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox)\n",
474 | "}\n",
475 | "\n",
476 | "layout = [['iA', 'kA_c', 'iB', 'kB_c', 'iB_c'], ['iA', 'kA_f', 'iB', 'kB_f', 'iB_f']]\n",
477 | "\n",
478 | "grid = visualize(images_dict, layout, True, font)\n",
479 | "\n",
480 | "display(grid)\n",
481 | "print('\\nSource description:\\n\\n' + source_text_description.replace('. ', '.\\n'))\n",
482 | "print('\\nTarget description:\\n\\n' + target_text_description.replace('. ', '.\\n'))"
483 | ],
484 | "execution_count": null,
485 | "outputs": []
486 | },
487 | {
488 | "cell_type": "markdown",
489 | "metadata": {
490 | "id": "fvHwRacqEGBO"
491 | },
492 | "source": [
493 | "## Generate all *within distribution* samples\n",
494 | "\n",
495 | "This will generate all *within distribution* test samples for reproducibility.\n",
496 | "\n",
497 | "Note: Output will be compressed and downloaded as a zip archive for offline viewing.\n"
498 | ]
499 | },
500 | {
501 | "cell_type": "code",
502 | "metadata": {
503 | "id": "ie7X3hK9FPY5"
504 | },
505 | "source": [
506 | "layout = [\n",
507 | " ['iA', 'kA', 'iB', 'kB', 'iB_k0'],\n",
508 | " ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],\n",
509 | " ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],\n",
510 | " ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],\n",
511 | " ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']\n",
512 | "]\n",
513 | "\n",
514 | "for i in range(len(img_pairs_df2df)):\n",
515 | " fpA = img_pairs_df2df.iloc[i].imgA\n",
516 | " fpB = img_pairs_df2df.iloc[i].imgB\n",
517 | " \n",
518 | " source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
519 | " target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
520 | " \n",
521 | " source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
522 | " target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
523 | " \n",
524 | " source_image = Image.open(f'{data_root}/{fpA}')\n",
525 | " target_image = Image.open(f'{data_root}/{fpB}')\n",
526 | " \n",
527 | " iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
528 | " out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
529 | " out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
530 | " \n",
531 | " images_dict = {\n",
532 | " 'iA': source_image.crop(bbox),\n",
533 | " 'iB': target_image.crop(bbox),\n",
534 | " 'iB_k0': iB_k.crop(bbox),\n",
535 | " 'iB_c1': out1['iB_c'].crop(bbox),\n",
536 | " 'iB_f1': out1['iB_f'].crop(bbox),\n",
537 | " 'iB_c2': out2['iB_c'].crop(bbox),\n",
538 | " 'iB_f2': out2['iB_f'].crop(bbox),\n",
539 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
540 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
541 | " 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
542 | " 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
543 | " 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
544 | " 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
545 | " 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
546 | " 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
547 | " }\n",
548 | " \n",
549 | " grid = visualize(images_dict, layout, True, font)\n",
550 | " grid.save(f'{save_root_df2df}/{file_id(fpA)}____{file_id(fpB)}.png')\n",
551 | " print(f'\\r[DF2DF] Testing TIPS inference pipeline... {i+1}/{len(img_pairs_df2df)}', end='')\n",
552 | "\n",
553 | "print('')\n",
554 | "\n",
555 | "!zip -rq tips_output_df2df.zip $save_root_df2df\n",
556 | "\n",
557 | "files.download('tips_output_df2df.zip')"
558 | ],
559 | "execution_count": null,
560 | "outputs": []
561 | },
562 | {
563 | "cell_type": "markdown",
564 | "metadata": {
565 | "id": "WcQ16H0BJNvn"
566 | },
567 | "source": [
568 | "## Generate all *out of distribution* samples\n",
569 | "\n",
570 | "This will generate all *out of distribution* test samples for reproducibility.\n",
571 | "\n",
572 | "Note: Output will be compressed and downloaded as a zip archive for offline viewing.\n"
573 | ]
574 | },
575 | {
576 | "cell_type": "code",
577 | "metadata": {
578 | "id": "7oiaNr0FJNv9"
579 | },
580 | "source": [
581 | "layout = [\n",
582 | " ['iA', 'kA', 'iB', 'kB', 'iB_k0'],\n",
583 | " ['iA', 'kA', 'iB', 'kB_c1', 'iB_c1'],\n",
584 | " ['iA', 'kA', 'iB', 'kB_f1', 'iB_f1'],\n",
585 | " ['iA', 'kA_c2', 'iB', 'kB_c2', 'iB_c2'],\n",
586 | " ['iA', 'kA_f2', 'iB', 'kB_f2', 'iB_f2']\n",
587 | "]\n",
588 | "\n",
589 | "for i in range(len(img_pairs_df2rw)):\n",
590 | " fpA = img_pairs_df2rw.iloc[i].imgA\n",
591 | " fpB = img_pairs_df2rw.iloc[i].imgB\n",
592 | " \n",
593 | " source_text_encoding = encodings.loc[file_id(fpA)].values[0:84].astype(np.float32)\n",
594 | " target_text_encoding = encodings.loc[file_id(fpB)].values[0:84].astype(np.float32)\n",
595 | " \n",
596 | " source_keypoints = keypoints.loc[file_id(fpA)].values[2:38].astype(np.int32)\n",
597 | " target_keypoints = keypoints.loc[file_id(fpB)].values[2:38].astype(np.int32)\n",
598 | " \n",
599 | " source_image = Image.open(f'{data_root}/{fpA}')\n",
600 | " target_image = Image.open(f'{data_root}/{fpB}')\n",
601 | " \n",
602 | " iB_k = tips.benchmark(source_image, source_keypoints, target_keypoints)\n",
603 | " out1 = tips.pipeline(source_image, source_keypoints, target_text_encoding, z)\n",
604 | " out2 = tips.pipeline_full(source_image, source_text_encoding, target_text_encoding, z)\n",
605 | " \n",
606 | " images_dict = {\n",
607 | " 'iA': source_image.crop(bbox),\n",
608 | " 'iB': target_image.crop(bbox),\n",
609 | " 'iB_k0': iB_k.crop(bbox),\n",
610 | " 'iB_c1': out1['iB_c'].crop(bbox),\n",
611 | " 'iB_f1': out1['iB_f'].crop(bbox),\n",
612 | " 'iB_c2': out2['iB_c'].crop(bbox),\n",
613 | " 'iB_f2': out2['iB_f'].crop(bbox),\n",
614 | " 'kA': Image.fromarray(visualize_skeletons([source_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
615 | " 'kB': Image.fromarray(visualize_skeletons([target_keypoints], head_color=(100, 255, 100))).crop(bbox),\n",
616 | " 'kA_c2': Image.fromarray(visualize_skeletons([out2['kA_c']], head_color=(255, 100, 100))).crop(bbox),\n",
617 | " 'kA_f2': Image.fromarray(visualize_skeletons([out2['kA_f']], head_color=(100, 100, 255))).crop(bbox),\n",
618 | " 'kB_c1': Image.fromarray(visualize_skeletons([out1['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
619 | " 'kB_f1': Image.fromarray(visualize_skeletons([out1['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
620 | " 'kB_c2': Image.fromarray(visualize_skeletons([out2['kB_c']], head_color=(255, 100, 100))).crop(bbox),\n",
621 | " 'kB_f2': Image.fromarray(visualize_skeletons([out2['kB_f']], head_color=(100, 100, 255))).crop(bbox),\n",
622 | " }\n",
623 | " \n",
624 | " grid = visualize(images_dict, layout, True, font)\n",
625 | " grid.save(f'{save_root_df2rw}/{file_id(fpA)}____{file_id(fpB)}.png')\n",
626 | " print(f'\\r[DF2RW] Testing TIPS inference pipeline... {i+1}/{len(img_pairs_df2rw)}', end='')\n",
627 | "\n",
628 | "print('')\n",
629 | "\n",
630 | "!zip -rq tips_output_df2rw.zip $save_root_df2rw\n",
631 | "\n",
632 | "files.download('tips_output_df2rw.zip')"
633 | ],
634 | "execution_count": null,
635 | "outputs": []
636 | },
637 | {
638 | "cell_type": "markdown",
639 | "metadata": {
640 | "id": "IH66_yKYR6v6"
641 | },
642 | "source": [
643 | "# ***Thank you for checking out TIPS!***\n"
644 | ]
645 | }
646 | ]
647 | }
--------------------------------------------------------------------------------
/pose2pose/README.md:
--------------------------------------------------------------------------------
1 | #### Code: https://github.com/prasunroy/pose-transfer
--------------------------------------------------------------------------------
/refinenet/dataloader.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | from torch.utils.data import Dataset, DataLoader
4 | from torchvision import transforms
5 |
6 |
7 | class RefineNetDataset(Dataset):
8 |
9 | def __init__(self, keypoints_data, noise_range=(-1, 1), transform=None):
10 | super(RefineNetDataset, self).__init__()
11 | self.keypoints = pd.read_csv(keypoints_data).values[:, 3:39].reshape(-1, 18, 2)[:, [0, 14, 15, 16, 17], :].astype(np.float32)
12 | self.keypoints = np.float32([k for k in self.keypoints if not np.allclose(k[0], np.float32([-1.0, -1.0]))])
13 | self.noise = np.random.randint(noise_range[0], noise_range[1], self.keypoints.shape).astype(np.float32)
14 | self.noise = np.where(self.keypoints >= 0, self.noise, 0)
15 | self.noise[:, 0, :] = 0
16 | self.noisy_keypoints = self.keypoints + self.noise
17 | self.translations = np.float32([np.where(k == [-1.0, -1.0], -1.0, k[0]) for k in self.keypoints])
18 | self.transform = transform or transforms.ToTensor()
19 |
20 | def __len__(self):
21 | return len(self.keypoints)
22 |
23 | def __getitem__(self, index):
24 | keypoints = self.keypoints[index].reshape(-1, 2)
25 | keypoints = np.where(keypoints == [-1.0, -1.0], 0.0, keypoints-keypoints[0])
26 | noisy_keypoints = self.noisy_keypoints[index].reshape(-1, 2)
27 | noisy_keypoints = np.where(noisy_keypoints == [-1.0, -1.0], 0.0, noisy_keypoints-noisy_keypoints[0])
28 | return {
29 | 'x': self.transform(noisy_keypoints.reshape(1, -1)),
30 | 'y': self.transform(keypoints.reshape(1, -1)),
31 | 't': self.translations[index]
32 | }
33 |
34 |
35 | def create_dataloader(keypoints_data, noise_range=(-1, 1), transform=None,
36 | batch_size=1, shuffle=False, num_workers=0, pin_memory=False):
37 | dataset = RefineNetDataset(keypoints_data, noise_range, transform)
38 | return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle,
39 | num_workers=num_workers, pin_memory=pin_memory)
40 |
--------------------------------------------------------------------------------
/refinenet/refinenet.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | class RefineNet(nn.Module):
6 |
7 | def __init__(self, in_features, out_features, bias=True):
8 | super(RefineNet, self).__init__()
9 | self.linear1 = nn.Linear(in_features, 128, bias=bias)
10 | self.linear2 = nn.Linear(128, 128, bias=bias)
11 | self.linear3 = nn.Linear(128, 128, bias=bias)
12 | self.linear4 = nn.Linear(128, out_features, bias=bias)
13 |
14 | def forward(self, x):
15 | y = torch.relu(self.linear1(x))
16 | y = torch.relu(self.linear2(y))
17 | y = torch.relu(self.linear3(y))
18 | y = torch.tanh(self.linear4(y))
19 | return y
20 |
--------------------------------------------------------------------------------
/refinenet/test.py:
--------------------------------------------------------------------------------
1 | # imports
2 | import numpy as np
3 | import os
4 | import pandas as pd
5 | import torch
6 | from refinenet import RefineNet
7 | from PIL import Image
8 | from visualization import visualize_skeletons
9 |
10 |
11 | # configurations
12 | # -----------------------------------------------------------------------------
13 | run_id = 'xxxx-xx-xx-xx-xx-xx' # from ../output/refinenet/xxxx-xx-xx-xx-xx-xx
14 |
15 | data_root = '../datasets/DF-PASS'
16 | test_images = f'{data_root}/test_img_list.csv'
17 | keypoints_data_test = f'{data_root}/test_img_keypoints.csv'
18 | model_state_dict = f'../output/refinenet/{run_id}/refinenet_best.pth'
19 | output_dir = f'../output/refinenet/{run_id}/test'
20 | noise_range = (-5, 5)
21 | use_gpu = True
22 | # -----------------------------------------------------------------------------
23 |
24 |
25 | # get file id of an image
26 | def get_file_id(fp):
27 | return os.path.splitext(os.path.normpath(fp))[0].replace('/', '').replace('\\', '')
28 |
29 |
30 | # create model
31 | model = RefineNet(10, 10, bias=True)
32 | if use_gpu and torch.cuda.is_available():
33 | model.cuda()
34 | model.load_state_dict(torch.load(model_state_dict))
35 | model.eval()
36 |
37 |
38 | # load data
39 | images = pd.read_csv(test_images)
40 | keypoints_data = pd.read_csv(keypoints_data_test, index_col='file_id')
41 |
42 |
43 | if not os.path.isdir(output_dir):
44 | os.makedirs(output_dir)
45 |
46 |
47 | skipped = 0
48 | success = 0
49 |
50 | for i in range(len(images)):
51 | fp = images.iloc[i].img
52 | file_id = get_file_id(fp)
53 | kp = keypoints_data.loc[file_id].values[2:38].reshape(-1, 2)[[0, 14, 15, 16, 17], :].astype(np.int32)
54 | if np.allclose(kp[0], [-1, -1]):
55 | skipped += 1
56 | continue
57 | z = np.random.randint(noise_range[0], noise_range[1], kp.shape)
58 | z = np.where(kp == [-1, -1], 0, z)
59 | z[0, :] = 0
60 | noisy_kp = kp + z
61 | x = torch.tensor(np.where(noisy_kp == [-1, -1], 0, noisy_kp-noisy_kp[0]).reshape(1, 1, 1, -1).astype(np.float32)) / 50
62 | if use_gpu and torch.cuda.is_available():
63 | x = x.cuda()
64 | with torch.no_grad():
65 | p = model(x)
66 | p = (p.detach().cpu().squeeze().numpy() * 50).astype(np.int32).reshape(-1, 2)
67 | p = np.where(kp == [-1, -1], -1, p+kp[0])
68 | kp_18x = keypoints_data.loc[file_id].values[2:38].reshape(-1, 2).astype(np.int32)
69 | kp_18x[[0, 14, 15, 16, 17], :] = noisy_kp
70 | kp_18p = keypoints_data.loc[file_id].values[2:38].reshape(-1, 2).astype(np.int32)
71 | kp_18p[[0, 14, 15, 16, 17], :] = p
72 | kp_18y = keypoints_data.loc[file_id].values[2:38].reshape(-1, 2).astype(np.int32)
73 | kp_18y[[0, 14, 15, 16, 17], :] = kp
74 | img_x = Image.fromarray(visualize_skeletons([kp_18x], 3, 1, head_color=(255, 100, 100))).crop((40, 0, 216, 256))
75 | img_p = Image.fromarray(visualize_skeletons([kp_18p], 3, 1, head_color=(100, 100, 255))).crop((40, 0, 216, 256))
76 | img_y = Image.fromarray(visualize_skeletons([kp_18y], 3, 1, head_color=(100, 255, 100))).crop((40, 0, 216, 256))
77 | grid = Image.new('RGB', (528, 256))
78 | grid.paste(img_x, (0, 0))
79 | grid.paste(img_p, (176, 0))
80 | grid.paste(img_y, (352, 0))
81 | grid.save(f'{output_dir}/{file_id}.png')
82 | success += 1
83 | print(f'\rTesting RefineNet... {i+1}/{len(images)} [success: {success}] [skipped: {skipped}]', end='')
84 | print('')
85 |
--------------------------------------------------------------------------------
/refinenet/train.py:
--------------------------------------------------------------------------------
1 | # imports
2 | import datetime
3 | import json
4 | import os
5 | import torch
6 | from dataloader import create_dataloader
7 | from refinenet import RefineNet
8 |
9 |
10 | # configurations
11 | # -----------------------------------------------------------------------------
12 | dataset_name = 'DF-PASS'
13 | data_root = f'../datasets/{dataset_name}'
14 | keypoints_data_train = f'{data_root}/train_img_keypoints.csv'
15 | keypoints_data_test = f'{data_root}/test_img_keypoints.csv'
16 | output_dir = f'../output/refinenet/{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}'
17 | noise_range = (-5, 5)
18 | batch_size = 128
19 | num_epochs = 100
20 | learning_rate = 1e-2
21 | use_gpu = True
22 | # -----------------------------------------------------------------------------
23 |
24 |
25 | # create dataloaders
26 | train_dataloader = create_dataloader(keypoints_data_train, noise_range, batch_size=batch_size, shuffle=True)
27 | test_dataloader = create_dataloader(keypoints_data_test, noise_range, batch_size=batch_size, shuffle=False)
28 |
29 |
30 | # create model
31 | model = RefineNet(10, 10, bias=True)
32 | if use_gpu and torch.cuda.is_available():
33 | model.cuda()
34 |
35 |
36 | # create objective and optimizer
37 | mse = torch.nn.MSELoss()
38 | opt = torch.optim.SGD(model.parameters(), lr=learning_rate)
39 |
40 |
41 | # create output directory
42 | if not os.path.isdir(output_dir):
43 | os.makedirs(output_dir)
44 |
45 |
46 | w1 = len(str(num_epochs))
47 | w2 = len(str(len(train_dataloader)))
48 |
49 | history = {'train_loss': [], 'eval_loss': []}
50 | best_loss = None
51 |
52 | for i_epoch in range(num_epochs):
53 | # train
54 | model.train()
55 | total_loss = 0
56 | total_samples = 0
57 | for i_batch, data in enumerate(train_dataloader):
58 | x = data['x'] / 50.0
59 | y = data['y'] / 50.0
60 | if use_gpu and torch.cuda.is_available():
61 | x = x.cuda()
62 | y = y.cuda()
63 | pred = model(x)
64 | loss = mse(pred, y)
65 | opt.zero_grad()
66 | loss.backward()
67 | opt.step()
68 | total_loss += pred.size(0) * loss.item()
69 | total_samples += pred.size(0)
70 | train_loss = total_loss / total_samples
71 | print(f'\rEpoch: {i_epoch+1:{w1}d}/{num_epochs} | Batch: {i_batch+1:{w2}d}/{len(train_dataloader)} | Loss: {train_loss:.4f}', end='')
72 | # eval
73 | model.eval()
74 | total_loss = 0
75 | total_samples = 0
76 | for i_batch, data in enumerate(test_dataloader):
77 | x = data['x'] / 50.0
78 | y = data['y'] / 50.0
79 | if use_gpu and torch.cuda.is_available():
80 | x = x.cuda()
81 | y = y.cuda()
82 | with torch.no_grad():
83 | pred = model(x)
84 | loss = mse(pred.detach().cpu(), y.detach().cpu())
85 | total_loss += pred.size(0) * loss.item()
86 | total_samples += pred.size(0)
87 | eval_loss = total_loss / total_samples
88 | print(f' | Validation Loss: {eval_loss:.4f}', end='')
89 | # save model
90 | if best_loss is None or eval_loss < best_loss:
91 | best_loss = eval_loss
92 | torch.save(model.state_dict(), f'{output_dir}/refinenet_best.pth')
93 | print(' | New best!')
94 | else:
95 | print('')
96 | torch.save(model.state_dict(), f'{output_dir}/refinenet_last.pth')
97 | # save history
98 | history['train_loss'].append(train_loss)
99 | history['eval_loss'].append(eval_loss)
100 | with open(f'{output_dir}/history.json', 'w') as fp:
101 | json.dump(history, fp)
102 |
--------------------------------------------------------------------------------
/refinenet/visualization.py:
--------------------------------------------------------------------------------
1 | import cv2
2 | import numpy as np
3 |
4 |
5 | def _draw_circle(image, point, color, radius=1):
6 | x, y = point
7 | if x >= 0 and y >= 0:
8 | cv2.circle(image, (int(x), int(y)), radius, color, -1, cv2.LINE_AA)
9 | return image
10 |
11 |
12 | def _draw_line(image, point1, point2, color, thickness=1):
13 | x1, y1 = point1
14 | x2, y2 = point2
15 | if x1 >= 0 and y1 >= 0 and x2 >= 0 and y2 >= 0:
16 | cv2.line(image, (int(x1), int(y1)), (int(x2), int(y2)), color, thickness, cv2.LINE_AA)
17 | return image
18 |
19 |
20 | def draw_keypoints(image, keypoints, radius=1, head_color=(128, 128, 128), alpha=1.0):
21 | overlay = image.copy()
22 | for kp in keypoints:
23 | for i, (x, y) in enumerate(kp.reshape(-1, 2)):
24 | if i in [0, 14, 15, 16, 17]:
25 | overlay = _draw_circle(overlay, (x, y), head_color, radius)
26 | else:
27 | overlay = _draw_circle(overlay, (x, y), (128, 128, 128), radius)
28 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
29 |
30 |
31 | def draw_connections(image, keypoints, thickness=1, head_color=(128, 128, 128), alpha=1.0):
32 | overlay = image.copy()
33 | conns_h = [(0, 14), (0, 15), (14, 16), (15, 17)]
34 | conns_b = [(0, 1), (1, 2), (1, 5), (2, 8), (5, 11), (8, 11)]
35 | conns_l = [(5, 6), (6, 7), (11, 12), (12, 13)]
36 | conns_r = [(2, 3), (3, 4), (8, 9), (9, 10)]
37 | for kp in keypoints:
38 | kp = kp.reshape(-1, 2)
39 | for i, j in conns_h:
40 | overlay = _draw_line(overlay, kp[i], kp[j], head_color, thickness)
41 | for i, j in conns_b:
42 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
43 | for i, j in conns_l:
44 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
45 | for i, j in conns_r:
46 | overlay = _draw_line(overlay, kp[i], kp[j], (128, 128, 128), thickness)
47 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
48 |
49 |
50 | def visualize_skeletons(keypoints, keypoint_radius=1, connection_thickness=1,
51 | head_color=(128, 128, 128), grid_size=(256, 256), alpha=1.0):
52 | image = np.zeros((grid_size[1], grid_size[0], 3), dtype=np.uint8) + 255
53 | image = draw_connections(image, keypoints, connection_thickness, head_color, alpha)
54 | image = draw_keypoints(image, keypoints, keypoint_radius, head_color, alpha)
55 | return image
56 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | --extra-index-url https://download.pytorch.org/whl/cu118
2 | torch==2.0.0+cu118
3 | torchvision==0.15.1+cu118
4 | tensorboard==2.12.1
5 | numpy==1.23.5
6 | scipy==1.10.0
7 | opencv-contrib-python==4.7.0.72
8 | pillow==9.4.0
9 | scikit-image==0.20.0
10 | pandas==1.5.3
11 | lpips==0.1.4
12 | flask==2.2.5
13 | flask-cors==3.0.10
14 | notebook==7.1.3
15 | git+https://github.com/prasunroy/openpose-pytorch.git
16 |
--------------------------------------------------------------------------------
/text2pose/data/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/text2pose/data/__init__.py
--------------------------------------------------------------------------------
/text2pose/data/dataloader.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import os
3 | import pandas as pd
4 | from torch.utils.data import Dataset, DataLoader
5 | from torchvision import transforms
6 |
7 |
8 | class Text2PoseDataset(Dataset):
9 |
10 | def __init__(self, img_list, text_encoding_data, pose_heatmaps_dir,
11 | text_transform=None, pose_transform=None):
12 | super(Text2PoseDataset, self).__init__()
13 | self._img_list = pd.read_csv(img_list)
14 | self._text_encoding_data = pd.read_csv(text_encoding_data, index_col='file_id')
15 | self._pose_heatmaps_dir = pose_heatmaps_dir
16 | self._text_transform = text_transform or transforms.ToTensor()
17 | self._pose_transform = pose_transform or transforms.ToTensor()
18 |
19 | def __len__(self):
20 | return len(self._img_list)
21 |
22 | def __getitem__(self, index):
23 | imgA = self._img_list.iloc[index].img
24 | fidA = os.path.splitext(imgA)[0].replace('/', '').replace('\\', '')
25 | textA = self._text_encoding_data.loc[fidA].values[:84].astype(np.float32).reshape(1, -1)
26 | poseA = np.load(f'{self._pose_heatmaps_dir}/{fidA}.npz')['arr_0']
27 | while True:
28 | imgB = self._img_list.iloc[np.random.randint(0, self.__len__())].img
29 | fidB = os.path.splitext(imgB)[0].replace('/', '').replace('\\', '')
30 | textB = self._text_encoding_data.loc[fidB].values[:84].astype(np.float32).reshape(1, -1)
31 | if (textB == textA).all():
32 | continue
33 | break
34 | poseA = self._pose_transform(poseA)
35 | textA = self._text_transform(textA)
36 | textB = self._text_transform(textB)
37 | return {'fidA': fidA, 'poseA': poseA, 'textA': textA, 'textB': textB}
38 |
39 |
40 | def create_dataloader(img_list, text_encoding_data, pose_heatmaps_dir,
41 | text_transform=None, pose_transform=None,
42 | batch_size=1, shuffle=False, num_workers=0, pin_memory=False):
43 | dataset = Text2PoseDataset(img_list, text_encoding_data, pose_heatmaps_dir, text_transform, pose_transform)
44 | return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, pin_memory=pin_memory)
45 |
--------------------------------------------------------------------------------
/text2pose/generate_heatmaps.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import os
3 | import pandas as pd
4 | from utils.heatmap import create_gaussian_heatmap, create_isotropic_image
5 |
6 |
7 | def generate_heatmaps(out_dir, keypoint_data, scale=1.0, spread=1.0):
8 | if not os.path.isdir(out_dir):
9 | os.makedirs(out_dir)
10 | kp_data = pd.read_csv(keypoint_data)
11 | for i in range(len(kp_data)):
12 | file_id = kp_data.iloc[i]['file_id']
13 | grid_size = np.max(kp_data.iloc[i, 1:3].values.astype(np.int32))
14 | grid_size = np.int32(np.round(scale * grid_size))
15 | keypoints = kp_data.iloc[i, 3:39].values.astype(np.int32).reshape(-1, 2)
16 | keypoints = np.int32(np.floor(scale * keypoints))
17 | n = keypoints.shape[0]
18 | heatmaps = np.zeros((grid_size, grid_size, n), dtype=np.float32)
19 | for k in range(n):
20 | if keypoints[k, 0] == -1 or keypoints[k, 1] == -1:
21 | continue
22 | gaussian_heatmap = create_gaussian_heatmap(grid_size, keypoints[k], spread)[0]
23 | isotropic_heatmap = create_isotropic_image(gaussian_heatmap)[1]
24 | heatmaps[:, :, k] = isotropic_heatmap.astype(np.float32) / 255.0
25 | np.savez_compressed(f'{out_dir}/{file_id}.npz', heatmaps)
26 | print(f'\rGenerating heatmaps... {i+1}/{len(kp_data)} [{(i+1)*100.0/len(kp_data):.0f}%]', end='')
27 | print('')
28 |
29 |
30 | if __name__ == '__main__':
31 | out_dir = '../datasets/DF-PASS/gaussian_heatmaps'
32 | keypoint_data_train = '../datasets/DF-PASS/train_img_keypoints.csv'
33 | keypoint_data_test = '../datasets/DF-PASS/test_img_keypoints.csv'
34 | generate_heatmaps(out_dir, keypoint_data_train, 0.25, 0.1)
35 | generate_heatmaps(out_dir, keypoint_data_test, 0.25, 0.1)
36 |
--------------------------------------------------------------------------------
/text2pose/models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/text2pose/models/__init__.py
--------------------------------------------------------------------------------
/text2pose/models/base_model.py:
--------------------------------------------------------------------------------
1 | import os
2 | import torch
3 | import torch.nn as nn
4 | from abc import ABC, abstractmethod
5 | from collections import OrderedDict
6 |
7 |
8 | class BaseModel(ABC):
9 |
10 | def __init__(self):
11 | self.models = []
12 | self.losses = []
13 | self.gpuids = []
14 | self.device = None
15 | self.setup()
16 |
17 | def setup(self, verbose=False):
18 | assert isinstance(self.models, list) or isinstance(self.models, tuple)
19 | assert isinstance(self.losses, list) or isinstance(self.losses, tuple)
20 | assert isinstance(self.gpuids, list) or isinstance(self.gpuids, tuple)
21 | self.models = [name for name in self.models if isinstance(name, str)]
22 | self.losses = [name for name in self.losses if isinstance(name, str)]
23 | self.gpuids = [index for index in self.gpuids if torch.cuda.is_available() \
24 | and index in range(0, torch.cuda.device_count())]
25 | self.device = torch.device(f'cuda:{self.gpuids[0]}') if len(self.gpuids) > 0 else torch.device('cpu')
26 | if verbose:
27 | if len(self.gpuids) > 0:
28 | for index in self.gpuids:
29 | print(f'[INFO] Using device: GPU{index} -> {torch.cuda.get_device_name(index)}')
30 | else:
31 | print('[INFO] Using device: CPU')
32 |
33 | @abstractmethod
34 | def set_inputs(self, *inputs):
35 | pass
36 |
37 | @abstractmethod
38 | def forward(self):
39 | pass
40 |
41 | @abstractmethod
42 | def backward(self):
43 | pass
44 |
45 | def optimize_parameters(self):
46 | self.forward()
47 | self.backward()
48 |
49 | def get_losses(self):
50 | loss_dict = OrderedDict()
51 | for name in self.losses:
52 | loss_dict[name] = getattr(self, name).item()
53 | return loss_dict
54 |
55 | def print_networks(self, verbose=False):
56 | print('-'*80)
57 | for name in self.models:
58 | network = getattr(self, name)
59 | n_params = 0
60 | for param in network.parameters():
61 | n_params += param.numel()
62 | if verbose:
63 | print(network)
64 | print(f'[INFO] Total parameters of network {name}: {n_params/1e6:.2f}M')
65 | print('-'*80)
66 |
67 | def init_networks(self, init_type='normal', init_gain=0.02, verbose=False):
68 | def init_params(m):
69 | classname = m.__class__.__name__
70 | if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
71 | if init_type == 'normal':
72 | nn.init.normal_(m.weight.data, 0.0, init_gain)
73 | elif init_type == 'orthogonal':
74 | nn.init.orthogonal_(m.weight.data, init_gain)
75 | elif init_type == 'xavier_normal':
76 | nn.init.xavier_normal_(m.weight.data, init_gain)
77 | elif init_type == 'kaiming_normal':
78 | nn.init.kaiming_normal_(m.weight.data)
79 | else:
80 | raise NotImplementedError(init_type)
81 | if hasattr(m, 'bias') and m.bias is not None:
82 | nn.init.constant_(m.bias.data, 0.0)
83 | elif classname.find('BatchNorm2d') != -1:
84 | nn.init.normal_(m.weight.data, 1.0, init_gain)
85 | nn.init.constant_(m.bias.data, 0.0)
86 |
87 | for name in self.models:
88 | network = getattr(self, name)
89 | if len(self.gpuids) > 0 and torch.cuda.is_available():
90 | network.to(self.gpuids[0])
91 | network = nn.DataParallel(network, self.gpuids)
92 | network.apply(init_params)
93 | setattr(self, name, network)
94 | if verbose:
95 | print(f'[INFO] Network {name} initialized')
96 |
97 | def save_networks(self, root, suffix, verbose=False):
98 | if not os.path.isdir(root):
99 | os.makedirs(root)
100 | for name in self.models:
101 | network = getattr(self, name)
102 | filepath = os.path.join(root, f'{name}_{suffix}.pth')
103 | if isinstance(network, torch.nn.DataParallel):
104 | torch.save(network.module.cpu().state_dict(), filepath)
105 | network.cuda(self.gpuids[0])
106 | else:
107 | torch.save(network.cpu().state_dict(), filepath)
108 | if verbose:
109 | print(f'[INFO] Network {name} weights saved to {filepath}')
110 |
111 | def load_networks(self, root, suffix, verbose=False):
112 | for name in self.models:
113 | network = getattr(self, name)
114 | if isinstance(network, torch.nn.DataParallel):
115 | network = network.module
116 | filepath = os.path.join(root, f'{name}_{suffix}.pth')
117 | network.load_state_dict(torch.load(filepath, map_location=self.device))
118 | if verbose:
119 | print(f'[INFO] Network {name} weights loaded from {filepath}')
120 |
121 | def set_requires_grad(self, network_names, requires_grad=True):
122 | assert isinstance(network_names, list) or isinstance(network_names, tuple)
123 | network_names = [name for name in network_names if isinstance(name, str)]
124 | for name in network_names:
125 | network = getattr(self, name)
126 | for param in network.parameters():
127 | param.requires_grad = requires_grad
128 |
--------------------------------------------------------------------------------
/text2pose/models/netD.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | def linear(in_features, out_features, bias=True):
6 | return nn.Sequential(
7 | nn.Linear(in_features, out_features, bias=bias),
8 | nn.LeakyReLU(inplace=True)
9 | )
10 |
11 |
12 | def conv1x1(in_channels, out_channels, bias=False):
13 | return nn.Sequential(
14 | nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=bias),
15 | nn.LeakyReLU(inplace=True)
16 | )
17 |
18 |
19 | def downconv2x_hidden(in_channels, out_channels, bias=False):
20 | return nn.Sequential(
21 | nn.Conv2d(in_channels, out_channels, 4, 2, 1, bias=bias),
22 | nn.LeakyReLU(inplace=True)
23 | )
24 |
25 |
26 | def downconv4x_output(in_channels, out_channels, bias=False):
27 | return nn.Conv2d(in_channels, out_channels, 4, 4, 0, bias=bias)
28 |
29 |
30 | class NetD(nn.Module):
31 |
32 | def __init__(self, heatmap_channels, embed_dim, ndf=32):
33 | super(NetD, self).__init__()
34 | self.heatmap_down1 = downconv2x_hidden(heatmap_channels, ndf)
35 | self.heatmap_down2 = downconv2x_hidden(ndf, ndf*2)
36 | self.heatmap_down3 = downconv2x_hidden(ndf*2, ndf*4)
37 | self.heatmap_down4 = downconv2x_hidden(ndf*4, ndf*8)
38 | self.embed_linear = linear(embed_dim, ndf*4)
39 | self.combined_conv = conv1x1(ndf*12, ndf*8)
40 | self.combined_down = downconv4x_output(ndf*8, 1)
41 |
42 | def forward(self, x1, x2):
43 | x1_heatmap = self.heatmap_down1(x1)
44 | x1_heatmap = self.heatmap_down2(x1_heatmap)
45 | x1_heatmap = self.heatmap_down3(x1_heatmap)
46 | x1_heatmap = self.heatmap_down4(x1_heatmap)
47 |
48 | x2_embed = x2.view(x2.size(0), 1, -1)
49 | x2_embed = self.embed_linear(x2_embed)
50 | x2_embed = x2_embed.view(x2_embed.size(0), -1, 1, 1)
51 |
52 | x2_tiled = torch.tile(x2_embed, (x1_heatmap.size(2), x1_heatmap.size(3)))
53 |
54 | combined = torch.cat((x1_heatmap, x2_tiled), dim=1)
55 |
56 | y = self.combined_conv(combined)
57 | y = self.combined_down(y)
58 |
59 | return y
60 |
--------------------------------------------------------------------------------
/text2pose/models/netG.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | def linear(in_features, out_features, bias=True):
6 | return nn.Sequential(
7 | nn.Linear(in_features, out_features, bias=bias),
8 | nn.LeakyReLU(inplace=True)
9 | )
10 |
11 |
12 | def upconv4x(in_channels, out_channels, bias=False):
13 | return nn.Sequential(
14 | nn.ConvTranspose2d(in_channels, out_channels, 4, 4, 0, bias=bias),
15 | nn.BatchNorm2d(out_channels),
16 | nn.ReLU(inplace=True)
17 | )
18 |
19 |
20 | def upconv2x_hidden(in_channels, out_channels, bias=False):
21 | return nn.Sequential(
22 | nn.ConvTranspose2d(in_channels, out_channels, 4, 2, 1, bias=bias),
23 | nn.BatchNorm2d(out_channels),
24 | nn.ReLU(inplace=True)
25 | )
26 |
27 |
28 | def upconv2x_output(in_channels, out_channels, bias=False):
29 | return nn.Sequential(
30 | nn.ConvTranspose2d(in_channels, out_channels, 4, 2, 1, bias=bias),
31 | nn.Tanh()
32 | )
33 |
34 |
35 | class NetG(nn.Module):
36 |
37 | def __init__(self, noise_dim, embed_dim, heatmap_channels, ngf=32):
38 | super(NetG, self).__init__()
39 | self.embed_linear = linear(embed_dim, noise_dim)
40 | self.combined_up1 = upconv4x(noise_dim*2, ngf*8)
41 | self.combined_up2 = upconv2x_hidden(ngf*8, ngf*4)
42 | self.combined_up3 = upconv2x_hidden(ngf*4, ngf*2)
43 | self.combined_up4 = upconv2x_hidden(ngf*2, ngf)
44 | self.combined_up5 = upconv2x_output(ngf, heatmap_channels)
45 |
46 | def forward(self, x1, x2):
47 | x1_noise = x1.view(x1.size(0), -1, 1, 1)
48 |
49 | x2_embed = x2.view(x2.size(0), 1, -1)
50 | x2_embed = self.embed_linear(x2_embed)
51 | x2_embed = x2_embed.view(x2_embed.size(0), -1, 1, 1)
52 |
53 | combined = torch.cat((x1_noise, x2_embed), dim=1)
54 |
55 | y = self.combined_up1(combined)
56 | y = self.combined_up2(y)
57 | y = self.combined_up3(y)
58 | y = self.combined_up4(y)
59 | y = self.combined_up5(y)
60 |
61 | return y
62 |
--------------------------------------------------------------------------------
/text2pose/text2pose_model.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | from models.base_model import BaseModel
4 | from models.netG import NetG
5 | from models.netD import NetD
6 | from utils.visualization import translate_heatmap, visualize_heatmap
7 |
8 |
9 | class Text2PoseModel(BaseModel):
10 |
11 | def __init__(self, gpuids=None, noise_dim=128, embed_dim=84, heatmap_channels=18, gradient_penalty=True):
12 | super(Text2PoseModel, self).__init__()
13 | self.models = ['netG', 'netD']
14 | self.losses = ['lossG1', 'lossG2', 'lossG', 'lossD1', 'lossD2', 'penalty', 'lossD']
15 | self.gpuids = gpuids if isinstance(gpuids, list) or isinstance(gpuids, tuple) else []
16 | self.device = None
17 |
18 | self.setup(verbose=True)
19 |
20 | self.noise_dim = noise_dim
21 | self.embed_dim = embed_dim
22 | self.heatmap_channels = heatmap_channels
23 | self.gradient_penalty = gradient_penalty
24 | self.lossG1 = torch.zeros(1)
25 | self.lossG2 = torch.zeros(1)
26 | self.lossG = torch.zeros(1)
27 |
28 | self.netG = NetG(noise_dim, embed_dim, heatmap_channels)
29 | self.netD = NetD(heatmap_channels, embed_dim)
30 |
31 | self.init_networks(verbose=True)
32 |
33 | if self.gradient_penalty:
34 | self.optimizerG = torch.optim.Adam(self.netG.parameters(), lr=0.0001, betas=(0, 0.9))
35 | self.optimizerD = torch.optim.Adam(self.netD.parameters(), lr=0.0001, betas=(0, 0.9))
36 | else:
37 | self.optimizerG = torch.optim.RMSprop(self.netG.parameters(), lr=0.00005)
38 | self.optimizerD = torch.optim.RMSprop(self.netD.parameters(), lr=0.00005)
39 |
40 | self.iters = 0
41 |
42 | def set_inputs(self, inputs):
43 | self.real_pose_x = inputs['poseA'].to(self.device)
44 | self.real_text_h1 = inputs['textA'].to(self.device)
45 | self.real_text_h2 = inputs['textB'].to(self.device)
46 |
47 | def forward(self):
48 | pass
49 |
50 | def backward(self):
51 | pass
52 |
53 | def optimize_parameters(self, d_iters=5, c=0.01, lambda_gp=10):
54 | self.iters += 1
55 | batch_size = self.real_pose_x.size(0)
56 |
57 | z = torch.randn(batch_size, self.noise_dim, 1, 1).to(self.device)
58 |
59 | # update netD
60 | self.set_requires_grad(['netD'], True)
61 | self.optimizerD.zero_grad()
62 |
63 | fake_pose_zh1 = self.netG(z, self.real_text_h1).detach()
64 |
65 | pred_fake_zh1 = self.netD(fake_pose_zh1, self.real_text_h1).squeeze()
66 | pred_real_xh1 = self.netD(self.real_pose_x, self.real_text_h1).squeeze()
67 | # pred_real_xh2 = self.netD(self.real_pose_x, self.real_text_h2).squeeze()
68 |
69 | self.lossD1 = -(torch.mean(pred_real_xh1) - torch.mean(pred_fake_zh1))
70 | self.lossD2 = torch.zeros(1) # -(torch.mean(pred_real_xh1) - torch.mean(pred_real_xh2))
71 |
72 | if self.gradient_penalty:
73 | self.penalty = self.compute_gradient_penalty(self.real_pose_x, self.real_text_h1, fake_pose_zh1)
74 | self.lossD = self.lossD1 + lambda_gp * self.penalty # self.lossD1 + self.lossD2 + lambda_gp * self.penalty
75 | else:
76 | self.penalty = torch.zeros(1)
77 | self.lossD = self.lossD1 # self.lossD1 + self.lossD2
78 |
79 | self.lossD.backward()
80 | self.optimizerD.step()
81 |
82 | if not self.gradient_penalty:
83 | for param in self.netD.parameters():
84 | param.data.clamp_(-c, c)
85 |
86 | # update netG once every d_iters updates of netD
87 | if self.iters % d_iters == 0:
88 | self.set_requires_grad(['netD'], False)
89 | self.optimizerG.zero_grad()
90 |
91 | fake_pose_zh1 = self.netG(z, self.real_text_h1)
92 | pred_fake_zh1 = self.netD(fake_pose_zh1, self.real_text_h1).squeeze()
93 |
94 | interp_real_text_h1h2 = 0.5 * (self.real_text_h1 + self.real_text_h2)
95 | interp_fake_pose_zh1h2 = self.netG(z, interp_real_text_h1h2)
96 | pred_fake_zh1h2 = self.netD(interp_fake_pose_zh1h2, interp_real_text_h1h2)
97 |
98 | self.lossG1 = -torch.mean(pred_fake_zh1)
99 | self.lossG2 = -torch.mean(pred_fake_zh1h2)
100 | self.lossG = self.lossG1 + self.lossG2
101 |
102 | self.lossG.backward()
103 | self.optimizerG.step()
104 |
105 | def compute_visuals(self, padding=1, confidence_cutoff=0.2):
106 | mode = self.netG.training
107 | self.netG.eval()
108 | batch_size = self.real_pose_x.size(0)
109 | z = torch.randn(batch_size, self.noise_dim, 1, 1).to(self.device)
110 | with torch.no_grad():
111 | fake_pose_zh1 = self.netG(z, self.real_text_h1)
112 | real_pose = self.real_pose_x.detach().cpu().permute(0, 2, 3, 1).numpy()
113 | real_pose = (real_pose + 1.0) / 2.0
114 | fake_pose = fake_pose_zh1.detach().cpu().permute(0, 2, 3, 1).numpy()
115 | fake_pose = (fake_pose + 1.0) / 2.0
116 | grid_image = np.zeros((batch_size*256 + (batch_size+1)*padding, 2*256 + 3*padding, 3), dtype=np.uint8)
117 | for i, (real, fake) in enumerate(zip(real_pose, fake_pose)):
118 | real = translate_heatmap(real, (256, 256))
119 | real = visualize_heatmap(real, confidence_cutoff=confidence_cutoff)
120 | fake = translate_heatmap(fake, (256, 256))
121 | fake = visualize_heatmap(fake, confidence_cutoff=confidence_cutoff)
122 | grid_image[padding + i*(256+padding) : (i+1)*(256+padding), padding : 256+padding] = real
123 | grid_image[padding + i*(256+padding) : (i+1)*(256+padding), 256+2*padding : 2*(256+padding)] = fake
124 | self.netG.train(mode)
125 | return grid_image
126 |
127 | def compute_gradient_penalty(self, real_poses, real_texts, fake_poses):
128 | batch_size = real_poses.size(0)
129 | alpha = torch.rand(batch_size, 1, 1, 1).to(self.device)
130 | interp_poses = (alpha * real_poses + (1 - alpha) * fake_poses).requires_grad_(True)
131 | pred = self.netD(interp_poses, real_texts).view(-1, 1)
132 | ones = torch.ones(batch_size, 1).to(self.device).requires_grad_(False)
133 | gradients = torch.autograd.grad(
134 | outputs=pred,
135 | inputs=interp_poses,
136 | grad_outputs=ones,
137 | retain_graph=True,
138 | create_graph=True,
139 | only_inputs=True,
140 | allow_unused=False
141 | )[0]
142 | gradients = gradients.view(gradients.size(0), -1)
143 | gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
144 | return gradient_penalty
145 |
--------------------------------------------------------------------------------
/text2pose/train.py:
--------------------------------------------------------------------------------
1 | # imports
2 | import cv2
3 | import datetime
4 | import time
5 | import torchvision
6 | from torch.utils.tensorboard import SummaryWriter
7 | from data.dataloader import create_dataloader
8 | from text2pose_model import Text2PoseModel
9 |
10 |
11 | # configurations
12 | # -----------------------------------------------------------------------------
13 | dataset_name = 'DF-PASS'
14 |
15 | dataset_root = f'../datasets/{dataset_name}'
16 | pose_heatmaps_dir = f'{dataset_root}/gaussian_heatmaps'
17 | text_encoding_data = f'{dataset_root}/encodings.csv'
18 | img_list_train = f'{dataset_root}/train_img_list.csv'
19 | img_list_test = f'{dataset_root}/test_img_list.csv'
20 |
21 | gpu_ids = [0]
22 |
23 | noise_dim = 128
24 | embed_dim = 84
25 | heatmap_channels = 18
26 | gradient_penalty = True
27 |
28 | batch_size_train = 32
29 | batch_size_test = 8
30 | n_epoch = 1000
31 | out_freq = 500
32 | d_iters = 5
33 |
34 | ckpt_id = None
35 | ckpt_dir = None
36 |
37 | run_info = f'[gp={gradient_penalty}][d_iters={d_iters}]'
38 | out_path = '../output/text2pose'
39 | # -----------------------------------------------------------------------------
40 |
41 |
42 | # create timestamp and infostamp
43 | timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
44 | infostamp = f'_{run_info.strip()}' if run_info.strip() else ''
45 |
46 | # create tensorboard logger
47 | logger = SummaryWriter(f'{out_path}/runs/{timestamp}{infostamp}')
48 |
49 | # create transforms
50 | text_transform = torchvision.transforms.Compose([
51 | torchvision.transforms.ToTensor(),
52 | torchvision.transforms.Normalize((0.5,), (0.5,))
53 | ])
54 | pose_transform = torchvision.transforms.Compose([
55 | torchvision.transforms.ToTensor(),
56 | torchvision.transforms.Normalize((0.5,), (0.5,))
57 | ])
58 |
59 | # create dataloaders
60 | train_dataloader = create_dataloader(img_list_train, text_encoding_data, pose_heatmaps_dir,
61 | text_transform, pose_transform, batch_size_train, shuffle=True)
62 | test_dataloader = create_dataloader(img_list_test, text_encoding_data, pose_heatmaps_dir,
63 | text_transform, pose_transform, batch_size_test, shuffle=False)
64 |
65 | # create fixed batch for testing
66 | fixed_test_batch = next(iter(test_dataloader))
67 |
68 | # create model
69 | model = Text2PoseModel(gpu_ids, noise_dim, embed_dim, heatmap_channels, gradient_penalty)
70 | model.print_networks(verbose=False)
71 |
72 | # load pretrained weights into model
73 | if ckpt_id and ckpt_dir:
74 | model.load_networks(ckpt_dir, ckpt_id, verbose=True)
75 |
76 | # train model
77 | n_batch = len(train_dataloader)
78 | w_batch = len(str(n_batch))
79 | w_epoch = len(str(n_epoch))
80 | n_iters = 0
81 |
82 | for epoch in range(n_epoch):
83 | for batch, data in enumerate(train_dataloader):
84 | time_0 = time.time()
85 | model.set_inputs(data)
86 | model.optimize_parameters(d_iters=d_iters)
87 | losses = model.get_losses()
88 | loss_G = losses['lossG']
89 | loss_D = losses['lossD']
90 | time_1 = time.time()
91 |
92 | print(f'[TRAIN] Epoch: {epoch+1:{w_epoch}d}/{n_epoch} | Batch: {batch+1:{w_batch}d}/{n_batch} |',
93 | f'LossG: {loss_G:7.4f} | LossD: {loss_D:7.4f} | Time: {round(time_1-time_0, 2):.2f} sec |')
94 |
95 | if (n_iters % out_freq == 0) or (batch+1 == n_batch and epoch+1 == n_epoch):
96 | model.save_networks(f'{out_path}/ckpt/{timestamp}{infostamp}', n_iters, verbose=True)
97 | for loss_name, loss in losses.items():
98 | loss_group = 'LossG' if loss_name.startswith('lossG') else 'LossD'
99 | logger.add_scalar(f'{loss_group}/{loss_name}', loss, n_iters)
100 | model.set_inputs(fixed_test_batch)
101 | visuals = model.compute_visuals(padding=4)
102 | # logger.add_image(f'Iteration_{n_iters}', visuals, n_iters)
103 | cv2.imwrite(f'{out_path}/runs/{timestamp}{infostamp}/iteration_{n_iters}.png', visuals)
104 |
105 | n_iters += 1
106 |
--------------------------------------------------------------------------------
/text2pose/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/prasunroy/tips/80a71f67a21ab2913ba9386e8323d492b40081e8/text2pose/utils/__init__.py
--------------------------------------------------------------------------------
/text2pose/utils/heatmap.py:
--------------------------------------------------------------------------------
1 | """
2 | References:
3 | [1] https://github.com/clovaai/CRAFT-pytorch/issues/3#issuecomment-504548693
4 | [2] https://colab.research.google.com/drive/1TQ1-BTisMYZHIRVVNpVwDFPviXYMhT7A
5 |
6 | """
7 |
8 |
9 | import cv2
10 | import numpy as np
11 |
12 |
13 | # probability density function that returns a probability value within the range [0, 1]
14 | # from a Gaussian distribution with mean = 0 and standard deviation = 1 (Normal distribution)
15 | def gaussian(x):
16 | return np.exp(-(x**2)/2)
17 |
18 |
19 | # estimate heatmap as the Gaussian probability distribution function of the Euclidean distance
20 | # from a given center on a 2D plane
21 | def create_gaussian_heatmap(grid_size=512, center=(256, 256), spread=1.0):
22 | k, (x, y), c = grid_size, center, spread
23 | assert k > 0 and x in range(k) and y in range(k) and c > 0.0 and c <= 1.0
24 | distmap = np.zeros((k, k), dtype=np.float32)
25 | for i in range(k):
26 | for j in range(k):
27 | distmap[i, j] = np.linalg.norm(np.float32([x-j, y-i])) / np.float32(c * k/2)
28 | heatmap = gaussian(distmap)
29 | return heatmap, distmap
30 |
31 |
32 | # create isotropic image representation of a given probability distribution
33 | def create_isotropic_image(distribution):
34 | isotropic_gray = np.uint8(np.clip(255 * distribution, 0, 255))
35 | isotropic_cmap = cv2.applyColorMap(isotropic_gray, cv2.COLORMAP_JET)
36 | return isotropic_cmap, isotropic_gray
37 |
38 |
39 | # find location of the highest probability in a given heatmap
40 | def find_heatmap_peak(heatmap, confidence_cutoff=0.0):
41 | probability_max = np.max(heatmap)
42 | if probability_max > confidence_cutoff:
43 | y, x = np.where(heatmap == probability_max)
44 | y, x = y[0], x[0]
45 | else:
46 | y, x = -1, -1
47 | return probability_max, (x, y)
48 |
--------------------------------------------------------------------------------
/text2pose/utils/visualization.py:
--------------------------------------------------------------------------------
1 | import cv2
2 | import numpy as np
3 | from .heatmap import create_isotropic_image, find_heatmap_peak
4 |
5 |
6 | def _draw_circle(image, point, color, radius=1):
7 | x, y = point
8 | if x >= 0 and y >= 0:
9 | cv2.circle(image, (int(x), int(y)), radius, color, -1, cv2.LINE_AA)
10 | return image
11 |
12 |
13 | def _draw_line(image, point1, point2, color, thickness=1):
14 | x1, y1 = point1
15 | x2, y2 = point2
16 | if x1 >= 0 and y1 >= 0 and x2 >= 0 and y2 >= 0:
17 | cv2.line(image, (int(x1), int(y1)), (int(x2), int(y2)), color, thickness, cv2.LINE_AA)
18 | return image
19 |
20 |
21 | def draw_keypoints(image, keypoints, radius=1, alpha=1.0):
22 | overlay = image.copy()
23 | for kp in keypoints:
24 | for x, y in kp.reshape(-1, 2):
25 | overlay = _draw_circle(overlay, (x, y), (0, 255, 0), radius)
26 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
27 |
28 |
29 | def draw_connections(image, keypoints, thickness=1, alpha=1.0):
30 | overlay = image.copy()
31 | conns_h = [(0, 14), (0, 15), (14, 16), (15, 17)]
32 | conns_b = [(0, 1), (1, 2), (1, 5), (2, 8), (5, 11), (8, 11)]
33 | conns_l = [(5, 6), (6, 7), (11, 12), (12, 13)]
34 | conns_r = [(2, 3), (3, 4), (8, 9), (9, 10)]
35 | for kp in keypoints:
36 | kp = kp.reshape(-1, 2)
37 | for i, j in conns_h:
38 | overlay = _draw_line(overlay, kp[i], kp[j], (0, 255, 255), thickness)
39 | for i, j in conns_b:
40 | overlay = _draw_line(overlay, kp[i], kp[j], (0, 255, 255), thickness)
41 | for i, j in conns_l:
42 | overlay = _draw_line(overlay, kp[i], kp[j], (255, 255, 0), thickness)
43 | for i, j in conns_r:
44 | overlay = _draw_line(overlay, kp[i], kp[j], (255, 0, 255), thickness)
45 | return cv2.addWeighted(overlay, alpha, image, 1.0 - alpha, 0)
46 |
47 |
48 | def translate_heatmap(heatmap, target_size):
49 | h1, w1 = heatmap.shape[:2]
50 | h2, w2 = target_size[1], target_size[0]
51 | scales = np.float32([w2/w1, h2/h1])
52 | kp_old = []
53 | for i in range(heatmap.shape[2]):
54 | kp_old.append(find_heatmap_peak(heatmap[:, :, i], 0.0)[1])
55 | kp_old = np.int32(kp_old)
56 | kp_new = np.int32(np.where(kp_old >= 0, scales * kp_old, 1.0 * kp_old))
57 | shifts = kp_new - kp_old
58 | heatmap_new = np.zeros((h2, w2, heatmap.shape[2]), dtype=heatmap.dtype)
59 | for i in range(heatmap.shape[2]):
60 | ys, xs = np.where(heatmap[:, :, i] > 0)
61 | coords_old = np.int32(np.concatenate((xs.reshape(-1, 1), ys.reshape(-1, 1)), axis=1))
62 | coords_new = coords_old + shifts[i]
63 | for (x1, y1), (x2, y2) in zip(coords_old, coords_new):
64 | if x2 in range(w2) and y2 in range(h2):
65 | heatmap_new[y2, x2, i] = heatmap[y1, x1, i]
66 | return heatmap_new
67 |
68 |
69 | def visualize_heatmap(heatmap, distribution=True, keypoints=True, connections=True, confidence_cutoff=0.5,
70 | keypoint_radius=1, connection_thickness=1, alpha=1.0):
71 | image = np.zeros((heatmap.shape[0], heatmap.shape[1], 3), dtype=np.uint8)
72 | if distribution:
73 | proba = np.sum(heatmap, axis=2)
74 | proba /= np.max(proba)
75 | image = create_isotropic_image(proba)[0]
76 | if keypoints or connections:
77 | kp = []
78 | for i in range(heatmap.shape[2]):
79 | kp.append(find_heatmap_peak(heatmap[:, :, i], confidence_cutoff)[1])
80 | kp = np.int32([kp])
81 | if connections:
82 | image = draw_connections(image, kp, connection_thickness, alpha)
83 | if keypoints:
84 | image = draw_keypoints(image, kp, keypoint_radius, alpha)
85 | return image
86 |
--------------------------------------------------------------------------------