├── .travis.yml ├── Data ├── net-data │ ├── 256_256_resfcn256_weight.index │ └── mmod_human_face_detector.dat └── uv-data │ ├── canonical_vertices.npy │ ├── face_ind.txt │ ├── triangles.txt │ ├── uv_face.png │ ├── uv_face_eyes.png │ ├── uv_face_mask.png │ ├── uv_kpt_ind.txt │ ├── uv_kpt_mask.png │ └── uv_weight_mask.png ├── Docs └── images │ ├── alignment.jpg │ ├── depth.jpg │ ├── eye.jpg │ ├── pose.jpg │ ├── prnet.gif │ ├── reconstruct.jpg │ └── swapping.jpg ├── LICENSE ├── README.md ├── TestImages ├── 0.jpg ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg ├── 5.jpg ├── AFLW2000 │ ├── image00050.jpg │ ├── image00050.mat │ ├── image00081.jpg │ ├── image00081.mat │ ├── image00251.jpg │ ├── image00251.mat │ ├── image00430.jpg │ ├── image00430.mat │ ├── image00475.jpg │ ├── image00475.mat │ ├── image00514.jpg │ ├── image00514.mat │ ├── image00516.jpg │ ├── image00516.mat │ ├── image01038.jpg │ ├── image01038.mat │ ├── image01322.jpg │ ├── image01322.mat │ ├── image02283.jpg │ ├── image02283.mat │ ├── image02420.jpg │ ├── image02420.mat │ ├── image02545.jpg │ ├── image02545.mat │ ├── image02616.jpg │ ├── image02616.mat │ ├── image03324.jpg │ ├── image03324.mat │ ├── image03614.jpg │ ├── image03614.mat │ ├── image04331.jpg │ └── image04331.mat └── trump.jpg ├── api.py ├── demo.py ├── demo_texture.py ├── predictor.py ├── requirements.txt ├── run_basics.py └── utils ├── __init__.py ├── cv_plot.py ├── estimate_pose.py ├── render.py ├── render_app.py ├── rotate_vertices.py └── write.py /.travis.yml: -------------------------------------------------------------------------------- 1 | group: travis_latest 2 | language: python 3 | cache: pip 4 | python: 5 | - 2.7 6 | - 3.6 7 | #- nightly 8 | #- pypy 9 | #- pypy3 10 | matrix: 11 | allow_failures: 12 | - python: nightly 13 | - python: pypy 14 | - python: pypy3 15 | install: 16 | - pip install -r requirements.txt 17 | - pip install flake8 # pytest # add another testing frameworks later 18 | before_script: 19 | # stop the build if there are Python syntax errors or undefined names 20 | - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics 21 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 22 | - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 23 | script: 24 | - ls 25 | - wget "https://drive.google.com/uc?export=download&confirm=D2ug&id=1UoE-XuW1SDLUjZmJPkIZ1MLxvQFgmTFH" 26 | - ls 27 | - python run_basics.py # Can run only with python and tensorflow 28 | notifications: 29 | on_success: change 30 | on_failure: change # `always` will be the setting once code changes slow down 31 | -------------------------------------------------------------------------------- /Data/net-data/256_256_resfcn256_weight.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/net-data/256_256_resfcn256_weight.index -------------------------------------------------------------------------------- /Data/net-data/mmod_human_face_detector.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/net-data/mmod_human_face_detector.dat -------------------------------------------------------------------------------- /Data/uv-data/canonical_vertices.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/canonical_vertices.npy -------------------------------------------------------------------------------- /Data/uv-data/uv_face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/uv_face.png -------------------------------------------------------------------------------- /Data/uv-data/uv_face_eyes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/uv_face_eyes.png -------------------------------------------------------------------------------- /Data/uv-data/uv_face_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/uv_face_mask.png -------------------------------------------------------------------------------- /Data/uv-data/uv_kpt_ind.txt: -------------------------------------------------------------------------------- 1 | 1.500000000000000000e+01 2.200000000000000000e+01 2.600000000000000000e+01 3.200000000000000000e+01 4.500000000000000000e+01 6.700000000000000000e+01 9.100000000000000000e+01 1.120000000000000000e+02 1.280000000000000000e+02 1.430000000000000000e+02 1.640000000000000000e+02 1.880000000000000000e+02 2.100000000000000000e+02 2.230000000000000000e+02 2.290000000000000000e+02 2.330000000000000000e+02 2.400000000000000000e+02 5.800000000000000000e+01 7.100000000000000000e+01 8.500000000000000000e+01 9.700000000000000000e+01 1.060000000000000000e+02 1.490000000000000000e+02 1.580000000000000000e+02 1.700000000000000000e+02 1.840000000000000000e+02 1.970000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.280000000000000000e+02 1.170000000000000000e+02 1.220000000000000000e+02 1.280000000000000000e+02 1.330000000000000000e+02 1.380000000000000000e+02 7.800000000000000000e+01 8.600000000000000000e+01 9.500000000000000000e+01 1.020000000000000000e+02 9.600000000000000000e+01 8.700000000000000000e+01 1.530000000000000000e+02 1.600000000000000000e+02 1.690000000000000000e+02 1.770000000000000000e+02 1.680000000000000000e+02 1.590000000000000000e+02 1.080000000000000000e+02 1.160000000000000000e+02 1.240000000000000000e+02 1.280000000000000000e+02 1.310000000000000000e+02 1.390000000000000000e+02 1.460000000000000000e+02 1.370000000000000000e+02 1.320000000000000000e+02 1.280000000000000000e+02 1.230000000000000000e+02 1.180000000000000000e+02 1.100000000000000000e+02 1.220000000000000000e+02 1.280000000000000000e+02 1.330000000000000000e+02 1.450000000000000000e+02 1.320000000000000000e+02 1.280000000000000000e+02 1.230000000000000000e+02 2 | 9.600000000000000000e+01 1.180000000000000000e+02 1.410000000000000000e+02 1.650000000000000000e+02 1.830000000000000000e+02 1.900000000000000000e+02 1.880000000000000000e+02 1.870000000000000000e+02 1.930000000000000000e+02 1.870000000000000000e+02 1.880000000000000000e+02 1.900000000000000000e+02 1.830000000000000000e+02 1.650000000000000000e+02 1.410000000000000000e+02 1.180000000000000000e+02 9.600000000000000000e+01 4.900000000000000000e+01 4.200000000000000000e+01 3.900000000000000000e+01 4.000000000000000000e+01 4.200000000000000000e+01 4.200000000000000000e+01 4.000000000000000000e+01 3.900000000000000000e+01 4.200000000000000000e+01 4.900000000000000000e+01 5.900000000000000000e+01 7.300000000000000000e+01 8.600000000000000000e+01 9.600000000000000000e+01 1.110000000000000000e+02 1.130000000000000000e+02 1.150000000000000000e+02 1.130000000000000000e+02 1.110000000000000000e+02 6.700000000000000000e+01 6.000000000000000000e+01 6.100000000000000000e+01 6.500000000000000000e+01 6.800000000000000000e+01 6.900000000000000000e+01 6.500000000000000000e+01 6.100000000000000000e+01 6.000000000000000000e+01 6.700000000000000000e+01 6.900000000000000000e+01 6.800000000000000000e+01 1.420000000000000000e+02 1.310000000000000000e+02 1.270000000000000000e+02 1.280000000000000000e+02 1.270000000000000000e+02 1.310000000000000000e+02 1.420000000000000000e+02 1.480000000000000000e+02 1.500000000000000000e+02 1.500000000000000000e+02 1.500000000000000000e+02 1.480000000000000000e+02 1.410000000000000000e+02 1.350000000000000000e+02 1.340000000000000000e+02 1.350000000000000000e+02 1.420000000000000000e+02 1.430000000000000000e+02 1.420000000000000000e+02 1.430000000000000000e+02 3 | -------------------------------------------------------------------------------- /Data/uv-data/uv_kpt_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/uv_kpt_mask.png -------------------------------------------------------------------------------- /Data/uv-data/uv_weight_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Data/uv-data/uv_weight_mask.png -------------------------------------------------------------------------------- /Docs/images/alignment.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/alignment.jpg -------------------------------------------------------------------------------- /Docs/images/depth.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/depth.jpg -------------------------------------------------------------------------------- /Docs/images/eye.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/eye.jpg -------------------------------------------------------------------------------- /Docs/images/pose.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/pose.jpg -------------------------------------------------------------------------------- /Docs/images/prnet.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/prnet.gif -------------------------------------------------------------------------------- /Docs/images/reconstruct.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/reconstruct.jpg -------------------------------------------------------------------------------- /Docs/images/swapping.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/Docs/images/swapping.jpg -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Yao Feng 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network 2 | 3 |

4 | 5 |

6 | 7 | 8 | 9 | This is an official python implementation of PRN. 10 | 11 | PRN is a method to jointly regress dense alignment and 3D face shape in an end-to-end manner. More examples on Multi-PIE and 300VW can be seen in [YouTube](https://youtu.be/tXTgLSyIha8) . 12 | 13 | The main features are: 14 | 15 | * **End-to-End** our method can directly regress the 3D facial structure and dense alignment from a single image bypassing 3DMM fitting. 16 | 17 | * **Multi-task** By regressing position map, the 3D geometry along with semantic meaning can be obtained. Thus, we can effortlessly complete the tasks of dense alignment, monocular 3D face reconstruction, pose estimation, etc. 18 | 19 | * **Faster than real-time** The method can run at over 100fps(with GTX 1080) to regress a position map. 20 | 21 | * **Robust** Tested on facial images in unconstrained conditions. Our method is robust to poses, illuminations and occlusions. 22 | 23 | 24 | 25 | ## Applications 26 | 27 | ### Basics(Evaluated in paper) 28 | 29 | * #### Face Alignment 30 | 31 | Dense alignment of both visible and non-visible points(including 68 key points). 32 | 33 | And the **visibility** of points(1 for visible and 0 for non-visible). 34 | 35 | ![alignment](Docs/images/alignment.jpg) 36 | 37 | * #### 3D Face Reconstruction 38 | 39 | Get the 3D vertices and corresponding colours from a single image. Save the result as mesh data(.obj), which can be opened with [Meshlab](http://www.meshlab.net/) or Microsoft [3D Builder](https://developer.microsoft.com/en-us/windows/hardware/3d-print/3d-builder-resources). Notice that, the texture of non-visible area is distorted due to self-occlusion. 40 | 41 | **New**: 42 | 43 | 1. you can choose to output mesh with its original pose(default) or with front view(which means all output meshes are aligned) 44 | 2. obj file can now also written with texture map(with specified texture size), and you can set non-visible texture to 0. 45 | 46 | 47 | 48 | ![alignment](Docs/images/reconstruct.jpg) 49 | 50 | 51 | 52 | ### More(To be added) 53 | 54 | * #### 3D Pose Estimation 55 | 56 | Rather than only use 68 key points to calculate the camera matrix(easily effected by expression and poses), we use all vertices(more than 40K) to calculate a more accurate pose. 57 | 58 | #### ![pose](Docs/images/pose.jpg) 59 | 60 | * #### Depth image 61 | 62 | ![pose](Docs/images/depth.jpg) 63 | 64 | * #### Texture Editing 65 | 66 | * Data Augmentation/Selfie Editing 67 | 68 | modify special parts of input face, eyes for example: 69 | 70 | ![pose](Docs/images/eye.jpg) 71 | 72 | * Face Swapping 73 | 74 | replace the texture with another, then warp it to original pose and use Poisson editing to blend images. 75 | 76 | ![pose](Docs/images/swapping.jpg) 77 | 78 | 79 | 80 | 81 | 82 | 83 | ## Getting Started 84 | 85 | ### Prerequisite 86 | 87 | * Python 2.7 (numpy, skimage, scipy) 88 | 89 | * TensorFlow >= 1.4 90 | 91 | Optional: 92 | 93 | * dlib (for detecting face. You do not have to install if you can provide bounding box information. ) 94 | 95 | * opencv2 (for showing results) 96 | 97 | GPU is highly recommended. The run time is ~0.01s with GPU(GeForce GTX 1080) and ~0.2s with CPU(Intel(R) Xeon(R) CPU E5-2640 v4 @ 2.40GHz). 98 | 99 | ### Usage 100 | 101 | 1. Clone the repository 102 | 103 | ```bash 104 | git clone https://github.com/YadiraF/PRNet 105 | cd PRNet 106 | ``` 107 | 108 | 2. Download the PRN trained model at [BaiduDrive](https://pan.baidu.com/s/10vuV7m00OHLcsihaC-Adsw) or [GoogleDrive](https://drive.google.com/file/d/1UoE-XuW1SDLUjZmJPkIZ1MLxvQFgmTFH/view?usp=sharing), and put it into `Data/net-data` 109 | 110 | 3. Run the test code.(test AFLW2000 images) 111 | 112 | `python run_basics.py #Can run only with python and tensorflow` 113 | 114 | 4. Run with your own images 115 | 116 | `python demo.py -i -o --isDlib True ` 117 | 118 | run `python demo.py --help` for more details. 119 | 120 | 5. For Texture Editing Apps: 121 | 122 | `python demo_texture.py -i image_path_1 -r image_path_2 -o output_path ` 123 | 124 | run `python demo_texture.py --help` for more details. 125 | 126 | 127 | 128 | ## Training 129 | 130 | The core idea of the paper is: 131 | 132 | Using position map to represent face geometry&alignment information, then learning this with an Encoder-Decoder Network. 133 | 134 | So, the training steps: 135 | 136 | 1. generate position map ground truth. 137 | 138 | the example of generating position map of 300W_LP dataset can be seen in [generate_posmap_300WLP](https://github.com/YadiraF/face3d/blob/master/examples/8_generate_posmap_300WLP.py) 139 | 140 | 2. an encoder-decoder network to learn mapping from rgb image to position map. 141 | 142 | the weight mask can be found in the folder `Data/uv-data` 143 | 144 | What you can custom: 145 | 146 | 1. the UV space of position map. 147 | 148 | you can change the parameterization method, or change the resolution of UV space. 149 | 150 | 2. the backbone of encoder-decoder network 151 | 152 | this demo uses residual blocks. VGG, mobile-net are also ok. 153 | 154 | 3. the weight mask 155 | 156 | you can change the weight to focus more on which part your project need more. 157 | 158 | 4. the training data 159 | 160 | if you have scanned 3d face, it's better to train PRN with your own data. Before that, you may need use ICP to align your face meshes. 161 | 162 | 163 | 164 | ## FQA 165 | 166 | 1. How to **speed up**? 167 | 168 | a. network inference part 169 | 170 | you can train a smaller network or use a smaller position map as input. 171 | 172 | b. render part 173 | 174 | you can refer to [c++ version](https://github.com/YadiraF/face3d/blob/master/face3d/mesh/render.py). 175 | 176 | c. other parts like detecting face, writing obj 177 | 178 | the best way is to rewrite them in c++. 179 | 180 | 2. How to improve the **precision**? 181 | 182 | a. geometry precision. 183 | 184 | Due to the restriction of training data, the precision of reconstructed face from this demo has little detail. You can train the network with your own detailed data or do post-processing like shape-from-shading to add details. 185 | 186 | b. texture precision. 187 | 188 | I just added an option to specify the texture size. When the texture size > face size in original image, and render new facial image with [texture mapping](https://github.com/YadiraF/face3d/blob/04869dcee1455d1fa5b157f165a6878c550cf695/face3d/mesh/render.py), there will be little resample error. 189 | 190 | 191 | 192 | ## Changelog 193 | 194 | * 2018/7/19 add training part. can specify the resolution of the texture map. 195 | * 2018/5/10 add texture editing examples(for data augmentation, face swapping) 196 | * 2018/4/28 add visibility of vertices, output obj file with texture map, depth image 197 | * 2018/4/26 can output mesh with front view 198 | * 2018/3/28 add pose estimation 199 | * 2018/3/12 first release(3d reconstruction and dense alignment) 200 | 201 | 202 | 203 | ## License 204 | 205 | Code: under MIT license. 206 | 207 | Trained model file: please see [issue 28](https://github.com/YadiraF/PRNet/issues/28), thank [Kyle McDonald](https://github.com/kylemcdonald) for his answer. 208 | 209 | 210 | 211 | ## Citation 212 | 213 | If you use this code, please consider citing: 214 | 215 | ``` 216 | @inProceedings{feng2018prn, 217 | title = {Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network}, 218 | author = {Yao Feng and Fan Wu and Xiaohu Shao and Yanfeng Wang and Xi Zhou}, 219 | booktitle = {ECCV}, 220 | year = {2018} 221 | } 222 | ``` 223 | 224 | 225 | 226 | ## Contacts 227 | 228 | Please contact _fengyao@sjtu.edu.cn_ or open an issue for any questions or suggestions. 229 | 230 | Thanks! (●'◡'●) 231 | 232 | 233 | 234 | ## Acknowledgements 235 | 236 | - Thanks [BFM team](https://faces.dmi.unibas.ch/bfm/), [Xiangyu Zhu](http://www.cbsr.ia.ac.cn/users/xiangyuzhu/projects/3DDFA/main.htm), and [Anil Bas](https://github.com/anilbas/3DMMasSTN) for sharing 3D data. 237 | - Thanks Patrik Huber for sharing his work [eos](https://github.com/patrikhuber/eos), which helps me a lot in studying 3D Face Reconstruction. 238 | - Thanks the authors of [3DMMasSTN](https://github.com/anilbas/3DMMasSTN), [DenseReg](https://github.com/ralpguler/DenseReg), [3dmm_cnn](https://github.com/anhttran/3dmm_cnn), [vrn](https://github.com/AaronJackson/vrn), [pix2vertex](https://github.com/matansel/pix2vertex), [face-alignment](https://github.com/1adrianb/face-alignment) for making their excellent works publicly available. 239 | -------------------------------------------------------------------------------- /TestImages/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/0.jpg -------------------------------------------------------------------------------- /TestImages/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/1.jpg -------------------------------------------------------------------------------- /TestImages/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/2.jpg -------------------------------------------------------------------------------- /TestImages/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/3.jpg -------------------------------------------------------------------------------- /TestImages/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/4.jpg -------------------------------------------------------------------------------- /TestImages/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/5.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00050.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00050.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00050.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00050.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00081.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00081.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00081.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00081.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00251.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00251.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00251.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00251.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00430.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00430.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00430.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00430.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00475.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00475.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00475.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00475.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00514.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00514.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00514.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00514.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00516.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00516.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image00516.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image00516.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image01038.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image01038.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image01038.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image01038.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image01322.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image01322.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image01322.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image01322.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02283.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02283.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02283.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02283.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02420.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02420.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02420.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02420.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02545.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02545.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02545.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02545.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02616.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02616.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image02616.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image02616.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image03324.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image03324.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image03324.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image03324.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image03614.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image03614.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image03614.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image03614.mat -------------------------------------------------------------------------------- /TestImages/AFLW2000/image04331.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image04331.jpg -------------------------------------------------------------------------------- /TestImages/AFLW2000/image04331.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/AFLW2000/image04331.mat -------------------------------------------------------------------------------- /TestImages/trump.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/TestImages/trump.jpg -------------------------------------------------------------------------------- /api.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | from skimage.io import imread, imsave 4 | from skimage.transform import estimate_transform, warp 5 | from time import time 6 | 7 | from predictor import PosPrediction 8 | 9 | class PRN: 10 | ''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network 11 | Args: 12 | is_dlib(bool, optional): If true, dlib is used for detecting faces. 13 | prefix(str, optional): If run at another folder, the absolute path is needed to load the data. 14 | ''' 15 | def __init__(self, is_dlib = False, prefix = '.'): 16 | 17 | # resolution of input and output image size. 18 | self.resolution_inp = 256 19 | self.resolution_op = 256 20 | 21 | #---- load detectors 22 | if is_dlib: 23 | import dlib 24 | detector_path = os.path.join(prefix, 'Data/net-data/mmod_human_face_detector.dat') 25 | self.face_detector = dlib.cnn_face_detection_model_v1( 26 | detector_path) 27 | 28 | #---- load PRN 29 | self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op) 30 | prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight') 31 | if not os.path.isfile(prn_path + '.data-00000-of-00001'): 32 | print("please download PRN trained model first.") 33 | exit() 34 | self.pos_predictor.restore(prn_path) 35 | 36 | # uv file 37 | self.uv_kpt_ind = np.loadtxt(prefix + '/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt 38 | self.face_ind = np.loadtxt(prefix + '/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map 39 | self.triangles = np.loadtxt(prefix + '/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3 40 | 41 | self.uv_coords = self.generate_uv_coords() 42 | 43 | def generate_uv_coords(self): 44 | resolution = self.resolution_op 45 | uv_coords = np.meshgrid(range(resolution),range(resolution)) 46 | uv_coords = np.transpose(np.array(uv_coords), [1,2,0]) 47 | uv_coords = np.reshape(uv_coords, [resolution**2, -1]); 48 | uv_coords = uv_coords[self.face_ind, :] 49 | uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1]))) 50 | return uv_coords 51 | 52 | def dlib_detect(self, image): 53 | return self.face_detector(image, 1) 54 | 55 | def net_forward(self, image): 56 | ''' The core of out method: regress the position map of a given image. 57 | Args: 58 | image: (256,256,3) array. value range: 0~1 59 | Returns: 60 | pos: the 3D position map. (256, 256, 3) array. 61 | ''' 62 | return self.pos_predictor.predict(image) 63 | 64 | def process(self, input, image_info = None): 65 | ''' process image with crop operation. 66 | Args: 67 | input: (h,w,3) array or str(image path). image value range:1~255. 68 | image_info(optional): the bounding box information of faces. if None, will use dlib to detect face. 69 | 70 | Returns: 71 | pos: the 3D position map. (256, 256, 3). 72 | ''' 73 | if isinstance(input, str): 74 | try: 75 | image = imread(input) 76 | except IOError: 77 | print("error opening file: ", input) 78 | return None 79 | else: 80 | image = input 81 | 82 | if image.ndim < 3: 83 | image = np.tile(image[:,:,np.newaxis], [1,1,3]) 84 | 85 | if image_info is not None: 86 | if np.max(image_info.shape) > 4: # key points to get bounding box 87 | kpt = image_info 88 | if kpt.shape[0] > 3: 89 | kpt = kpt.T 90 | left = np.min(kpt[0, :]); right = np.max(kpt[0, :]); 91 | top = np.min(kpt[1,:]); bottom = np.max(kpt[1,:]) 92 | else: # bounding box 93 | bbox = image_info 94 | left = bbox[0]; right = bbox[1]; top = bbox[2]; bottom = bbox[3] 95 | old_size = (right - left + bottom - top)/2 96 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0]) 97 | size = int(old_size*1.6) 98 | else: 99 | detected_faces = self.dlib_detect(image) 100 | if len(detected_faces) == 0: 101 | print('warning: no detected face') 102 | return None 103 | 104 | d = detected_faces[0].rect ## only use the first detected face (assume that each input image only contains one face) 105 | left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom() 106 | old_size = (right - left + bottom - top)/2 107 | center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14]) 108 | size = int(old_size*1.58) 109 | 110 | # crop image 111 | src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]]) 112 | DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]]) 113 | tform = estimate_transform('similarity', src_pts, DST_PTS) 114 | 115 | image = image/255. 116 | cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp)) 117 | 118 | # run our net 119 | #st = time() 120 | cropped_pos = self.net_forward(cropped_image) 121 | #print 'net time:', time() - st 122 | 123 | # restore 124 | cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T 125 | z = cropped_vertices[2,:].copy()/tform.params[0,0] 126 | cropped_vertices[2,:] = 1 127 | vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices) 128 | vertices = np.vstack((vertices[:2,:], z)) 129 | pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3]) 130 | 131 | return pos 132 | 133 | def get_landmarks(self, pos): 134 | ''' 135 | Args: 136 | pos: the 3D position map. shape = (256, 256, 3). 137 | Returns: 138 | kpt: 68 3D landmarks. shape = (68, 3). 139 | ''' 140 | kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :] 141 | return kpt 142 | 143 | 144 | def get_vertices(self, pos): 145 | ''' 146 | Args: 147 | pos: the 3D position map. shape = (256, 256, 3). 148 | Returns: 149 | vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here. 150 | ''' 151 | all_vertices = np.reshape(pos, [self.resolution_op**2, -1]) 152 | vertices = all_vertices[self.face_ind, :] 153 | 154 | return vertices 155 | 156 | def get_colors_from_texture(self, texture): 157 | ''' 158 | Args: 159 | texture: the texture map. shape = (256, 256, 3). 160 | Returns: 161 | colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here. 162 | ''' 163 | all_colors = np.reshape(texture, [self.resolution_op**2, -1]) 164 | colors = all_colors[self.face_ind, :] 165 | 166 | return colors 167 | 168 | 169 | def get_colors(self, image, vertices): 170 | ''' 171 | Args: 172 | pos: the 3D position map. shape = (256, 256, 3). 173 | Returns: 174 | colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here. 175 | ''' 176 | [h, w, _] = image.shape 177 | vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x 178 | vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y 179 | ind = np.round(vertices).astype(np.int32) 180 | colors = image[ind[:,1], ind[:,0], :] # n x 3 181 | 182 | return colors 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | from glob import glob 4 | import scipy.io as sio 5 | from skimage.io import imread, imsave 6 | from skimage.transform import rescale, resize 7 | from time import time 8 | import argparse 9 | import ast 10 | 11 | from api import PRN 12 | 13 | from utils.estimate_pose import estimate_pose 14 | from utils.rotate_vertices import frontalize 15 | from utils.render_app import get_visibility, get_uv_mask, get_depth_image 16 | from utils.write import write_obj_with_colors, write_obj_with_texture 17 | 18 | def main(args): 19 | if args.isShow or args.isTexture: 20 | import cv2 21 | from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box 22 | 23 | # ---- init PRN 24 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU 25 | prn = PRN(is_dlib = args.isDlib) 26 | 27 | # ------------- load data 28 | image_folder = args.inputDir 29 | save_folder = args.outputDir 30 | if not os.path.exists(save_folder): 31 | os.mkdir(save_folder) 32 | 33 | types = ('*.jpg', '*.png') 34 | image_path_list= [] 35 | for files in types: 36 | image_path_list.extend(glob(os.path.join(image_folder, files))) 37 | total_num = len(image_path_list) 38 | 39 | for i, image_path in enumerate(image_path_list): 40 | 41 | name = image_path.strip().split('/')[-1][:-4] 42 | 43 | # read image 44 | image = imread(image_path) 45 | [h, w, c] = image.shape 46 | if c>3: 47 | image = image[:,:,:3] 48 | 49 | # the core: regress position map 50 | if args.isDlib: 51 | max_size = max(image.shape[0], image.shape[1]) 52 | if max_size> 1000: 53 | image = rescale(image, 1000./max_size) 54 | image = (image*255).astype(np.uint8) 55 | pos = prn.process(image) # use dlib to detect face 56 | else: 57 | if image.shape[0] == image.shape[1]: 58 | image = resize(image, (256,256)) 59 | pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 60 | else: 61 | box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box 62 | pos = prn.process(image, box) 63 | 64 | image = image/255. 65 | if pos is None: 66 | continue 67 | 68 | if args.is3d or args.isMat or args.isPose or args.isShow: 69 | # 3D vertices 70 | vertices = prn.get_vertices(pos) 71 | if args.isFront: 72 | save_vertices = frontalize(vertices) 73 | else: 74 | save_vertices = vertices.copy() 75 | save_vertices[:,1] = h - 1 - save_vertices[:,1] 76 | 77 | if args.isImage: 78 | imsave(os.path.join(save_folder, name + '.jpg'), image) 79 | 80 | if args.is3d: 81 | # corresponding colors 82 | colors = prn.get_colors(image, vertices) 83 | 84 | if args.isTexture: 85 | if args.texture_size != 256: 86 | pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) 87 | else: 88 | pos_interpolated = pos.copy() 89 | texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) 90 | if args.isMask: 91 | vertices_vis = get_visibility(vertices, prn.triangles, h, w) 92 | uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) 93 | uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) 94 | texture = texture*uv_mask[:,:,np.newaxis] 95 | write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) 96 | else: 97 | write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) 98 | 99 | if args.isDepth: 100 | depth_image = get_depth_image(vertices, prn.triangles, h, w, True) 101 | depth = get_depth_image(vertices, prn.triangles, h, w) 102 | imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) 103 | sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth':depth}) 104 | 105 | if args.isMat: 106 | sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles}) 107 | 108 | if args.isKpt or args.isShow: 109 | # get landmarks 110 | kpt = prn.get_landmarks(pos) 111 | np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) 112 | 113 | if args.isPose or args.isShow: 114 | # estimate pose 115 | camera_matrix, pose = estimate_pose(vertices) 116 | np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) 117 | np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) 118 | 119 | np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) 120 | 121 | if args.isShow: 122 | # ---------- Plot 123 | image_pose = plot_pose_box(image, camera_matrix, kpt) 124 | cv2.imshow('sparse alignment', plot_kpt(image, kpt)) 125 | cv2.imshow('dense alignment', plot_vertices(image, vertices)) 126 | cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) 127 | cv2.waitKey(0) 128 | 129 | 130 | if __name__ == '__main__': 131 | parser = argparse.ArgumentParser(description='Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network') 132 | 133 | parser.add_argument('-i', '--inputDir', default='TestImages/', type=str, 134 | help='path to the input directory, where input images are stored.') 135 | parser.add_argument('-o', '--outputDir', default='TestImages/results', type=str, 136 | help='path to the output directory, where results(obj,txt files) will be stored.') 137 | parser.add_argument('--gpu', default='0', type=str, 138 | help='set gpu id, -1 for CPU') 139 | parser.add_argument('--isDlib', default=True, type=ast.literal_eval, 140 | help='whether to use dlib for detecting face, default is True, if False, the input image should be cropped in advance') 141 | parser.add_argument('--is3d', default=True, type=ast.literal_eval, 142 | help='whether to output 3D face(.obj). default save colors.') 143 | parser.add_argument('--isMat', default=False, type=ast.literal_eval, 144 | help='whether to save vertices,color,triangles as mat for matlab showing') 145 | parser.add_argument('--isKpt', default=False, type=ast.literal_eval, 146 | help='whether to output key points(.txt)') 147 | parser.add_argument('--isPose', default=False, type=ast.literal_eval, 148 | help='whether to output estimated pose(.txt)') 149 | parser.add_argument('--isShow', default=False, type=ast.literal_eval, 150 | help='whether to show the results with opencv(need opencv)') 151 | parser.add_argument('--isImage', default=False, type=ast.literal_eval, 152 | help='whether to save input image') 153 | # update in 2017/4/10 154 | parser.add_argument('--isFront', default=False, type=ast.literal_eval, 155 | help='whether to frontalize vertices(mesh)') 156 | # update in 2017/4/25 157 | parser.add_argument('--isDepth', default=False, type=ast.literal_eval, 158 | help='whether to output depth image') 159 | # update in 2017/4/27 160 | parser.add_argument('--isTexture', default=False, type=ast.literal_eval, 161 | help='whether to save texture in obj file') 162 | parser.add_argument('--isMask', default=False, type=ast.literal_eval, 163 | help='whether to set invisible pixels(due to self-occlusion) in texture as 0') 164 | # update in 2017/7/19 165 | parser.add_argument('--texture_size', default=256, type=int, 166 | help='size of texture map, default is 256. need isTexture is True') 167 | main(parser.parse_args()) 168 | -------------------------------------------------------------------------------- /demo_texture.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | from glob import glob 4 | import scipy.io as sio 5 | from skimage.io import imread, imsave 6 | from skimage.transform import rescale, resize 7 | from time import time 8 | import argparse 9 | import ast 10 | import matplotlib.pyplot as plt 11 | import argparse 12 | 13 | from api import PRN 14 | from utils.render import render_texture 15 | import cv2 16 | 17 | 18 | def texture_editing(prn, args): 19 | # read image 20 | image = imread(args.image_path) 21 | [h, w, _] = image.shape 22 | 23 | #-- 1. 3d reconstruction -> get texture. 24 | pos = prn.process(image) 25 | vertices = prn.get_vertices(pos) 26 | image = image/255. 27 | texture = cv2.remap(image, pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) 28 | 29 | #-- 2. Texture Editing 30 | Mode = args.mode 31 | # change part of texture(for data augumentation/selfie editing. Here modify eyes for example) 32 | if Mode == 0: 33 | # load eye mask 34 | uv_face_eye = imread('Data/uv-data/uv_face_eyes.png', as_grey=True)/255. 35 | uv_face = imread('Data/uv-data/uv_face.png', as_grey=True)/255. 36 | eye_mask = (abs(uv_face_eye - uv_face) > 0).astype(np.float32) 37 | 38 | # texture from another image or a processed texture 39 | ref_image = imread(args.ref_path) 40 | ref_pos = prn.process(ref_image) 41 | ref_image = ref_image/255. 42 | ref_texture = cv2.remap(ref_image, ref_pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) 43 | 44 | # modify texture 45 | new_texture = texture*(1 - eye_mask[:,:,np.newaxis]) + ref_texture*eye_mask[:,:,np.newaxis] 46 | 47 | # change whole face(face swap) 48 | elif Mode == 1: 49 | # texture from another image or a processed texture 50 | ref_image = imread(args.ref_path) 51 | ref_pos = prn.process(ref_image) 52 | ref_image = ref_image/255. 53 | ref_texture = cv2.remap(ref_image, ref_pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) 54 | ref_vertices = prn.get_vertices(ref_pos) 55 | new_texture = ref_texture#(texture + ref_texture)/2. 56 | 57 | else: 58 | print('Wrong Mode! Mode should be 0 or 1.') 59 | exit() 60 | 61 | 62 | #-- 3. remap to input image.(render) 63 | vis_colors = np.ones((vertices.shape[0], 1)) 64 | face_mask = render_texture(vertices.T, vis_colors.T, prn.triangles.T, h, w, c = 1) 65 | face_mask = np.squeeze(face_mask > 0).astype(np.float32) 66 | 67 | new_colors = prn.get_colors_from_texture(new_texture) 68 | new_image = render_texture(vertices.T, new_colors.T, prn.triangles.T, h, w, c = 3) 69 | new_image = image*(1 - face_mask[:,:,np.newaxis]) + new_image*face_mask[:,:,np.newaxis] 70 | 71 | # Possion Editing for blending image 72 | vis_ind = np.argwhere(face_mask>0) 73 | vis_min = np.min(vis_ind, 0) 74 | vis_max = np.max(vis_ind, 0) 75 | center = (int((vis_min[1] + vis_max[1])/2+0.5), int((vis_min[0] + vis_max[0])/2+0.5)) 76 | output = cv2.seamlessClone((new_image*255).astype(np.uint8), (image*255).astype(np.uint8), (face_mask*255).astype(np.uint8), center, cv2.NORMAL_CLONE) 77 | 78 | # save output 79 | imsave(args.output_path, output) 80 | print('Done.') 81 | 82 | 83 | if __name__ == '__main__': 84 | parser = argparse.ArgumentParser(description='Texture Editing by PRN') 85 | 86 | parser.add_argument('-i', '--image_path', default='TestImages/AFLW2000/image00081.jpg', type=str, 87 | help='path to input image') 88 | parser.add_argument('-r', '--ref_path', default='TestImages/trump.jpg', type=str, 89 | help='path to reference image(texture ref)') 90 | parser.add_argument('-o', '--output_path', default='TestImages/output.jpg', type=str, 91 | help='path to save output') 92 | parser.add_argument('--mode', default=1, type=int, 93 | help='ways to edit texture. 0 for modifying parts, 1 for changing whole') 94 | parser.add_argument('--gpu', default='0', type=str, 95 | help='set gpu id, -1 for CPU') 96 | 97 | # ---- init PRN 98 | os.environ['CUDA_VISIBLE_DEVICES'] = parser.parse_args().gpu # GPU number, -1 for CPU 99 | prn = PRN(is_dlib = True) 100 | 101 | texture_editing(prn, parser.parse_args()) -------------------------------------------------------------------------------- /predictor.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import tensorflow.contrib.layers as tcl 3 | from tensorflow.contrib.framework import arg_scope 4 | import numpy as np 5 | 6 | def resBlock(x, num_outputs, kernel_size = 4, stride=1, activation_fn=tf.nn.relu, normalizer_fn=tcl.batch_norm, scope=None): 7 | assert num_outputs%2==0 #num_outputs must be divided by channel_factor(2 here) 8 | with tf.variable_scope(scope, 'resBlock'): 9 | shortcut = x 10 | if stride != 1 or x.get_shape()[3] != num_outputs: 11 | shortcut = tcl.conv2d(shortcut, num_outputs, kernel_size=1, stride=stride, 12 | activation_fn=None, normalizer_fn=None, scope='shortcut') 13 | x = tcl.conv2d(x, num_outputs/2, kernel_size=1, stride=1, padding='SAME') 14 | x = tcl.conv2d(x, num_outputs/2, kernel_size=kernel_size, stride=stride, padding='SAME') 15 | x = tcl.conv2d(x, num_outputs, kernel_size=1, stride=1, activation_fn=None, padding='SAME', normalizer_fn=None) 16 | 17 | x += shortcut 18 | x = normalizer_fn(x) 19 | x = activation_fn(x) 20 | return x 21 | 22 | 23 | class resfcn256(object): 24 | def __init__(self, resolution_inp = 256, resolution_op = 256, channel = 3, name = 'resfcn256'): 25 | self.name = name 26 | self.channel = channel 27 | self.resolution_inp = resolution_inp 28 | self.resolution_op = resolution_op 29 | 30 | def __call__(self, x, is_training = True): 31 | with tf.variable_scope(self.name) as scope: 32 | with arg_scope([tcl.batch_norm], is_training=is_training, scale=True): 33 | with arg_scope([tcl.conv2d, tcl.conv2d_transpose], activation_fn=tf.nn.relu, 34 | normalizer_fn=tcl.batch_norm, 35 | biases_initializer=None, 36 | padding='SAME', 37 | weights_regularizer=tcl.l2_regularizer(0.0002)): 38 | size = 16 39 | # x: s x s x 3 40 | se = tcl.conv2d(x, num_outputs=size, kernel_size=4, stride=1) # 256 x 256 x 16 41 | se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=2) # 128 x 128 x 32 42 | se = resBlock(se, num_outputs=size * 2, kernel_size=4, stride=1) # 128 x 128 x 32 43 | se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=2) # 64 x 64 x 64 44 | se = resBlock(se, num_outputs=size * 4, kernel_size=4, stride=1) # 64 x 64 x 64 45 | se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=2) # 32 x 32 x 128 46 | se = resBlock(se, num_outputs=size * 8, kernel_size=4, stride=1) # 32 x 32 x 128 47 | se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=2) # 16 x 16 x 256 48 | se = resBlock(se, num_outputs=size * 16, kernel_size=4, stride=1) # 16 x 16 x 256 49 | se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=2) # 8 x 8 x 512 50 | se = resBlock(se, num_outputs=size * 32, kernel_size=4, stride=1) # 8 x 8 x 512 51 | 52 | pd = tcl.conv2d_transpose(se, size * 32, 4, stride=1) # 8 x 8 x 512 53 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=2) # 16 x 16 x 256 54 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256 55 | pd = tcl.conv2d_transpose(pd, size * 16, 4, stride=1) # 16 x 16 x 256 56 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=2) # 32 x 32 x 128 57 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128 58 | pd = tcl.conv2d_transpose(pd, size * 8, 4, stride=1) # 32 x 32 x 128 59 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=2) # 64 x 64 x 64 60 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64 61 | pd = tcl.conv2d_transpose(pd, size * 4, 4, stride=1) # 64 x 64 x 64 62 | 63 | pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=2) # 128 x 128 x 32 64 | pd = tcl.conv2d_transpose(pd, size * 2, 4, stride=1) # 128 x 128 x 32 65 | pd = tcl.conv2d_transpose(pd, size, 4, stride=2) # 256 x 256 x 16 66 | pd = tcl.conv2d_transpose(pd, size, 4, stride=1) # 256 x 256 x 16 67 | 68 | pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3 69 | pd = tcl.conv2d_transpose(pd, 3, 4, stride=1) # 256 x 256 x 3 70 | pos = tcl.conv2d_transpose(pd, 3, 4, stride=1, activation_fn = tf.nn.sigmoid)#, padding='SAME', weights_initializer=tf.random_normal_initializer(0, 0.02)) 71 | 72 | return pos 73 | @property 74 | def vars(self): 75 | return [var for var in tf.global_variables() if self.name in var.name] 76 | 77 | 78 | class PosPrediction(): 79 | def __init__(self, resolution_inp = 256, resolution_op = 256): 80 | # -- hyper settings 81 | self.resolution_inp = resolution_inp 82 | self.resolution_op = resolution_op 83 | self.MaxPos = resolution_inp*1.1 84 | 85 | # network type 86 | self.network = resfcn256(self.resolution_inp, self.resolution_op) 87 | 88 | # net forward 89 | self.x = tf.placeholder(tf.float32, shape=[None, self.resolution_inp, self.resolution_inp, 3]) 90 | self.x_op = self.network(self.x, is_training = False) 91 | self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) 92 | 93 | def restore(self, model_path): 94 | tf.train.Saver(self.network.vars).restore(self.sess, model_path) 95 | 96 | def predict(self, image): 97 | pos = self.sess.run(self.x_op, 98 | feed_dict = {self.x: image[np.newaxis, :,:,:]}) 99 | pos = np.squeeze(pos) 100 | return pos*self.MaxPos 101 | 102 | def predict_batch(self, images): 103 | pos = self.sess.run(self.x_op, 104 | feed_dict = {self.x: images}) 105 | return pos*self.MaxPos 106 | 107 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.14.3 2 | scikit-image 3 | scipy 4 | tensorflow 5 | -------------------------------------------------------------------------------- /run_basics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | from glob import glob 4 | import scipy.io as sio 5 | from skimage.io import imread, imsave 6 | from time import time 7 | 8 | from api import PRN 9 | from utils.write import write_obj_with_colors 10 | 11 | # ---- init PRN 12 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' # GPU number, -1 for CPU 13 | prn = PRN(is_dlib = False) 14 | 15 | 16 | # ------------- load data 17 | image_folder = 'TestImages/AFLW2000/' 18 | save_folder = 'TestImages/AFLW2000_results' 19 | if not os.path.exists(save_folder): 20 | os.mkdir(save_folder) 21 | 22 | types = ('*.jpg', '*.png') 23 | image_path_list= [] 24 | for files in types: 25 | image_path_list.extend(glob(os.path.join(image_folder, files))) 26 | total_num = len(image_path_list) 27 | 28 | for i, image_path in enumerate(image_path_list): 29 | # read image 30 | image = imread(image_path) 31 | 32 | # the core: regress position map 33 | if 'AFLW2000' in image_path: 34 | mat_path = image_path.replace('jpg', 'mat') 35 | info = sio.loadmat(mat_path) 36 | kpt = info['pt3d_68'] 37 | pos = prn.process(image, kpt) # kpt information is only used for detecting face and cropping image 38 | else: 39 | pos = prn.process(image) # use dlib to detect face 40 | 41 | # -- Basic Applications 42 | # get landmarks 43 | kpt = prn.get_landmarks(pos) 44 | # 3D vertices 45 | vertices = prn.get_vertices(pos) 46 | # corresponding colors 47 | colors = prn.get_colors(image, vertices) 48 | 49 | # -- save 50 | name = image_path.strip().split('/')[-1][:-4] 51 | np.savetxt(os.path.join(save_folder, name + '.txt'), kpt) 52 | write_obj_with_colors(os.path.join(save_folder, name + '.obj'), vertices, prn.triangles, colors) #save 3d face(can open with meshlab) 53 | 54 | sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles}) 55 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yfeng95/PRNet/fc12fe5e1f1462bdea52409b213d0cf1c8cf6c5b/utils/__init__.py -------------------------------------------------------------------------------- /utils/cv_plot.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | end_list = np.array([17, 22, 27, 42, 48, 31, 36, 68], dtype = np.int32) - 1 5 | def plot_kpt(image, kpt): 6 | ''' Draw 68 key points 7 | Args: 8 | image: the input image 9 | kpt: (68, 3). 10 | ''' 11 | image = image.copy() 12 | kpt = np.round(kpt).astype(np.int32) 13 | for i in range(kpt.shape[0]): 14 | st = kpt[i, :2] 15 | image = cv2.circle(image,(st[0], st[1]), 1, (0,0,255), 2) 16 | if i in end_list: 17 | continue 18 | ed = kpt[i + 1, :2] 19 | image = cv2.line(image, (st[0], st[1]), (ed[0], ed[1]), (255, 255, 255), 1) 20 | return image 21 | 22 | 23 | def plot_vertices(image, vertices): 24 | image = image.copy() 25 | vertices = np.round(vertices).astype(np.int32) 26 | for i in range(0, vertices.shape[0], 2): 27 | st = vertices[i, :2] 28 | image = cv2.circle(image,(st[0], st[1]), 1, (255,0,0), -1) 29 | return image 30 | 31 | 32 | def plot_pose_box(image, P, kpt, color=(0, 255, 0), line_width=2): 33 | ''' Draw a 3D box as annotation of pose. Ref:https://github.com/yinguobing/head-pose-estimation/blob/master/pose_estimator.py 34 | Args: 35 | image: the input image 36 | P: (3, 4). Affine Camera Matrix. 37 | kpt: (68, 3). 38 | ''' 39 | image = image.copy() 40 | 41 | point_3d = [] 42 | rear_size = 90 43 | rear_depth = 0 44 | point_3d.append((-rear_size, -rear_size, rear_depth)) 45 | point_3d.append((-rear_size, rear_size, rear_depth)) 46 | point_3d.append((rear_size, rear_size, rear_depth)) 47 | point_3d.append((rear_size, -rear_size, rear_depth)) 48 | point_3d.append((-rear_size, -rear_size, rear_depth)) 49 | 50 | front_size = 105 51 | front_depth = 110 52 | point_3d.append((-front_size, -front_size, front_depth)) 53 | point_3d.append((-front_size, front_size, front_depth)) 54 | point_3d.append((front_size, front_size, front_depth)) 55 | point_3d.append((front_size, -front_size, front_depth)) 56 | point_3d.append((-front_size, -front_size, front_depth)) 57 | point_3d = np.array(point_3d, dtype=np.float).reshape(-1, 3) 58 | 59 | # Map to 2d image points 60 | point_3d_homo = np.hstack((point_3d, np.ones([point_3d.shape[0],1]))) #n x 4 61 | point_2d = point_3d_homo.dot(P.T)[:,:2] 62 | point_2d[:,:2] = point_2d[:,:2] - np.mean(point_2d[:4,:2], 0) + np.mean(kpt[:27,:2], 0) 63 | point_2d = np.int32(point_2d.reshape(-1, 2)) 64 | 65 | # Draw all the lines 66 | cv2.polylines(image, [point_2d], True, color, line_width, cv2.LINE_AA) 67 | cv2.line(image, tuple(point_2d[1]), tuple( 68 | point_2d[6]), color, line_width, cv2.LINE_AA) 69 | cv2.line(image, tuple(point_2d[2]), tuple( 70 | point_2d[7]), color, line_width, cv2.LINE_AA) 71 | cv2.line(image, tuple(point_2d[3]), tuple( 72 | point_2d[8]), color, line_width, cv2.LINE_AA) 73 | 74 | return image -------------------------------------------------------------------------------- /utils/estimate_pose.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import cos, sin, atan2, asin 3 | 4 | 5 | def isRotationMatrix(R): 6 | ''' checks if a matrix is a valid rotation matrix(whether orthogonal or not) 7 | ''' 8 | Rt = np.transpose(R) 9 | shouldBeIdentity = np.dot(Rt, R) 10 | I = np.identity(3, dtype = R.dtype) 11 | n = np.linalg.norm(I - shouldBeIdentity) 12 | return n < 1e-6 13 | 14 | 15 | def matrix2angle(R): 16 | ''' compute three Euler angles from a Rotation Matrix. Ref: http://www.gregslabaugh.net/publications/euler.pdf 17 | Args: 18 | R: (3,3). rotation matrix 19 | Returns: 20 | x: yaw 21 | y: pitch 22 | z: roll 23 | ''' 24 | # assert(isRotationMatrix(R)) 25 | 26 | if R[2,0] !=1 or R[2,0] != -1: 27 | x = asin(R[2,0]) 28 | y = atan2(R[2,1]/cos(x), R[2,2]/cos(x)) 29 | z = atan2(R[1,0]/cos(x), R[0,0]/cos(x)) 30 | 31 | else:# Gimbal lock 32 | z = 0 #can be anything 33 | if R[2,0] == -1: 34 | x = np.pi/2 35 | y = z + atan2(R[0,1], R[0,2]) 36 | else: 37 | x = -np.pi/2 38 | y = -z + atan2(-R[0,1], -R[0,2]) 39 | 40 | return x, y, z 41 | 42 | 43 | def P2sRt(P): 44 | ''' decompositing camera matrix P. 45 | Args: 46 | P: (3, 4). Affine Camera Matrix. 47 | Returns: 48 | s: scale factor. 49 | R: (3, 3). rotation matrix. 50 | t2d: (2,). 2d translation. 51 | ''' 52 | t2d = P[:2, 3] 53 | R1 = P[0:1, :3] 54 | R2 = P[1:2, :3] 55 | s = (np.linalg.norm(R1) + np.linalg.norm(R2))/2.0 56 | r1 = R1/np.linalg.norm(R1) 57 | r2 = R2/np.linalg.norm(R2) 58 | r3 = np.cross(r1, r2) 59 | 60 | R = np.concatenate((r1, r2, r3), 0) 61 | return s, R, t2d 62 | 63 | 64 | def compute_similarity_transform(points_static, points_to_transform): 65 | #http://nghiaho.com/?page_id=671 66 | p0 = np.copy(points_static).T 67 | p1 = np.copy(points_to_transform).T 68 | 69 | t0 = -np.mean(p0, axis=1).reshape(3,1) 70 | t1 = -np.mean(p1, axis=1).reshape(3,1) 71 | t_final = t1 -t0 72 | 73 | p0c = p0+t0 74 | p1c = p1+t1 75 | 76 | covariance_matrix = p0c.dot(p1c.T) 77 | U,S,V = np.linalg.svd(covariance_matrix) 78 | R = U.dot(V) 79 | if np.linalg.det(R) < 0: 80 | R[:,2] *= -1 81 | 82 | rms_d0 = np.sqrt(np.mean(np.linalg.norm(p0c, axis=0)**2)) 83 | rms_d1 = np.sqrt(np.mean(np.linalg.norm(p1c, axis=0)**2)) 84 | 85 | s = (rms_d0/rms_d1) 86 | P = np.c_[s*np.eye(3).dot(R), t_final] 87 | return P 88 | 89 | def estimate_pose(vertices): 90 | canonical_vertices = np.load('Data/uv-data/canonical_vertices.npy') 91 | P = compute_similarity_transform(vertices, canonical_vertices) 92 | _,R,_ = P2sRt(P) # decompose affine matrix to s, R, t 93 | pose = matrix2angle(R) 94 | 95 | return P, pose 96 | -------------------------------------------------------------------------------- /utils/render.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Author: YadiraF 3 | Mail: fengyao@sjtu.edu.cn 4 | ''' 5 | import numpy as np 6 | 7 | def isPointInTri(point, tri_points): 8 | ''' Judge whether the point is in the triangle 9 | Method: 10 | http://blackpawn.com/texts/pointinpoly/ 11 | Args: 12 | point: [u, v] or [x, y] 13 | tri_points: three vertices(2d points) of a triangle. 2 coords x 3 vertices 14 | Returns: 15 | bool: true for in triangle 16 | ''' 17 | tp = tri_points 18 | 19 | # vectors 20 | v0 = tp[:,2] - tp[:,0] 21 | v1 = tp[:,1] - tp[:,0] 22 | v2 = point - tp[:,0] 23 | 24 | # dot products 25 | dot00 = np.dot(v0.T, v0) 26 | dot01 = np.dot(v0.T, v1) 27 | dot02 = np.dot(v0.T, v2) 28 | dot11 = np.dot(v1.T, v1) 29 | dot12 = np.dot(v1.T, v2) 30 | 31 | # barycentric coordinates 32 | if dot00*dot11 - dot01*dot01 == 0: 33 | inverDeno = 0 34 | else: 35 | inverDeno = 1/(dot00*dot11 - dot01*dot01) 36 | 37 | u = (dot11*dot02 - dot01*dot12)*inverDeno 38 | v = (dot00*dot12 - dot01*dot02)*inverDeno 39 | 40 | # check if point in triangle 41 | return (u >= 0) & (v >= 0) & (u + v < 1) 42 | 43 | def get_point_weight(point, tri_points): 44 | ''' Get the weights of the position 45 | Methods: https://gamedev.stackexchange.com/questions/23743/whats-the-most-efficient-way-to-find-barycentric-coordinates 46 | -m1.compute the area of the triangles formed by embedding the point P inside the triangle 47 | -m2.Christer Ericson's book "Real-Time Collision Detection". faster, so I used this. 48 | Args: 49 | point: [u, v] or [x, y] 50 | tri_points: three vertices(2d points) of a triangle. 2 coords x 3 vertices 51 | Returns: 52 | w0: weight of v0 53 | w1: weight of v1 54 | w2: weight of v3 55 | ''' 56 | tp = tri_points 57 | # vectors 58 | v0 = tp[:,2] - tp[:,0] 59 | v1 = tp[:,1] - tp[:,0] 60 | v2 = point - tp[:,0] 61 | 62 | # dot products 63 | dot00 = np.dot(v0.T, v0) 64 | dot01 = np.dot(v0.T, v1) 65 | dot02 = np.dot(v0.T, v2) 66 | dot11 = np.dot(v1.T, v1) 67 | dot12 = np.dot(v1.T, v2) 68 | 69 | # barycentric coordinates 70 | if dot00*dot11 - dot01*dot01 == 0: 71 | inverDeno = 0 72 | else: 73 | inverDeno = 1/(dot00*dot11 - dot01*dot01) 74 | 75 | u = (dot11*dot02 - dot01*dot12)*inverDeno 76 | v = (dot00*dot12 - dot01*dot02)*inverDeno 77 | 78 | w0 = 1 - u - v 79 | w1 = v 80 | w2 = u 81 | 82 | return w0, w1, w2 83 | 84 | 85 | def render_texture(vertices, colors, triangles, h, w, c = 3): 86 | ''' render mesh by z buffer 87 | Args: 88 | vertices: 3 x nver 89 | colors: 3 x nver 90 | triangles: 3 x ntri 91 | h: height 92 | w: width 93 | ''' 94 | # initial 95 | image = np.zeros((h, w, c)) 96 | 97 | depth_buffer = np.zeros([h, w]) - 999999. 98 | # triangle depth: approximate the depth to the average value of z in each vertex(v0, v1, v2), since the vertices are closed to each other 99 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 100 | tri_tex = (colors[:, triangles[0,:]] + colors[:,triangles[1,:]] + colors[:, triangles[2,:]])/3. 101 | 102 | for i in range(triangles.shape[1]): 103 | tri = triangles[:, i] # 3 vertex indices 104 | 105 | # the inner bounding box 106 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 107 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 108 | 109 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 110 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 111 | 112 | if umax depth_buffer[v, u] and isPointInTri([u,v], vertices[:2, tri]): 118 | depth_buffer[v, u] = tri_depth[i] 119 | image[v, u, :] = tri_tex[:, i] 120 | return image 121 | 122 | 123 | def map_texture(src_image, src_vertices, dst_vertices, dst_triangle_buffer, triangles, h, w, c = 3, mapping_type = 'bilinear'): 124 | ''' 125 | Args: 126 | triangles: 3 x ntri 127 | 128 | # src 129 | src_image: height x width x nchannels 130 | src_vertices: 3 x nver 131 | 132 | # dst 133 | dst_vertices: 3 x nver 134 | dst_triangle_buffer: height x width. the triangle index of each pixel in dst image 135 | 136 | Returns: 137 | dst_image: height x width x nchannels 138 | 139 | ''' 140 | [sh, sw, sc] = src_image.shape 141 | dst_image = np.zeros((h, w, c)) 142 | for y in range(h): 143 | for x in range(w): 144 | tri_ind = dst_triangle_buffer[y,x] 145 | if tri_ind < 0: # no tri in dst image 146 | continue 147 | #if src_triangles_vis[tri_ind]: # the corresponding triangle in src image is invisible 148 | # continue 149 | 150 | # then. For this triangle index, map corresponding pixels(in triangles) in src image to dst image 151 | # Two Methods: 152 | # M1. Calculate the corresponding affine matrix from src triangle to dst triangle. Then find the corresponding src position of this dst pixel. 153 | # -- ToDo 154 | # M2. Calculate the relative position of three vertices in dst triangle, then find the corresponding src position relative to three src vertices. 155 | tri = triangles[:, tri_ind] 156 | # dst weight, here directly use the center to approximate because the tri is small 157 | # if tri_ind < 366: 158 | # print tri_ind 159 | w0, w1, w2 = get_point_weight([x, y], dst_vertices[:2, tri]) 160 | # else: 161 | # w0 = w1 = w2 = 1./3 162 | # src 163 | src_texel = w0*src_vertices[:2, tri[0]] + w1*src_vertices[:2, tri[1]] + w2*src_vertices[:2, tri[2]] # 164 | # 165 | if src_texel[0] < 0 or src_texel[0]> sw-1 or src_texel[1]<0 or src_texel[1] > sh-1: 166 | dst_image[y, x, :] = 0 167 | continue 168 | # As the coordinates of the transformed pixel in the image will most likely not lie on a texel, we have to choose how to 169 | # calculate the pixel colors depending on the next texels 170 | # there are three different texture interpolation methods: area, bilinear and nearest neighbour 171 | # print y, x, src_texel 172 | # nearest neighbour 173 | if mapping_type == 'nearest': 174 | dst_image[y, x, :] = src_image[int(round(src_texel[1])), int(round(src_texel[0])), :] 175 | # bilinear 176 | elif mapping_type == 'bilinear': 177 | # next 4 pixels 178 | ul = src_image[int(np.floor(src_texel[1])), int(np.floor(src_texel[0])), :] 179 | ur = src_image[int(np.floor(src_texel[1])), int(np.ceil(src_texel[0])), :] 180 | dl = src_image[int(np.ceil(src_texel[1])), int(np.floor(src_texel[0])), :] 181 | dr = src_image[int(np.ceil(src_texel[1])), int(np.ceil(src_texel[0])), :] 182 | 183 | yd = src_texel[1] - np.floor(src_texel[1]) 184 | xd = src_texel[0] - np.floor(src_texel[0]) 185 | dst_image[y, x, :] = ul*(1-xd)*(1-yd) + ur*xd*(1-yd) + dl*(1-xd)*yd + dr*xd*yd 186 | 187 | return dst_image 188 | 189 | 190 | def get_depth_buffer(vertices, triangles, h, w): 191 | ''' 192 | Args: 193 | vertices: 3 x nver 194 | triangles: 3 x ntri 195 | h: height 196 | w: width 197 | Returns: 198 | depth_buffer: height x width 199 | ToDo: 200 | whether to add x, y by 0.5? the center of the pixel? 201 | m3. like somewhere is wrong 202 | # Each triangle has 3 vertices & Each vertex has 3 coordinates x, y, z. 203 | # Here, the bigger the z, the fronter the point. 204 | ''' 205 | # initial 206 | depth_buffer = np.zeros([h, w]) - 999999. #+ np.min(vertices[2,:]) - 999999. # set the initial z to the farest position 207 | 208 | ## calculate the depth(z) of each triangle 209 | #-m1. z = the center of shpere(through 3 vertices) 210 | #center3d = (vertices[:, triangles[0,:]] + vertices[:,triangles[1,:]] + vertices[:, triangles[2,:]])/3. 211 | #tri_depth = np.sum(center3d**2, axis = 0) 212 | #-m2. z = the center of z(v0, v1, v2) 213 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 214 | 215 | for i in range(triangles.shape[1]): 216 | tri = triangles[:, i] # 3 vertex indices 217 | 218 | # the inner bounding box 219 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 220 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 221 | 222 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 223 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 224 | 225 | if umax depth_buffer[v, u]: # and is_pointIntri([u,v], vertices[:2, tri]): 234 | depth_buffer[v, u] = tri_depth[i] 235 | 236 | return depth_buffer 237 | 238 | 239 | def get_triangle_buffer(vertices, triangles, h, w): 240 | ''' 241 | Args: 242 | vertices: 3 x nver 243 | triangles: 3 x ntri 244 | h: height 245 | w: width 246 | Returns: 247 | depth_buffer: height x width 248 | ToDo: 249 | whether to add x, y by 0.5? the center of the pixel? 250 | m3. like somewhere is wrong 251 | # Each triangle has 3 vertices & Each vertex has 3 coordinates x, y, z. 252 | # Here, the bigger the z, the fronter the point. 253 | ''' 254 | # initial 255 | depth_buffer = np.zeros([h, w]) - 999999. #+ np.min(vertices[2,:]) - 999999. # set the initial z to the farest position 256 | triangle_buffer = np.zeros_like(depth_buffer, dtype = np.int32) - 1 # if -1, the pixel has no triangle correspondance 257 | 258 | ## calculate the depth(z) of each triangle 259 | #-m1. z = the center of shpere(through 3 vertices) 260 | #center3d = (vertices[:, triangles[0,:]] + vertices[:,triangles[1,:]] + vertices[:, triangles[2,:]])/3. 261 | #tri_depth = np.sum(center3d**2, axis = 0) 262 | #-m2. z = the center of z(v0, v1, v2) 263 | tri_depth = (vertices[2, triangles[0,:]] + vertices[2,triangles[1,:]] + vertices[2, triangles[2,:]])/3. 264 | 265 | for i in range(triangles.shape[1]): 266 | tri = triangles[:, i] # 3 vertex indices 267 | 268 | # the inner bounding box 269 | umin = max(int(np.ceil(np.min(vertices[0,tri]))), 0) 270 | umax = min(int(np.floor(np.max(vertices[0,tri]))), w-1) 271 | 272 | vmin = max(int(np.ceil(np.min(vertices[1,tri]))), 0) 273 | vmax = min(int(np.floor(np.max(vertices[1,tri]))), h-1) 274 | 275 | if umax depth_buffer[v, u] and isPointInTri([u,v], vertices[:2, tri]): 284 | depth_buffer[v, u] = tri_depth[i] 285 | triangle_buffer[v, u] = i 286 | 287 | return triangle_buffer 288 | 289 | 290 | def vis_of_vertices(vertices, triangles, h, w, depth_buffer = None): 291 | ''' 292 | Args: 293 | vertices: 3 x nver 294 | triangles: 3 x ntri 295 | depth_buffer: height x width 296 | Returns: 297 | vertices_vis: nver. the visibility of each vertex 298 | ''' 299 | if depth_buffer == None: 300 | depth_buffer = get_depth_buffer(vertices, triangles, h, w) 301 | 302 | vertices_vis = np.zeros(vertices.shape[1], dtype = bool) 303 | 304 | depth_tmp = np.zeros_like(depth_buffer) - 99999 305 | for i in range(vertices.shape[1]): 306 | vertex = vertices[:, i] 307 | 308 | if np.floor(vertex[0]) < 0 or np.ceil(vertex[0]) > w-1 or np.floor(vertex[1]) < 0 or np.ceil(vertex[1]) > h-1: 309 | continue 310 | 311 | # bilinear interp 312 | # ul = depth_buffer[int(np.floor(vertex[1])), int(np.floor(vertex[0]))] 313 | # ur = depth_buffer[int(np.floor(vertex[1])), int(np.ceil(vertex[0]))] 314 | # dl = depth_buffer[int(np.ceil(vertex[1])), int(np.floor(vertex[0]))] 315 | # dr = depth_buffer[int(np.ceil(vertex[1])), int(np.ceil(vertex[0]))] 316 | 317 | # yd = vertex[1] - np.floor(vertex[1]) 318 | # xd = vertex[0] - np.floor(vertex[0]) 319 | 320 | # vertex_depth = ul*(1-xd)*(1-yd) + ur*xd*(1-yd) + dl*(1-xd)*yd + dr*xd*yd 321 | 322 | # nearest 323 | px = int(np.round(vertex[0])) 324 | py = int(np.round(vertex[1])) 325 | 326 | # if (vertex[2] > depth_buffer[ul[0], ul[1]]) & (vertex[2] > depth_buffer[ur[0], ur[1]]) & (vertex[2] > depth_buffer[dl[0], dl[1]]) & (vertex[2] > depth_buffer[dr[0], dr[1]]): 327 | if vertex[2] < depth_tmp[py, px]: 328 | continue 329 | 330 | # if vertex[2] > depth_buffer[py, px]: 331 | # vertices_vis[i] = True 332 | # depth_tmp[py, px] = vertex[2] 333 | # elif np.abs(vertex[2] - depth_buffer[py, px]) < 1: 334 | # vertices_vis[i] = True 335 | 336 | threshold = 2 # need to be optimized. 337 | if np.abs(vertex[2] - depth_buffer[py, px]) < threshold: 338 | # if np.abs(vertex[2] - vertex_depth) < threshold: 339 | vertices_vis[i] = True 340 | depth_tmp[py, px] = vertex[2] 341 | 342 | return vertices_vis 343 | -------------------------------------------------------------------------------- /utils/render_app.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from utils.render import vis_of_vertices, render_texture 3 | from scipy import ndimage 4 | 5 | def get_visibility(vertices, triangles, h, w): 6 | triangles = triangles.T 7 | vertices_vis = vis_of_vertices(vertices.T, triangles, h, w) 8 | vertices_vis = vertices_vis.astype(bool) 9 | for k in range(2): 10 | tri_vis = vertices_vis[triangles[0,:]] | vertices_vis[triangles[1,:]] | vertices_vis[triangles[2,:]] 11 | ind = triangles[:, tri_vis] 12 | vertices_vis[ind] = True 13 | # for k in range(2): 14 | # tri_vis = vertices_vis[triangles[0,:]] & vertices_vis[triangles[1,:]] & vertices_vis[triangles[2,:]] 15 | # ind = triangles[:, tri_vis] 16 | # vertices_vis[ind] = True 17 | vertices_vis = vertices_vis.astype(np.float32) #1 for visible and 0 for non-visible 18 | return vertices_vis 19 | 20 | def get_uv_mask(vertices_vis, triangles, uv_coords, h, w, resolution): 21 | triangles = triangles.T 22 | vertices_vis = vertices_vis.astype(np.float32) 23 | uv_mask = render_texture(uv_coords.T, vertices_vis[np.newaxis, :], triangles, resolution, resolution, 1) 24 | uv_mask = np.squeeze(uv_mask > 0) 25 | uv_mask = ndimage.binary_closing(uv_mask) 26 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 27 | uv_mask = ndimage.binary_closing(uv_mask) 28 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 29 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 30 | uv_mask = ndimage.binary_erosion(uv_mask, structure = np.ones((4,4))) 31 | uv_mask = uv_mask.astype(np.float32) 32 | 33 | return np.squeeze(uv_mask) 34 | 35 | def get_depth_image(vertices, triangles, h, w, isShow = False): 36 | z = vertices[:, 2:] 37 | if isShow: 38 | z = z/max(z) 39 | depth_image = render_texture(vertices.T, z.T, triangles.T, h, w, 1) 40 | return np.squeeze(depth_image) -------------------------------------------------------------------------------- /utils/rotate_vertices.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # import scipy.io as 4 | def frontalize(vertices): 5 | canonical_vertices = np.load('Data/uv-data/canonical_vertices.npy') 6 | 7 | vertices_homo = np.hstack((vertices, np.ones([vertices.shape[0],1]))) #n x 4 8 | P = np.linalg.lstsq(vertices_homo, canonical_vertices)[0].T # Affine matrix. 3 x 4 9 | front_vertices = vertices_homo.dot(P.T) 10 | 11 | return front_vertices 12 | -------------------------------------------------------------------------------- /utils/write.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage.io import imsave 3 | import os 4 | 5 | def write_asc(path, vertices): 6 | ''' 7 | Args: 8 | vertices: shape = (nver, 3) 9 | ''' 10 | if path.split('.')[-1] == 'asc': 11 | np.savetxt(path, vertices) 12 | else: 13 | np.savetxt(path + '.asc', vertices) 14 | 15 | 16 | def write_obj_with_colors(obj_name, vertices, triangles, colors): 17 | ''' Save 3D face model with texture represented by colors. 18 | Args: 19 | obj_name: str 20 | vertices: shape = (nver, 3) 21 | colors: shape = (nver, 3) 22 | triangles: shape = (ntri, 3) 23 | ''' 24 | triangles = triangles.copy() 25 | triangles += 1 # meshlab start with 1 26 | 27 | if obj_name.split('.')[-1] != 'obj': 28 | obj_name = obj_name + '.obj' 29 | 30 | # write obj 31 | with open(obj_name, 'w') as f: 32 | 33 | # write vertices & colors 34 | for i in range(vertices.shape[0]): 35 | # s = 'v {} {} {} \n'.format(vertices[0,i], vertices[1,i], vertices[2,i]) 36 | s = 'v {} {} {} {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2], colors[i, 0], colors[i, 1], colors[i, 2]) 37 | f.write(s) 38 | 39 | # write f: ver ind/ uv ind 40 | [k, ntri] = triangles.shape 41 | for i in range(triangles.shape[0]): 42 | # s = 'f {} {} {}\n'.format(triangles[i, 0], triangles[i, 1], triangles[i, 2]) 43 | s = 'f {} {} {}\n'.format(triangles[i, 2], triangles[i, 1], triangles[i, 0]) 44 | f.write(s) 45 | 46 | 47 | def write_obj_with_texture(obj_name, vertices, triangles, texture, uv_coords): 48 | ''' Save 3D face model with texture represented by texture map. 49 | Ref: https://github.com/patrikhuber/eos/blob/bd00155ebae4b1a13b08bf5a991694d682abbada/include/eos/core/Mesh.hpp 50 | Args: 51 | obj_name: str 52 | vertices: shape = (nver, 3) 53 | triangles: shape = (ntri, 3) 54 | texture: shape = (256,256,3) 55 | uv_coords: shape = (nver, 3) max value<=1 56 | ''' 57 | if obj_name.split('.')[-1] != 'obj': 58 | obj_name = obj_name + '.obj' 59 | mtl_name = obj_name.replace('.obj', '.mtl') 60 | texture_name = obj_name.replace('.obj', '_texture.png') 61 | 62 | triangles = triangles.copy() 63 | triangles += 1 # mesh lab start with 1 64 | 65 | # write obj 66 | with open(obj_name, 'w') as f: 67 | # first line: write mtlib(material library) 68 | s = "mtllib {}\n".format(os.path.abspath(mtl_name)) 69 | f.write(s) 70 | 71 | # write vertices 72 | for i in range(vertices.shape[0]): 73 | s = 'v {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2]) 74 | f.write(s) 75 | 76 | # write uv coords 77 | for i in range(uv_coords.shape[0]): 78 | s = 'vt {} {}\n'.format(uv_coords[i,0], 1 - uv_coords[i,1]) 79 | f.write(s) 80 | 81 | f.write("usemtl FaceTexture\n") 82 | 83 | # write f: ver ind/ uv ind 84 | for i in range(triangles.shape[0]): 85 | # s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,0], triangles[i,0], triangles[i,1], triangles[i,1], triangles[i,2], triangles[i,2]) 86 | s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,2], triangles[i,2], triangles[i,1], triangles[i,1], triangles[i,0], triangles[i,0]) 87 | f.write(s) 88 | 89 | # write mtl 90 | with open(mtl_name, 'w') as f: 91 | f.write("newmtl FaceTexture\n") 92 | s = 'map_Kd {}\n'.format(os.path.abspath(texture_name)) # map to image 93 | f.write(s) 94 | 95 | # write texture as png 96 | imsave(texture_name, texture) 97 | 98 | 99 | def write_obj_with_colors_texture(obj_name, vertices, colors, triangles, texture, uv_coords): 100 | ''' Save 3D face model with texture. 101 | Ref: https://github.com/patrikhuber/eos/blob/bd00155ebae4b1a13b08bf5a991694d682abbada/include/eos/core/Mesh.hpp 102 | Args: 103 | obj_name: str 104 | vertices: shape = (nver, 3) 105 | colors: shape = (nver, 3) 106 | triangles: shape = (ntri, 3) 107 | texture: shape = (256,256,3) 108 | uv_coords: shape = (nver, 3) max value<=1 109 | ''' 110 | if obj_name.split('.')[-1] != 'obj': 111 | obj_name = obj_name + '.obj' 112 | mtl_name = obj_name.replace('.obj', '.mtl') 113 | texture_name = obj_name.replace('.obj', '_texture.png') 114 | 115 | triangles = triangles.copy() 116 | triangles += 1 # mesh lab start with 1 117 | 118 | # write obj 119 | with open(obj_name, 'w') as f: 120 | # first line: write mtlib(material library) 121 | s = "mtllib {}\n".format(os.path.abspath(mtl_name)) 122 | f.write(s) 123 | 124 | # write vertices 125 | for i in range(vertices.shape[0]): 126 | s = 'v {} {} {} {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2], colors[i, 0], colors[i, 1], colors[i, 2]) 127 | f.write(s) 128 | 129 | # write uv coords 130 | for i in range(uv_coords.shape[0]): 131 | s = 'vt {} {}\n'.format(uv_coords[i,0], 1 - uv_coords[i,1]) 132 | f.write(s) 133 | 134 | f.write("usemtl FaceTexture\n") 135 | 136 | # write f: ver ind/ uv ind 137 | for i in range(triangles.shape[0]): 138 | # s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,0], triangles[i,0], triangles[i,1], triangles[i,1], triangles[i,2], triangles[i,2]) 139 | s = 'f {}/{} {}/{} {}/{}\n'.format(triangles[i,2], triangles[i,2], triangles[i,1], triangles[i,1], triangles[i,0], triangles[i,0]) 140 | f.write(s) 141 | 142 | # write mtl 143 | with open(mtl_name, 'w') as f: 144 | f.write("newmtl FaceTexture\n") 145 | s = 'map_Kd {}\n'.format(os.path.abspath(texture_name)) # map to image 146 | f.write(s) 147 | 148 | # write texture as png 149 | imsave(texture_name, texture) --------------------------------------------------------------------------------