├── .idea ├── .gitignore ├── InfoSwap-master.iml ├── deployment.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── LICENSE.md ├── README.md ├── USAGE.md ├── data ├── src │ └── Anna-Popplewell.png └── tar │ ├── 0034.png │ ├── 0268.png │ └── 0300.png ├── examples ├── 1024.jpg ├── films.jpg └── main_fig4.jpg ├── inference_demo.py ├── modules ├── aii_generator.py ├── decoder1024.py ├── decoder512.py ├── discriminator.py ├── encoder128.py ├── iib.py └── weights128 │ ├── readout_layer0.pth │ ├── readout_layer1.pth │ ├── readout_layer10.pth │ ├── readout_layer2.pth │ ├── readout_layer3.pth │ ├── readout_layer4.pth │ ├── readout_layer5.pth │ ├── readout_layer6.pth │ ├── readout_layer7.pth │ ├── readout_layer8.pth │ └── readout_layer9.pth ├── preprocess ├── __init__.py ├── mtcnn.py └── mtcnn_pytorch │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── caffe_models │ ├── det1.caffemodel │ ├── det1.prototxt │ ├── det2.caffemodel │ ├── det2.prototxt │ ├── det3.caffemodel │ ├── det3.prototxt │ ├── det4.caffemodel │ └── det4.prototxt │ ├── extract_weights_from_caffe_models.py │ └── src │ ├── __init__.py │ ├── align_trans.py │ ├── box_utils.py │ ├── detector.py │ ├── first_stage.py │ ├── get_nets.py │ ├── matlab_cp2tform.py │ ├── visualization_utils.py │ └── weights │ ├── onet.npy │ ├── pnet.npy │ └── rnet.npy ├── requirements.txt └── utils.py /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Datasource local storage ignored files 5 | /dataSources/ 6 | /dataSources.local.xml 7 | # Editor-based HTTP Client requests 8 | /httpRequests/ 9 | -------------------------------------------------------------------------------- /.idea/InfoSwap-master.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 102 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | ## Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0) 3 | 4 | _Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible._ 5 | 6 | ##### _Using Creative Commons Public Licenses_ 7 | 8 | Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. 9 | 10 | **_Considerations for licensors:_** Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](https://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). 11 | 12 | **Considerations for the public:** By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More_considerations 13 | for the public](https://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). 14 | 15 | 16 | ### Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License 17 | 18 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. 19 | 20 | 21 | #### Section 1 -- Definitions. 22 | 23 | a. **Adapted Material** means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. 24 | 25 | b. **Adapter's License** means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. 26 | 27 | c. **BY-NC-SA Compatible License** means a license listed at [creativecommons.org/compatiblelicenses](https://creativecommons.org/compatiblelicenses), approved by Creative Commons as essentially the equivalent of this Public License. 28 | 29 | d. **Copyright and Similar Rights** means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section [2(b)(1)-(2)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s2b) are not Copyright and Similar Rights. 30 | 31 | e. **Effective Technological Measures** means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. 32 | 33 | f. **Exceptions and Limitations** means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. 34 | 35 | g. **License Elements** means the license attributes listed in the name of a Creative Commons Public License. The License Elements of this Public License are Attribution, NonCommercial, and ShareAlike. 36 | 37 | h. **Licensed Material** means the artistic or literary work, database, or other material to which the Licensor applied this Public License. 38 | 39 | i. **Licensed Rights** means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. 40 | 41 | j. **Licensor** means the individual(s) or entity(ies) granting rights under this Public License. 42 | 43 | k. **NonCommercial** means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. 44 | 45 | l. **Share** means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. 46 | 47 | m. **Sui Generis Database Rights** means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. 48 | 49 | n. **You** means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. 50 | 51 | 52 | #### Section 2 -- Scope. 53 | 54 | ##### a. License grant. 55 | 56 | 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: 57 | 58 | A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and 59 | 60 | B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only. 61 | 62 | 2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 63 | 64 | 3. Term. The term of this Public License is specified in Section [6(a)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s6a). 65 | 66 | 4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section [2(a)(4)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s2a4) never produces Adapted Material. 67 | 68 | 5. Downstream recipients. 69 | 70 | A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. 71 | 72 | B. Additional offer from the Licensor – Adapted Material. Every recipient of Adapted Material from You automatically receives an offer from the Licensor to exercise the Licensed Rights in the Adapted Material under the conditions of the Adapter’s License You apply. 73 | 74 | C. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 75 | 76 | 6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section [3(a)(1)(A)(i)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3a1Ai). 77 | 78 | ##### b. Other rights. 79 | 80 | 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 81 | 82 | 2. Patent and trademark rights are not licensed under this Public License. 83 | 84 | 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes. 85 | 86 | 87 | #### Section 3 -- License Conditions. 88 | 89 | Your exercise of the Licensed Rights is expressly made subject to the following conditions. 90 | 91 | ##### a. Attribution. 92 | 93 | 1. If You Share the Licensed Material (including in modified form), You must: 94 | 95 | A. retain the following if it is supplied by the Licensor with the Licensed Material: 96 | 97 | i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); 98 | 99 | ii. a copyright notice; 100 | 101 | iii. a notice that refers to this Public License; 102 | 103 | iv. a notice that refers to the disclaimer of warranties; 104 | 105 | v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; 106 | 107 | B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and 108 | 109 | C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 110 | 111 | 2. You may satisfy the conditions in Section [3(a)(1)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3a1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 112 | 113 | 3. If requested by the Licensor, You must remove any of the information required by Section [3(a)(1)(A)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3a1A) to the extent reasonably practicable. 114 | 115 | ##### b. ShareAlike. 116 | 117 | In addition to the conditions in Section [3(a)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3a), if You Share Adapted Material You produce, the following conditions also apply. 118 | 119 | 1. The Adapter’s License You apply must be a Creative Commons license with the same License Elements, this version or later, or a BY-NC-SA Compatible License. 120 | 121 | 2. You must include the text of, or the URI or hyperlink to, the Adapter's License You apply. You may satisfy this condition in any reasonable manner based on the medium, means, and context in which You Share Adapted Material. 122 | 123 | 3. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, Adapted Material that restrict exercise of the rights granted under the Adapter's License You apply. 124 | 125 | #### Section 4 -- Sui Generis Database Rights. 126 | 127 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: 128 | 129 | a. for the avoidance of doubt, Section [2(a)(1)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s2a1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; 130 | 131 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material, including for purposes of Section [3(b)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3b); and 132 | 133 | c. You must comply with the conditions in Section [3(a)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s3a) if You Share all or a substantial portion of the contents of the database. 134 | 135 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. 136 | 137 | 138 | #### Section 5 -- Disclaimer of Warranties and Limitation of Liability. 139 | 140 | **a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.** 141 | 142 | **b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.** 143 | 144 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. 145 | 146 | 147 | #### Section 6 -- Term and Termination. 148 | 149 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. 150 | 151 | 152 | b. Where Your right to use the Licensed Material has terminated under Section [6(a)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s6a), it reinstates: 153 | 154 | 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 155 | 156 | 2. upon express reinstatement by the Licensor. 157 | 158 | For the avoidance of doubt, this Section [6(b)](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s6b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. 159 | 160 | c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. 161 | 162 | d. Sections [1](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s1), [5](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s5), [6](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s6), [7](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s7), and [8](https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode#s8) survive termination of this Public License. 163 | 164 | 165 | #### Section 7 -- Other Terms and Conditions. 166 | 167 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. 168 | 169 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. 170 | 171 | 172 | #### Section 8 -- Interpretation. 173 | 174 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. 175 | 176 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. 177 | 178 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. 179 | 180 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. 181 | 182 | --- 183 | 184 | Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” The text of the Creative Commons public licenses is dedicated to the public domain under the [_CC0 Public Domain Dedication_](https://creativecommons.org/publicdomain/zero/1.0/legalcode). Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](https://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. 185 | 186 | Creative Commons may be contacted at [creativecommons.org](https://creativecommons.org/). -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## InfoSwap: Information Bottleneck Disentanglement for Identity Swapping 2 | 3 | ### License 4 | 5 | Copyright (C) 2021, CRIPAC, NLPR, CASIA. All rights reserved. Licensed under the CC BY-NC-SA 4.0 license ([https://creativecommons.org/licenses/by-nc-sa/4.0/](https://creativecommons.org/licenses/by-nc-sa/4.0/)). 6 | 7 | ### Code usage 8 | 9 | Please check out the [user manual page](USAGE.md). 10 | 11 | ### Paper 12 | 13 | [Gege Gao](https://scholar.google.com/citations?user=nYYIYaUAAAAJ), [Huaibo Huang](https://scholar.google.com/citations?user=XMvLciUAAAAJ), [Chaoyou Fu](https://scholar.google.com/citations?user=4A1xYQwAAAAJ), Zhaoyang Li, [Ran He](https://scholar.google.com/citations?user=ayrg9AUAAAAJ), "[Information Bottleneck Disentanglement for Identity Swapping](https://openaccess.thecvf.com/content/CVPR2021/html/Gao_Information_Bottleneck_Disentanglement_for_Identity_Swapping_CVPR_2021_paper.html)", CVPR 2021 14 | 15 | ### Results Across Large Gaps: 16 | 17 | ![](examples/main_fig4.jpg) 18 | 19 | ### Results of 1024x1024 Pixels: 20 | 21 | ![](examples/1024.jpg) 22 | 23 | ### Results in Film Scenes: 24 | 25 | ![](examples/films.jpg) 26 | 27 | 28 | ### Citation 29 | 30 | If you find this code useful for your research, please cite our paper: 31 | 32 | ``` 33 | @InProceedings{Gao_2021_CVPR, 34 | author = {Gao, Gege and Huang, Huaibo and Fu, Chaoyou and Li, Zhaoyang and He, Ran}, 35 | title = {Information Bottleneck Disentanglement for Identity Swapping}, 36 | booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, 37 | month = {June}, 38 | year = {2021}, 39 | pages = {3404-3413} 40 | } 41 | ``` -------------------------------------------------------------------------------- /USAGE.md: -------------------------------------------------------------------------------- 1 | ## InfoSwap: Information Bottleneck Disentanglement for Identity Swapping (Official PyTorch Implementation) 2 | 3 | ### License 4 | 5 | Copyright (C) 2021, CRIPAC, NLPR, CASIA. All rights reserved. Licensed under the CC BY-NC-SA 4.0 license ([https://creativecommons.org/licenses/by-nc-sa/4.0/](https://creativecommons.org/licenses/by-nc-sa/4.0/)). 6 | 7 | ### Getting started 8 | 9 | #### Requirements 10 | See [requirements.txt](./requirements.txt), tested on Linux platforms. 11 | 12 | For pre-trained models, please [send a request email](mailto:grace.heseri@gmail.com) with subject "APPLY FOR MODELS" to us and describe **in detail** your purpose of using the models. Please inform us your name and institution, and use an email address **certified** by your research institution (e.g., @ia.ac.cn) to send this request, as we need to confirm that our models will not be used for any potential commercial purposes. Thanks for understanding! 13 | 14 | #### Example Usage 15 | 16 | Clone this repo: 17 | 18 | ```shell script 19 | git clone https://github.com/GGGHSL/InfoSwap-master.git 20 | cd InfoSwap-master 21 | ``` 22 | 23 | Run the following command: 24 | ```shell script 25 | python inference_demo.py -src [YOUR SOURCE IMAGE] -tar [YOUR DIR OF TARGET IMAGES] -save [YOUR SAVE DIR] --ib_mode [CHOICES: smooth, no_smooth] 26 | ``` 27 | The results are stored in `results_[INFERENCE_DATE]` folder. 28 | -------------------------------------------------------------------------------- /data/src/Anna-Popplewell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/data/src/Anna-Popplewell.png -------------------------------------------------------------------------------- /data/tar/0034.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/data/tar/0034.png -------------------------------------------------------------------------------- /data/tar/0268.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/data/tar/0268.png -------------------------------------------------------------------------------- /data/tar/0300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/data/tar/0300.png -------------------------------------------------------------------------------- /examples/1024.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/examples/1024.jpg -------------------------------------------------------------------------------- /examples/films.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/examples/films.jpg -------------------------------------------------------------------------------- /examples/main_fig4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/examples/main_fig4.jpg -------------------------------------------------------------------------------- /inference_demo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import time 4 | import argparse 5 | import logging 6 | import torch 7 | import numpy as np 8 | from PIL import Image 9 | import matplotlib.pyplot as plt 10 | import torch.nn.functional as F 11 | import torchvision.transforms as transforms 12 | from datetime import datetime, timedelta 13 | 14 | from utils import laplacian_blending, make_image 15 | from modules.encoder128 import Backbone128 16 | from modules.iib import IIB 17 | from modules.aii_generator import AII512 18 | from modules.decoder512 import UnetDecoder512 19 | from preprocess.mtcnn import MTCNN 20 | 21 | mtcnn = MTCNN() 22 | TRANSFORMS = transforms.Compose([ 23 | transforms.Resize((512, 512), interpolation=2), 24 | transforms.ToTensor(), 25 | transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) 26 | ]) 27 | 28 | 29 | def to_np(t: torch.Tensor): 30 | t = t.detach() 31 | if t.is_cuda: 32 | t = t.cpu() 33 | return t.numpy() 34 | 35 | 36 | def inference(src_img_path, tar_dir, save_dir): 37 | """ 38 | :param src_img_path: path to a source image 39 | :param tar_dir: path to the dir of target images 40 | :return: no return 41 | """ 42 | os.makedirs(save_dir, exist_ok=True) 43 | test_date = str(datetime.strptime(time.strftime( 44 | "%a, %d %b %Y %H:%M:%S", time.localtime()), "%a, %d %b %Y %H:%M:%S") + timedelta(hours=12)).split(' ')[ 45 | 0] 46 | save_dir = os.path.join(save_dir, test_date) 47 | os.makedirs(save_dir, exist_ok=True) 48 | 49 | logger = logging.getLogger('inference') 50 | logger.setLevel(logging.DEBUG) 51 | logger.propagate = True 52 | train_handler = logging.FileHandler(filename=os.path.join(save_dir, f'similarity_{test_date}.log')) 53 | train_formatter = logging.Formatter('%(message)s') 54 | train_handler.setFormatter(train_formatter) 55 | logger.addHandler(train_handler) 56 | 57 | if tar_dir.endswith('.png') or tar_dir.endswith('.jpg'): 58 | tar_list = [tar_dir, ] 59 | else: 60 | tmp_list = [f for f in os.listdir(tar_dir) if f.endswith('jpg') or f.endswith('png')] 61 | tar_list = sorted(tmp_list) 62 | M = len(tar_list) 63 | 64 | """ load pre-calculated mean and std: """ 65 | param_dict = [] 66 | for i in range(N + 1): 67 | state = torch.load(f'./modules/weights128/readout_layer{i}.pth', map_location=device) 68 | n_samples = state['n_samples'].float() 69 | std = torch.sqrt(state['s'] / (n_samples - 1)).to(device) 70 | neuron_nonzero = state['neuron_nonzero'].float() 71 | active_neurons = (neuron_nonzero / n_samples) > 0.01 72 | param_dict.append([state['m'].to(device), std, active_neurons]) 73 | 74 | """ inference: """ 75 | Xs = cv2.imread(src_img_path) 76 | Xs = Image.fromarray(Xs) 77 | face_s = mtcnn.align_multi(Xs, min_face_size=64., thresholds=[0.6, 0.7, 0.8], factor=0.707, crop_size=(512, 512)) 78 | if face_s is not None: 79 | Xs = face_s[0] 80 | else: 81 | print('s') 82 | Xs = None 83 | Xs = TRANSFORMS(Xs).unsqueeze(0) 84 | Xs = Xs.to(device) 85 | 86 | for idx in range(M): 87 | tar_img_path = os.path.join(tar_dir, tar_list[idx]) 88 | prefix = tar_list[idx].split('.')[0] 89 | suffix = tar_img_path.split('.')[-1] 90 | save_path = os.path.join(save_dir, prefix, '_gen.', suffix) 91 | if os.path.exists(save_path): 92 | continue 93 | 94 | with torch.no_grad(): 95 | '''(1) load Xt: ''' 96 | print(tar_img_path, end=', ') 97 | xt = cv2.imread(tar_img_path) 98 | print(xt.shape) 99 | 100 | Xt = Image.fromarray(xt) 101 | out = mtcnn.align_multi(Xt, min_face_size=64., thresholds=[0.6, 0.7, 0.7], 102 | crop_size=(512, 512), reverse=True) 103 | if out is not None: 104 | faces, tfm_invs, boxes = out 105 | if faces is not None: 106 | ss = 0 107 | fi = 0 108 | for j in range(len(boxes)): 109 | box = boxes[j] 110 | w = box[2] - box[0] + 1.0 111 | h = box[3] - box[1] + 1.0 112 | s = w * h 113 | if s > ss: 114 | ss = s 115 | fi = j 116 | Xt = faces[fi] 117 | tfm_inv = tfm_invs[fi] 118 | else: 119 | try: 120 | mini = 20. 121 | th1, th2, th3 = 0.6, 0.6, 0.6 122 | while out is None: 123 | out = mtcnn.align_multi(Xt, min_face_size=mini, thresholds=[th1, th2, th3], 124 | crop_size=(512, 512), reverse=True) 125 | if out is not None: 126 | faces, tfm_invs, boxes = out 127 | ss = 0 128 | fi = 0 129 | for j in range(len(boxes)): 130 | box = boxes[j] 131 | w = box[2] - box[0] + 1.0 132 | h = box[3] - box[1] + 1.0 133 | s = w * h 134 | if s > ss: 135 | ss = s 136 | fi = j 137 | Xt = faces[fi] 138 | tfm_inv = tfm_invs[fi] 139 | else: 140 | th1 *= 0.8 141 | th2 *= 0.8 142 | th2 *= 0.8 143 | mini *= 0.8 144 | except Exception as e: 145 | print(e) 146 | plt.imsave(save_path, cv2.cvtColor(xt.astype(np.uint8), cv2.COLOR_RGB2BGR)) 147 | plt.close() 148 | continue 149 | 150 | '''(2) generate Y: ''' 151 | B = 1 152 | Xt = TRANSFORMS(Xt).unsqueeze(0).to(device) 153 | X_id = encoder( 154 | F.interpolate(torch.cat((Xs, Xt), dim=0)[:, :, 37:475, 37:475], size=[128, 128], 155 | mode='bilinear', align_corners=True), 156 | cache_feats=True 157 | ) 158 | # 01 Get Inter-features After One Feed-Forward: 159 | # batch size is 2 * B, [:B] for Xs and [B:] for Xt 160 | min_std = torch.tensor(0.01, device=device) 161 | readout_feats = [(encoder.features[i] - param_dict[i][0]) / torch.max(param_dict[i][1], min_std) 162 | for i in range(N + 1)] 163 | 164 | # 02 information restriction: 165 | X_id_restrict = torch.zeros_like(X_id).to(device) # [2*B, 512] 166 | Xt_feats, X_lambda = [], [] 167 | Xt_lambda = [] 168 | Rs_params, Rt_params = [], [] 169 | for i in range(N): 170 | R = encoder.features[i] # [2*B, Cr, Hr, Wr] 171 | Z, lambda_, _ = getattr(iib, f'iba_{i}')( 172 | R, readout_feats, 173 | m_r=param_dict[i][0], std_r=param_dict[i][1], 174 | active_neurons=param_dict[i][2], 175 | ) 176 | X_id_restrict += encoder.restrict_forward(Z, i) 177 | 178 | Rs, Rt = R[:B], R[B:] 179 | lambda_s, lambda_t = lambda_[:B], lambda_[B:] 180 | 181 | m_s = torch.mean(Rs, dim=0) # [C, H, W] 182 | std_s = torch.mean(Rs, dim=0) 183 | Rs_params.append([m_s, std_s]) 184 | 185 | eps_s = torch.randn(size=Rt.shape).to(Rt.device) * std_s + m_s 186 | feat_t = Rt * (1. - lambda_t) + lambda_t * eps_s 187 | 188 | Xt_feats.append(feat_t) # only related with lambda 189 | Xt_lambda.append(lambda_t) 190 | 191 | X_id_restrict /= float(N) 192 | Xs_id = X_id_restrict[:B] 193 | Xt_feats[0] = Xt 194 | Xt_attr, Xt_attr_lamb = decoder(Xt_feats, lambs=Xt_lambda, use_lambda=True) 195 | 196 | Y = G(Xs_id, Xt_attr, Xt_attr_lamb) 197 | encoder.features = [] 198 | 199 | # log identity similarities: 200 | Y_id_gt = encoder( 201 | F.interpolate(Y[:, :, 37:475, 37:475], size=[128, 128], mode='bilinear', align_corners=True), 202 | cache_feats=False 203 | ) 204 | Xs_id_gt, Xt_id_gt = X_id[:B], X_id[B:] 205 | msg = '' 206 | msg += "cos=%.3f | " % torch.cosine_similarity(Xs_id_gt, Xt_id_gt, 207 | dim=1).mean().detach().cpu().numpy() 208 | msg += "cos=%.3f | " % torch.cosine_similarity(Xt_id_gt, Y_id_gt, 209 | dim=1).mean().detach().cpu().numpy() 210 | msg += "cos=%.3f | " % torch.cosine_similarity(Xs_id_gt, Y_id_gt, 211 | dim=1).mean().detach().cpu().numpy() 212 | logger.info(msg) 213 | 214 | '''(3) save Y: ''' 215 | I = [Xs, Xt, Y] 216 | image = make_image(I, 1) 217 | save_path_Y = os.path.join(save_dir, prefix + '_gen_Y.' + suffix) 218 | # print("save path Y: ", save_path_Y) 219 | cv2.imwrite(save_path_Y, image.transpose([1, 2, 0]), 220 | [int(cv2.IMWRITE_PNG_COMPRESSION), 0]) 221 | 222 | img_Y = (Y[0].cpu().numpy().transpose([1, 2, 0]) * 0.5 + 0.5) * 255 223 | img_Y = img_Y.astype(np.uint8) 224 | H, W, _ = xt.shape 225 | frame = cv2.warpAffine(img_Y.astype(np.float32), tfm_inv.astype(np.float32), 226 | dsize=(int(W), int(H)), borderValue=0) 227 | 228 | mask = np.zeros(img_Y.shape, img_Y.dtype) 229 | mask[37:475, 90:422, :] = 1 # 90:422 230 | mask = cv2.warpAffine(mask, 231 | tfm_inv.astype(np.float32), dsize=(int(W), int(H)), 232 | borderValue=0) # can not set cv2.BORDER_TRANSPARENT ! 233 | try: 234 | src = np.array([255., 255., 1.]).reshape(3, 1) 235 | x, y = np.matmul(tfm_inv, src) 236 | print(x, y) 237 | 238 | m = np.zeros(img_Y.shape, img_Y.dtype) 239 | m[40:472, 80:432, :] = 1 # 90:432 240 | m = cv2.warpAffine( 241 | m, tfm_inv.astype(np.float32), 242 | dsize=(int(W), int(H)), borderValue=0) 243 | print(m.shape) 244 | res_possion = cv2.seamlessClone(frame.astype(np.uint8), xt.astype(np.uint8), m.astype(np.uint8)*255, 245 | p=(x, y), flags=cv2.NORMAL_CLONE) 246 | # plt.imshow(cv2.cvtColor(res_possion.astype(np.uint8), cv2.COLOR_RGB2BGR)) 247 | plt.imsave(save_path, cv2.cvtColor(res_possion.astype(np.uint8), cv2.COLOR_RGB2BGR)) 248 | # plt.show() 249 | # plt.close() 250 | except Exception as e: 251 | print(e) 252 | res = laplacian_blending(A=frame, B=xt, m=mask) 253 | # plt.imshow(cv2.cvtColor(res.astype(np.uint8), cv2.COLOR_RGB2BGR)) 254 | plt.imsave(save_path, cv2.cvtColor(res.astype(np.uint8), cv2.COLOR_RGB2BGR)) 255 | # plt.show() 256 | # plt.close() 257 | 258 | if __name__ == '__main__': 259 | torch.backends.cudnn.benchmark = True 260 | ROOT = { 261 | 'smooth': {'root': './checkpoints_512/w_kernel_smooth', 'path': 'ckpt_ks_*.pth'}, 262 | 'no_smooth': {'root': './checkpoints_512/wo_kernel_smooth', 'path': 'ckpt_*.pth'} 263 | } 264 | 265 | p = argparse.ArgumentParser( 266 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 267 | ) 268 | p.add_argument('-ib', '--ib_mode', type=str, choices=list(ROOT.keys())) 269 | p.add_argument('-src', '--src_path', type=str, default='data/src/Anna-Popplewell.png') 270 | p.add_argument('-tar', '--tar_dir', type=str, default='data/tar') 271 | p.add_argument('-save', '--save_dir', type=str, default='./results') 272 | args = p.parse_args() 273 | 274 | """ Prepare Models: """ 275 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 276 | root = ROOT[args.ib_mode]['root'] 277 | path = ROOT[args.ib_mode]['path'] 278 | 279 | pathG = path.replace('*', 'G') 280 | pathE = path.replace('*', 'E') 281 | pathI = path.replace('*', 'I') 282 | 283 | encoder = Backbone128(50, 0.6, 'ir_se').eval().to(device) 284 | state_dict = torch.load('modules/model_128_ir_se50.pth', map_location=device) 285 | encoder.load_state_dict(state_dict, strict=True) 286 | 287 | G = AII512().eval().to(device) 288 | decoder = UnetDecoder512().eval().to(device) 289 | 290 | # Define Information Bottlenecks: 291 | N = 10 292 | _ = encoder(torch.rand(1, 3, 128, 128).to(device), cache_feats=True) 293 | _readout_feats = encoder.features[:(N + 1)] # one layer deeper than the z_attrs needed 294 | in_c = sum(map(lambda f: f.shape[-3], _readout_feats)) 295 | out_c_list = [_readout_feats[i].shape[-3] for i in range(N)] 296 | 297 | iib = IIB(in_c, out_c_list, device, smooth=args.ib_mode=='smooth', kernel_size=1) 298 | iib = iib.eval() 299 | 300 | G.load_state_dict(torch.load(os.path.join(root, pathG), map_location=device), strict=True) 301 | print("Successfully load G!") 302 | decoder.load_state_dict(torch.load(os.path.join(root, pathE), map_location=device), strict=True) 303 | print("Successfully load Decoder!") 304 | # 3) load IIB: 305 | iib.load_state_dict(torch.load(os.path.join(root, pathI), map_location=device), 306 | strict=args.ib_mode=='smooth') 307 | print("Successfully load IIB!") 308 | 309 | with torch.no_grad(): 310 | inference(args.src_path, args.tar_dir, args.save_dir) 311 | -------------------------------------------------------------------------------- /modules/aii_generator.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from utils import ConvexUpsample 5 | from torch.nn import Sequential 6 | 7 | 8 | def weight_init(m): 9 | if isinstance(m, nn.Linear): 10 | m.weight.data.normal_(0, 0.001) 11 | m.bias.data.zero_() 12 | if isinstance(m, nn.Conv2d): 13 | # nn.init.xavier_normal_(m.weight.data) 14 | nn.init.kaiming_normal_(m.weight.data) 15 | 16 | if isinstance(m, nn.ConvTranspose2d): 17 | # nn.init.xavier_normal_(m.weight.data) 18 | nn.init.kaiming_normal_(m.weight.data) 19 | 20 | 21 | class AIILayerLambda(nn.Module): 22 | def __init__(self, c_h, c_attr, c_id, c_lamb): 23 | super(AIILayerLambda, self).__init__() 24 | self.attr_c = c_attr 25 | self.c_id = c_id 26 | self.c_h = c_h 27 | 28 | self.conv1 = nn.Conv2d(c_attr, c_h, kernel_size=1, stride=1, padding=0, bias=True) 29 | self.conv2 = nn.Conv2d(c_attr, c_h, kernel_size=1, stride=1, padding=0, bias=True) 30 | self.fc1 = nn.Linear(c_id, c_h) 31 | self.fc2 = nn.Linear(c_id, c_h) 32 | self.norm = nn.InstanceNorm2d(c_h, affine=False) 33 | 34 | self.conv_2h = nn.Conv2d(c_h + c_lamb, 1, kernel_size=1, stride=1, padding=0, bias=True) 35 | 36 | def forward(self, h_in, z_attr, z_id, lamb): 37 | h = self.norm(h_in) 38 | 39 | # together calculate: 40 | if lamb is None: 41 | M = torch.sigmoid(self.conv_2h(h)) 42 | else: 43 | M = torch.sigmoid(self.conv_2h(torch.cat((h, lamb), dim=1))) 44 | 45 | gamma_attr = self.conv1(z_attr) 46 | beta_attr = self.conv2(z_attr) 47 | A = gamma_attr * h + beta_attr 48 | 49 | gamma_id = self.fc1(z_id) 50 | beta_id = self.fc2(z_id) 51 | gamma_id = gamma_id.reshape(h.shape[0], self.c_h, 1, 1).expand_as(h) # broadcast 52 | beta_id = beta_id.reshape(h.shape[0], self.c_h, 1, 1).expand_as(h) 53 | I = gamma_id * h + beta_id 54 | 55 | out = (torch.ones_like(M).to(M.device) - M) * A + M * I 56 | return out 57 | 58 | 59 | class AIIResBlkLambda(nn.Module): 60 | def __init__(self, cin, cout, c_attr, c_id, c_lamb): 61 | super(AIIResBlkLambda, self).__init__() 62 | self.cin = cin 63 | self.cout = cout 64 | 65 | self.AAD1 = AIILayerLambda(cin, c_attr, c_id, c_lamb) # out channel == cin 66 | self.conv1 = nn.Conv2d(cin, cin, kernel_size=3, stride=1, padding=1, bias=False) 67 | self.relu1 = nn.LeakyReLU(negative_slope=1e-2, inplace=True) 68 | 69 | self.AAD2 = AIILayerLambda(cin, c_attr, c_id, c_lamb) 70 | self.conv2 = nn.Conv2d(cin, cout, kernel_size=3, stride=1, padding=1, bias=False) 71 | self.relu2 = nn.LeakyReLU(negative_slope=1e-2, inplace=True) 72 | 73 | if cin != cout: 74 | self.AAD3 = AIILayerLambda(cin, c_attr, c_id, c_lamb) 75 | self.conv3 = nn.Conv2d(cin, cout, kernel_size=3, stride=1, padding=1, bias=False) 76 | self.relu3 = nn.LeakyReLU(negative_slope=1e-2, inplace=True) 77 | 78 | def forward(self, h, z_attr, z_id, lamb): 79 | x = self.AAD1(h, z_attr, z_id, lamb) 80 | x = self.relu1(x) 81 | x = self.conv1(x) 82 | 83 | x = self.AAD2(x, z_attr, z_id, lamb) 84 | x = self.relu2(x) 85 | x = self.conv2(x) 86 | 87 | if self.cin != self.cout: 88 | h = self.AAD3(h, z_attr, z_id, lamb) 89 | h = self.relu3(h) 90 | h = self.conv3(h) 91 | 92 | x = x + h 93 | 94 | return x 95 | 96 | 97 | class AII512(nn.Module): 98 | def __init__(self, c_id=512): 99 | super(AII512, self).__init__() 100 | self.up1 = nn.ConvTranspose2d(c_id, 1024, kernel_size=2, stride=1, padding=0) 101 | self.up2 = nn.ConvTranspose2d(1024, 1024, kernel_size=2, stride=2, padding=0) 102 | 103 | self.AADBlk1 = AIIResBlkLambda(1024, 1024, c_attr=1024, c_id=c_id, c_lamb=1024) 104 | self.AADBlk2 = AIIResBlkLambda(1024, 1024, 2048, c_id, c_lamb=1024) 105 | self.AADBlk3 = AIIResBlkLambda(1024, 1024, 1024, c_id, c_lamb=512) 106 | self.AADBlk4 = AIIResBlkLambda(1024, 512, 512, c_id, c_lamb=256) 107 | self.AADBlk5 = AIIResBlkLambda(512, 256, 256, c_id, c_lamb=128) 108 | self.AADBlk6 = AIIResBlkLambda(256, 128, 128, c_id, c_lamb=64) 109 | self.AADBlk7 = AIIResBlkLambda(128, 64, 64, c_id, c_lamb=32) 110 | # self.last_no_lamb = last_no_lamb 111 | self.AADBlk8 = AIIResBlkLambda(64, 3, 64, c_id, c_lamb=32) 112 | 113 | self.deconv = nn.ConvTranspose2d(in_channels=32, out_channels=3, kernel_size=4, stride=2, padding=1, bias=False) 114 | 115 | self.apply(weight_init) 116 | 117 | def forward(self, z_id, z_attr, lamb): 118 | m = self.up1(z_id.reshape(z_id.shape[0], -1, 1, 1)) # [B, 1024, 2, 2] for 256 generation 119 | m = self.up2(m) # [B, 1024, 4, 4] for 512 generation 120 | # print(m.shape) 121 | # if m.shape[-1] != z_attr[0].shape[-1]: 122 | # m = self.up3(m) # [B, 1024, 8, 8] for 1024 generation 123 | 124 | m = self.AADBlk1(m, z_attr[0], z_id, lamb[0]) 125 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 126 | # print(m.shape) 127 | 128 | m = self.AADBlk2(m, z_attr[1], z_id, lamb[1]) 129 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 130 | # print(m.shape) 131 | 132 | m = self.AADBlk3(m, z_attr[2], z_id, lamb[2]) 133 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 134 | # print(m.shape) 135 | 136 | m = self.AADBlk4(m, z_attr[3], z_id, lamb[3]) 137 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 138 | # print(m.shape) 139 | 140 | m = self.AADBlk5(m, z_attr[4], z_id, lamb[4]) 141 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 142 | # print(m.shape) 143 | 144 | m = self.AADBlk6(m, z_attr[5], z_id, lamb[5]) 145 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 146 | # print(m.shape) 147 | 148 | m = self.AADBlk7(m, z_attr[6], z_id, lamb[6]) 149 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 150 | # print(m.shape) 151 | 152 | y = self.AADBlk8(m, z_attr[7], z_id, lamb[7]) 153 | # print(y.shape) 154 | 155 | return torch.tanh(y) 156 | 157 | 158 | class AII256(nn.Module): 159 | def __init__(self, c_id=512, last_no_lamb=False, use_lamb=False): 160 | super(AII256, self).__init__() 161 | self.up1 = nn.ConvTranspose2d(c_id, 1024, kernel_size=2, stride=1, padding=0) 162 | # self.up2 = nn.ConvTranspose2d(1024, 1024, kernel_size=2, stride=2, padding=0) 163 | 164 | self.use_lamb = use_lamb 165 | if use_lamb: 166 | self.AADBlk1 = AIIResBlkLambda(1024, 1024, c_attr=1024, c_id=c_id, c_lamb=1024) 167 | self.AADBlk2 = AIIResBlkLambda(1024, 1024, 2048, c_id, c_lamb=1024) 168 | self.AADBlk3 = AIIResBlkLambda(1024, 1024, 1024, c_id, c_lamb=512) 169 | self.AADBlk4 = AIIResBlkLambda(1024, 512, 512, c_id, c_lamb=256) 170 | self.AADBlk5 = AIIResBlkLambda(512, 256, 256, c_id, c_lamb=128) 171 | self.AADBlk6 = AIIResBlkLambda(256, 128, 128, c_id, c_lamb=64) 172 | self.AADBlk7 = AIIResBlkLambda(128, 64, 64, c_id, c_lamb=64) 173 | self.AADBlk8 = AIIResBlkLambda(64, 3, 64, c_id, c_lamb=64) 174 | else: 175 | self.AADBlk1 = AIIResBlkLambda(1024, 1024, c_attr=1024, c_id=c_id, c_lamb=0) 176 | self.AADBlk2 = AIIResBlkLambda(1024, 1024, 2048, c_id, 0) 177 | self.AADBlk3 = AIIResBlkLambda(1024, 1024, 1024, c_id, 0) 178 | self.AADBlk4 = AIIResBlkLambda(1024, 512, 512, c_id, 0) 179 | self.AADBlk5 = AIIResBlkLambda(512, 256, 256, c_id, 0) 180 | self.AADBlk6 = AIIResBlkLambda(256, 128, 128, c_id, 0) 181 | self.AADBlk7 = AIIResBlkLambda(128, 64, 64, c_id, 0) 182 | self.AADBlk8 = AIIResBlkLambda(64, 3, 64, c_id, 0) 183 | 184 | self.deconv = nn.ConvTranspose2d(in_channels=32, out_channels=3, kernel_size=4, stride=2, padding=1, bias=False) 185 | 186 | self.apply(weight_init) 187 | 188 | def forward(self, z_id, z_attr, lamb): 189 | m = self.up1(z_id.reshape(z_id.shape[0], -1, 1, 1)) # [B, 1024, 2, 2] for 256 generation 190 | if not self.use_lamb: 191 | lamb = [None for _ in range(8)] 192 | 193 | m = self.AADBlk1(m, z_attr[0], z_id, lamb[0]) 194 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 195 | # print(m.shape) 196 | 197 | m = self.AADBlk2(m, z_attr[1], z_id, lamb[1]) 198 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 199 | # print(m.shape) 200 | 201 | m = self.AADBlk3(m, z_attr[2], z_id, lamb[2]) 202 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 203 | # print(m.shape) 204 | 205 | m = self.AADBlk4(m, z_attr[3], z_id, lamb[3]) 206 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 207 | # print(m.shape) 208 | 209 | m = self.AADBlk5(m, z_attr[4], z_id, lamb[4]) 210 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 211 | # print(m.shape) 212 | 213 | m = self.AADBlk6(m, z_attr[5], z_id, lamb[5]) 214 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 215 | # print(m.shape) 216 | 217 | m = self.AADBlk7(m, z_attr[6], z_id, lamb[6]) 218 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 219 | # print(m.shape) 220 | 221 | y = self.AADBlk8(m, z_attr[7], z_id, lamb[7]) 222 | 223 | return torch.tanh(y) 224 | 225 | 226 | class AII1024(nn.Module): 227 | def __init__(self, c_id=512, last_no_lamb=False): 228 | super(AII1024, self).__init__() 229 | self.up1 = nn.ConvTranspose2d(c_id, 1024, kernel_size=2, stride=1, padding=0) 230 | self.up2 = nn.ConvTranspose2d(1024, 1024, kernel_size=2, stride=2, padding=0) 231 | self.up3 = nn.ConvTranspose2d(1024, 1024, kernel_size=2, stride=2, padding=0) 232 | 233 | self.AADBlk1 = AIIResBlkLambda(1024, 1024, c_attr=1024, c_id=c_id, c_lamb=1024) 234 | self.AADBlk2 = AIIResBlkLambda(1024, 1024, 2048, c_id, c_lamb=1024) 235 | self.AADBlk3 = AIIResBlkLambda(1024, 1024, 1024, c_id, c_lamb=512) 236 | self.AADBlk4 = AIIResBlkLambda(1024, 512, 512, c_id, c_lamb=256) 237 | self.AADBlk5 = AIIResBlkLambda(512, 256, 256, c_id, c_lamb=128) 238 | self.AADBlk6 = AIIResBlkLambda(256, 128, 128, c_id, c_lamb=64) 239 | self.AADBlk7 = AIIResBlkLambda(128, 64, 64, c_id, c_lamb=32) 240 | self.AADBlk_8 = AIIResBlkLambda(64, 64, 64, c_id, c_lamb=32) 241 | self.AADBlk9 = AIIResBlkLambda(64, 3, 64, c_id, c_lamb=32) 242 | 243 | self.apply(weight_init) 244 | 245 | def forward(self, z_id, z_attr, lamb): 246 | m = self.up1(z_id.reshape(z_id.shape[0], -1, 1, 1)) # B, 1024, 2, 2 247 | # print(m.shape) 248 | m = self.up2(m) # [B, 1024, 4, 4] 249 | # print(m.shape) 250 | if m.shape[-1] != z_attr[0].shape[-1]: 251 | m = self.up3(m) # [B, 1024, 8, 8] for 1024 generation 252 | 253 | m = self.AADBlk1(m, z_attr[0], z_id, lamb[0]) 254 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 255 | # print(m.shape) 256 | 257 | m = self.AADBlk2(m, z_attr[1], z_id, lamb[1]) 258 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 259 | # print(m.shape) 260 | 261 | m = self.AADBlk3(m, z_attr[2], z_id, lamb[2]) 262 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 263 | # print(m.shape) 264 | 265 | m = self.AADBlk4(m, z_attr[3], z_id, lamb[3]) 266 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 267 | # print(m.shape) 268 | 269 | m = self.AADBlk5(m, z_attr[4], z_id, lamb[4]) 270 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 271 | # print(m.shape) 272 | 273 | m = self.AADBlk6(m, z_attr[5], z_id, lamb[5]) 274 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 275 | # print(m.shape) 276 | 277 | m = self.AADBlk7(m, z_attr[6], z_id, lamb[6]) 278 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) 279 | # print(m.shape) 280 | 281 | m = self.AADBlk_8(m, z_attr[7], z_id, lamb[7]) # 64x512x512 282 | m = F.interpolate(m, scale_factor=2, mode='bilinear', align_corners=True) # 283 | 284 | y = self.AADBlk9(m, z_attr[8], z_id, lamb[8]) 285 | 286 | return torch.tanh(y) 287 | -------------------------------------------------------------------------------- /modules/decoder1024.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import Sequential 5 | 6 | 7 | def weight_init(m): 8 | if isinstance(m, nn.Linear): 9 | m.weight.data.normal_(0, 0.001) 10 | m.bias.data.zero_() 11 | if isinstance(m, nn.Conv2d): 12 | # nn.init.xavier_normal_(m.weight.data) 13 | nn.init.kaiming_normal_(m.weight.data) 14 | 15 | if isinstance(m, nn.ConvTranspose2d): 16 | # nn.init.xavier_normal_(m.weight.data) 17 | nn.init.kaiming_normal_(m.weight.data) 18 | 19 | 20 | def conv4x4(in_c, out_c, norm=nn.BatchNorm2d): 21 | return Sequential( 22 | nn.Conv2d(in_channels=in_c, out_channels=out_c, kernel_size=4, stride=2, padding=1, bias=False), 23 | norm(out_c), 24 | nn.LeakyReLU(0.1, inplace=True) 25 | ) 26 | 27 | 28 | class deconv4x4(nn.Module): 29 | def __init__(self, in_c, out_c, norm=nn.BatchNorm2d): 30 | super(deconv4x4, self).__init__() 31 | self.deconv = nn.ConvTranspose2d(in_channels=in_c, out_channels=out_c, kernel_size=4, stride=2, padding=1, bias=False) 32 | self.bn = norm(out_c) 33 | self.lrelu = nn.LeakyReLU(0.1, inplace=True) 34 | 35 | def forward(self, input, skip): 36 | x = self.deconv(input) 37 | x = self.bn(x) 38 | x = self.lrelu(x) 39 | return torch.cat((x, skip), dim=1) 40 | 41 | 42 | class UnetDecoder1024(nn.Module): 43 | def __init__(self): 44 | super(UnetDecoder1024, self).__init__() 45 | # if img_1024: 46 | # self.conv0 = nn.Conv2d(3, 32, 3, 1, 1, bias=False) 47 | # else: 48 | # self.conv_0 = nn.Conv2d(3, 32, 4, 2, 1, bias=False) 49 | # self.conv0 = self.conv_0 50 | self.conv_0 = nn.Conv2d(3, 32, 4, 2, 1, bias=False) 51 | 52 | self.conv_1 = nn.Sequential(nn.PixelShuffle(upscale_factor=2), 53 | nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1, bias=False)) 54 | # self.conv1 = nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1, bias=False) 55 | 56 | self.conv_2 = nn.Conv2d(64, 64, 3, 1, 1, bias=False) 57 | self.conv_3 = nn.Conv2d(128, 128, 3, 1, 1, bias=False) 58 | self.conv_4 = nn.Conv2d(256, 256, 3, 1, 1, bias=False) 59 | self.conv_5 = nn.Conv2d(256, 512, 4, 2, 1, bias=False) # 512 60 | self.conv_6 = nn.Conv2d(512, 1024, 4, 2, 1, bias=False) # 512 61 | 62 | self.conv7 = conv4x4(1024, 1024) 63 | self.deconv1 = deconv4x4(1024, 1024) 64 | self.deconv2 = deconv4x4(2048, 512) 65 | self.deconv3 = deconv4x4(1024, 256) 66 | self.deconv4 = deconv4x4(512, 128) 67 | self.deconv5 = deconv4x4(256, 64) 68 | self.deconv6 = deconv4x4(128, 32) 69 | self.deconv7 = deconv4x4(64, 32) 70 | 71 | self.apply(weight_init) 72 | 73 | def forward(self, img, feats, lambs, use_lambda=True): 74 | """ 75 | :param img: 3x1024x1024 76 | :param feats: Arcface intermediate features 77 | :param lambs: 78 | :param use_lambda: 79 | :return: decoded features 80 | """ 81 | # if self.first_use_img: 82 | # feat1 = self.conv_0(feats[0]) 83 | # # 3x512x512 -> 32x256x256 84 | # else: 85 | # feat1 = self.conv_1(feats[0]) 86 | # # 64x128x128 -> 32x256x256 87 | feat0 = self.conv_0(img) 88 | # 3x1024x1024 -> 32x512x512 89 | 90 | feat1 = self.conv_1(feats[0]) 91 | # 64x128x128 -> 32x256x256 92 | 93 | feat2 = self.conv_2(F.interpolate(feats[1], scale_factor=2, mode='bilinear', align_corners=True)) 94 | # 64x64x64 --up+conv--> 64x128x128 95 | 96 | feat3 = torch.cat((feats[2], feats[3]), dim=1) 97 | feat3 = self.conv_3(feat3) 98 | # 64x64x64|cat|64x64x64 -> 128x64x64 99 | 100 | feat4 = torch.cat((feats[4], feats[5]), dim=1) 101 | feat4 = self.conv_4(feat4) 102 | # 128x32x32|cat|128x32x32 -> 256x32x32 103 | 104 | feat5 = self.conv_5(torch.cat((feats[6], feats[7]), dim=1)) 105 | # print(feat5.shape) 106 | # 128x32x32|cat|128x32x32 -> 512x16x16 107 | 108 | feat6 = self.conv_6(torch.cat((feats[8], feats[9]), dim=1)) 109 | # 256x16x16|cat|256x16x16 -> 1024x8x8 110 | 111 | z_attr1 = self.conv7(feat6) 112 | # print(z_attr1.shape) 113 | # 1024x4x4 114 | 115 | z_attr2 = self.deconv1(z_attr1, feat6) 116 | # print(z_attr2.shape) 117 | # 2048x8x8 118 | z_attr3 = self.deconv2(z_attr2, feat5) 119 | # 1024x16x16 120 | z_attr4 = self.deconv3(z_attr3, feat4) 121 | # 512x32x32 122 | z_attr5 = self.deconv4(z_attr4, feat3) 123 | # 256x64x64 124 | z_attr6 = self.deconv5(z_attr5, feat2) 125 | # print(z_attr6.shape) 126 | # 128x128x128 127 | z_attr7 = self.deconv6(z_attr6, feat1) # z_attr6 --> 32x256x256, then || feat1 128 | # 64x256x256 129 | 130 | # z_attr8 = F.interpolate(z_attr7, scale_factor=2, mode='bilinear', align_corners=True) 131 | z_attr8 = self.deconv7(z_attr7, feat0) # z_attr7 --> 32x512x512 || 32x512x512 132 | # 64x512x512 133 | 134 | z_attr9 = F.interpolate(z_attr8, scale_factor=2, mode='bilinear', align_corners=True) 135 | # 64x1024x1024 136 | 137 | if not use_lambda: 138 | return z_attr1, z_attr2, z_attr3, z_attr4, z_attr5, z_attr6, z_attr7, z_attr8, z_attr9 139 | else: 140 | lamb1 = self.conv_1(lambs[0]) # 32x256x256 141 | 142 | lamb0 = F.interpolate(lamb1, scale_factor=2, mode='bilinear', align_corners=True) # 32x512x512 143 | lamb_img = F.interpolate(lamb0, scale_factor=2, mode='bilinear', align_corners=True) # 32x1024x1024 144 | 145 | lamb2 = self.conv_2(F.interpolate(lambs[1], scale_factor=2, mode='bilinear', align_corners=True)) # 64x128x128 146 | 147 | lamb3 = torch.cat((lambs[2], lambs[3]), dim=1) 148 | lamb3 = self.conv_3(lamb3) # 128x64x64 149 | 150 | lamb4 = torch.cat((lambs[4], lambs[5]), dim=1) 151 | lamb4 = self.conv_4(lamb4) # 256x32x32 152 | 153 | lamb5 = self.conv_5(torch.cat((lambs[6], lambs[7]), dim=1)) # 512x16x16 154 | lamb6 = self.conv_6(torch.cat((lambs[8], lambs[9]), dim=1)) # 1024x8x8 155 | lamb7 = self.conv7(lamb6) # 1024x4x4 156 | return [[z_attr1, z_attr2, z_attr3, z_attr4, z_attr5, z_attr6, z_attr7, z_attr8, z_attr9], 157 | [lamb7, lamb6, lamb5, lamb4, lamb3, lamb2, lamb1, lamb0, lamb_img]] 158 | 159 | -------------------------------------------------------------------------------- /modules/decoder512.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch.nn import Sequential 5 | 6 | 7 | def weight_init(m): 8 | if isinstance(m, nn.Linear): 9 | m.weight.data.normal_(0, 0.001) 10 | m.bias.data.zero_() 11 | if isinstance(m, nn.Conv2d): 12 | # nn.init.xavier_normal_(m.weight.data) 13 | nn.init.kaiming_normal_(m.weight.data) 14 | 15 | if isinstance(m, nn.ConvTranspose2d): 16 | # nn.init.xavier_normal_(m.weight.data) 17 | nn.init.kaiming_normal_(m.weight.data) 18 | 19 | 20 | def conv4x4(in_c, out_c, norm=nn.BatchNorm2d): 21 | return Sequential( 22 | nn.Conv2d(in_channels=in_c, out_channels=out_c, 23 | kernel_size=4, stride=2, padding=1, bias=False), 24 | norm(out_c), 25 | nn.LeakyReLU(0.1, inplace=True) 26 | ) 27 | 28 | 29 | class deconv4x4(nn.Module): 30 | def __init__(self, in_c, out_c, norm=nn.BatchNorm2d): 31 | super(deconv4x4, self).__init__() 32 | self.deconv = nn.ConvTranspose2d(in_channels=in_c, out_channels=out_c, 33 | kernel_size=4, stride=2, padding=1, bias=False) 34 | self.bn = norm(out_c) 35 | self.lrelu = nn.LeakyReLU(0.1, inplace=True) 36 | 37 | def forward(self, input, skip): 38 | x = self.deconv(input) 39 | x = self.bn(x) 40 | x = self.lrelu(x) 41 | return torch.cat((x, skip), dim=1) 42 | 43 | 44 | class UnetDecoder512(nn.Module): 45 | def __init__(self, first_use_img=True): 46 | super(UnetDecoder512, self).__init__() 47 | self.conv_0 = nn.Conv2d(3, 32, 4, 2, 1, bias=False) 48 | self.first_use_img = first_use_img 49 | 50 | self.conv_1 = nn.Sequential(nn.PixelShuffle(upscale_factor=2), 51 | nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1, bias=False)) 52 | self.conv_2 = nn.Conv2d(64, 64, 3, 1, 1, bias=False) 53 | self.conv_3 = nn.Conv2d(128, 128, 3, 1, 1, bias=False) 54 | self.conv_4 = nn.Conv2d(256, 256, 3, 1, 1, bias=False) 55 | self.conv_5 = nn.Conv2d(256, 512, 4, 2, 1, bias=False) # 512 56 | self.conv_6 = nn.Conv2d(512, 1024, 4, 2, 1, bias=False) # 512 57 | 58 | self.conv7 = conv4x4(1024, 1024) 59 | self.deconv1 = deconv4x4(1024, 1024) 60 | self.deconv2 = deconv4x4(2048, 512) 61 | self.deconv3 = deconv4x4(1024, 256) 62 | self.deconv4 = deconv4x4(512, 128) 63 | self.deconv5 = deconv4x4(256, 64) 64 | self.deconv6 = deconv4x4(128, 32) 65 | 66 | self.apply(weight_init) 67 | 68 | def forward(self, feats, lambs, use_lambda=True): 69 | """ 70 | :param feats: Arcface intermediate features 71 | :return: decoded features 72 | """ 73 | if self.first_use_img: 74 | feat1 = self.conv_0(feats[0]) 75 | # 3x512x512 -> 32x256x256 76 | else: 77 | feat1 = self.conv_1(feats[0]) 78 | # 64x128x128 -> 32x256x256 79 | 80 | feat2 = self.conv_2(F.interpolate(feats[1], scale_factor=2, mode='bilinear', align_corners=True)) 81 | # 64x128x128 82 | 83 | feat3 = torch.cat((feats[2], feats[3]), dim=1) 84 | feat3 = self.conv_3(feat3) 85 | # 128x64x64 86 | 87 | feat4 = torch.cat((feats[4], feats[5]), dim=1) 88 | feat4 = self.conv_4(feat4) 89 | # 256x32x32 90 | 91 | feat5 = self.conv_5(torch.cat((feats[6], feats[7]), dim=1)) 92 | # 512x16x16 93 | 94 | feat6 = self.conv_6(torch.cat((feats[8], feats[9]), dim=1)) 95 | # 1024x8x8 96 | 97 | z_attr1 = self.conv7(feat6) 98 | # 1024x4x4 99 | 100 | z_attr2 = self.deconv1(z_attr1, feat6) 101 | # 2048x8x8 102 | z_attr3 = self.deconv2(z_attr2, feat5) 103 | # 1024x16x16 104 | z_attr4 = self.deconv3(z_attr3, feat4) 105 | # 512x32x32 106 | z_attr5 = self.deconv4(z_attr4, feat3) 107 | # 256x64x64 108 | z_attr6 = self.deconv5(z_attr5, feat2) 109 | # print(z_attr6.shape) 110 | # 128x128x128 111 | z_attr7 = self.deconv6(z_attr6, feat1) # z_attr6 --> 32x256x256, then || feat1 112 | # 64x256x256 113 | z_attr8 = F.interpolate(z_attr7, scale_factor=2, mode='bilinear', align_corners=True) 114 | # 64x512x512 115 | 116 | if not use_lambda: 117 | return z_attr1, z_attr2, z_attr3, z_attr4, z_attr5, z_attr6, z_attr7, z_attr8 118 | else: 119 | lamb1 = lambs[0] 120 | lamb1 = self.conv_1(lamb1) # 32x256x256 121 | lamb0 = F.interpolate(lamb1, scale_factor=2, mode='bilinear', align_corners=True) # 32x512x512 122 | lamb2 = self.conv_2(F.interpolate(lambs[1], scale_factor=2, mode='bilinear', align_corners=True)) # 64x128x128 123 | 124 | lamb3 = torch.cat((lambs[2], lambs[3]), dim=1) 125 | lamb3 = self.conv_3(lamb3) # 128x64x64 126 | 127 | lamb4 = torch.cat((lambs[4], lambs[5]), dim=1) 128 | lamb4 = self.conv_4(lamb4) # 256x32x32 129 | 130 | lamb5 = self.conv_5(torch.cat((lambs[6], lambs[7]), dim=1)) # 512x16x16 131 | lamb6 = self.conv_6(torch.cat((lambs[8], lambs[9]), dim=1)) # 1024x8x8 132 | lamb7 = self.conv7(lamb6) # 1024x4x4 133 | return [[z_attr1, z_attr2, z_attr3, z_attr4, z_attr5, z_attr6, z_attr7, z_attr8], 134 | [lamb7, lamb6, lamb5, lamb4, lamb3, lamb2, lamb1, lamb0]] -------------------------------------------------------------------------------- /modules/discriminator.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import numpy as np 3 | 4 | 5 | class NLayerDiscriminator(nn.Module): 6 | def __init__(self, input_nc, ndf=64, n_layers=6, norm_layer=nn.BatchNorm2d, use_sigmoid=False, getIntermFeat=False): 7 | super(NLayerDiscriminator, self).__init__() 8 | self.getIntermFeat = getIntermFeat 9 | self.n_layers = n_layers 10 | 11 | kw = 4 12 | padw = int(np.ceil((kw-1.0)/2)) 13 | sequence = [[nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]] 14 | 15 | nf = ndf 16 | for n in range(1, n_layers): 17 | nf_prev = nf 18 | nf = min(nf * 2, 512) 19 | sequence += [[ 20 | nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=2, padding=padw), 21 | norm_layer(nf), nn.LeakyReLU(0.2, True) 22 | ]] 23 | 24 | nf_prev = nf 25 | nf = min(nf * 2, 512) 26 | sequence += [[ 27 | nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=1, padding=padw), 28 | norm_layer(nf), 29 | nn.LeakyReLU(0.2, True) 30 | ]] 31 | 32 | sequence += [[nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw)]] 33 | 34 | if use_sigmoid: 35 | sequence += [[nn.Sigmoid()]] 36 | 37 | if getIntermFeat: 38 | for n in range(len(sequence)): 39 | setattr(self, 'encoder'+str(n), nn.Sequential(*sequence[n])) 40 | else: 41 | sequence_stream = [] 42 | for n in range(len(sequence)): 43 | sequence_stream += sequence[n] 44 | self.model = nn.Sequential(*sequence_stream) 45 | 46 | def forward(self, x): 47 | if self.getIntermFeat: 48 | res = [x] 49 | for n in range(self.n_layers+2): 50 | model = getattr(self, 'encoder'+str(n)) 51 | res.append(model(res[-1])) 52 | return res[1:] 53 | else: 54 | return self.model(x) 55 | 56 | 57 | class MultiscaleDiscriminator(nn.Module): 58 | 59 | def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, 60 | use_sigmoid=False, num_D=3, getIntermFeat=False): 61 | super(MultiscaleDiscriminator, self).__init__() 62 | self.num_D = num_D 63 | self.n_layers = n_layers 64 | self.getIntermFeat = getIntermFeat 65 | 66 | for i in range(num_D): 67 | netD = NLayerDiscriminator(input_nc, ndf, n_layers, norm_layer, use_sigmoid, getIntermFeat) 68 | if getIntermFeat: 69 | for j in range(n_layers + 2): 70 | setattr(self, 'scale' + str(i) + '_layer' + str(j), getattr(netD, 'encoder' + str(j))) 71 | else: 72 | setattr(self, 'layer' + str(i), netD.model) 73 | 74 | self.downsample = nn.AvgPool2d(3, stride=2, padding=[1, 1], count_include_pad=False) 75 | 76 | def singleD_forward(self, model, input): 77 | if self.getIntermFeat: 78 | result = [input] 79 | for i in range(len(model)): 80 | result.append(model[i](result[-1])) 81 | return result[1:] 82 | else: 83 | return [model(input)] 84 | 85 | def forward(self, input): 86 | num_D = self.num_D 87 | result = [] 88 | 89 | input_downsampled = input 90 | for i in range(num_D): 91 | if self.getIntermFeat: 92 | model = [getattr(self, 'scale' + str(num_D - 1 - i) + '_layer' + str(j)) for j in 93 | range(self.n_layers + 2)] 94 | else: 95 | model = getattr(self, 'layer' + str(num_D - 1 - i)) 96 | 97 | out = self.singleD_forward(model, input_downsampled) 98 | result.append(out) 99 | 100 | if i != (num_D - 1): 101 | input_downsampled = self.downsample(input_downsampled) 102 | 103 | return result 104 | 105 | 106 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /modules/encoder128.py: -------------------------------------------------------------------------------- 1 | from torch.nn import Linear, Conv2d, BatchNorm1d, BatchNorm2d, InstanceNorm2d ,PReLU, ReLU, Sigmoid, Dropout2d, Dropout, AvgPool2d, MaxPool2d, AdaptiveAvgPool2d, Module, Parameter 2 | import torch.nn.functional as F 3 | import torch 4 | from collections import namedtuple 5 | import math 6 | import pdb 7 | from torch.nn import Sequential 8 | 9 | 10 | class Flatten(Module): 11 | def forward(self, input): 12 | return input.view(input.size(0), -1) 13 | 14 | 15 | def l2_norm(input, axis=1): 16 | norm = torch.norm(input, 2, axis, True) 17 | output = torch.div(input, norm) 18 | return output 19 | 20 | 21 | class SEModule(Module): 22 | def __init__(self, channels, reduction): 23 | super(SEModule, self).__init__() 24 | self.avg_pool = AdaptiveAvgPool2d(1) 25 | self.fc1 = Conv2d( 26 | channels, channels // reduction, kernel_size=1, padding=0, bias=False) 27 | self.relu = ReLU(inplace=True) 28 | self.fc2 = Conv2d( 29 | channels // reduction, channels, kernel_size=1, padding=0, bias=False) 30 | self.sigmoid = Sigmoid() 31 | 32 | def forward(self, x): 33 | module_input = x 34 | x = self.avg_pool(x) 35 | x = self.fc1(x) 36 | x = self.relu(x) 37 | x = self.fc2(x) 38 | x = self.sigmoid(x) 39 | return module_input * x 40 | 41 | 42 | class bottleneck_IR_SE(Module): 43 | def __init__(self, in_channel, depth, stride): 44 | super(bottleneck_IR_SE, self).__init__() 45 | if in_channel == depth: 46 | self.shortcut_layer = MaxPool2d(1, stride) 47 | else: 48 | self.shortcut_layer = Sequential( 49 | Conv2d(in_channel, depth, (1, 1), stride, bias=False), 50 | BatchNorm2d(depth)) 51 | self.res_layer = Sequential( 52 | BatchNorm2d(in_channel), 53 | Conv2d(in_channel, depth, (3, 3), (1, 1), 1, bias=False), 54 | PReLU(depth), 55 | Conv2d(depth, depth, (3, 3), stride, 1, bias=False), 56 | BatchNorm2d(depth), 57 | SEModule(depth, 16) 58 | ) 59 | 60 | def forward(self, x): 61 | shortcut = self.shortcut_layer(x) 62 | res = self.res_layer(x) 63 | return res + shortcut 64 | 65 | 66 | class Bottleneck(namedtuple('Block', ['in_channel', 'depth', 'stride'])): 67 | '''A named tuple describing a ResNet block.''' 68 | 69 | 70 | def get_block(in_channel, depth, num_units, stride=2): 71 | block = [bottleneck_IR_SE(in_channel, depth, stride)] + [bottleneck_IR_SE(depth, depth, 1) for i in range(num_units - 1)] 72 | # return Sequential(*block) 73 | return block 74 | 75 | 76 | def get_blocks(num_layers): 77 | blocks = [] 78 | if num_layers == 50: 79 | blocks += get_block(in_channel=64, depth=64, num_units=3) # MaxPool2d 80 | blocks += get_block(in_channel=64, depth=128, num_units=4) 81 | blocks += get_block(in_channel=128, depth=256, num_units=14) 82 | blocks += get_block(in_channel=256, depth=512, num_units=3) 83 | elif num_layers == 100: 84 | blocks = [ 85 | get_block(in_channel=64, depth=64, num_units=3), # MaxPool2d 86 | get_block(in_channel=64, depth=128, num_units=13), 87 | get_block(in_channel=128, depth=256, num_units=30), 88 | get_block(in_channel=256, depth=512, num_units=3) 89 | ] 90 | elif num_layers == 152: 91 | blocks = [ 92 | get_block(in_channel=64, depth=64, num_units=3), # MaxPool2d 93 | get_block(in_channel=64, depth=128, num_units=8), 94 | get_block(in_channel=128, depth=256, num_units=36), 95 | get_block(in_channel=256, depth=512, num_units=3) 96 | ] 97 | return Sequential(*blocks) 98 | 99 | 100 | class Backbone(Module): 101 | def __init__(self, num_layers, drop_ratio, mode='ir'): 102 | super(Backbone, self).__init__() 103 | assert num_layers in [50, 100, 152], 'num_layers should be 50,100, or 152' 104 | assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' 105 | blocks = get_blocks(num_layers) # list 106 | self.features = [] 107 | self.input_layer = Sequential(Conv2d(3, 64, (3, 3), 1, 1, bias=False), 108 | BatchNorm2d(64), 109 | PReLU(64)) 110 | 111 | self.body = blocks 112 | 113 | self.output_layer = Sequential(BatchNorm2d(512), 114 | Dropout(drop_ratio), 115 | Flatten(), 116 | Linear(512 * 7 * 7, 512), 117 | BatchNorm1d(512)) 118 | 119 | def forward(self, x, cache_feats=False, train_header=False): 120 | self.features = [] 121 | 122 | if x.dim() == 3: # need to be 4-dimensions 123 | x = x.unsqueeze(0) 124 | 125 | x = self.input_layer(x) 126 | if cache_feats: 127 | self.features.append(x) 128 | # print(x.shape) 129 | 130 | for i, m in enumerate(self.body.children()): 131 | # print(m) 132 | x = m(x) 133 | # print(x.shape) 134 | if cache_feats: 135 | self.features.append(x) 136 | 137 | if train_header: 138 | return x 139 | else: 140 | x = self.output_layer(x) 141 | return l2_norm(x) 142 | 143 | 144 | class Backbone128(Module): # 50, 0.6, 'ir_se' 145 | def __init__(self, num_layers, drop_ratio, mode='ir_se'): 146 | super(Backbone128, self).__init__() 147 | assert num_layers in [50, 100, 152], 'num_layers should be 50,100, or 152' 148 | assert mode in ['ir', 'ir_se'], 'mode should be ir or ir_se' 149 | blocks = get_blocks(num_layers) # list 150 | self.features = [] 151 | self.input_layer = Sequential(Conv2d(3, 64, (3, 3), 1, 1, bias=False), 152 | BatchNorm2d(64), 153 | PReLU(64)) 154 | 155 | self.body = blocks 156 | 157 | self.output_layer128 = Sequential(BatchNorm2d(512), 158 | Dropout(drop_ratio), 159 | Flatten(), 160 | Linear(512 * 8 * 8, 512), 161 | BatchNorm1d(512)) 162 | 163 | def forward(self, x, cache_feats=False, train_header=False): 164 | if x.dim() == 3: # need to be 4-dimensions 165 | x = x.unsqueeze(0) 166 | self.features = [] 167 | 168 | x = self.input_layer(x) 169 | if cache_feats: 170 | self.features.append(x) 171 | # print(x.shape) 172 | 173 | for i, m in enumerate(self.body.children()): 174 | x = m(x) 175 | # print(x.shape) 176 | if cache_feats: 177 | self.features.append(x) 178 | 179 | if train_header: 180 | return x 181 | else: 182 | x = self.output_layer128(x) 183 | return l2_norm(x) 184 | 185 | def restrict_forward(self, z, index): 186 | """ 187 | Execute the Information Bottleneck 188 | :param z: the feature with unwanted information being filtered out 189 | :param index: which inter-feature to be replaced 190 | :return: new id vector 191 | """ 192 | if index == 0: # replace the output of input layer 193 | for i, m in enumerate(self.body.children()): 194 | z = m(z) 195 | else: # index > 0: 196 | for i, m in enumerate(self.body.children()): 197 | if i + 1 > index: 198 | z = m(z) 199 | z = self.output_layer128(z) 200 | return l2_norm(z) 201 | 202 | 203 | class Header128(Module): 204 | def __init__(self, drop_ratio): 205 | super(Header128, self).__init__() 206 | self.output_layer128 = Sequential(BatchNorm2d(512), 207 | Dropout(drop_ratio), 208 | Flatten(), 209 | Linear(512 * 8 * 8, 512), 210 | BatchNorm1d(512)) 211 | 212 | def forward(self, x): 213 | x = self.output_layer128(x) 214 | return l2_norm(x) 215 | -------------------------------------------------------------------------------- /modules/iib.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The code for calculating mutual information refers to the implementation of this paper: 3 | @inproceedings{ 4 | schulz2020iba, 5 | title={Restricting the Flow: Information Bottlenecks for Attribution}, 6 | author={Schulz, Karl and Sixt, Leon and Tombari, Federico and Landgraf, Tim}, 7 | booktitle={International Conference on Learning Representations}, 8 | year={2020}, 9 | url={https://openreview.net/forum?id=S1xWh1rYwB} 10 | } 11 | ''' 12 | 13 | import math 14 | import torch 15 | import torch.nn as nn 16 | import torch.nn.functional as F 17 | 18 | 19 | def _kl_div(r, lambda_, mean_r, std_r): 20 | """ Return the feature-wise KL-divergence of p(z|x) and q(z) 21 | # The equation in the paper is: 22 | # Z = λ * R + (1 - λ) * ε) 23 | # where ε ~ N(μ_r, σ_r**2), 24 | # and given R the distribution of Z ~ N(λ * R, ((1 - λ)*σ_r)**2) (λ * R is constant variable) 25 | # 26 | # As the KL-Divergence stays the same when both distributions are scaled, 27 | # normalizing Z such that Q'(Z) = N(0, 1) by using σ(R), μ(R). 28 | # Then for Gaussian Distribution: 29 | # I(R, z) = KL[P(z|R)||Q(z)] = KL[P'(z|R)||N(0, 1)] 30 | # = 0.5 * ( - log[det(noise)] - k + tr(noise_cov) + μ^T·μ ) 31 | """ 32 | r_norm = (r - mean_r) / std_r 33 | var_z = (1 - lambda_) ** 2 34 | log_var_z = torch.log(var_z) 35 | mu_z = r_norm * lambda_ 36 | 37 | capacity = -0.5 * (1 + log_var_z - mu_z ** 2 - var_z) 38 | return capacity 39 | 40 | 41 | class SpatialGaussianKernel(nn.Module): 42 | """ A simple convolutional layer with fixed gaussian kernels, used to smoothen the input """ 43 | def __init__(self, kernel_size, sigma, channels, device): 44 | super().__init__() 45 | self.sigma = sigma 46 | self.kernel_size = kernel_size 47 | assert kernel_size % 2 == 1, "kernel_size must be an odd number (for padding), {} given".format(self.kernel_size) 48 | variance = sigma ** 2. 49 | x_cord = torch.arange(kernel_size, dtype=torch.float, device=device) # 1, 2, 3, 4 50 | x_grid = x_cord.repeat(kernel_size).view(kernel_size, kernel_size) # 1, 2, 3 \ 1, 2, 3 \ 1, 2, 3 51 | y_grid = x_grid.t() # 1, 1, 1 \ 2, 2, 2 \ 3, 3, 3 52 | xy_grid = torch.stack([x_grid, y_grid], dim=-1) 53 | mean_xy = (kernel_size - 1) / 2. 54 | kernel_2d = (1. / (2. * math.pi * variance)) * torch.exp( 55 | -torch.sum((xy_grid - mean_xy) ** 2., dim=-1) / 56 | (2 * variance) 57 | ) 58 | kernel_2d = kernel_2d / kernel_2d.sum() 59 | kernel_3d = kernel_2d.expand(channels, 1, -1, -1) # expand in channel dimension 60 | self.conv = nn.Conv2d(in_channels=channels, out_channels=channels, 61 | padding=0, kernel_size=kernel_size, 62 | groups=channels, bias=False) 63 | self.conv.weight.data.copy_(kernel_3d) 64 | self.conv.to(device) 65 | self.conv.weight.requires_grad = False 66 | self.pad = nn.ReflectionPad2d(int((kernel_size - 1) / 2)) 67 | 68 | def forward(self, x): 69 | return self.conv(self.pad(x)) 70 | 71 | 72 | class IBLayer(nn.Module): 73 | def __init__(self, in_c, out_c, device, smooth=True, kernel_size=1, sigma=1.): 74 | """ 75 | Insert Information Bottleneck at one inter-feature 76 | :param in_c: sum of the channels of ‘readout_feats’ 77 | :param out_c: the channel of ‘attach_feat’ 78 | :param kernel_size: param of Gaussian Smooth 79 | """ 80 | super(IBLayer, self).__init__() 81 | 82 | self.conv1 = nn.Conv2d(in_channels=in_c, out_channels=in_c//2, kernel_size=1) 83 | self.conv2 = nn.Conv2d(in_channels=in_c//2, out_channels=out_c*2, kernel_size=1) 84 | self.conv3 = nn.Conv2d(in_channels=out_c*2, out_channels=out_c, kernel_size=1) 85 | self.relu = nn.ReLU(inplace=False) 86 | self.sigmoid = nn.Sigmoid() 87 | 88 | with torch.no_grad(): 89 | nn.init.constant_(self.conv3.bias, 5.0) 90 | self.conv3.weight *= 1e-3 91 | 92 | self._alpha_bound = 5 93 | 94 | # Smoothing layer 95 | if smooth: 96 | if kernel_size is not None: 97 | # Construct static convolution layer with gaussian kernel 98 | sigma = kernel_size * 0.25 # Cover 2 stds in both directions 99 | self.smooth = SpatialGaussianKernel(kernel_size, sigma, out_c, device) 100 | elif sigma is not None and sigma > 0: 101 | # Construct static conv layer with gaussian kernel 102 | kernel_size = int(round(2 * sigma)) * 2 + 1 # Cover 2.5 stds in both directions 103 | self.smooth = SpatialGaussianKernel(kernel_size, sigma, out_c, device) 104 | else: 105 | self.smooth = None 106 | 107 | self.to(device) 108 | 109 | def forward(self, R, readout_feats, active_neurons, m_r=None, std_r=None, info_mean=True): 110 | # channel-wise mean and std: [Cr, Hr, Wr] 111 | m_r = torch.mean(R, dim=0) if m_r is None else m_r 112 | std_r = torch.std(R, dim=0) if std_r is None else std_r 113 | 114 | # --- 1. Get the smoothed mask 'lambda' for one attach feature 115 | readout = [] 116 | for idx, feat in enumerate(readout_feats): 117 | # Reshape as spatial shape of the attach feature: [B, Cf, Hr, Wr] 118 | feat = F.interpolate(feat, size=(R.shape[-2], R.shape[-1]), mode='bilinear', align_corners=True) 119 | readout.append(feat) 120 | readout = torch.cat(readout, dim=1) # [B, sum(Cf), Hr, Wr] 121 | 122 | # Pass through the readout network to obtain alpha 123 | alpha = self.conv1(readout) 124 | alpha = self.relu(alpha) 125 | alpha = self.conv2(alpha) 126 | alpha = self.relu(alpha) 127 | alpha = self.conv3(alpha) # [B, Cr, Hr, Wr] 128 | 129 | # Keep alphas in a meaningful range during training 130 | alpha = alpha.clamp(-self._alpha_bound, self._alpha_bound) 131 | lambda_ = self.sigmoid(alpha) # TODO: 132 | 133 | # Smoothing step 134 | lambda_ = self.smooth(lambda_) if self.smooth is not None else lambda_ 135 | 136 | # --- 2. Get restricted latent feature Z 137 | eps = torch.randn(size=R.shape).to(R.device) * std_r + m_r # [B, Cr, Hr, Wr] 138 | Z = R * lambda_ + (torch.ones_like(R).to(R.device) - lambda_) * eps 139 | 140 | info_capacity = _kl_div(R, lambda_, m_r, std_r) * active_neurons # [Cr, Hr, Wr] 141 | Z *= active_neurons 142 | 143 | info = info_capacity.mean(dim=0) if info_mean else info_capacity 144 | 145 | return Z, lambda_, info 146 | 147 | 148 | class IIB(nn.Module): 149 | def __init__(self, in_c, out_c_list, device, smooth=True, kernel_size=1): 150 | super(IIB, self).__init__() 151 | self.N = len(out_c_list) 152 | for i in range(self.N): 153 | setattr(self, f'iba_{i}', IBLayer(in_c, out_c=out_c_list[i], device=device, 154 | smooth=smooth, kernel_size=kernel_size)) 155 | 156 | self.to(device) 157 | self.device = device 158 | 159 | def forward(self, model, B, N, readout_feats, param_dict=None): 160 | """ 161 | Multi-average information bottleneck 162 | :param model: 163 | :param B: first [:B] samples are for Xs, last [B:] samples are for Xt 164 | :param N: 165 | :param readout_feats: 166 | :param param_dict: 167 | :return: 168 | """ 169 | Info = 0. 170 | # X_id_restrict = torch.zeros_like(X_id).to(self.device) # [2*B, 512] 171 | X_id_restrict = torch.zeros([2*B, 512]).to(self.device) 172 | Xt_feats = [] 173 | Xs_feats = [] 174 | X_lambda = [] 175 | 176 | Rs_params = [] 177 | Rt_params = [] 178 | 179 | for i in range(N): # multi-average information bottleneck 180 | R = readout_feats[i] # [2*B, Cr, Hr, Wr] 181 | Z, lambda_, info = getattr(IIB, f'iba_{i}')(R, readout_feats, 182 | m_r=param_dict[i][0], std_r=param_dict[i][1]) 183 | 184 | # (1) loss 185 | X_id_restrict += model.restrict_forward(Z, i) # [2*B, 512] 186 | Info += info.mean() 187 | 188 | # (2) attributes 189 | Rs, Rt = R[:B], R[B:] 190 | lambda_s, lambda_t = lambda_[:B], lambda_[B:] 191 | 192 | m_s = torch.mean(Rs, dim=0) # [C, H, W] 193 | std_s = torch.mean(Rs, dim=0) 194 | Rs_params.append([m_s, std_s]) 195 | eps_s = torch.randn(size=Rt.shape).to(Rt.device) * std_s + m_s 196 | feat_t = Rt * (1. - lambda_t) + lambda_t * eps_s 197 | Xt_feats.append(feat_t) # only related with lambda 198 | 199 | m_t = torch.mean(Rt, dim=0) # [C, H, W] 200 | std_t = torch.mean(Rt, dim=0) 201 | Rt_params.append([m_t, std_t]) 202 | eps_t = torch.randn(size=Rs.shape).to(Rs.device) * std_t + m_t 203 | feat_s = Rs * (1. - lambda_s) + lambda_s * eps_t 204 | Xs_feats.append(feat_s) # only related with lambda 205 | 206 | X_lambda.append(lambda_) 207 | 208 | X_id_restrict /= float(N) 209 | Info /= float(N) 210 | 211 | return X_id_restrict, Info, Xs_feats, Xt_feats, X_lambda, Rs_params, Rt_params 212 | 213 | 214 | def get_restricted_id_attrs(model, readout_feats, lambdas, R_params, lamb_detach=True, calc_id=True, calc_Zid=False): 215 | feats = [] 216 | Zs = [] 217 | N = len(readout_feats) 218 | id = None 219 | for i, R in enumerate(readout_feats): 220 | R = readout_feats[i] 221 | lamb = lambdas[i].detach() if lamb_detach else lambdas[i] 222 | m, std = R_params[i] 223 | 224 | eps = torch.randn(size=R.shape).to(R.device) * std + m 225 | feat = R * (1. - lamb) + lamb * eps # to prevent updating iba via lambda 226 | feats.append(feat) 227 | if calc_Zid: 228 | Z = R * lamb + eps * (1. - lamb) 229 | Zs.append(Z) 230 | 231 | if calc_id: 232 | if id is None: 233 | id = model.restrict_forward(feat, i) 234 | else: 235 | id += model.restrict_forward(feat, i) 236 | if calc_id: 237 | id /= float(N) 238 | if calc_Zid: 239 | return id, feats, Zs 240 | else: 241 | return id, feats 242 | -------------------------------------------------------------------------------- /modules/weights128/readout_layer0.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer0.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer1.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer1.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer10.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer10.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer2.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer2.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer3.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer3.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer4.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer4.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer5.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer5.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer6.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer6.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer7.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer7.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer8.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer8.pth -------------------------------------------------------------------------------- /modules/weights128/readout_layer9.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/modules/weights128/readout_layer9.pth -------------------------------------------------------------------------------- /preprocess/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/__init__.py -------------------------------------------------------------------------------- /preprocess/mtcnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from PIL import Image 5 | from preprocess.mtcnn_pytorch.src.get_nets import PNet, RNet, ONet 6 | from preprocess.mtcnn_pytorch.src.box_utils import nms, calibrate_box, get_image_boxes, convert_to_square 7 | from preprocess.mtcnn_pytorch.src.first_stage import run_first_stage 8 | from preprocess.mtcnn_pytorch.src.align_trans import get_reference_facial_points, warp_and_crop_face 9 | 10 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 11 | 12 | 13 | class MTCNN(nn.Module): 14 | def __init__(self, square=True): 15 | super().__init__() 16 | self.pnet = PNet().to(device) 17 | self.rnet = RNet().to(device) 18 | self.onet = ONet().to(device) 19 | self.pnet.eval() 20 | self.rnet.eval() 21 | self.onet.eval() 22 | # crop_size = (112, 112): 23 | self.reference = get_reference_facial_points(default_square=square) 24 | 25 | def align(self, img, crop_size=(112, 112), return_trans_inv=False): 26 | # TODO: 换关键点检测模型 27 | _, landmarks = self.detect_faces(img) 28 | if len(landmarks) == 0: 29 | return None if not return_trans_inv else (None, None) 30 | facial5points = [[landmarks[0][j], landmarks[0][j+5]] for j in range(5)] 31 | """ 32 | # 取关键点列的前3个点的坐标, 计算Similarity Transform Matrix, 进行warp和crop 33 | """ 34 | warped_face = warp_and_crop_face(np.array(img), facial5points, self.reference, crop_size=crop_size, 35 | return_trans_inv=return_trans_inv) 36 | if return_trans_inv: 37 | return Image.fromarray(warped_face[0]), warped_face[1] 38 | else: 39 | return Image.fromarray(warped_face) 40 | 41 | def align_fully(self, img, crop_size=(112, 112), return_trans_inv=False, ori=[0, 1, 3], fast_mode=True): 42 | ori_size = img.copy() 43 | h = img.size[1] 44 | w = img.size[0] 45 | sw = 320. if fast_mode else w 46 | scale = sw / w 47 | img = img.resize((int(w*scale), int(h*scale))) 48 | candi = [] 49 | for i in ori: 50 | if len(candi) > 0: 51 | break 52 | if i > 0: 53 | rimg = img.transpose(i+1) 54 | else: 55 | rimg = img 56 | box, landmarks = self.detect_faces(rimg, min_face_size=sw/10, thresholds=[0.6, 0.7, 0.7]) 57 | landmarks /= scale 58 | if len(landmarks) == 0: 59 | continue 60 | if i == 0: 61 | f5p = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] 62 | elif i == 1: 63 | f5p = [[w-1-landmarks[0][j+5], landmarks[0][j]] for j in range(5)] 64 | elif i == 2: 65 | f5p = [[w-1-landmarks[0][j], h-1-landmarks[0][j+5]] for j in range(5)] 66 | elif i == 3: 67 | f5p = [[landmarks[0][j + 5], h-1-landmarks[0][j]] for j in range(5)] 68 | candi.append((box[0][4], f5p)) 69 | if len(candi) == 0: 70 | return None if not return_trans_inv else (None, None) 71 | while len(candi) > 1: 72 | if candi[0][0] > candi[1][0]: 73 | del candi[1] 74 | else: 75 | del candi[0] 76 | facial5points = candi[0][1] 77 | warped_face = warp_and_crop_face(np.array(ori_size), facial5points, self.reference, crop_size=crop_size, 78 | return_trans_inv=return_trans_inv) 79 | if return_trans_inv: 80 | return Image.fromarray(warped_face[0]), warped_face[1] 81 | else: 82 | return Image.fromarray(warped_face) 83 | 84 | def align_multi(self, img, limit=None, min_face_size=64.0, crop_size=(112, 112), thresholds=[0.6, 0.7, 0.8], factor=0.707, reverse=False, mode=None): 85 | boxes, landmarks = self.detect_faces(img, min_face_size, thresholds=thresholds, factor=factor) 86 | if len(landmarks) == 0: 87 | return None 88 | if limit: 89 | boxes = boxes[:limit] 90 | landmarks = landmarks[:limit] 91 | faces = [] 92 | tfm_invs = [] 93 | for landmark in landmarks: 94 | facial5points = [[landmark[j], landmark[j+5]] for j in range(5)] 95 | # print(facial5points) # in the original image (un-cropped) 96 | # crop_size = (256, 256) 97 | warped_face, tfm_inv = warp_and_crop_face(np.array(img), facial5points, self.reference, crop_size=crop_size, return_trans_inv=True, mode=mode) 98 | faces.append(Image.fromarray(warped_face)) 99 | 100 | tfm_invs.append(tfm_inv) 101 | if reverse: 102 | return faces, tfm_invs, boxes 103 | else: 104 | return faces 105 | 106 | def get_landmarks(self, img, min_face_size=32, crop_size=(256, 256), fast_mode=False, ori=[0,1,3]): 107 | ori_size = img.copy() 108 | h = img.size[1] 109 | w = img.size[0] 110 | sw = 640. if fast_mode else w 111 | scale = sw / w 112 | img = img.resize((int(w*scale), int(h*scale))) 113 | min_face_size = min_face_size if not fast_mode else sw/20 114 | candi = [] 115 | boxes = np.zeros([0, 5]) 116 | for i in ori: 117 | if i > 0: 118 | rimg = img.transpose(i+1) 119 | else: 120 | rimg = img 121 | box, landmarks = self.detect_faces(rimg, min_face_size=min_face_size, thresholds=[0.6, 0.7, 0.7]) 122 | landmarks /= scale 123 | if len(landmarks) == 0: 124 | continue 125 | if i == 0: 126 | f5p = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] 127 | elif i == 1: 128 | f5p = [[w-1-landmarks[0][j+5], landmarks[0][j]] for j in range(5)] 129 | x1 = w-1-box[:, 1] 130 | y1 = box[:, 0] 131 | x2 = w-1-box[:, 3] 132 | y2 = box[:, 2] 133 | box[:, :4] = np.stack((x2, y1, x1, y2), axis=1) 134 | elif i == 2: 135 | f5p = [[w-1-landmarks[0][j], h-1-landmarks[0][j+5]] for j in range(5)] 136 | x1 = w-1-box[:, 0] 137 | y1 = h-1-box[:, 1] 138 | x2 = w-1-box[:, 2] 139 | y2 = h-1-box[:, 3] 140 | box[:, :4] = np.stack((x2, y2, x1, y1), axis=1) 141 | elif i == 3: 142 | f5p = [[landmarks[0][j + 5], h-1-landmarks[0][j]] for j in range(5)] 143 | x1 = box[:, 1] 144 | y1 = h-1-box[:, 0] 145 | x2 = box[:, 3] 146 | y2 = h-1-box[:, 2] 147 | box[:, :4] = np.stack((x1, y2, x2, y1), axis=1) 148 | candi.append(f5p) 149 | boxes = np.concatenate((boxes, box), axis=0) 150 | # pick = nms(boxes) 151 | faces = [] 152 | for idx, facial5points in enumerate(candi): 153 | # if idx not in pick: 154 | # continue 155 | warped_face = warp_and_crop_face(np.array(ori_size), facial5points, self.reference, crop_size=crop_size, 156 | return_trans_inv=False) 157 | faces.append((warped_face, facial5points)) 158 | return faces 159 | 160 | def detect_faces(self, image, min_face_size=64.0, 161 | thresholds=[0.6, 0.7, 0.8], 162 | nms_thresholds=[0.7, 0.7, 0.7], 163 | factor= 0.707): 164 | """ 165 | Arguments: 166 | image: an instance of PIL.Image. 167 | min_face_size: a float number. 168 | thresholds: a list of length 3. 169 | nms_thresholds: a list of length 3. 170 | 171 | Returns: 172 | two float numpy arrays of shapes [n_boxes, 4] and [n_boxes, 10], 173 | bounding boxes and facial landmarks. 174 | """ 175 | 176 | # BUILD AN IMAGE PYRAMID 177 | width, height = image.size 178 | min_length = min(height, width) 179 | 180 | min_detection_size = 12 181 | # factor = 0.707 # sqrt(0.5) 182 | 183 | # scales for scaling the image 184 | scales = [] 185 | 186 | # scales the image so that 187 | # minimum size that we can detect equals to 188 | # minimum face size that we want to detect 189 | m = min_detection_size/min_face_size 190 | min_length *= m 191 | 192 | factor_count = 0 193 | while min_length > min_detection_size: 194 | scales.append(m*factor**factor_count) 195 | min_length *= factor 196 | factor_count += 1 197 | 198 | # STAGE 1 199 | 200 | # it will be returned 201 | bounding_boxes = [] 202 | 203 | with torch.no_grad(): 204 | # run P-Net on different scales 205 | for s in scales: 206 | boxes = run_first_stage(image, self.pnet, scale=s, threshold=thresholds[0]) 207 | bounding_boxes.append(boxes) 208 | 209 | # collect boxes (and offsets, and scores) from different scales 210 | bounding_boxes = [i for i in bounding_boxes if i is not None] 211 | if len(bounding_boxes) == 0: 212 | return np.zeros([0]), np.zeros([0]) 213 | bounding_boxes = np.vstack(bounding_boxes) 214 | 215 | keep = nms(bounding_boxes[:, 0:5], nms_thresholds[0]) 216 | bounding_boxes = bounding_boxes[keep] 217 | 218 | # use offsets predicted by pnet to transform bounding boxes 219 | bounding_boxes = calibrate_box(bounding_boxes[:, 0:5], bounding_boxes[:, 5:]) 220 | # shape [n_boxes, 5] 221 | 222 | bounding_boxes = convert_to_square(bounding_boxes) 223 | bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) 224 | 225 | # STAGE 2 226 | 227 | img_boxes = get_image_boxes(bounding_boxes, image, size=24) 228 | img_boxes = torch.FloatTensor(img_boxes).to(device) 229 | 230 | output = self.rnet(img_boxes) 231 | offsets = output[0].cpu().data.numpy() # shape [n_boxes, 4] 232 | probs = output[1].cpu().data.numpy() # shape [n_boxes, 2] 233 | 234 | keep = np.where(probs[:, 1] > thresholds[1])[0] 235 | bounding_boxes = bounding_boxes[keep] 236 | bounding_boxes[:, 4] = probs[keep, 1].reshape((-1,)) 237 | offsets = offsets[keep] 238 | 239 | keep = nms(bounding_boxes, nms_thresholds[1]) 240 | bounding_boxes = bounding_boxes[keep] 241 | bounding_boxes = calibrate_box(bounding_boxes, offsets[keep]) 242 | bounding_boxes = convert_to_square(bounding_boxes) 243 | bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) 244 | 245 | # STAGE 3 246 | 247 | img_boxes = get_image_boxes(bounding_boxes, image, size=48) 248 | if len(img_boxes) == 0: 249 | return np.zeros([0]), np.zeros([0]) 250 | img_boxes = torch.FloatTensor(img_boxes).to(device) 251 | output = self.onet(img_boxes) 252 | landmarks = output[0].cpu().data.numpy() # shape [n_boxes, 10] 253 | offsets = output[1].cpu().data.numpy() # shape [n_boxes, 4] 254 | probs = output[2].cpu().data.numpy() # shape [n_boxes, 2] 255 | 256 | keep = np.where(probs[:, 1] > thresholds[2])[0] 257 | bounding_boxes = bounding_boxes[keep] 258 | bounding_boxes[:, 4] = probs[keep, 1].reshape((-1,)) 259 | offsets = offsets[keep] 260 | landmarks = landmarks[keep] 261 | 262 | # compute landmark points 263 | width = bounding_boxes[:, 2] - bounding_boxes[:, 0] + 1.0 264 | height = bounding_boxes[:, 3] - bounding_boxes[:, 1] + 1.0 265 | xmin, ymin = bounding_boxes[:, 0], bounding_boxes[:, 1] 266 | landmarks[:, 0:5] = np.expand_dims(xmin, 1) + np.expand_dims(width, 1)*landmarks[:, 0:5] 267 | landmarks[:, 5:10] = np.expand_dims(ymin, 1) + np.expand_dims(height, 1)*landmarks[:, 5:10] 268 | 269 | bounding_boxes = calibrate_box(bounding_boxes, offsets) 270 | keep = nms(bounding_boxes, nms_thresholds[2], mode='min') 271 | bounding_boxes = bounding_boxes[keep] 272 | landmarks = landmarks[keep] 273 | 274 | return bounding_boxes, landmarks 275 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | __pycache__ 3 | 4 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Dan Antoshchenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/README.md: -------------------------------------------------------------------------------- 1 | # MTCNN 2 | 3 | `pytorch` implementation of **inference stage** of face detection algorithm described in 4 | [Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks](https://arxiv.org/abs/1604.02878). 5 | 6 | ## Example 7 | ![example of a face detection](images/example.png) 8 | 9 | ## How to use it 10 | Just download the repository and then do this 11 | ```python 12 | from src import detect_faces 13 | from PIL import Image 14 | 15 | image = Image.open('image.jpg') 16 | bounding_boxes, landmarks = detect_faces(image) 17 | ``` 18 | For examples see `test_on_images.ipynb`. 19 | 20 | ## Requirements 21 | * pytorch 0.2 22 | * Pillow, numpy 23 | 24 | ## Credit 25 | This implementation is heavily inspired by: 26 | * [pangyupo/mxnet_mtcnn_face_detection](https://github.com/pangyupo/mxnet_mtcnn_face_detection) 27 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det1.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/caffe_models/det1.caffemodel -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det1.prototxt: -------------------------------------------------------------------------------- 1 | name: "PNet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 12 6 | input_dim: 12 7 | 8 | layer { 9 | name: "conv1" 10 | type: "Convolution" 11 | bottom: "data" 12 | top: "conv1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 0 20 | } 21 | convolution_param { 22 | num_output: 10 23 | kernel_size: 3 24 | stride: 1 25 | weight_filler { 26 | type: "xavier" 27 | } 28 | bias_filler { 29 | type: "constant" 30 | value: 0 31 | } 32 | } 33 | } 34 | layer { 35 | name: "PReLU1" 36 | type: "PReLU" 37 | bottom: "conv1" 38 | top: "conv1" 39 | } 40 | layer { 41 | name: "pool1" 42 | type: "Pooling" 43 | bottom: "conv1" 44 | top: "pool1" 45 | pooling_param { 46 | pool: MAX 47 | kernel_size: 2 48 | stride: 2 49 | } 50 | } 51 | 52 | layer { 53 | name: "conv2" 54 | type: "Convolution" 55 | bottom: "pool1" 56 | top: "conv2" 57 | param { 58 | lr_mult: 1 59 | decay_mult: 1 60 | } 61 | param { 62 | lr_mult: 2 63 | decay_mult: 0 64 | } 65 | convolution_param { 66 | num_output: 16 67 | kernel_size: 3 68 | stride: 1 69 | weight_filler { 70 | type: "xavier" 71 | } 72 | bias_filler { 73 | type: "constant" 74 | value: 0 75 | } 76 | } 77 | } 78 | layer { 79 | name: "PReLU2" 80 | type: "PReLU" 81 | bottom: "conv2" 82 | top: "conv2" 83 | } 84 | 85 | layer { 86 | name: "conv3" 87 | type: "Convolution" 88 | bottom: "conv2" 89 | top: "conv3" 90 | param { 91 | lr_mult: 1 92 | decay_mult: 1 93 | } 94 | param { 95 | lr_mult: 2 96 | decay_mult: 0 97 | } 98 | convolution_param { 99 | num_output: 32 100 | kernel_size: 3 101 | stride: 1 102 | weight_filler { 103 | type: "xavier" 104 | } 105 | bias_filler { 106 | type: "constant" 107 | value: 0 108 | } 109 | } 110 | } 111 | layer { 112 | name: "PReLU3" 113 | type: "PReLU" 114 | bottom: "conv3" 115 | top: "conv3" 116 | } 117 | 118 | 119 | layer { 120 | name: "conv4-1" 121 | type: "Convolution" 122 | bottom: "conv3" 123 | top: "conv4-1" 124 | param { 125 | lr_mult: 1 126 | decay_mult: 1 127 | } 128 | param { 129 | lr_mult: 2 130 | decay_mult: 0 131 | } 132 | convolution_param { 133 | num_output: 2 134 | kernel_size: 1 135 | stride: 1 136 | weight_filler { 137 | type: "xavier" 138 | } 139 | bias_filler { 140 | type: "constant" 141 | value: 0 142 | } 143 | } 144 | } 145 | 146 | layer { 147 | name: "conv4-2" 148 | type: "Convolution" 149 | bottom: "conv3" 150 | top: "conv4-2" 151 | param { 152 | lr_mult: 1 153 | decay_mult: 1 154 | } 155 | param { 156 | lr_mult: 2 157 | decay_mult: 0 158 | } 159 | convolution_param { 160 | num_output: 4 161 | kernel_size: 1 162 | stride: 1 163 | weight_filler { 164 | type: "xavier" 165 | } 166 | bias_filler { 167 | type: "constant" 168 | value: 0 169 | } 170 | } 171 | } 172 | layer { 173 | name: "prob1" 174 | type: "Softmax" 175 | bottom: "conv4-1" 176 | top: "prob1" 177 | } 178 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det2.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/caffe_models/det2.caffemodel -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det2.prototxt: -------------------------------------------------------------------------------- 1 | name: "RNet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 24 6 | input_dim: 24 7 | 8 | 9 | ########################## 10 | ###################### 11 | layer { 12 | name: "conv1" 13 | type: "Convolution" 14 | bottom: "data" 15 | top: "conv1" 16 | param { 17 | lr_mult: 0 18 | decay_mult: 0 19 | } 20 | param { 21 | lr_mult: 0 22 | decay_mult: 0 23 | } 24 | convolution_param { 25 | num_output: 28 26 | kernel_size: 3 27 | stride: 1 28 | weight_filler { 29 | type: "xavier" 30 | } 31 | bias_filler { 32 | type: "constant" 33 | value: 0 34 | } 35 | } 36 | } 37 | layer { 38 | name: "prelu1" 39 | type: "PReLU" 40 | bottom: "conv1" 41 | top: "conv1" 42 | propagate_down: true 43 | } 44 | layer { 45 | name: "pool1" 46 | type: "Pooling" 47 | bottom: "conv1" 48 | top: "pool1" 49 | pooling_param { 50 | pool: MAX 51 | kernel_size: 3 52 | stride: 2 53 | } 54 | } 55 | 56 | layer { 57 | name: "conv2" 58 | type: "Convolution" 59 | bottom: "pool1" 60 | top: "conv2" 61 | param { 62 | lr_mult: 0 63 | decay_mult: 0 64 | } 65 | param { 66 | lr_mult: 0 67 | decay_mult: 0 68 | } 69 | convolution_param { 70 | num_output: 48 71 | kernel_size: 3 72 | stride: 1 73 | weight_filler { 74 | type: "xavier" 75 | } 76 | bias_filler { 77 | type: "constant" 78 | value: 0 79 | } 80 | } 81 | } 82 | layer { 83 | name: "prelu2" 84 | type: "PReLU" 85 | bottom: "conv2" 86 | top: "conv2" 87 | propagate_down: true 88 | } 89 | layer { 90 | name: "pool2" 91 | type: "Pooling" 92 | bottom: "conv2" 93 | top: "pool2" 94 | pooling_param { 95 | pool: MAX 96 | kernel_size: 3 97 | stride: 2 98 | } 99 | } 100 | #################################### 101 | 102 | ################################## 103 | layer { 104 | name: "conv3" 105 | type: "Convolution" 106 | bottom: "pool2" 107 | top: "conv3" 108 | param { 109 | lr_mult: 0 110 | decay_mult: 0 111 | } 112 | param { 113 | lr_mult: 0 114 | decay_mult: 0 115 | } 116 | convolution_param { 117 | num_output: 64 118 | kernel_size: 2 119 | stride: 1 120 | weight_filler { 121 | type: "xavier" 122 | } 123 | bias_filler { 124 | type: "constant" 125 | value: 0 126 | } 127 | } 128 | } 129 | layer { 130 | name: "prelu3" 131 | type: "PReLU" 132 | bottom: "conv3" 133 | top: "conv3" 134 | propagate_down: true 135 | } 136 | ############################### 137 | 138 | ############################### 139 | 140 | layer { 141 | name: "conv4" 142 | type: "InnerProduct" 143 | bottom: "conv3" 144 | top: "conv4" 145 | param { 146 | lr_mult: 0 147 | decay_mult: 0 148 | } 149 | param { 150 | lr_mult: 0 151 | decay_mult: 0 152 | } 153 | inner_product_param { 154 | num_output: 128 155 | weight_filler { 156 | type: "xavier" 157 | } 158 | bias_filler { 159 | type: "constant" 160 | value: 0 161 | } 162 | } 163 | } 164 | layer { 165 | name: "prelu4" 166 | type: "PReLU" 167 | bottom: "conv4" 168 | top: "conv4" 169 | } 170 | 171 | layer { 172 | name: "conv5-1" 173 | type: "InnerProduct" 174 | bottom: "conv4" 175 | top: "conv5-1" 176 | param { 177 | lr_mult: 0 178 | decay_mult: 0 179 | } 180 | param { 181 | lr_mult: 0 182 | decay_mult: 0 183 | } 184 | inner_product_param { 185 | num_output: 2 186 | #kernel_size: 1 187 | #stride: 1 188 | weight_filler { 189 | type: "xavier" 190 | } 191 | bias_filler { 192 | type: "constant" 193 | value: 0 194 | } 195 | } 196 | } 197 | layer { 198 | name: "conv5-2" 199 | type: "InnerProduct" 200 | bottom: "conv4" 201 | top: "conv5-2" 202 | param { 203 | lr_mult: 1 204 | decay_mult: 1 205 | } 206 | param { 207 | lr_mult: 2 208 | decay_mult: 1 209 | } 210 | inner_product_param { 211 | num_output: 4 212 | #kernel_size: 1 213 | #stride: 1 214 | weight_filler { 215 | type: "xavier" 216 | } 217 | bias_filler { 218 | type: "constant" 219 | value: 0 220 | } 221 | } 222 | } 223 | layer { 224 | name: "prob1" 225 | type: "Softmax" 226 | bottom: "conv5-1" 227 | top: "prob1" 228 | } -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det3.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/caffe_models/det3.caffemodel -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det3.prototxt: -------------------------------------------------------------------------------- 1 | name: "ONet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 48 6 | input_dim: 48 7 | ################################## 8 | layer { 9 | name: "conv1" 10 | type: "Convolution" 11 | bottom: "data" 12 | top: "conv1" 13 | param { 14 | lr_mult: 1 15 | decay_mult: 1 16 | } 17 | param { 18 | lr_mult: 2 19 | decay_mult: 1 20 | } 21 | convolution_param { 22 | num_output: 32 23 | kernel_size: 3 24 | stride: 1 25 | weight_filler { 26 | type: "xavier" 27 | } 28 | bias_filler { 29 | type: "constant" 30 | value: 0 31 | } 32 | } 33 | } 34 | layer { 35 | name: "prelu1" 36 | type: "PReLU" 37 | bottom: "conv1" 38 | top: "conv1" 39 | } 40 | layer { 41 | name: "pool1" 42 | type: "Pooling" 43 | bottom: "conv1" 44 | top: "pool1" 45 | pooling_param { 46 | pool: MAX 47 | kernel_size: 3 48 | stride: 2 49 | } 50 | } 51 | layer { 52 | name: "conv2" 53 | type: "Convolution" 54 | bottom: "pool1" 55 | top: "conv2" 56 | param { 57 | lr_mult: 1 58 | decay_mult: 1 59 | } 60 | param { 61 | lr_mult: 2 62 | decay_mult: 1 63 | } 64 | convolution_param { 65 | num_output: 64 66 | kernel_size: 3 67 | stride: 1 68 | weight_filler { 69 | type: "xavier" 70 | } 71 | bias_filler { 72 | type: "constant" 73 | value: 0 74 | } 75 | } 76 | } 77 | 78 | layer { 79 | name: "prelu2" 80 | type: "PReLU" 81 | bottom: "conv2" 82 | top: "conv2" 83 | } 84 | layer { 85 | name: "pool2" 86 | type: "Pooling" 87 | bottom: "conv2" 88 | top: "pool2" 89 | pooling_param { 90 | pool: MAX 91 | kernel_size: 3 92 | stride: 2 93 | } 94 | } 95 | 96 | layer { 97 | name: "conv3" 98 | type: "Convolution" 99 | bottom: "pool2" 100 | top: "conv3" 101 | param { 102 | lr_mult: 1 103 | decay_mult: 1 104 | } 105 | param { 106 | lr_mult: 2 107 | decay_mult: 1 108 | } 109 | convolution_param { 110 | num_output: 64 111 | kernel_size: 3 112 | weight_filler { 113 | type: "xavier" 114 | } 115 | bias_filler { 116 | type: "constant" 117 | value: 0 118 | } 119 | } 120 | } 121 | layer { 122 | name: "prelu3" 123 | type: "PReLU" 124 | bottom: "conv3" 125 | top: "conv3" 126 | } 127 | layer { 128 | name: "pool3" 129 | type: "Pooling" 130 | bottom: "conv3" 131 | top: "pool3" 132 | pooling_param { 133 | pool: MAX 134 | kernel_size: 2 135 | stride: 2 136 | } 137 | } 138 | layer { 139 | name: "conv4" 140 | type: "Convolution" 141 | bottom: "pool3" 142 | top: "conv4" 143 | param { 144 | lr_mult: 1 145 | decay_mult: 1 146 | } 147 | param { 148 | lr_mult: 2 149 | decay_mult: 1 150 | } 151 | convolution_param { 152 | num_output: 128 153 | kernel_size: 2 154 | weight_filler { 155 | type: "xavier" 156 | } 157 | bias_filler { 158 | type: "constant" 159 | value: 0 160 | } 161 | } 162 | } 163 | layer { 164 | name: "prelu4" 165 | type: "PReLU" 166 | bottom: "conv4" 167 | top: "conv4" 168 | } 169 | 170 | 171 | layer { 172 | name: "conv5" 173 | type: "InnerProduct" 174 | bottom: "conv4" 175 | top: "conv5" 176 | param { 177 | lr_mult: 1 178 | decay_mult: 1 179 | } 180 | param { 181 | lr_mult: 2 182 | decay_mult: 1 183 | } 184 | inner_product_param { 185 | #kernel_size: 3 186 | num_output: 256 187 | weight_filler { 188 | type: "xavier" 189 | } 190 | bias_filler { 191 | type: "constant" 192 | value: 0 193 | } 194 | } 195 | } 196 | 197 | layer { 198 | name: "drop5" 199 | type: "Dropout" 200 | bottom: "conv5" 201 | top: "conv5" 202 | dropout_param { 203 | dropout_ratio: 0.25 204 | } 205 | } 206 | layer { 207 | name: "prelu5" 208 | type: "PReLU" 209 | bottom: "conv5" 210 | top: "conv5" 211 | } 212 | 213 | 214 | layer { 215 | name: "conv6-1" 216 | type: "InnerProduct" 217 | bottom: "conv5" 218 | top: "conv6-1" 219 | param { 220 | lr_mult: 1 221 | decay_mult: 1 222 | } 223 | param { 224 | lr_mult: 2 225 | decay_mult: 1 226 | } 227 | inner_product_param { 228 | #kernel_size: 1 229 | num_output: 2 230 | weight_filler { 231 | type: "xavier" 232 | } 233 | bias_filler { 234 | type: "constant" 235 | value: 0 236 | } 237 | } 238 | } 239 | layer { 240 | name: "conv6-2" 241 | type: "InnerProduct" 242 | bottom: "conv5" 243 | top: "conv6-2" 244 | param { 245 | lr_mult: 1 246 | decay_mult: 1 247 | } 248 | param { 249 | lr_mult: 2 250 | decay_mult: 1 251 | } 252 | inner_product_param { 253 | #kernel_size: 1 254 | num_output: 4 255 | weight_filler { 256 | type: "xavier" 257 | } 258 | bias_filler { 259 | type: "constant" 260 | value: 0 261 | } 262 | } 263 | } 264 | layer { 265 | name: "conv6-3" 266 | type: "InnerProduct" 267 | bottom: "conv5" 268 | top: "conv6-3" 269 | param { 270 | lr_mult: 1 271 | decay_mult: 1 272 | } 273 | param { 274 | lr_mult: 2 275 | decay_mult: 1 276 | } 277 | inner_product_param { 278 | #kernel_size: 1 279 | num_output: 10 280 | weight_filler { 281 | type: "xavier" 282 | } 283 | bias_filler { 284 | type: "constant" 285 | value: 0 286 | } 287 | } 288 | } 289 | layer { 290 | name: "prob1" 291 | type: "Softmax" 292 | bottom: "conv6-1" 293 | top: "prob1" 294 | } 295 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det4.caffemodel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/caffe_models/det4.caffemodel -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/caffe_models/det4.prototxt: -------------------------------------------------------------------------------- 1 | name: "LNet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 15 5 | input_dim: 24 6 | input_dim: 24 7 | 8 | layer { 9 | name: "slicer_data" 10 | type: "Slice" 11 | bottom: "data" 12 | top: "data241" 13 | top: "data242" 14 | top: "data243" 15 | top: "data244" 16 | top: "data245" 17 | slice_param { 18 | axis: 1 19 | slice_point: 3 20 | slice_point: 6 21 | slice_point: 9 22 | slice_point: 12 23 | } 24 | } 25 | layer { 26 | name: "conv1_1" 27 | type: "Convolution" 28 | bottom: "data241" 29 | top: "conv1_1" 30 | param { 31 | lr_mult: 1 32 | decay_mult: 1 33 | } 34 | param { 35 | lr_mult: 2 36 | decay_mult: 1 37 | } 38 | convolution_param { 39 | num_output: 28 40 | kernel_size: 3 41 | stride: 1 42 | weight_filler { 43 | type: "xavier" 44 | } 45 | bias_filler { 46 | type: "constant" 47 | value: 0 48 | } 49 | } 50 | 51 | } 52 | layer { 53 | name: "prelu1_1" 54 | type: "PReLU" 55 | bottom: "conv1_1" 56 | top: "conv1_1" 57 | 58 | } 59 | layer { 60 | name: "pool1_1" 61 | type: "Pooling" 62 | bottom: "conv1_1" 63 | top: "pool1_1" 64 | pooling_param { 65 | pool: MAX 66 | kernel_size: 3 67 | stride: 2 68 | } 69 | } 70 | 71 | layer { 72 | name: "conv2_1" 73 | type: "Convolution" 74 | bottom: "pool1_1" 75 | top: "conv2_1" 76 | param { 77 | lr_mult: 1 78 | decay_mult: 1 79 | } 80 | param { 81 | lr_mult: 2 82 | decay_mult: 1 83 | } 84 | convolution_param { 85 | num_output: 48 86 | kernel_size: 3 87 | stride: 1 88 | weight_filler { 89 | type: "xavier" 90 | } 91 | bias_filler { 92 | type: "constant" 93 | value: 0 94 | } 95 | } 96 | 97 | } 98 | layer { 99 | name: "prelu2_1" 100 | type: "PReLU" 101 | bottom: "conv2_1" 102 | top: "conv2_1" 103 | } 104 | layer { 105 | name: "pool2_1" 106 | type: "Pooling" 107 | bottom: "conv2_1" 108 | top: "pool2_1" 109 | pooling_param { 110 | pool: MAX 111 | kernel_size: 3 112 | stride: 2 113 | } 114 | 115 | } 116 | layer { 117 | name: "conv3_1" 118 | type: "Convolution" 119 | bottom: "pool2_1" 120 | top: "conv3_1" 121 | param { 122 | lr_mult: 1 123 | decay_mult: 1 124 | } 125 | param { 126 | lr_mult: 2 127 | decay_mult: 1 128 | } 129 | convolution_param { 130 | num_output: 64 131 | kernel_size: 2 132 | stride: 1 133 | weight_filler { 134 | type: "xavier" 135 | } 136 | bias_filler { 137 | type: "constant" 138 | value: 0 139 | } 140 | } 141 | 142 | } 143 | layer { 144 | name: "prelu3_1" 145 | type: "PReLU" 146 | bottom: "conv3_1" 147 | top: "conv3_1" 148 | } 149 | ########################## 150 | layer { 151 | name: "conv1_2" 152 | type: "Convolution" 153 | bottom: "data242" 154 | top: "conv1_2" 155 | param { 156 | lr_mult: 1 157 | decay_mult: 1 158 | } 159 | param { 160 | lr_mult: 2 161 | decay_mult: 1 162 | } 163 | convolution_param { 164 | num_output: 28 165 | kernel_size: 3 166 | stride: 1 167 | weight_filler { 168 | type: "xavier" 169 | } 170 | bias_filler { 171 | type: "constant" 172 | value: 0 173 | } 174 | } 175 | 176 | } 177 | layer { 178 | name: "prelu1_2" 179 | type: "PReLU" 180 | bottom: "conv1_2" 181 | top: "conv1_2" 182 | 183 | } 184 | layer { 185 | name: "pool1_2" 186 | type: "Pooling" 187 | bottom: "conv1_2" 188 | top: "pool1_2" 189 | pooling_param { 190 | pool: MAX 191 | kernel_size: 3 192 | stride: 2 193 | } 194 | } 195 | 196 | layer { 197 | name: "conv2_2" 198 | type: "Convolution" 199 | bottom: "pool1_2" 200 | top: "conv2_2" 201 | param { 202 | lr_mult: 1 203 | decay_mult: 1 204 | } 205 | param { 206 | lr_mult: 2 207 | decay_mult: 1 208 | } 209 | convolution_param { 210 | num_output: 48 211 | kernel_size: 3 212 | stride: 1 213 | weight_filler { 214 | type: "xavier" 215 | } 216 | bias_filler { 217 | type: "constant" 218 | value: 0 219 | } 220 | } 221 | 222 | } 223 | layer { 224 | name: "prelu2_2" 225 | type: "PReLU" 226 | bottom: "conv2_2" 227 | top: "conv2_2" 228 | } 229 | layer { 230 | name: "pool2_2" 231 | type: "Pooling" 232 | bottom: "conv2_2" 233 | top: "pool2_2" 234 | pooling_param { 235 | pool: MAX 236 | kernel_size: 3 237 | stride: 2 238 | } 239 | 240 | } 241 | layer { 242 | name: "conv3_2" 243 | type: "Convolution" 244 | bottom: "pool2_2" 245 | top: "conv3_2" 246 | param { 247 | lr_mult: 1 248 | decay_mult: 1 249 | } 250 | param { 251 | lr_mult: 2 252 | decay_mult: 1 253 | } 254 | convolution_param { 255 | num_output: 64 256 | kernel_size: 2 257 | stride: 1 258 | weight_filler { 259 | type: "xavier" 260 | } 261 | bias_filler { 262 | type: "constant" 263 | value: 0 264 | } 265 | } 266 | 267 | } 268 | layer { 269 | name: "prelu3_2" 270 | type: "PReLU" 271 | bottom: "conv3_2" 272 | top: "conv3_2" 273 | } 274 | ########################## 275 | ########################## 276 | layer { 277 | name: "conv1_3" 278 | type: "Convolution" 279 | bottom: "data243" 280 | top: "conv1_3" 281 | param { 282 | lr_mult: 1 283 | decay_mult: 1 284 | } 285 | param { 286 | lr_mult: 2 287 | decay_mult: 1 288 | } 289 | convolution_param { 290 | num_output: 28 291 | kernel_size: 3 292 | stride: 1 293 | weight_filler { 294 | type: "xavier" 295 | } 296 | bias_filler { 297 | type: "constant" 298 | value: 0 299 | } 300 | } 301 | 302 | } 303 | layer { 304 | name: "prelu1_3" 305 | type: "PReLU" 306 | bottom: "conv1_3" 307 | top: "conv1_3" 308 | 309 | } 310 | layer { 311 | name: "pool1_3" 312 | type: "Pooling" 313 | bottom: "conv1_3" 314 | top: "pool1_3" 315 | pooling_param { 316 | pool: MAX 317 | kernel_size: 3 318 | stride: 2 319 | } 320 | } 321 | 322 | layer { 323 | name: "conv2_3" 324 | type: "Convolution" 325 | bottom: "pool1_3" 326 | top: "conv2_3" 327 | param { 328 | lr_mult: 1 329 | decay_mult: 1 330 | } 331 | param { 332 | lr_mult: 2 333 | decay_mult: 1 334 | } 335 | convolution_param { 336 | num_output: 48 337 | kernel_size: 3 338 | stride: 1 339 | weight_filler { 340 | type: "xavier" 341 | } 342 | bias_filler { 343 | type: "constant" 344 | value: 0 345 | } 346 | } 347 | 348 | } 349 | layer { 350 | name: "prelu2_3" 351 | type: "PReLU" 352 | bottom: "conv2_3" 353 | top: "conv2_3" 354 | } 355 | layer { 356 | name: "pool2_3" 357 | type: "Pooling" 358 | bottom: "conv2_3" 359 | top: "pool2_3" 360 | pooling_param { 361 | pool: MAX 362 | kernel_size: 3 363 | stride: 2 364 | } 365 | 366 | } 367 | layer { 368 | name: "conv3_3" 369 | type: "Convolution" 370 | bottom: "pool2_3" 371 | top: "conv3_3" 372 | param { 373 | lr_mult: 1 374 | decay_mult: 1 375 | } 376 | param { 377 | lr_mult: 2 378 | decay_mult: 1 379 | } 380 | convolution_param { 381 | num_output: 64 382 | kernel_size: 2 383 | stride: 1 384 | weight_filler { 385 | type: "xavier" 386 | } 387 | bias_filler { 388 | type: "constant" 389 | value: 0 390 | } 391 | } 392 | 393 | } 394 | layer { 395 | name: "prelu3_3" 396 | type: "PReLU" 397 | bottom: "conv3_3" 398 | top: "conv3_3" 399 | } 400 | ########################## 401 | ########################## 402 | layer { 403 | name: "conv1_4" 404 | type: "Convolution" 405 | bottom: "data244" 406 | top: "conv1_4" 407 | param { 408 | lr_mult: 1 409 | decay_mult: 1 410 | } 411 | param { 412 | lr_mult: 2 413 | decay_mult: 1 414 | } 415 | convolution_param { 416 | num_output: 28 417 | kernel_size: 3 418 | stride: 1 419 | weight_filler { 420 | type: "xavier" 421 | } 422 | bias_filler { 423 | type: "constant" 424 | value: 0 425 | } 426 | } 427 | 428 | } 429 | layer { 430 | name: "prelu1_4" 431 | type: "PReLU" 432 | bottom: "conv1_4" 433 | top: "conv1_4" 434 | 435 | } 436 | layer { 437 | name: "pool1_4" 438 | type: "Pooling" 439 | bottom: "conv1_4" 440 | top: "pool1_4" 441 | pooling_param { 442 | pool: MAX 443 | kernel_size: 3 444 | stride: 2 445 | } 446 | } 447 | 448 | layer { 449 | name: "conv2_4" 450 | type: "Convolution" 451 | bottom: "pool1_4" 452 | top: "conv2_4" 453 | param { 454 | lr_mult: 1 455 | decay_mult: 1 456 | } 457 | param { 458 | lr_mult: 2 459 | decay_mult: 1 460 | } 461 | convolution_param { 462 | num_output: 48 463 | kernel_size: 3 464 | stride: 1 465 | weight_filler { 466 | type: "xavier" 467 | } 468 | bias_filler { 469 | type: "constant" 470 | value: 0 471 | } 472 | } 473 | 474 | } 475 | layer { 476 | name: "prelu2_4" 477 | type: "PReLU" 478 | bottom: "conv2_4" 479 | top: "conv2_4" 480 | } 481 | layer { 482 | name: "pool2_4" 483 | type: "Pooling" 484 | bottom: "conv2_4" 485 | top: "pool2_4" 486 | pooling_param { 487 | pool: MAX 488 | kernel_size: 3 489 | stride: 2 490 | } 491 | 492 | } 493 | layer { 494 | name: "conv3_4" 495 | type: "Convolution" 496 | bottom: "pool2_4" 497 | top: "conv3_4" 498 | param { 499 | lr_mult: 1 500 | decay_mult: 1 501 | } 502 | param { 503 | lr_mult: 2 504 | decay_mult: 1 505 | } 506 | convolution_param { 507 | num_output: 64 508 | kernel_size: 2 509 | stride: 1 510 | weight_filler { 511 | type: "xavier" 512 | } 513 | bias_filler { 514 | type: "constant" 515 | value: 0 516 | } 517 | } 518 | 519 | } 520 | layer { 521 | name: "prelu3_4" 522 | type: "PReLU" 523 | bottom: "conv3_4" 524 | top: "conv3_4" 525 | } 526 | ########################## 527 | ########################## 528 | layer { 529 | name: "conv1_5" 530 | type: "Convolution" 531 | bottom: "data245" 532 | top: "conv1_5" 533 | param { 534 | lr_mult: 1 535 | decay_mult: 1 536 | } 537 | param { 538 | lr_mult: 2 539 | decay_mult: 1 540 | } 541 | convolution_param { 542 | num_output: 28 543 | kernel_size: 3 544 | stride: 1 545 | weight_filler { 546 | type: "xavier" 547 | } 548 | bias_filler { 549 | type: "constant" 550 | value: 0 551 | } 552 | } 553 | 554 | } 555 | layer { 556 | name: "prelu1_5" 557 | type: "PReLU" 558 | bottom: "conv1_5" 559 | top: "conv1_5" 560 | 561 | } 562 | layer { 563 | name: "pool1_5" 564 | type: "Pooling" 565 | bottom: "conv1_5" 566 | top: "pool1_5" 567 | pooling_param { 568 | pool: MAX 569 | kernel_size: 3 570 | stride: 2 571 | } 572 | } 573 | 574 | layer { 575 | name: "conv2_5" 576 | type: "Convolution" 577 | bottom: "pool1_5" 578 | top: "conv2_5" 579 | param { 580 | lr_mult: 1 581 | decay_mult: 1 582 | } 583 | param { 584 | lr_mult: 2 585 | decay_mult: 1 586 | } 587 | convolution_param { 588 | num_output: 48 589 | kernel_size: 3 590 | stride: 1 591 | weight_filler { 592 | type: "xavier" 593 | } 594 | bias_filler { 595 | type: "constant" 596 | value: 0 597 | } 598 | } 599 | 600 | } 601 | layer { 602 | name: "prelu2_5" 603 | type: "PReLU" 604 | bottom: "conv2_5" 605 | top: "conv2_5" 606 | } 607 | layer { 608 | name: "pool2_5" 609 | type: "Pooling" 610 | bottom: "conv2_5" 611 | top: "pool2_5" 612 | pooling_param { 613 | pool: MAX 614 | kernel_size: 3 615 | stride: 2 616 | } 617 | 618 | } 619 | layer { 620 | name: "conv3_5" 621 | type: "Convolution" 622 | bottom: "pool2_5" 623 | top: "conv3_5" 624 | param { 625 | lr_mult: 1 626 | decay_mult: 1 627 | } 628 | param { 629 | lr_mult: 2 630 | decay_mult: 1 631 | } 632 | convolution_param { 633 | num_output: 64 634 | kernel_size: 2 635 | stride: 1 636 | weight_filler { 637 | type: "xavier" 638 | } 639 | bias_filler { 640 | type: "constant" 641 | value: 0 642 | } 643 | } 644 | 645 | } 646 | layer { 647 | name: "prelu3_5" 648 | type: "PReLU" 649 | bottom: "conv3_5" 650 | top: "conv3_5" 651 | } 652 | ########################## 653 | layer { 654 | name: "concat" 655 | bottom: "conv3_1" 656 | bottom: "conv3_2" 657 | bottom: "conv3_3" 658 | bottom: "conv3_4" 659 | bottom: "conv3_5" 660 | top: "conv3" 661 | type: "Concat" 662 | concat_param { 663 | axis: 1 664 | } 665 | } 666 | ########################## 667 | layer { 668 | name: "fc4" 669 | type: "InnerProduct" 670 | bottom: "conv3" 671 | top: "fc4" 672 | param { 673 | lr_mult: 1 674 | decay_mult: 1 675 | } 676 | param { 677 | lr_mult: 2 678 | decay_mult: 1 679 | } 680 | inner_product_param { 681 | num_output: 256 682 | weight_filler { 683 | type: "xavier" 684 | } 685 | bias_filler { 686 | type: "constant" 687 | value: 0 688 | } 689 | } 690 | 691 | } 692 | layer { 693 | name: "prelu4" 694 | type: "PReLU" 695 | bottom: "fc4" 696 | top: "fc4" 697 | } 698 | ############################ 699 | layer { 700 | name: "fc4_1" 701 | type: "InnerProduct" 702 | bottom: "fc4" 703 | top: "fc4_1" 704 | param { 705 | lr_mult: 1 706 | decay_mult: 1 707 | } 708 | param { 709 | lr_mult: 2 710 | decay_mult: 1 711 | } 712 | inner_product_param { 713 | num_output: 64 714 | weight_filler { 715 | type: "xavier" 716 | } 717 | bias_filler { 718 | type: "constant" 719 | value: 0 720 | } 721 | } 722 | 723 | } 724 | layer { 725 | name: "prelu4_1" 726 | type: "PReLU" 727 | bottom: "fc4_1" 728 | top: "fc4_1" 729 | } 730 | layer { 731 | name: "fc5_1" 732 | type: "InnerProduct" 733 | bottom: "fc4_1" 734 | top: "fc5_1" 735 | param { 736 | lr_mult: 1 737 | decay_mult: 1 738 | } 739 | param { 740 | lr_mult: 2 741 | decay_mult: 1 742 | } 743 | inner_product_param { 744 | num_output: 2 745 | weight_filler { 746 | type: "xavier" 747 | #type: "constant" 748 | #value: 0 749 | } 750 | bias_filler { 751 | type: "constant" 752 | value: 0 753 | } 754 | } 755 | } 756 | 757 | 758 | ######################### 759 | layer { 760 | name: "fc4_2" 761 | type: "InnerProduct" 762 | bottom: "fc4" 763 | top: "fc4_2" 764 | param { 765 | lr_mult: 1 766 | decay_mult: 1 767 | } 768 | param { 769 | lr_mult: 2 770 | decay_mult: 1 771 | } 772 | inner_product_param { 773 | num_output: 64 774 | weight_filler { 775 | type: "xavier" 776 | } 777 | bias_filler { 778 | type: "constant" 779 | value: 0 780 | } 781 | } 782 | 783 | } 784 | layer { 785 | name: "prelu4_2" 786 | type: "PReLU" 787 | bottom: "fc4_2" 788 | top: "fc4_2" 789 | } 790 | layer { 791 | name: "fc5_2" 792 | type: "InnerProduct" 793 | bottom: "fc4_2" 794 | top: "fc5_2" 795 | param { 796 | lr_mult: 1 797 | decay_mult: 1 798 | } 799 | param { 800 | lr_mult: 2 801 | decay_mult: 1 802 | } 803 | inner_product_param { 804 | num_output: 2 805 | weight_filler { 806 | type: "xavier" 807 | #type: "constant" 808 | #value: 0 809 | } 810 | bias_filler { 811 | type: "constant" 812 | value: 0 813 | } 814 | } 815 | } 816 | 817 | ######################### 818 | layer { 819 | name: "fc4_3" 820 | type: "InnerProduct" 821 | bottom: "fc4" 822 | top: "fc4_3" 823 | param { 824 | lr_mult: 1 825 | decay_mult: 1 826 | } 827 | param { 828 | lr_mult: 2 829 | decay_mult: 1 830 | } 831 | inner_product_param { 832 | num_output: 64 833 | weight_filler { 834 | type: "xavier" 835 | } 836 | bias_filler { 837 | type: "constant" 838 | value: 0 839 | } 840 | } 841 | 842 | } 843 | layer { 844 | name: "prelu4_3" 845 | type: "PReLU" 846 | bottom: "fc4_3" 847 | top: "fc4_3" 848 | } 849 | layer { 850 | name: "fc5_3" 851 | type: "InnerProduct" 852 | bottom: "fc4_3" 853 | top: "fc5_3" 854 | param { 855 | lr_mult: 1 856 | decay_mult: 1 857 | } 858 | param { 859 | lr_mult: 2 860 | decay_mult: 1 861 | } 862 | inner_product_param { 863 | num_output: 2 864 | weight_filler { 865 | type: "xavier" 866 | #type: "constant" 867 | #value: 0 868 | } 869 | bias_filler { 870 | type: "constant" 871 | value: 0 872 | } 873 | } 874 | } 875 | 876 | ######################### 877 | layer { 878 | name: "fc4_4" 879 | type: "InnerProduct" 880 | bottom: "fc4" 881 | top: "fc4_4" 882 | param { 883 | lr_mult: 1 884 | decay_mult: 1 885 | } 886 | param { 887 | lr_mult: 2 888 | decay_mult: 1 889 | } 890 | inner_product_param { 891 | num_output: 64 892 | weight_filler { 893 | type: "xavier" 894 | } 895 | bias_filler { 896 | type: "constant" 897 | value: 0 898 | } 899 | } 900 | 901 | } 902 | layer { 903 | name: "prelu4_4" 904 | type: "PReLU" 905 | bottom: "fc4_4" 906 | top: "fc4_4" 907 | } 908 | layer { 909 | name: "fc5_4" 910 | type: "InnerProduct" 911 | bottom: "fc4_4" 912 | top: "fc5_4" 913 | param { 914 | lr_mult: 1 915 | decay_mult: 1 916 | } 917 | param { 918 | lr_mult: 2 919 | decay_mult: 1 920 | } 921 | inner_product_param { 922 | num_output: 2 923 | weight_filler { 924 | type: "xavier" 925 | #type: "constant" 926 | #value: 0 927 | } 928 | bias_filler { 929 | type: "constant" 930 | value: 0 931 | } 932 | } 933 | } 934 | 935 | ######################### 936 | layer { 937 | name: "fc4_5" 938 | type: "InnerProduct" 939 | bottom: "fc4" 940 | top: "fc4_5" 941 | param { 942 | lr_mult: 1 943 | decay_mult: 1 944 | } 945 | param { 946 | lr_mult: 2 947 | decay_mult: 1 948 | } 949 | inner_product_param { 950 | num_output: 64 951 | weight_filler { 952 | type: "xavier" 953 | } 954 | bias_filler { 955 | type: "constant" 956 | value: 0 957 | } 958 | } 959 | 960 | } 961 | layer { 962 | name: "prelu4_5" 963 | type: "PReLU" 964 | bottom: "fc4_5" 965 | top: "fc4_5" 966 | } 967 | layer { 968 | name: "fc5_5" 969 | type: "InnerProduct" 970 | bottom: "fc4_5" 971 | top: "fc5_5" 972 | param { 973 | lr_mult: 1 974 | decay_mult: 1 975 | } 976 | param { 977 | lr_mult: 2 978 | decay_mult: 1 979 | } 980 | inner_product_param { 981 | num_output: 2 982 | weight_filler { 983 | type: "xavier" 984 | #type: "constant" 985 | #value: 0 986 | } 987 | bias_filler { 988 | type: "constant" 989 | value: 0 990 | } 991 | } 992 | } 993 | 994 | ######################### 995 | 996 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/extract_weights_from_caffe_models.py: -------------------------------------------------------------------------------- 1 | import caffe 2 | import numpy as np 3 | 4 | """ 5 | The purpose of this script is to convert pretrained weights taken from 6 | official implementation here: 7 | https://github.com/kpzhang93/MTCNN_face_detection_alignment/tree/master/code/codes/MTCNNv2 8 | to required format. 9 | 10 | In a nutshell, it just renames and transposes some of the weights. 11 | You don't have to use this script because weights are already in `src/weights`. 12 | """ 13 | 14 | 15 | def get_all_weights(net): 16 | all_weights = {} 17 | for p in net.params: 18 | if 'conv' in p: 19 | name = 'features.' + p 20 | if '-' in p: 21 | s = list(p) 22 | s[-2] = '_' 23 | s = ''.join(s) 24 | all_weights[s + '.weight'] = net.params[p][0].data 25 | all_weights[s + '.bias'] = net.params[p][1].data 26 | elif len(net.params[p][0].data.shape) == 4: 27 | all_weights[name + '.weight'] = net.params[p][0].data.transpose((0, 1, 3, 2)) 28 | all_weights[name + '.bias'] = net.params[p][1].data 29 | else: 30 | all_weights[name + '.weight'] = net.params[p][0].data 31 | all_weights[name + '.bias'] = net.params[p][1].data 32 | elif 'prelu' in p.lower(): 33 | all_weights['features.' + p.lower() + '.weight'] = net.params[p][0].data 34 | return all_weights 35 | 36 | 37 | # P-Net 38 | net = caffe.Net('caffe_models/det1.prototxt', 'caffe_models/det1.caffemodel', caffe.TEST) 39 | np.save('src/weights/pnet.npy', get_all_weights(net)) 40 | 41 | # R-Net 42 | net = caffe.Net('caffe_models/det2.prototxt', 'caffe_models/det2.caffemodel', caffe.TEST) 43 | np.save('src/weights/rnet.npy', get_all_weights(net)) 44 | 45 | # O-Net 46 | net = caffe.Net('caffe_models/det3.prototxt', 'caffe_models/det3.caffemodel', caffe.TEST) 47 | np.save('src/weights/onet.npy', get_all_weights(net)) 48 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/__init__.py: -------------------------------------------------------------------------------- 1 | from .visualization_utils import show_bboxes 2 | from .detector import detect_faces 3 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/align_trans.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Apr 24 15:43:29 2017 4 | @author: zhaoy 5 | """ 6 | import numpy as np 7 | import cv2 8 | 9 | # from scipy.linalg import lstsq 10 | # from scipy.ndimage import geometric_transform # , map_coordinates 11 | 12 | from mtcnn_pytorch.src.matlab_cp2tform import get_similarity_transform_for_cv2 13 | 14 | # reference facial points, a list of coordinates (x,y) 15 | REFERENCE_FACIAL_POINTS = [ 16 | [30.29459953, 51.69630051], 17 | [65.53179932, 51.50139999], 18 | [48.02519989, 71.73660278], 19 | [33.54930115, 92.3655014], 20 | [62.72990036, 92.20410156] 21 | ] 22 | 23 | NEW_REFERENCE_FACIAL_POINTS = [ 24 | [25.29459953, 51.69630051], 25 | [60.53179932, 51.50139999], 26 | [43.02519989, 71.73660278], 27 | [28.54930115, 92.3655014 ], 28 | [57.72990036, 92.20410156] 29 | ] 30 | 31 | DEFAULT_CROP_SIZE = (96, 112) 32 | 33 | 34 | class FaceWarpException(Exception): 35 | def __str__(self): 36 | return 'In File {}:{}'.format( 37 | __file__, super.__str__(self)) 38 | 39 | 40 | def get_reference_facial_points(output_size=None, 41 | inner_padding_factor=0.0, 42 | outer_padding=(0, 0), 43 | default_square=False): 44 | """ 45 | Function: 46 | ---------- 47 | get reference 5 key points according to crop settings: 48 | 0. Set default crop_size: 49 | if default_square: 50 | crop_size = (112, 112) 51 | else: 52 | crop_size = (96, 112) 53 | 1. Pad the crop_size by inner_padding_factor in each side; 54 | 2. Resize crop_size into (output_size - outer_padding*2), 55 | pad into output_size with outer_padding; 56 | 3. Output reference_5point; 57 | Parameters: 58 | ---------- 59 | @output_size: (w, h) or None 60 | size of aligned face image 61 | @inner_padding_factor: (w_factor, h_factor) 62 | padding factor for inner (w, h) 63 | @outer_padding: (w_pad, h_pad) 64 | each row is a pair of coordinates (x, y) 65 | @default_square: True or False 66 | if True: 67 | default crop_size = (112, 112) 68 | else: 69 | default crop_size = (96, 112); 70 | !!! make sure, if output_size is not None: 71 | (output_size - outer_padding) 72 | = some_scale * (default crop_size * (1.0 + inner_padding_factor)) 73 | Returns: 74 | ---------- 75 | @reference_5point: 5x2 np.array 76 | each row is a pair of transformed coordinates (x, y) 77 | """ 78 | #print('\n===> get_reference_facial_points():') 79 | 80 | #print('---> Params:') 81 | #print(' output_size: ', output_size) 82 | #print(' inner_padding_factor: ', inner_padding_factor) 83 | #print(' outer_padding:', outer_padding) 84 | #print(' default_square: ', default_square) 85 | 86 | tmp_5pts = np.array(REFERENCE_FACIAL_POINTS) 87 | tmp_crop_size = np.array(DEFAULT_CROP_SIZE) 88 | 89 | # 0) make the inner region a square 90 | if default_square: 91 | size_diff = max(tmp_crop_size) - tmp_crop_size 92 | tmp_5pts += size_diff / 2 93 | tmp_crop_size += size_diff 94 | 95 | #print('---> default:') 96 | #print(' crop_size = ', tmp_crop_size) 97 | #print(' reference_5pts = ', tmp_5pts) 98 | 99 | if (output_size and 100 | output_size[0] == tmp_crop_size[0] and 101 | output_size[1] == tmp_crop_size[1]): 102 | # print('output_size == DEFAULT_CROP_SIZE {}: return default reference points'.format(tmp_crop_size)) 103 | return tmp_5pts 104 | 105 | if (inner_padding_factor == 0 and 106 | outer_padding == (0, 0)): 107 | if output_size is None: 108 | #print('No paddings to do: return default reference points') 109 | return tmp_5pts 110 | else: 111 | raise FaceWarpException( 112 | 'No paddings to do, output_size must be None or {}'.format(tmp_crop_size)) 113 | 114 | # check output size 115 | if not (0 <= inner_padding_factor <= 1.0): 116 | raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)') 117 | 118 | if ((inner_padding_factor > 0 or outer_padding[0] > 0 or outer_padding[1] > 0) 119 | and output_size is None): 120 | output_size = tmp_crop_size * \ 121 | (1 + inner_padding_factor * 2).astype(np.int32) 122 | output_size += np.array(outer_padding) 123 | #print(' deduced from paddings, output_size = ', output_size) 124 | 125 | if not (outer_padding[0] < output_size[0] 126 | and outer_padding[1] < output_size[1]): 127 | raise FaceWarpException('Not (outer_padding[0] < output_size[0]' 128 | 'and outer_padding[1] < output_size[1])') 129 | 130 | # 1) pad the inner region according inner_padding_factor 131 | #print('---> STEP1: pad the inner region according inner_padding_factor') 132 | if inner_padding_factor > 0: 133 | size_diff = tmp_crop_size * inner_padding_factor * 2 134 | tmp_5pts += size_diff / 2 135 | tmp_crop_size += np.round(size_diff).astype(np.int32) 136 | 137 | #print(' crop_size = ', tmp_crop_size) 138 | #print(' reference_5pts = ', tmp_5pts) 139 | 140 | # 2) resize the padded inner region 141 | #print('---> STEP2: resize the padded inner region') 142 | size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2 143 | #print(' crop_size = ', tmp_crop_size) 144 | #print(' size_bf_outer_pad = ', size_bf_outer_pad) 145 | 146 | if size_bf_outer_pad[0] * tmp_crop_size[1] != size_bf_outer_pad[1] * tmp_crop_size[0]: 147 | raise FaceWarpException('Must have (output_size - outer_padding)' 148 | '= some_scale * (crop_size * (1.0 + inner_padding_factor)') 149 | 150 | scale_factor = size_bf_outer_pad[0].astype(np.float32) / tmp_crop_size[0] 151 | #print(' resize scale_factor = ', scale_factor) 152 | tmp_5pts = tmp_5pts * scale_factor 153 | # size_diff = tmp_crop_size * (scale_factor - min(scale_factor)) 154 | # tmp_5pts = tmp_5pts + size_diff / 2 155 | tmp_crop_size = size_bf_outer_pad 156 | #print(' crop_size = ', tmp_crop_size) 157 | #print(' reference_5pts = ', tmp_5pts) 158 | 159 | # 3) add outer_padding to make output_size 160 | reference_5point = tmp_5pts + np.array(outer_padding) 161 | tmp_crop_size = output_size 162 | #print('---> STEP3: add outer_padding to make output_size') 163 | #print(' crop_size = ', tmp_crop_size) 164 | #print(' reference_5pts = ', tmp_5pts) 165 | 166 | #print('===> end get_reference_facial_points\n') 167 | 168 | return reference_5point 169 | 170 | 171 | def get_affine_transform_matrix(src_pts, dst_pts): 172 | """ 173 | Function: 174 | ---------- 175 | get affine transform matrix 'tfm' from src_pts to dst_pts 176 | Parameters: 177 | ---------- 178 | @src_pts: Kx2 np.array 179 | source points matrix, each row is a pair of coordinates (x, y) 180 | @dst_pts: Kx2 np.array 181 | destination points matrix, each row is a pair of coordinates (x, y) 182 | Returns: 183 | ---------- 184 | @tfm: 2x3 np.array 185 | transform matrix from src_pts to dst_pts 186 | """ 187 | 188 | tfm = np.float32([[1, 0, 0], [0, 1, 0]]) 189 | n_pts = src_pts.shape[0] 190 | ones = np.ones((n_pts, 1), src_pts.dtype) 191 | src_pts_ = np.hstack([src_pts, ones]) 192 | dst_pts_ = np.hstack([dst_pts, ones]) 193 | 194 | # #print(('src_pts_:\n' + str(src_pts_)) 195 | # #print(('dst_pts_:\n' + str(dst_pts_)) 196 | 197 | A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_) 198 | 199 | # #print(('np.linalg.lstsq return A: \n' + str(A)) 200 | # #print(('np.linalg.lstsq return res: \n' + str(res)) 201 | # #print(('np.linalg.lstsq return rank: \n' + str(rank)) 202 | # #print(('np.linalg.lstsq return s: \n' + str(s)) 203 | 204 | if rank == 3: 205 | tfm = np.float32([ 206 | [A[0, 0], A[1, 0], A[2, 0]], 207 | [A[0, 1], A[1, 1], A[2, 1]] 208 | ]) 209 | elif rank == 2: 210 | tfm = np.float32([ 211 | [A[0, 0], A[1, 0], 0], 212 | [A[0, 1], A[1, 1], 0] 213 | ]) 214 | 215 | return tfm 216 | 217 | 218 | def warp_and_crop_face(src_img, 219 | facial_pts, 220 | reference_pts=None, 221 | crop_size=(96, 112), 222 | align_type='smilarity', 223 | return_trans_inv=False, mode=None): 224 | """ 225 | Function: 226 | ---------- 227 | apply affine transform 'trans' to uv 228 | Parameters: 229 | ---------- 230 | @src_img: 3x3 np.array 231 | input image 232 | @facial_pts: could be 233 | 1)a list of K coordinates (x,y) 234 | or 235 | 2) Kx2 or 2xK np.array 236 | each row or col is a pair of coordinates (x, y) 237 | @reference_pts: could be 238 | 1) a list of K coordinates (x,y) 239 | or 240 | 2) Kx2 or 2xK np.array 241 | each row or col is a pair of coordinates (x, y) 242 | or 243 | 3) None 244 | if None, use default reference facial points 245 | @crop_size: (w, h) 246 | output face image size 247 | @align_type: transform type, could be one of 248 | 1) 'similarity': use similarity transform 249 | 2) 'cv2_affine': use the first 3 points to do affine transform, 250 | by calling cv2.getAffineTransform() 251 | 3) 'affine': use all points to do affine transform 252 | Returns: 253 | ---------- 254 | @face_img: output face image with size (w, h) = @crop_size 255 | """ 256 | 257 | if reference_pts is None: 258 | if crop_size[0] == 96 and crop_size[1] == 112: 259 | reference_pts = REFERENCE_FACIAL_POINTS 260 | else: 261 | default_square = False 262 | inner_padding_factor = 0 263 | outer_padding = (0, 0) 264 | output_size = crop_size 265 | 266 | reference_pts = get_reference_facial_points(output_size, 267 | inner_padding_factor, 268 | outer_padding, 269 | default_square) 270 | 271 | # reference_pts: 272 | # [[38.29459953 51.69630051] 273 | # [73.53179932 51.50139999] 274 | # [56.02519989 71.73660278] 275 | # [41.54930115 92.3655014 ] 276 | # [70.72990036 92.20410156]] 277 | ref_pts = np.float32(reference_pts) 278 | 279 | if crop_size[0] == 96 and crop_size[1] == 112: 280 | ref_pts[:, 0] = (ref_pts[:, 0] - crop_size[0] / 2) * 0.85 + crop_size[0] / 2 281 | ref_pts[:, 1] = (ref_pts[:, 1] - crop_size[1] / 2) * 0.85 + crop_size[1] / 2 282 | else: 283 | ref_pts = (ref_pts - 112/2)*0.85 + 112/2 284 | ref_pts *= crop_size[0]/112. 285 | 286 | ref_pts_shp = ref_pts.shape 287 | if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: 288 | raise FaceWarpException( 289 | 'reference_pts.shape must be (K,2) or (2,K) and K>2') 290 | if ref_pts_shp[0] == 2: 291 | ref_pts = ref_pts.T 292 | 293 | src_pts = np.float32(facial_pts) 294 | src_pts_shp = src_pts.shape 295 | if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: 296 | raise FaceWarpException( 297 | 'facial_pts.shape must be (K,2) or (2,K) and K>2') 298 | if src_pts_shp[0] == 2: 299 | src_pts = src_pts.T 300 | 301 | # #print('--->src_pts:\n', src_pts 302 | # #print('--->ref_pts\n', ref_pts 303 | 304 | if src_pts.shape != ref_pts.shape: 305 | raise FaceWarpException( 306 | 'facial_pts and reference_pts must have the same shape') 307 | 308 | if align_type is 'cv2_affine': 309 | tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) 310 | # cv2.getPerspectiveTransform() 311 | # #print(('cv2.getAffineTransform() returns tfm=\n' + str(tfm)) 312 | elif align_type is 'affine': 313 | tfm = get_affine_transform_matrix(src_pts, ref_pts) 314 | # #print(('get_affine_transform_matrix() returns tfm=\n' + str(tfm)) 315 | else: 316 | tfm, tfm_inv = get_similarity_transform_for_cv2(src_pts, ref_pts) 317 | # #print(('get_similarity_transform_for_cv2() returns tfm=\n' + str(tfm)) 318 | 319 | # #print('--->Transform matrix: ' 320 | # #print(('type(tfm):' + str(type(tfm))) 321 | # #print(('tfm.dtype:' + str(tfm.dtype)) 322 | # #print( tfm 323 | 324 | face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1])) 325 | 326 | if return_trans_inv and align_type is 'smilarity': 327 | return face_img, tfm_inv 328 | else: 329 | return face_img 330 | 331 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/box_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from PIL import Image 3 | 4 | 5 | def nms(boxes, overlap_threshold=0.5, mode='union'): 6 | """Non-maximum suppression. 7 | 8 | Arguments: 9 | boxes: a float numpy array of shape [n, 5], 10 | where each row is (xmin, ymin, xmax, ymax, score). 11 | overlap_threshold: a float number. 12 | mode: 'union' or 'min'. 13 | 14 | Returns: 15 | list with indices of the selected boxes 16 | """ 17 | 18 | # if there are no boxes, return the empty list 19 | if len(boxes) == 0: 20 | return [] 21 | 22 | # list of picked indices 23 | pick = [] 24 | 25 | # grab the coordinates of the bounding boxes 26 | x1, y1, x2, y2, score = [boxes[:, i] for i in range(5)] 27 | 28 | area = (x2 - x1 + 1.0)*(y2 - y1 + 1.0) 29 | ids = np.argsort(score) # in increasing order 30 | 31 | while len(ids) > 0: 32 | 33 | # grab index of the largest value 34 | last = len(ids) - 1 35 | i = ids[last] 36 | pick.append(i) 37 | 38 | # compute intersections 39 | # of the box with the largest score 40 | # with the rest of boxes 41 | 42 | # left top corner of intersection boxes 43 | ix1 = np.maximum(x1[i], x1[ids[:last]]) 44 | iy1 = np.maximum(y1[i], y1[ids[:last]]) 45 | 46 | # right bottom corner of intersection boxes 47 | ix2 = np.minimum(x2[i], x2[ids[:last]]) 48 | iy2 = np.minimum(y2[i], y2[ids[:last]]) 49 | 50 | # width and height of intersection boxes 51 | w = np.maximum(0.0, ix2 - ix1 + 1.0) 52 | h = np.maximum(0.0, iy2 - iy1 + 1.0) 53 | 54 | # intersections' areas 55 | inter = w * h 56 | if mode == 'min': 57 | overlap = inter/np.minimum(area[i], area[ids[:last]]) 58 | elif mode == 'union': 59 | # intersection over union (IoU) 60 | overlap = inter/(area[i] + area[ids[:last]] - inter) 61 | 62 | # delete all boxes where overlap is too big 63 | ids = np.delete( 64 | ids, 65 | np.concatenate([[last], np.where(overlap > overlap_threshold)[0]]) 66 | ) 67 | 68 | return pick 69 | 70 | 71 | def convert_to_square(bboxes): 72 | """Convert bounding boxes to a square form. 73 | 74 | Arguments: 75 | bboxes: a float numpy array of shape [n, 5]. 76 | 77 | Returns: 78 | a float numpy array of shape [n, 5], 79 | squared bounding boxes. 80 | """ 81 | 82 | square_bboxes = np.zeros_like(bboxes) 83 | x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] 84 | h = y2 - y1 + 1.0 85 | w = x2 - x1 + 1.0 86 | max_side = np.maximum(h, w) 87 | square_bboxes[:, 0] = x1 + w*0.5 - max_side*0.5 88 | square_bboxes[:, 1] = y1 + h*0.5 - max_side*0.5 89 | square_bboxes[:, 2] = square_bboxes[:, 0] + max_side - 1.0 90 | square_bboxes[:, 3] = square_bboxes[:, 1] + max_side - 1.0 91 | return square_bboxes 92 | 93 | 94 | def calibrate_box(bboxes, offsets): 95 | """Transform bounding boxes to be more like true bounding boxes. 96 | 'offsets' is one of the outputs of the nets. 97 | 98 | Arguments: 99 | bboxes: a float numpy array of shape [n, 5]. 100 | offsets: a float numpy array of shape [n, 4]. 101 | 102 | Returns: 103 | a float numpy array of shape [n, 5]. 104 | """ 105 | x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] 106 | w = x2 - x1 + 1.0 107 | h = y2 - y1 + 1.0 108 | w = np.expand_dims(w, 1) 109 | h = np.expand_dims(h, 1) 110 | 111 | # this is what happening here: 112 | # tx1, ty1, tx2, ty2 = [offsets[:, i] for i in range(4)] 113 | # x1_true = x1 + tx1*w 114 | # y1_true = y1 + ty1*h 115 | # x2_true = x2 + tx2*w 116 | # y2_true = y2 + ty2*h 117 | # below is just more compact form of this 118 | 119 | # are offsets always such that 120 | # x1 < x2 and y1 < y2 ? 121 | 122 | translation = np.hstack([w, h, w, h])*offsets 123 | bboxes[:, 0:4] = bboxes[:, 0:4] + translation 124 | return bboxes 125 | 126 | 127 | def get_image_boxes(bounding_boxes, img, size=24): 128 | """Cut out boxes from the image. 129 | 130 | Arguments: 131 | bounding_boxes: a float numpy array of shape [n, 5]. 132 | img: an instance of PIL.Image. 133 | size: an integer, size of cutouts. 134 | 135 | Returns: 136 | a float numpy array of shape [n, 3, size, size]. 137 | """ 138 | 139 | num_boxes = len(bounding_boxes) 140 | width, height = img.size 141 | 142 | [dy, edy, dx, edx, y, ey, x, ex, w, h] = correct_bboxes(bounding_boxes, width, height) 143 | img_boxes = np.zeros((num_boxes, 3, size, size), 'float32') 144 | 145 | for i in range(num_boxes): 146 | img_box = np.zeros((h[i], w[i], 3), 'uint8') 147 | 148 | img_array = np.asarray(img, 'uint8') 149 | img_box[dy[i]:(edy[i] + 1), dx[i]:(edx[i] + 1), :] =\ 150 | img_array[y[i]:(ey[i] + 1), x[i]:(ex[i] + 1), :] 151 | 152 | # resize 153 | img_box = Image.fromarray(img_box) 154 | img_box = img_box.resize((size, size), Image.BILINEAR) 155 | img_box = np.asarray(img_box, 'float32') 156 | 157 | img_boxes[i, :, :, :] = _preprocess(img_box) 158 | 159 | return img_boxes 160 | 161 | 162 | def correct_bboxes(bboxes, width, height): 163 | """Crop boxes that are too big and get coordinates 164 | with respect to cutouts. 165 | 166 | Arguments: 167 | bboxes: a float numpy array of shape [n, 5], 168 | where each row is (xmin, ymin, xmax, ymax, score). 169 | width: a float number. 170 | height: a float number. 171 | 172 | Returns: 173 | dy, dx, edy, edx: a int numpy arrays of shape [n], 174 | coordinates of the boxes with respect to the cutouts. 175 | y, x, ey, ex: a int numpy arrays of shape [n], 176 | corrected ymin, xmin, ymax, xmax. 177 | h, w: a int numpy arrays of shape [n], 178 | just heights and widths of boxes. 179 | 180 | in the following order: 181 | [dy, edy, dx, edx, y, ey, x, ex, w, h]. 182 | """ 183 | 184 | x1, y1, x2, y2 = [bboxes[:, i] for i in range(4)] 185 | w, h = x2 - x1 + 1.0, y2 - y1 + 1.0 186 | num_boxes = bboxes.shape[0] 187 | 188 | # 'e' stands for end 189 | # (x, y) -> (ex, ey) 190 | x, y, ex, ey = x1, y1, x2, y2 191 | 192 | # we need to cut out a box from the image. 193 | # (x, y, ex, ey) are corrected coordinates of the box 194 | # in the image. 195 | # (dx, dy, edx, edy) are coordinates of the box in the cutout 196 | # from the image. 197 | dx, dy = np.zeros((num_boxes,)), np.zeros((num_boxes,)) 198 | edx, edy = w.copy() - 1.0, h.copy() - 1.0 199 | 200 | # if box's bottom right corner is too far right 201 | ind = np.where(ex > width - 1.0)[0] 202 | edx[ind] = w[ind] + width - 2.0 - ex[ind] 203 | ex[ind] = width - 1.0 204 | 205 | # if box's bottom right corner is too low 206 | ind = np.where(ey > height - 1.0)[0] 207 | edy[ind] = h[ind] + height - 2.0 - ey[ind] 208 | ey[ind] = height - 1.0 209 | 210 | # if box's top left corner is too far left 211 | ind = np.where(x < 0.0)[0] 212 | dx[ind] = 0.0 - x[ind] 213 | x[ind] = 0.0 214 | 215 | # if box's top left corner is too high 216 | ind = np.where(y < 0.0)[0] 217 | dy[ind] = 0.0 - y[ind] 218 | y[ind] = 0.0 219 | 220 | return_list = [dy, edy, dx, edx, y, ey, x, ex, w, h] 221 | return_list = [i.astype('int32') for i in return_list] 222 | 223 | return return_list 224 | 225 | 226 | def _preprocess(img): 227 | """Preprocessing step before feeding the network. 228 | 229 | Arguments: 230 | img: a float numpy array of shape [h, w, c]. 231 | 232 | Returns: 233 | a float numpy array of shape [1, c, h, w]. 234 | """ 235 | img = img.transpose((2, 0, 1)) 236 | img = np.expand_dims(img, 0) 237 | img = (img - 127.5)*0.0078125 238 | return img 239 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/detector.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch.autograd import Variable 4 | from .get_nets import PNet, RNet, ONet 5 | from .box_utils import nms, calibrate_box, get_image_boxes, convert_to_square 6 | from .first_stage import run_first_stage 7 | 8 | 9 | def detect_faces(image, min_face_size=20.0, 10 | thresholds=[0.6, 0.7, 0.8], 11 | nms_thresholds=[0.7, 0.7, 0.7]): 12 | """ 13 | Arguments: 14 | image: an instance of PIL.Image. 15 | min_face_size: a float number. 16 | thresholds: a list of length 3. 17 | nms_thresholds: a list of length 3. 18 | 19 | Returns: 20 | two float numpy arrays of shapes [n_boxes, 4] and [n_boxes, 10], 21 | bounding boxes and facial landmarks. 22 | """ 23 | 24 | # LOAD MODELS 25 | pnet = PNet() 26 | rnet = RNet() 27 | onet = ONet() 28 | onet.eval() 29 | 30 | # BUILD AN IMAGE PYRAMID 31 | width, height = image.size 32 | min_length = min(height, width) 33 | 34 | min_detection_size = 12 35 | factor = 0.707 # sqrt(0.5) 36 | 37 | # scales for scaling the image 38 | scales = [] 39 | 40 | # scales the image so that 41 | # minimum size that we can detect equals to 42 | # minimum face size that we want to detect 43 | m = min_detection_size/min_face_size 44 | min_length *= m 45 | 46 | factor_count = 0 47 | while min_length > min_detection_size: 48 | scales.append(m*factor**factor_count) 49 | min_length *= factor 50 | factor_count += 1 51 | 52 | # STAGE 1 53 | 54 | # it will be returned 55 | bounding_boxes = [] 56 | 57 | with torch.no_grad(): 58 | # run P-Net on different scales 59 | for s in scales: 60 | boxes = run_first_stage(image, pnet, scale=s, threshold=thresholds[0]) 61 | bounding_boxes.append(boxes) 62 | 63 | # collect boxes (and offsets, and scores) from different scales 64 | bounding_boxes = [i for i in bounding_boxes if i is not None] 65 | bounding_boxes = np.vstack(bounding_boxes) 66 | 67 | keep = nms(bounding_boxes[:, 0:5], nms_thresholds[0]) 68 | bounding_boxes = bounding_boxes[keep] 69 | 70 | # use offsets predicted by pnet to transform bounding boxes 71 | bounding_boxes = calibrate_box(bounding_boxes[:, 0:5], bounding_boxes[:, 5:]) 72 | # shape [n_boxes, 5] 73 | 74 | bounding_boxes = convert_to_square(bounding_boxes) 75 | bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) 76 | 77 | # STAGE 2 78 | 79 | img_boxes = get_image_boxes(bounding_boxes, image, size=24) 80 | img_boxes = torch.FloatTensor(img_boxes) 81 | 82 | output = rnet(img_boxes) 83 | offsets = output[0].data.numpy() # shape [n_boxes, 4] 84 | probs = output[1].data.numpy() # shape [n_boxes, 2] 85 | 86 | keep = np.where(probs[:, 1] > thresholds[1])[0] 87 | bounding_boxes = bounding_boxes[keep] 88 | bounding_boxes[:, 4] = probs[keep, 1].reshape((-1,)) 89 | offsets = offsets[keep] 90 | 91 | keep = nms(bounding_boxes, nms_thresholds[1]) 92 | bounding_boxes = bounding_boxes[keep] 93 | bounding_boxes = calibrate_box(bounding_boxes, offsets[keep]) 94 | bounding_boxes = convert_to_square(bounding_boxes) 95 | bounding_boxes[:, 0:4] = np.round(bounding_boxes[:, 0:4]) 96 | 97 | # STAGE 3 98 | 99 | img_boxes = get_image_boxes(bounding_boxes, image, size=48) 100 | if len(img_boxes) == 0: 101 | return [], [] 102 | img_boxes = torch.FloatTensor(img_boxes) 103 | output = onet(img_boxes) 104 | landmarks = output[0].data.numpy() # shape [n_boxes, 10] 105 | offsets = output[1].data.numpy() # shape [n_boxes, 4] 106 | probs = output[2].data.numpy() # shape [n_boxes, 2] 107 | 108 | keep = np.where(probs[:, 1] > thresholds[2])[0] 109 | bounding_boxes = bounding_boxes[keep] 110 | bounding_boxes[:, 4] = probs[keep, 1].reshape((-1,)) 111 | offsets = offsets[keep] 112 | landmarks = landmarks[keep] 113 | 114 | # compute landmark points 115 | width = bounding_boxes[:, 2] - bounding_boxes[:, 0] + 1.0 116 | height = bounding_boxes[:, 3] - bounding_boxes[:, 1] + 1.0 117 | xmin, ymin = bounding_boxes[:, 0], bounding_boxes[:, 1] 118 | landmarks[:, 0:5] = np.expand_dims(xmin, 1) + np.expand_dims(width, 1)*landmarks[:, 0:5] 119 | landmarks[:, 5:10] = np.expand_dims(ymin, 1) + np.expand_dims(height, 1)*landmarks[:, 5:10] 120 | 121 | bounding_boxes = calibrate_box(bounding_boxes, offsets) 122 | keep = nms(bounding_boxes, nms_thresholds[2], mode='min') 123 | bounding_boxes = bounding_boxes[keep] 124 | landmarks = landmarks[keep] 125 | 126 | return bounding_boxes, landmarks 127 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/first_stage.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | import math 4 | from PIL import Image 5 | import numpy as np 6 | from .box_utils import nms, _preprocess 7 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 8 | # device = 'cpu' 9 | 10 | def run_first_stage(image, net, scale, threshold): 11 | """Run P-Net, generate bounding boxes, and do NMS. 12 | 13 | Arguments: 14 | image: an instance of PIL.Image. 15 | net: an instance of pytorch's nn.Module, P-Net. 16 | scale: a float number, 17 | scale width and height of the image by this number. 18 | threshold: a float number, 19 | threshold on the probability of a face when generating 20 | bounding boxes from predictions of the net. 21 | 22 | Returns: 23 | a float numpy array of shape [n_boxes, 9], 24 | bounding boxes with scores and offsets (4 + 1 + 4). 25 | """ 26 | 27 | # scale the image and convert it to a float array 28 | width, height = image.size 29 | sw, sh = math.ceil(width*scale), math.ceil(height*scale) 30 | img = image.resize((sw, sh), Image.BILINEAR) 31 | img = np.asarray(img, 'float32') 32 | 33 | img = torch.FloatTensor(_preprocess(img)).to(device) 34 | with torch.no_grad(): 35 | output = net(img) 36 | probs = output[1].cpu().data.numpy()[0, 1, :, :] 37 | offsets = output[0].cpu().data.numpy() 38 | # probs: probability of a face at each sliding window 39 | # offsets: transformations to true bounding boxes 40 | 41 | boxes = _generate_bboxes(probs, offsets, scale, threshold) 42 | if len(boxes) == 0: 43 | return None 44 | 45 | keep = nms(boxes[:, 0:5], overlap_threshold=0.5) 46 | return boxes[keep] 47 | 48 | 49 | def _generate_bboxes(probs, offsets, scale, threshold): 50 | """Generate bounding boxes at places 51 | where there is probably a face. 52 | 53 | Arguments: 54 | probs: a float numpy array of shape [n, m]. 55 | offsets: a float numpy array of shape [1, 4, n, m]. 56 | scale: a float number, 57 | width and height of the image were scaled by this number. 58 | threshold: a float number. 59 | 60 | Returns: 61 | a float numpy array of shape [n_boxes, 9] 62 | """ 63 | 64 | # applying P-Net is equivalent, in some sense, to 65 | # moving 12x12 window with stride 2 66 | stride = 2 67 | cell_size = 12 68 | 69 | # indices of boxes where there is probably a face 70 | inds = np.where(probs > threshold) 71 | 72 | if inds[0].size == 0: 73 | return np.array([]) 74 | 75 | # transformations of bounding boxes 76 | tx1, ty1, tx2, ty2 = [offsets[0, i, inds[0], inds[1]] for i in range(4)] 77 | # they are defined as: 78 | # w = x2 - x1 + 1 79 | # h = y2 - y1 + 1 80 | # x1_true = x1 + tx1*w 81 | # x2_true = x2 + tx2*w 82 | # y1_true = y1 + ty1*h 83 | # y2_true = y2 + ty2*h 84 | 85 | offsets = np.array([tx1, ty1, tx2, ty2]) 86 | score = probs[inds[0], inds[1]] 87 | 88 | # P-Net is applied to scaled images 89 | # so we need to rescale bounding boxes back 90 | bounding_boxes = np.vstack([ 91 | np.round((stride*inds[1] + 1.0)/scale), 92 | np.round((stride*inds[0] + 1.0)/scale), 93 | np.round((stride*inds[1] + 1.0 + cell_size)/scale), 94 | np.round((stride*inds[0] + 1.0 + cell_size)/scale), 95 | score, offsets 96 | ]) 97 | # why one is added? 98 | 99 | return bounding_boxes.T 100 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/get_nets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from collections import OrderedDict 5 | import numpy as np 6 | 7 | 8 | class Flatten(nn.Module): 9 | 10 | def __init__(self): 11 | super(Flatten, self).__init__() 12 | 13 | def forward(self, x): 14 | """ 15 | Arguments: 16 | x: a float tensor with shape [batch_size, c, h, w]. 17 | Returns: 18 | a float tensor with shape [batch_size, c*h*w]. 19 | """ 20 | 21 | # without this pretrained encoder isn't working 22 | x = x.transpose(3, 2).contiguous() 23 | 24 | return x.view(x.size(0), -1) 25 | 26 | 27 | class PNet(nn.Module): 28 | 29 | def __init__(self): 30 | 31 | super(PNet, self).__init__() 32 | 33 | # suppose we have input with size HxW, then 34 | # after first layer: H - 2, 35 | # after pool: ceil((H - 2)/2), 36 | # after second conv: ceil((H - 2)/2) - 2, 37 | # after last conv: ceil((H - 2)/2) - 4, 38 | # and the same for W 39 | 40 | self.features = nn.Sequential(OrderedDict([ 41 | ('conv1', nn.Conv2d(3, 10, 3, 1)), 42 | ('prelu1', nn.PReLU(10)), 43 | ('pool1', nn.MaxPool2d(2, 2, ceil_mode=True)), 44 | 45 | ('conv2', nn.Conv2d(10, 16, 3, 1)), 46 | ('prelu2', nn.PReLU(16)), 47 | 48 | ('conv3', nn.Conv2d(16, 32, 3, 1)), 49 | ('prelu3', nn.PReLU(32)) 50 | ])) 51 | 52 | self.conv4_1 = nn.Conv2d(32, 2, 1, 1) 53 | self.conv4_2 = nn.Conv2d(32, 4, 1, 1) 54 | 55 | weights = np.load('./preprocess/mtcnn_pytorch/src/weights/pnet.npy', 56 | allow_pickle=True)[()] 57 | for n, p in self.named_parameters(): 58 | p.data = torch.FloatTensor(weights[n]) 59 | 60 | def forward(self, x): 61 | """ 62 | Arguments: 63 | x: a float tensor with shape [batch_size, 3, h, w]. 64 | Returns: 65 | b: a float tensor with shape [batch_size, 4, h', w']. 66 | a: a float tensor with shape [batch_size, 2, h', w']. 67 | """ 68 | x = self.features(x) 69 | a = self.conv4_1(x) 70 | b = self.conv4_2(x) 71 | a = F.softmax(a, dim=-1) 72 | return b, a 73 | 74 | 75 | class RNet(nn.Module): 76 | 77 | def __init__(self): 78 | 79 | super(RNet, self).__init__() 80 | 81 | self.features = nn.Sequential(OrderedDict([ 82 | ('conv1', nn.Conv2d(3, 28, 3, 1)), 83 | ('prelu1', nn.PReLU(28)), 84 | ('pool1', nn.MaxPool2d(3, 2, ceil_mode=True)), 85 | 86 | ('conv2', nn.Conv2d(28, 48, 3, 1)), 87 | ('prelu2', nn.PReLU(48)), 88 | ('pool2', nn.MaxPool2d(3, 2, ceil_mode=True)), 89 | 90 | ('conv3', nn.Conv2d(48, 64, 2, 1)), 91 | ('prelu3', nn.PReLU(64)), 92 | 93 | ('flatten', Flatten()), 94 | ('conv4', nn.Linear(576, 128)), 95 | ('prelu4', nn.PReLU(128)) 96 | ])) 97 | 98 | self.conv5_1 = nn.Linear(128, 2) 99 | self.conv5_2 = nn.Linear(128, 4) 100 | 101 | weights = np.load('./preprocess/mtcnn_pytorch/src/weights/rnet.npy', 102 | allow_pickle=True)[()] 103 | for n, p in self.named_parameters(): 104 | p.data = torch.FloatTensor(weights[n]) 105 | 106 | def forward(self, x): 107 | """ 108 | Arguments: 109 | x: a float tensor with shape [batch_size, 3, h, w]. 110 | Returns: 111 | b: a float tensor with shape [batch_size, 4]. 112 | a: a float tensor with shape [batch_size, 2]. 113 | """ 114 | x = self.features(x) 115 | a = self.conv5_1(x) 116 | b = self.conv5_2(x) 117 | a = F.softmax(a, dim=-1) 118 | return b, a 119 | 120 | 121 | class ONet(nn.Module): 122 | 123 | def __init__(self): 124 | 125 | super(ONet, self).__init__() 126 | 127 | self.features = nn.Sequential(OrderedDict([ 128 | ('conv1', nn.Conv2d(3, 32, 3, 1)), 129 | ('prelu1', nn.PReLU(32)), 130 | ('pool1', nn.MaxPool2d(3, 2, ceil_mode=True)), 131 | 132 | ('conv2', nn.Conv2d(32, 64, 3, 1)), 133 | ('prelu2', nn.PReLU(64)), 134 | ('pool2', nn.MaxPool2d(3, 2, ceil_mode=True)), 135 | 136 | ('conv3', nn.Conv2d(64, 64, 3, 1)), 137 | ('prelu3', nn.PReLU(64)), 138 | ('pool3', nn.MaxPool2d(2, 2, ceil_mode=True)), 139 | 140 | ('conv4', nn.Conv2d(64, 128, 2, 1)), 141 | ('prelu4', nn.PReLU(128)), 142 | 143 | ('flatten', Flatten()), 144 | ('conv5', nn.Linear(1152, 256)), 145 | ('drop5', nn.Dropout(0.25)), 146 | ('prelu5', nn.PReLU(256)), 147 | ])) 148 | 149 | self.conv6_1 = nn.Linear(256, 2) 150 | self.conv6_2 = nn.Linear(256, 4) 151 | self.conv6_3 = nn.Linear(256, 10) 152 | 153 | weights = np.load('./preprocess/mtcnn_pytorch/src/weights/onet.npy', 154 | allow_pickle=True)[()] 155 | for n, p in self.named_parameters(): 156 | p.data = torch.FloatTensor(weights[n]) 157 | 158 | def forward(self, x): 159 | """ 160 | Arguments: 161 | x: a float tensor with shape [batch_size, 3, h, w]. 162 | Returns: 163 | c: a float tensor with shape [batch_size, 10]. 164 | b: a float tensor with shape [batch_size, 4]. 165 | a: a float tensor with shape [batch_size, 2]. 166 | """ 167 | x = self.features(x) 168 | a = self.conv6_1(x) 169 | b = self.conv6_2(x) 170 | c = self.conv6_3(x) 171 | a = F.softmax(a, dim = -1) 172 | return c, b, a 173 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/matlab_cp2tform.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Jul 11 06:54:28 2017 4 | 5 | @author: zhaoyafei 6 | """ 7 | 8 | import numpy as np 9 | from numpy.linalg import inv, norm, lstsq 10 | from numpy.linalg import matrix_rank as rank 11 | 12 | class MatlabCp2tormException(Exception): 13 | def __str__(self): 14 | return 'In File {}:{}'.format( 15 | __file__, super.__str__(self)) 16 | 17 | def tformfwd(trans, uv): 18 | """ 19 | Function: 20 | ---------- 21 | apply affine transform 'trans' to uv 22 | 23 | Parameters: 24 | ---------- 25 | @trans: 3x3 np.array 26 | transform matrix 27 | @uv: Kx2 np.array 28 | each row is a pair of coordinates (x, y) 29 | 30 | Returns: 31 | ---------- 32 | @xy: Kx2 np.array 33 | each row is a pair of transformed coordinates (x, y) 34 | """ 35 | uv = np.hstack(( 36 | uv, np.ones((uv.shape[0], 1)) 37 | )) 38 | xy = np.dot(uv, trans) 39 | xy = xy[:, 0:-1] 40 | return xy 41 | 42 | 43 | def tforminv(trans, uv): 44 | """ 45 | Function: 46 | ---------- 47 | apply the inverse of affine transform 'trans' to uv 48 | 49 | Parameters: 50 | ---------- 51 | @trans: 3x3 np.array 52 | transform matrix 53 | @uv: Kx2 np.array 54 | each row is a pair of coordinates (x, y) 55 | 56 | Returns: 57 | ---------- 58 | @xy: Kx2 np.array 59 | each row is a pair of inverse-transformed coordinates (x, y) 60 | """ 61 | Tinv = inv(trans) 62 | xy = tformfwd(Tinv, uv) 63 | return xy 64 | 65 | 66 | def findNonreflectiveSimilarity(uv, xy, options=None): 67 | 68 | options = {'K': 2} 69 | 70 | K = options['K'] 71 | M = xy.shape[0] 72 | x = xy[:, 0].reshape((-1, 1)) # use reshape to keep a column vector 73 | y = xy[:, 1].reshape((-1, 1)) # use reshape to keep a column vector 74 | # print('--->x, y:\n', x, y 75 | 76 | tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1)))) 77 | tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1)))) 78 | X = np.vstack((tmp1, tmp2)) 79 | # print('--->X.shape: ', X.shape 80 | # print('X:\n', X 81 | 82 | u = uv[:, 0].reshape((-1, 1)) # use reshape to keep a column vector 83 | v = uv[:, 1].reshape((-1, 1)) # use reshape to keep a column vector 84 | U = np.vstack((u, v)) 85 | # print('--->U.shape: ', U.shape 86 | # print('U:\n', U 87 | 88 | # We know that X * r = U 89 | if rank(X) >= 2 * K: 90 | r, _, _, _ = lstsq(X, U) 91 | r = np.squeeze(r) 92 | else: 93 | raise Exception('cp2tform:twoUniquePointsReq') 94 | 95 | # print('--->r:\n', r 96 | 97 | sc = r[0] 98 | ss = r[1] 99 | tx = r[2] 100 | ty = r[3] 101 | 102 | Tinv = np.array([ 103 | [sc, -ss, 0], 104 | [ss, sc, 0], 105 | [tx, ty, 1] 106 | ]) 107 | 108 | # print('--->Tinv:\n', Tinv 109 | 110 | T = inv(Tinv) 111 | # print('--->T:\n', T 112 | 113 | T[:, 2] = np.array([0, 0, 1]) 114 | 115 | return T, Tinv 116 | 117 | 118 | def findSimilarity(uv, xy, options=None): 119 | 120 | options = {'K': 2} 121 | 122 | # uv = np.array(uv) 123 | # xy = np.array(xy) 124 | 125 | # Solve for trans1 126 | trans1, trans1_inv = findNonreflectiveSimilarity(uv, xy, options) 127 | 128 | # Solve for trans2 129 | 130 | # manually reflect the xy data across the Y-axis 131 | xyR = xy 132 | xyR[:, 0] = -1 * xyR[:, 0] 133 | 134 | trans2r, trans2r_inv = findNonreflectiveSimilarity(uv, xyR, options) 135 | 136 | # manually reflect the tform to undo the reflection done on xyR 137 | TreflectY = np.array([ 138 | [-1, 0, 0], 139 | [0, 1, 0], 140 | [0, 0, 1] 141 | ]) 142 | 143 | trans2 = np.dot(trans2r, TreflectY) 144 | 145 | # Figure out if trans1 or trans2 is better 146 | xy1 = tformfwd(trans1, uv) 147 | norm1 = norm(xy1 - xy) 148 | 149 | xy2 = tformfwd(trans2, uv) 150 | norm2 = norm(xy2 - xy) 151 | 152 | if norm1 <= norm2: 153 | return trans1, trans1_inv 154 | else: 155 | trans2_inv = inv(trans2) 156 | return trans2, trans2_inv 157 | 158 | 159 | def get_similarity_transform(src_pts, dst_pts, reflective=True): 160 | """ 161 | Function: 162 | ---------- 163 | Find Similarity Transform Matrix 'trans': 164 | u = src_pts[:, 0] 165 | v = src_pts[:, 1] 166 | x = dst_pts[:, 0] 167 | y = dst_pts[:, 1] 168 | [x, y, 1] = [u, v, 1] * trans 169 | 170 | Parameters: 171 | ---------- 172 | @src_pts: Kx2 np.array 173 | source points, each row is a pair of coordinates (x, y) 174 | @dst_pts: Kx2 np.array 175 | destination points, each row is a pair of transformed 176 | coordinates (x, y) 177 | @reflective: True or False 178 | if True: 179 | use reflective similarity transform 180 | else: 181 | use non-reflective similarity transform 182 | 183 | Returns: 184 | ---------- 185 | @trans: 3x3 np.array 186 | transform matrix from uv to xy 187 | trans_inv: 3x3 np.array 188 | inverse of trans, transform matrix from xy to uv 189 | """ 190 | 191 | if reflective: 192 | trans, trans_inv = findSimilarity(src_pts, dst_pts) 193 | else: 194 | trans, trans_inv = findNonreflectiveSimilarity(src_pts, dst_pts) 195 | 196 | return trans, trans_inv 197 | 198 | 199 | def cvt_tform_mat_for_cv2(trans): 200 | """ 201 | Function: 202 | ---------- 203 | Convert Transform Matrix 'trans' into 'cv2_trans' which could be 204 | directly used by cv2.warpAffine(): 205 | u = src_pts[:, 0] 206 | v = src_pts[:, 1] 207 | x = dst_pts[:, 0] 208 | y = dst_pts[:, 1] 209 | [x, y].T = cv_trans * [u, v, 1].T 210 | 211 | Parameters: 212 | ---------- 213 | @trans: 3x3 np.array 214 | transform matrix from uv to xy 215 | 216 | Returns: 217 | ---------- 218 | @cv2_trans: 2x3 np.array 219 | transform matrix from src_pts to dst_pts, could be directly used 220 | for cv2.warpAffine() 221 | """ 222 | cv2_trans = trans[:, 0:2].T 223 | 224 | return cv2_trans 225 | 226 | 227 | def get_similarity_transform_for_cv2(src_pts, dst_pts, reflective=True): 228 | """ 229 | Function: 230 | ---------- 231 | Find Similarity Transform Matrix 'cv2_trans' which could be 232 | directly used by cv2.warpAffine(): 233 | u = src_pts[:, 0] 234 | v = src_pts[:, 1] 235 | x = dst_pts[:, 0] 236 | y = dst_pts[:, 1] 237 | [x, y].T = cv_trans * [u, v, 1].T 238 | 239 | Parameters: 240 | ---------- 241 | @src_pts: Kx2 np.array 242 | source points, each row is a pair of coordinates (x, y) 243 | @dst_pts: Kx2 np.array 244 | destination points, each row is a pair of transformed 245 | coordinates (x, y) 246 | reflective: True or False 247 | if True: 248 | use reflective similarity transform 249 | else: 250 | use non-reflective similarity transform 251 | 252 | Returns: 253 | ---------- 254 | @cv2_trans: 2x3 np.array 255 | transform matrix from src_pts to dst_pts, could be directly used 256 | for cv2.warpAffine() 257 | """ 258 | trans, trans_inv = get_similarity_transform(src_pts, dst_pts, reflective) 259 | cv2_trans = cvt_tform_mat_for_cv2(trans) 260 | cv2_trans_inv = cvt_tform_mat_for_cv2(trans_inv) 261 | 262 | return cv2_trans, cv2_trans_inv 263 | 264 | 265 | if __name__ == '__main__': 266 | """ 267 | u = [0, 6, -2] 268 | v = [0, 3, 5] 269 | x = [-1, 0, 4] 270 | y = [-1, -10, 4] 271 | 272 | # In Matlab, run: 273 | # 274 | # uv = [u'; v']; 275 | # xy = [x'; y']; 276 | # tform_sim=cp2tform(uv,xy,'similarity'); 277 | # 278 | # trans = tform_sim.tdata.T 279 | # ans = 280 | # -0.0764 -1.6190 0 281 | # 1.6190 -0.0764 0 282 | # -3.2156 0.0290 1.0000 283 | # trans_inv = tform_sim.tdata.Tinv 284 | # ans = 285 | # 286 | # -0.0291 0.6163 0 287 | # -0.6163 -0.0291 0 288 | # -0.0756 1.9826 1.0000 289 | # xy_m=tformfwd(tform_sim, u,v) 290 | # 291 | # xy_m = 292 | # 293 | # -3.2156 0.0290 294 | # 1.1833 -9.9143 295 | # 5.0323 2.8853 296 | # uv_m=tforminv(tform_sim, x,y) 297 | # 298 | # uv_m = 299 | # 300 | # 0.5698 1.3953 301 | # 6.0872 2.2733 302 | # -2.6570 4.3314 303 | """ 304 | u = [0, 6, -2] 305 | v = [0, 3, 5] 306 | x = [-1, 0, 4] 307 | y = [-1, -10, 4] 308 | 309 | uv = np.array((u, v)).T 310 | xy = np.array((x, y)).T 311 | 312 | print('\n--->uv:') 313 | print(uv) 314 | print('\n--->xy:') 315 | print(xy) 316 | 317 | trans, trans_inv = get_similarity_transform(uv, xy) 318 | 319 | print('\n--->trans matrix:') 320 | print(trans) 321 | 322 | print('\n--->trans_inv matrix:') 323 | print(trans_inv) 324 | 325 | print('\n---> apply transform to uv') 326 | print('\nxy_m = uv_augmented * trans') 327 | uv_aug = np.hstack(( 328 | uv, np.ones((uv.shape[0], 1)) 329 | )) 330 | xy_m = np.dot(uv_aug, trans) 331 | print(xy_m) 332 | 333 | print('\nxy_m = tformfwd(trans, uv)') 334 | xy_m = tformfwd(trans, uv) 335 | print(xy_m) 336 | 337 | print('\n---> apply inverse transform to xy') 338 | print('\nuv_m = xy_augmented * trans_inv') 339 | xy_aug = np.hstack(( 340 | xy, np.ones((xy.shape[0], 1)) 341 | )) 342 | uv_m = np.dot(xy_aug, trans_inv) 343 | print(uv_m) 344 | 345 | print('\nuv_m = tformfwd(trans_inv, xy)') 346 | uv_m = tformfwd(trans_inv, xy) 347 | print(uv_m) 348 | 349 | uv_m = tforminv(trans, xy) 350 | print('\nuv_m = tforminv(trans, xy)') 351 | print(uv_m) 352 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/visualization_utils.py: -------------------------------------------------------------------------------- 1 | from PIL import ImageDraw 2 | 3 | 4 | def show_bboxes(img, bounding_boxes, facial_landmarks=[]): 5 | """Draw bounding boxes and facial landmarks. 6 | 7 | Arguments: 8 | img: an instance of PIL.Image. 9 | bounding_boxes: a float numpy array of shape [n, 5]. 10 | facial_landmarks: a float numpy array of shape [n, 10]. 11 | 12 | Returns: 13 | an instance of PIL.Image. 14 | """ 15 | 16 | img_copy = img.copy() 17 | draw = ImageDraw.Draw(img_copy) 18 | 19 | for b in bounding_boxes: 20 | draw.rectangle([ 21 | (b[0], b[1]), (b[2], b[3]) 22 | ], outline='white') 23 | 24 | for p in facial_landmarks: 25 | for i in range(5): 26 | draw.ellipse([ 27 | (p[i] - 1.0, p[i + 5] - 1.0), 28 | (p[i] + 1.0, p[i + 5] + 1.0) 29 | ], outline='blue') 30 | 31 | return img_copy 32 | -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/weights/onet.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/src/weights/onet.npy -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/weights/pnet.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/src/weights/pnet.npy -------------------------------------------------------------------------------- /preprocess/mtcnn_pytorch/src/weights/rnet.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GGGHSL/InfoSwap-master/0484ef5fcb35bf811f2c3ec52ecf77ccade822db/preprocess/mtcnn_pytorch/src/weights/rnet.npy -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.9.0 2 | alabaster==0.7.12 3 | apipkg==1.5 4 | argh==0.26.2 5 | astor==0.8.1 6 | attrs==19.3.0 7 | Babel==2.8.0 8 | backcall==0.1.0 9 | bleach==3.1.1 10 | certifi==2019.11.28 11 | chardet==3.0.4 12 | cloudpickle==1.3.0 13 | coverage==5.0.3 14 | cycler==0.10.0 15 | decorator==4.4.2 16 | defusedxml==0.6.0 17 | docutils==0.16 18 | entrypoints==0.3 19 | execnet==1.7.1 20 | flake8==3.7.9 21 | gast==0.3.3 22 | grpcio==1.27.2 23 | h5py==2.10.0 24 | idna==2.9 25 | imageio==2.8.0 26 | imagesize==1.2.0 27 | importlib-metadata==1.5.0 28 | ipykernel==5.1.4 29 | ipython==7.13.0 30 | ipython-genutils==0.2.0 31 | ipywidgets==7.5.1 32 | jedi==0.16.0 33 | Jinja2==2.11.1 34 | jsonschema==3.2.0 35 | jupyter-client==6.0.0 36 | jupyter-core==4.6.3 37 | Keras==2.2.5 38 | Keras-Applications==1.0.8 39 | Keras-Preprocessing==1.1.0 40 | kiwisolver==1.1.0 41 | livereload==2.6.1 42 | Markdown==3.2.1 43 | MarkupSafe==1.1.1 44 | matplotlib==3.1.3 45 | mccabe==0.6.1 46 | mistune==0.8.4 47 | mock==4.0.1 48 | more-itertools==8.2.0 49 | nbconvert==5.6.1 50 | nbformat==5.0.4 51 | networkx==2.4 52 | notebook==6.0.3 53 | numpy==1.18.1 54 | packaging==20.1 55 | pandocfilters==1.4.2 56 | parso==0.6.2 57 | pathtools==0.1.2 58 | pep8==1.7.1 59 | pexpect==4.8.0 60 | pickleshare==0.7.5 61 | Pillow==7.0.0 62 | pluggy==0.13.1 63 | port-for==0.3.1 64 | prometheus-client==0.7.1 65 | prompt-toolkit==3.0.3 66 | protobuf==3.11.3 67 | ptyprocess==0.6.0 68 | py==1.8.1 69 | pycodestyle==2.5.0 70 | pydot==1.4.1 71 | pyflakes==2.1.1 72 | Pygments==2.5.2 73 | pyparsing==2.4.6 74 | pyrsistent==0.15.7 75 | pytest==5.3.5 76 | pytest-cache==1.0 77 | pytest-cov==2.8.1 78 | pytest-pep8==1.0.6 79 | python-dateutil==2.8.1 80 | pytz==2019.3 81 | PyWavelets==1.1.1 82 | PyYAML==5.3 83 | pyzmq==19.0.0 84 | requests==2.23.0 85 | scikit-image==0.16.2 86 | scipy==1.4.1 87 | Send2Trash==1.5.0 88 | six==1.14.0 89 | snowballstemmer==2.0.0 90 | Sphinx==2.4.3 91 | sphinx-autobuild==0.7.1 92 | sphinx-rtd-theme==0.4.3 93 | sphinxcontrib-applehelp==1.0.2 94 | sphinxcontrib-devhelp==1.0.2 95 | sphinxcontrib-htmlhelp==1.0.3 96 | sphinxcontrib-jsmath==1.0.1 97 | sphinxcontrib-qthelp==1.0.3 98 | sphinxcontrib-serializinghtml==1.1.4 99 | tensorboard==1.13.1 100 | tensorflow-estimator==1.13.0 101 | tensorflow==1.13.2 102 | tensorflow-probability==0.6.0 103 | termcolor==1.1.0 104 | terminado==0.8.3 105 | testpath==0.4.4 106 | torch==1.4.0 107 | torchvision==0.5.0 108 | tornado==6.0.3 109 | tqdm==4.43.0 110 | traitlets==4.3.3 111 | urllib3==1.25.8 112 | watchdog==0.10.2 113 | wcwidth==0.1.8 114 | webencodings==0.5.1 115 | Werkzeug==1.0.0 116 | widgetsnbextension==3.5.1 117 | zipp==3.0.0 --------------------------------------------------------------------------------