├── .deepsource.toml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md ├── dependabot.yml └── workflows │ └── pythonapp.yml ├── .gitignore ├── .whitesource ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── README.md ├── api ├── Api.py └── templates │ ├── login.html │ ├── train.html │ └── upload.html ├── conf ├── conf.json ├── dashboard.ini └── ssl │ ├── localhost.crt │ └── localhost.key ├── dataset ├── face_training_dataset.zip ├── face_training_dataset_little.zip └── model │ ├── 20191202_153034 │ ├── model.clf │ ├── model.dat │ └── model.json │ └── README.md ├── datastructure ├── Administrator.py ├── Classifier.py ├── Person.py └── Response.py ├── docker-compose.yml ├── main.py ├── requirements.txt ├── test ├── conf_test.json ├── test_admin.py ├── test_classifier.py └── test_images │ ├── bush_test.jpg │ ├── multi_face_test.jpg │ ├── no_face_test.png │ └── unknown_face.jpg ├── utils ├── add_users.py └── util.py └── wsgi.py /.deepsource.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | [[analyzers]] 4 | name = "python" 5 | enabled = true 6 | runtime_version = "3.x.x" 7 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | dataset/*.zip 2 | .vscode/ 3 | .idea/ 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: alessiosavi 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - Dataset link 28 | - Test data 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: tqdm 11 | versions: 12 | - 4.56.0 13 | - 4.56.1 14 | - 4.56.2 15 | - 4.57.0 16 | - 4.58.0 17 | - 4.59.0 18 | - dependency-name: pillow 19 | versions: 20 | - 8.1.0 21 | - 8.1.1 22 | - dependency-name: cython 23 | versions: 24 | - 0.29.22 25 | - dependency-name: numpy 26 | versions: 27 | - 1.19.5 28 | - 1.20.0 29 | - 1.20.1 30 | - dependency-name: scikit-learn 31 | versions: 32 | - 0.24.1 33 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | name: Python application 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up Python 3.7 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: 3.7 16 | - uses: actions/cache@v1 17 | with: 18 | path: ~/.cache/pip 19 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 20 | restore-keys: | 21 | ${{ runner.os }}-pip- 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install -r requirements.txt 26 | - name: Lint with flake8 27 | run: | 28 | pip install flake8 29 | # stop the build if there are Python syntax errors or undefined names 30 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 31 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 32 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 33 | - name: Test with pytest 34 | run: | 35 | python main.py & 36 | sleep 2 37 | cd test 38 | python -m unittest test_classifier.TestPredict 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 2 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 3 | 4 | # User-specific stuff 5 | .idea/**/workspace.xml 6 | .idea/**/tasks.xml 7 | .idea/**/usage.statistics.xml 8 | .idea/**/dictionaries 9 | .idea/**/shelf 10 | 11 | # Generated files 12 | .idea/**/contentModel.xml 13 | 14 | # Sensitive or high-churn files 15 | .idea/**/dataSources/ 16 | .idea/**/dataSources.ids 17 | .idea/**/dataSources.local.xml 18 | .idea/**/sqlDataSources.xml 19 | .idea/**/dynamic.xml 20 | .idea/**/uiDesigner.xml 21 | .idea/**/dbnavigator.xml 22 | 23 | # Gradle 24 | .idea/**/gradle.xml 25 | .idea/**/libraries 26 | 27 | # Gradle and Maven with auto-import 28 | # When using Gradle or Maven with auto-import, you should exclude module files, 29 | # since they will be recreated, and may cause churn. Uncomment if using 30 | # auto-import. 31 | # .idea/modules.xml 32 | # .idea/*.iml 33 | # .idea/modules 34 | # *.iml 35 | # *.ipr 36 | 37 | # CMake 38 | cmake-build-*/ 39 | 40 | # Mongo Explorer plugin 41 | .idea/**/mongoSettings.xml 42 | 43 | # File-based project format 44 | *.iws 45 | 46 | # IntelliJ 47 | out/ 48 | 49 | # mpeltonen/sbt-idea plugin 50 | .idea_modules/ 51 | 52 | # JIRA plugin 53 | atlassian-ide-plugin.xml 54 | 55 | # Cursive Clojure plugin 56 | .idea/replstate.xml 57 | 58 | # Crashlytics plugin (for Android Studio and IntelliJ) 59 | com_crashlytics_export_strings.xml 60 | crashlytics.properties 61 | crashlytics-build.properties 62 | fabric.properties 63 | 64 | # Editor-based Rest Client 65 | .idea/httpRequests 66 | 67 | # Android studio 3.1+ serialized cache file 68 | .idea/caches/build_file_checksums.ser 69 | 70 | 71 | .vscode/ 72 | 73 | # CUSTOM 74 | .idea 75 | log/ 76 | test/test_log/ 77 | dataset 78 | *.pyc 79 | flask_monitoringdashboard.db 80 | uploads 81 | conf 82 | __pycache__/ 83 | -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "checkRunSettings": { 3 | "vulnerableCheckRunConclusionLevel": "failure" 4 | }, 5 | "issueSettings": { 6 | "minSeverityLevel": "LOW" 7 | } 8 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at alessiosavibtc@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:latest 2 | 3 | # The latest alpine images don't have some tools like (`git` and `bash`). 4 | # Adding git, bash and openssh to the image 5 | RUN apt update && apt upgrade -y && apt install -y cmake build-essential libatlas-base-dev liblapack-dev 6 | 7 | LABEL maintainer="Alessio Savi " 8 | 9 | # Set the Current Working Directory inside the container 10 | WORKDIR /app 11 | 12 | # Copy the source from the current directory to the Working Directory inside the container 13 | COPY . /app 14 | 15 | RUN pip install -r requirements.txt 16 | 17 | # Expose port 8081 to the outside world 18 | EXPOSE 8081 19 | 20 | # Run the executable 21 | CMD ["python", "main.py"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 alessiosavi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyRecognizer 2 | 3 | A simple face recognition engine 4 | 5 | ![Python application](https://github.com/alessiosavi/PyRecognizer/workflows/Python%20application/badge.svg)[![License](https://img.shields.io/github/license/alessiosavi/PyRecognizer)](https://img.shields.io/github/license/alessiosavi/PyRecognizer) [![Version](https://img.shields.io/github/v/tag/alessiosavi/PyRecognizer)](https://img.shields.io/github/v/tag/alessiosavi/PyRecognizer) [![Code size](https://img.shields.io/github/languages/code-size/alessiosavi/PyRecognizer)](https://img.shields.io/github/languages/code-size/alessiosavi/PyRecognizer) [![Repo size](https://img.shields.io/github/repo-size/alessiosavi/PyRecognizer)](https://img.shields.io/github/repo-size/alessiosavi/PyRecognizer) [![Issue open](https://img.shields.io/github/issues/alessiosavi/PyRecognizer)](https://img.shields.io/github/issues/alessiosavi/PyRecognizer) 6 | [![Issue closed](https://img.shields.io/github/issues-closed/alessiosavi/PyRecognizer)](https://img.shields.io/github/issues-closed/alessiosavi/PyRecognizer)[![DeepSource](https://static.deepsource.io/deepsource-badge-light-mini.svg)](https://deepsource.io/gh/alessiosavi/PyRecognizer/?ref=repository-badge) 7 | 8 | ## Video guide for train/predict 9 | 10 | 11 | 12 | ## Model tuned for some celebrities 13 | 14 | The following list contains the name of the celebrity and the number of photos used for training, ordered by the number of photos 15 | 16 |
Celebrites list
 17 | George_W_Bush  530
 18 | Colin_Powell  236
 19 | Tony_Blair  144
 20 | Donald_Rumsfeld  121
 21 | Gerhard_Schroeder  109
 22 | Ariel_Sharon  77
 23 | Hugo_Chavez   71
 24 | Junichiro_Koizumi  60
 25 | Jean_Chretien  55
 26 | John_Ashcroft  53
 27 | Serena_Williams 	52
 28 | Jacques_Chirac  52
 29 | Vladimir_Putin  49
 30 | Luiz_Inacio_Lula_da_Silva 	48
 31 | Gloria_Macapagal_Arroyo  44
 32 | Jennifer_Capriati 	42
 33 | Arnold_Schwarzenegger  42
 34 | Lleyton_Hewitt 	41
 35 | Laura_Bush 	41
 36 | Hans_Blix  39
 37 | Alejandro_Toledo 	39
 38 | Nestor_Kirchner  37
 39 | Andre_Agassi  36
 40 | Alvaro_Uribe  35
 41 | Tom_Ridge  33
 42 | Silvio_Berlusconi  33
 43 | Megawati_Sukarnoputri  33
 44 | Vicente_Fox  32
 45 | Roh_Moo-hyun  32
 46 | Kofi_Annan  32
 47 | John_Negroponte  31
 48 | David_Beckham  31
 49 | Recep_Tayyip_Erdogan  30
 50 | Guillermo_Coria  30
 51 | Mahmoud_Abbas 	29
 52 | Bill_Clinton  29
 53 | Juan_Carlos_Ferrero  28
 54 | Jack_Straw 	28
 55 | Ricardo_Lagos  27
 56 | Rudolph_Giuliani  26
 57 | Gray_Davis  26
 58 | Tom_Daschle 	25
 59 | Winona_Ryder 	24
 60 | Jeremy_Greenstock  24
 61 | Atal_Bihari_Vajpayee  24
 62 | Tiger_Woods 	23
 63 | Saddam_Hussein  23
 64 | Jose_Maria_Aznar 	23
 65 | Pete_Sampras  22
 66 | Naomi_Watts 	22
 67 | Lindsay_Davenport  22
 68 | Hamid_Karzai 	22
 69 | George_Robertson  22
 70 | Jennifer_Lopez 	21
 71 | Jennifer_Aniston 	21
 72 | Carlos_Menem 	21
 73 | Amelie_Mauresmo 	21
 74 | Paul_Bremer 	20
 75 | Michael_Bloomberg 	20
 76 | Jiang_Zemin 	20
 77 | Igor_Ivanov 	20
 78 | Angelina_Jolie 	20
 79 | Tim_Henman 	19
 80 | Nicole_Kidman 	19
 81 | Julianne_Moore 	19
 82 | Joschka_Fischer 	19
 83 | John_Howard 	19
 84 | Carlos_Moya 	19
 85 | Abdullah_Gul 	19
 86 | Richard_Myers 	18
 87 | Pervez_Musharraf 	18
 88 | Michael_Schumacher 	18
 89 | Lance_Armstrong 	18
 90 | Fidel_Castro 	18
 91 | Venus_Williams 	17
 92 | Spencer_Abraham 	17
 93 | Renee_Zellweger 	17
 94 | John_Snow 	17
 95 | John_Kerry 	17
 96 | John_Bolton  17
 97 | Jean_Charest  17
 98 | Bill_Gates  17
 99 | Trent_Lott 	16
100 | Tommy_Franks  16
101 | Halle_Berry 	16
102 | Taha_Yassin_Ramadan  15
103 | Pierce_Brosnan 	15
104 | Norah_Jones  15
105 | Nancy_Pelosi 	15
106 | Mohammed_Al-Douri 	15
107 | Meryl_Streep 	15
108 | Julie_Gerberding 	15
109 | Hu_Jintao 	15
110 | Dominique_de_Villepin 	15
111 | Bill_Simon 	15
112 | Andy_Roddick 	15
113 | Yoriko_Kawaguchi 	14
114 | Roger_Federer 	14
115 | Mahathir_Mohamad 	14
116 | Kim_Clijsters 	14
117 | James_Blake 	14
118 | Hillary_Clinton 	14
119 | Eduardo_Duhalde 	14
120 | Dick_Cheney 	14
121 | David_Nalbandian 	14
122 | Britney_Spears 	14
123 | Wen_Jiabao 	13
124 | Salma_Hayek 	13
125 | Queen_Elizabeth_II 	13
126 | Lucio_Gutierrez 	13
127 | Joe_Lieberman 	13
128 | Jackie_Chan 	13
129 | Gordon_Brown 	13
130 | George_HW_Bush 	13
131 | Edmund_Stoiber 	13
132 | Charles_Moose 	13
133 | Ari_Fleischer 	13
134 | Rubens_Barrichello 	12
135 | Michael_Jackson 	12
136 | Keanu_Reeves 	12
137 | Jennifer_Garner 	12
138 | Jeb_Bush 	12
139 | Howard_Dean 	12
140 | Harrison_Ford 	12
141 | Gonzalo_Sanchez_de_Lozada 	12
142 | Anna_Kournikova 	12
143 | Adrien_Brody 	12
144 | Tang_Jiaxuan 	11
145 | Sergio_Vieira_De_Mello 	11
146 | Sergey_Lavrov 	11
147 | Richard_Gephardt 	11
148 | Paul_Burrell 	11
149 | Nicanor_Duarte_Frutos 	11
150 | Mike_Weir 	11
151 | Mark_Philippoussis 	11
152 | Kim_Ryong-sung 	11
153 | John_Paul_II 	11
154 | John_Allen_Muhammad 	11
155 | Jiri_Novak 	11
156 | James_Kelly 	11
157 | Condoleezza_Rice 	11
158 | Catherine_Zeta-Jones 	11
159 | Ann_Veneman 	11
160 | Walter_Mondale 	10
161 | Tommy_Thompson 	10
162 | Tom_Hanks 	10
163 | Tom_Cruise 	10
164 | Richard_Gere 	10
165 | Paul_Wolfowitz 	10
166 | Paradorn_Srichaphan 	10
167 | Muhammad_Ali 	10
168 | Mohammad_Khatami 	10
169 | Jean-David_Levitte 	10
170 | Javier_Solana 	10
171 | Jason_Kidd 	10
172 | Jacques_Rogge 	10
173 | Ian_Thorpe 	10
174 | Bill_McBride 	10
175 | Zhu_Rongji 	9
176 | Vaclav_Havel 	9
177 | Tung_Chee-hwa 	9
178 | Thomas_OBrien 	9
179 | Sylvester_Stallone 	9
180 | Richard_Armitage 	9
181 | Ray_Romano 	9
182 | Paul_ONeill 	9
183 | Li_Peng 	9
184 | Leonardo_DiCaprio 	9
185 | Kate_Hudson 	9
186 | Jose_Serra 	9
187 | John_Abizaid 	9
188 | Joan_Laporta 	9
189 | Jimmy_Carter 	9
190 | Jesse_Jackson 	9
191 | Jeong_Se-hyun 	9
192 | Hugh_Grant 	9
193 | Hosni_Mubarak 	9
194 | Heizo_Takenaka 	9
195 | George_Clooney 	9
196 | Fernando_Gonzalez 	9
197 | Colin_Farrell 	9
198 | Charles_Taylor 	9
199 | Bill_Graham 	9
200 | Bill_Frist 	9
201 | Yasser_Arafat 	8
202 | Yao_Ming 	8
203 | Shimon_Peres 	8
204 | Sheryl_Crow 	8
205 | Ron_Dittemore 	8
206 | Robert_Redford 	8
207 | Robert_Duvall 	8
208 | Robert_Blake 	8
209 | Richard_Virenque 	8
210 | Ralf_Schumacher 	8
211 | Paul_Martin 	8
212 | Naji_Sabri 	8
213 | Mohamed_ElBaradei 	8
214 | Michelle_Kwan 	8
215 | Michael_Chang 	8
216 | Maria_Shriver 	8
217 | Li_Zhaoxing 	8
218 | Kim_Dae-jung 	8
219 | Kevin_Costner 	8
220 | Justin_Timberlake 	8
221 | Juan_Pablo_Montoya 	8
222 | Jonathan_Edwards 	8
223 | John_Edwards 	8
224 | Jelena_Dokic 	8
225 | Gerry_Adams 	8
226 | Fernando_Henrique_Cardoso 	8
227 | Cesar_Gaviria 	8
228 | Celine_Dion 	8
229 | Bob_Hope 	8
230 | Antonio_Palocci 	8
231 | Ana_Palacio 	8
232 | Ali_Naimi 	8
233 | Al_Gore 	8
234 | Yashwant_Sinha 	7
235 | William_Ford_Jr 	7
236 | William_Donaldson 	7
237 | Vojislav_Kostunica 	7
238 | Vincent_Brooks 	7
239 | Steven_Spielberg 	7
240 | Sophia_Loren 	7
241 | Romano_Prodi 	7
242 | Robert_Zoellick 	7
243 | Pedro_Almodovar 	7
244 | Paul_McCartney 	7
245 | Oscar_De_La_Hoya 	7
246 | Norm_Coleman 	7
247 | Mike_Myers 	7
248 | Mike_Martz 	7
249 | Matthew_Perry 	7
250 | Martin_Scorsese 	7
251 | Mariah_Carey 	7
252 | Liza_Minnelli 	7
253 | Larry_Brown 	7
254 | Justine_Pasek 	7
255 | Jon_Gruden 	7
256 | John_Travolta 	7
257 | John_McCain 	7
258 | John_Manley 	7
259 | Jean-Pierre_Raffarin 	7
260 | Holly_Hunter 	7
261 | Gunter_Pleuger 	7
262 | Goldie_Hawn 	7
263 | Geoff_Hoon 	7
264 | Elton_John 	7
265 | Dennis_Kucinich 	7
266 | David_Wells 	7
267 | Bob_Stoops 	7
268 | Binyamin_Ben-Eliezer 	7
269 | Ben_Affleck 	7
270 | Ana_Guevara 	7
271 | Amelia_Vega 	7
272 | Al_Sharpton 	7
273 | Zinedine_Zidane 	6
274 | Yoko_Ono 	6
275 | Valery_Giscard_dEstaing 	6
276 | Valentino_Rossi 	6
277 | Tony_Stewart 	6
278 | Tommy_Haas 	6
279 | Thaksin_Shinawatra 	6
280 | Tariq_Aziz 	6
281 | Susan_Sarandon 	6
282 | Steve_Lavin 	6
283 | Silvan_Shalom 	6
284 | Sarah_Jessica_Parker 	6
285 | Sarah_Hughes 	6
286 | Roy_Moore 	6
287 | Roman_Polanski 	6
288 | Rob_Marshall 	6
289 | Robert_De_Niro 	6
290 | Rick_Perry 	6
291 | Ricardo_Sanchez 	6
292 | Paula_Radcliffe 	6
293 | Natalie_Coughlin 	6
294 | Monica_Seles 	6
295 | Mike_Krzyzewski 	6
296 | Michael_Douglas 	6
297 | Marco_Antonio_Barrera 	6
298 | Luis_Horna 	6
299 | Luis_Ernesto_Derbez_Bautista 	6
300 | Leonid_Kuchma 	6
301 | Kamal_Kharrazi 	6
302 | Jose_Manuel_Durao_Barroso 	6
303 | JK_Rowling  6
304 | Jim_Furyk 	6
305 | Jay_Garner 	6
306 | Jan_Ullrich 	6
307 | Gwyneth_Paltrow 	6
308 | Fujio_Cho  6
309 | Elsa_Zylberstein 	6
310 | Edward_Lu 	6
311 | Diana_Krall 	6
312 | Dennis_Hastert 	6
313 | Costas_Simitis 	6
314 | Clint_Eastwood 	6
315 | Clay_Aiken 	6
316 | Christine_Todd_Whitman 	6
317 | Charlton_Heston 	6
318 | Carmen_Electra 	6
319 | Cameron_Diaz 	6
320 | Calista_Flockhart 	6
321 | Bulent_Ecevit 	6
322 | Boris_Becker 	6
323 | Bob_Graham 	6
324 | Billy_Crystal 	6
325 | Arminio_Fraga 	6
326 | Angela_Bassett 	6
327 | Albert_Costa 	6
328 |
329 | 330 | ## Introduction 331 | 332 | This project is developed for have a plug-and-play facial recognition tool able to detect and recognize *__multiple__* faces from photos. 333 | It aim to be inter-operable with other tool. For this purpose, it expose REST api in order to interact with the internal face-recognition engine (train/tune/predict) and return the result of the prediction in a JSON format. 334 | 335 | It's written for be a basecode/project-template for future project where a more complicated facial detect + neural network have to be engaged. 336 | But is a complete face recognition tool that can be deployed on Docker. 337 | Currently it use a Multi Layer Perceptron (MLP) as neural network in order to predict the given faces. 338 | 339 | The tool is powered with `Flask_MonitoringDashboard` that expose some useful utilization/performance graph at the `/dashboard` endpoint 340 | 341 | ## Requirements 342 | 343 | - [face_recognition](https://github.com/ageitgey/face_recognition) Extract face point from image 344 | - [Flask](https://github.com/pallets/flask) The Python micro framework for building web applications 345 | - [Flask_MonitoringDashboard](https://github.com/flask-dashboard/Flask-MonitoringDashboard) Automatically monitor the evolving performance of Flask/Python web services 346 | - [numpy](https://github.com/numpy/numpy) The fundamental package for scientific computing with Python. 347 | - [olefile](https://github.com/decalage2/olefile) Parse, read and write Microsoft OLE2 files (deal with image) 348 | - [Pillow](https://github.com/python-pillow/Pillow) The friendly PIL fork (Python Imaging Library) 349 | - [py-bcrypt](https://code.google.com/archive/p/py-bcrypt/) Python wrapper of OpenBSD's Blowfish password hashing code 350 | - [redis-py](https://github.com/andymccurdy/redis-py) The Python interface to the Redis key-value store. 351 | - [scikit-learn](https://github.com/scikit-learn/scikit-learn) Machine learning in Python 352 | - [tqdm](https://github.com/tqdm/tqdm) A Fast, Extensible Progress Bar 353 | - [werkzeug](https://github.com/pallets/werkzeug) The comprehensive WSGI web application library 354 | 355 | ***NOTE***: If you encounter an error during `pip install -r requirements.txt`, it's possible that you have not installed `cmake`. `dlib` need `cmake`. 356 | You can install `cmake` using: 357 | 358 | - `apt install cmake -y` (Debian/Ubuntu). 359 | - `yum install cmake -y` (CentOS/Fedora/RedHat). 360 | 361 | ## Table Of Contents 362 | 363 | - [PyRecognizer](#pyrecognizer) 364 | - [Video guide for train/predict](#video-guide-for-trainpredict) 365 | - [Model tuned for some celebrities](#model-tuned-for-some-celebrities) 366 | - [Introduction](#introduction) 367 | - [Requirements](#requirements) 368 | - [Table Of Contents](#table-of-contents) 369 | - [Prerequisites](#prerequisites) 370 | - [Usage](#usage) 371 | - [In Details](#in-details) 372 | - [Example response](#example-response) 373 | - [Contributing](#contributing) 374 | - [Versioning](#versioning) 375 | - [Authors](#authors) 376 | - [License](#license) 377 | - [Acknowledgments](#acknowledgments) 378 | 379 | ## Prerequisites 380 | 381 | The software is coded in `Python`, into the `requirements.txt` file are saved the necessary dependencies. 382 | 383 | Create a virtual environment with you favorite `python` package manager 384 | 385 | ```bash 386 | # Create a new environment 387 | conda create -n PyRecognizer python=3.7.4 388 | # Activate the environment 389 | conda activate PyRecognizer 390 | # Install the necessary dependencies 391 | pip install -r requirements.txt 392 | ``` 393 | 394 | At this point all the necessary library for run the tool are ready, and you can run the software. 395 | 396 | ## Usage 397 | 398 | You can view the following example video in order to understand how to interact with the tool for the following process: 399 | 400 | - Create dataset from images 401 | - Predict image 402 | - Train/Tune the neural network 403 | 404 | [Video guide for train/predict](#video-guide-for-trainpredict) 405 | 406 | Before you can train the neural network with the photos, you need to create an archive that contains the image of the people's faces that you want to predict. 407 | 408 | - Save a bunch of images of the people that you need to recognize. 409 | - Copy the image in a folder. The name of that folder is important, cause it will be used as a label for the dataset (images) that contains during prediction. 410 | - Compress the folders in a `zip` file. 411 | 412 | Before train the neural network, you have to create a dataset with the people images that you want to recognize. 413 | If your dataset tree structure look likes the following tree dir, you can continue with training phase. 414 | 415 | ```text 416 | ├── bfegan 417 | └── ... 418 | ├── chris 419 | └── ... 420 | ├── dhawley 421 | └── ... 422 | ├── graeme 423 | └──... 424 | ├── heather 425 | └──... 426 | ``` 427 | 428 | In this case we have a dataset that contains the photos of 5 people (bfegan, dhawley, heather etc). 429 | Each directory, contains the photos related to the "target". 430 | 431 | You can find an example dataset at the following link: 432 | 433 | 434 | Some people in this dataset have only very few image. 435 | 436 | We can create a new one dataset using the following `bash` command, in order to extract only the people that contains more than 5 images: 437 | 438 | ```bash 439 | # Extract only the people that have more than 5 photos (-gt 5) 440 | for i in $(ls); do a=$(ls $i |wc -l); if [ "$a" -gt 5 ]; then echo $i ; fi ; done > people_ok 441 | # Create a directory for store the images 442 | mkdir -p /tmp/faces 443 | # Copy the filtered directory in the new one 444 | for i in $(cat people_ok | xargs echo -n) ; do cp -r $i /tmp/faces/ ; done 445 | ``` 446 | 447 | At this point the dataset is complete and you can continue with training/tuning. 448 | 449 | Backup and remove the already present model (if present,inside the `dataset/model` directory), the tool will understand that you want to train the model and will initialize a new MLP model. The model have the following name template: `%Y%m%d_%H%M%S`, related to the time that was generated. 450 | 451 | Open your browser at the `endpoint:port/train` specified in the configuration file (`conf/test.json`) and you will be redirect to the Administrator login page. 452 | **NOTE:** you can switch on/off the SSL, be sure to add `https` before the endpoint ip/hostname if it is enabled. 453 | **NOTE:** In order to access to the training/tuning page, you have to run the script in [utils/add_users.py](utils/add_users.py) for create an admin user, capable of manage the train/tune for the neural network. 454 | **NOTE:** A instance of `redis` have to be up and running if you want to train your custom neural network, cause the login will read the data from `redis`. 455 | 456 | At this point you can upload the dataset (the previous `zip` file) and wait for the training of the neural network. 457 | 458 | You can tail the log in `log/pyrecognizer.log` in order to understand the status of the training (`lnav` is your friends). 459 | 460 | Once completed, the browser page will be refreshed automatically and you can: 461 | 462 | - predict a new photos that the neural network haven't seen before, realated to the peoeple in the dataset. 463 | - reduce the treeshold and see how you are similar to a celebrity!. 464 | 465 | **NOTE:** The same procedure can be applied for `tune` the neural network. By this way, you are going to execute an exhaustive search over specified parameter values for the KNN classifier. And, obviously, is more time consuming and the neural network produced will be more precise. The endpoint is `/tune` instead of `/train` 466 | 467 | After `train/tune` phase, you have to modify the configuration file in order to use the new model. The model is saved in a new folder with the related timestamp (modify classifier -> timestamp in the configuration file) 468 | 469 | ## In Details 470 | 471 | ```bash 472 | tree 473 | . 474 | ├── api 475 | │   ├── Api.py # Code that contains the API endpoint logic 476 | │   └── templates # Folder that contains the HTML template for tune/train/predict 477 | │   ├── train.html 478 | │   └── upload.html 479 | ├── conf # Configuration folder 480 | │   ├── conf.json # Tool configuration file 481 | │   ├── dashboard.ini # File related to the Dashboard configuration 482 | │   ├── flask_monitoringdashboard.db # Dashboard database 483 | │   ├── ssl # SSL Certificates folder 484 | │   │   ├── localhost.crt 485 | │   │   └── localhost.key 486 | 487 | ├── dataset # Model folder + test dataset 488 | │   ├── face_training_dataset_little.zip # Model used for test train 489 | │   ├── face_training_dataset.zip 490 | │   └── model # Neural network model's folder 491 | │   ├── 20191123_171821 # Folder for the NN model 492 | │   │   ├── model.clf # Neural network dumped 493 | │   │   ├── model.dat # Data used for train/tune 494 | │   │   └── model.json # Hyperparmaters of the NN 495 | │   └── README.md 496 | ├── datastructure # Datastructure/Class used 497 | │   ├── Administrator.py # Class that handle the admin of the NN, for train/tune 498 | │   ├── Classifier.py # Class delegated to predict the photos 499 | │   ├── Person.py # Class delegated to handle the "stuff" related to loading people data 500 | │   └── Response.py # Class delegated to wrap the response 501 | ├── docker-compose.yml # docker-compose file for raise up the PyRecognizer (predict + train/tune) 502 | ├── Dockerfile # Dockerfile related to the PyRecognizer only (only predict) 503 | ├── LICENSE # License file 504 | ├── log # Log folder 505 | │   └── pyrecognizer.log 506 | ├── main.py # Main program to spawn the tool 507 | ├── README.md 508 | ├── requirements.txt # Dependencies file 509 | ├── uploads # Folder that contains the upload data 510 | ├── test # Test folder 511 | │   ├── conf_test.json 512 | │   ├── test_classifier.py # File with test cases 513 | │   ├── test_images # Test data 514 | │   │   ├── bush_test.jpg 515 | │   │   ├── multi_face_test.jpg 516 | │   │   └── unknown_face.jpg 517 | │   ├── test_log # Log of the test 518 | │   │   └── pyrecognizer.log 519 | │   └── uploads 520 | │   ├── predict 521 | │   ├── training 522 | │   ├── unknown 523 | │   └── upload 524 | │   ├── predict 525 | │   ├── training 526 | │   └── upload 527 | ├── utils 528 | │   ├── add_users.py # Python file for add a new user for train/tune the network 529 | │ └── util.py # Common methods 530 | └── wsgi.py 531 | ``` 532 | 533 | ## Example response 534 | 535 | - **Missing the photo in request** 536 | 537 | ```text 538 | { 539 | "response": { 540 | "data": null, 541 | "date": "2020-01-12 15:12:14.762526", 542 | "description": "You have sent a request without the photo to predict :/", 543 | "error": "NO_FILE_IN_REQUEST", 544 | "status": "KO" 545 | } 546 | } 547 | ``` 548 | 549 | - **Missing threshold parameter in request** 550 | 551 | ```text 552 | { 553 | "response": { 554 | "data": null, 555 | "date": "2020-01-12 15:12:14.769286", 556 | "description": "You have sent a request without the `threshold` parameter :/", 557 | "error": "THRESHOLD_NOT_PROVIDED", 558 | "status": "KO" 559 | } 560 | } 561 | ``` 562 | 563 | - **Threshold provided is a number not in the properly range** 564 | 565 | ```text 566 | { 567 | "response": { 568 | "data": null, 569 | "date": "2020-01-12 15:12:14.776730", 570 | "description": "Threshold have to be greater than 0 and lesser than 100!", 571 | "error": "THRESHOLD_ERROR_VALUE", 572 | "status": "KO" 573 | } 574 | } 575 | ``` 576 | 577 | - **File in request is not a valid one** 578 | 579 | ```text 580 | { 581 | "response": { 582 | "data": null, 583 | "date": "2019-11-23 18:10:11.038329", 584 | "description": "Seems that the file that you have tried to upload is not valid ...", 585 | "error": "FILE_NOT_VALID", 586 | "status": "KO" 587 | } 588 | } 589 | ``` 590 | 591 | - **Error parsing the threshold parameter** 592 | 593 | ```text 594 | { 595 | "response": { 596 | "data": null, 597 | "date": "2020-01-12 15:12:14.784154", 598 | "description": "Threshold is not an integer!", 599 | "error": "UNABLE_CAST_INT", 600 | "status": "KO" 601 | } 602 | } 603 | ``` 604 | 605 | - **Dataset upload is not valid** 606 | 607 | ```text 608 | { 609 | "response": { 610 | "data": null, 611 | "date": "2019-11-23 18:10:11.038329", 612 | "description": "Seems that the dataset is not valid", 613 | "error": "ERROR DURING LOADING DAT", 614 | "status": "KO" 615 | } 616 | } 617 | ``` 618 | 619 | - **Unable to detect a face** 620 | 621 | ```text 622 | { 623 | "response": { 624 | "data": null, 625 | "date": "2019-11-23 18:10:11.038329", 626 | "description": "Seems that in this images there is no face :/", 627 | "error": "FACE_NOT_FOUND", 628 | "status": "KO" 629 | } 630 | } 631 | ``` 632 | 633 | - **Face not recognized** 634 | 635 | ```text 636 | { 637 | "response": { 638 | "data": {}, 639 | "date": "2019-11-23 18:17:58.287413", 640 | "description": "FACE_NOT_RECOGNIZED", 641 | "error": null, 642 | "status": "OK" 643 | } 644 | } 645 | ``` 646 | 647 | - **Face recognized** 648 | 649 | ```text 650 | { 651 | "response": { 652 | "data": { 653 | "iroy": 0.5762745881923004 # Name of the person: confidence 654 | }, 655 | "date": "2019-11-23 18:23:01.762757", 656 | "description": "ijyibbvcgq.png", # Random string for view image prediction (visit /uploads/ijyibbvcgq.png) 657 | "error": null, 658 | "status": "OK" 659 | } 660 | } 661 | ``` 662 | 663 | - **Missing model's classifier** 664 | 665 | ```text 666 | { 667 | "response": { 668 | "data": null, 669 | "date": "2019-11-23 18:27:55.761851", 670 | "description": "CLASSIFIER_NOT_LOADED", 671 | "error": null, 672 | "status": "KO" 673 | } 674 | } 675 | ``` 676 | 677 | - **Login not successfully** 678 | 679 | ```text 680 | { 681 | "response": { 682 | "data": null, 683 | "date": "2019-11-23 18:27:55.761851", 684 | "description": "The password inserted is not valid!", 685 | "error": "PASSWORD_NOT_VALID", 686 | "status": "KO" 687 | } 688 | } 689 | ``` 690 | 691 | - **Unable to connect to redis** 692 | 693 | ```text 694 | { 695 | "response": { 696 | "data": null, 697 | "date": "2019-11-23 18:27:55.761851", 698 | "description": "Seems that the DB is not reachable!", 699 | "error": "UNABLE_CONNECT_REDIS_DB", 700 | "status": "KO" 701 | } 702 | } 703 | ``` 704 | 705 | 706 | ## Contributing 707 | 708 | - Feel free to open issue in order to __*require new functionality*__; 709 | - Feel free to open issue __*if you discover a bug*__; 710 | - New idea/request/concept are very appreciated!; 711 | 712 | ## Test 713 | 714 | In order to run the basic test case, you need to: 715 | - Spawn the `PyRecognizer` tool using `python main.py` 716 | - Change directory into the `test/` folder 717 | - Run `python -m unittest test_classifier.TestPredict` 718 | 719 | If you are the admin of the neural network, you can test the Admin related methods: 720 | - Spawn the docker image of a redis-db `docker run -dt -p 6379:6379 redis` 721 | - Change directory into the `test/` folder 722 | - Run `python -m unittest test_admin.TestAdmin` 723 | 724 | ## Versioning 725 | 726 | We use [SemVer](http://semver.org/) for versioning. 727 | 728 | ## Authors 729 | 730 | - **Alessio Savi** - *Initial work & Concept* - [Linkedin](https://www.linkedin.com/in/alessio-savi-2136b2188/) - [Github](https://github.com/alessiosavi/PyRecognizer) 731 | 732 | ## Contributors 733 | - **Alessio Savi** 734 | 735 | ## License 736 | 737 | This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details 738 | 739 | ## Acknowledgments 740 | 741 | Face data are sensible information. In order to mitigate the risk of stealing sensible data, the tool can run in SSL mode for avoid packet sniffing and secure every request using a CSRF mitigation 742 | -------------------------------------------------------------------------------- /api/Api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Custom function that will be wrapped for be HTTP compliant 4 | """ 5 | 6 | import os 7 | import shutil 8 | import time 9 | from datetime import datetime 10 | from logging import getLogger 11 | 12 | from datastructure.Response import Response 13 | from utils.util import print_prediction_on_image, random_string, retrieve_dataset 14 | 15 | log = getLogger() 16 | 17 | 18 | def predict_image(img_path, clf, PREDICTION_PATH, TMP_UNKNOWN, DETECTION_MODEL, JITTER, encoding_models, threshold=45): 19 | """ 20 | 21 | :param TMP_UNKNOWN: 22 | :param threshold: 23 | :param PREDICTION_PATH: global variable where image recognized are saved 24 | :param img_path: image that have to be predicted 25 | :param clf: classifier in charge to predict the image 26 | :return: Response dictionary jsonizable 27 | """ 28 | response = Response() 29 | if clf is None: 30 | log.error("predict_image | FATAL | Classifier is None!") 31 | prediction = None 32 | else: 33 | log.debug("predict_image | Predicting {}".format(img_path)) 34 | prediction = clf.predict(img_path, DETECTION_MODEL, JITTER, encoding_models, threshold) 35 | log.debug("predict_image | Result: {}".format(prediction)) 36 | 37 | # Manage error 38 | if prediction is None: 39 | response.error = "CLASSIFIER_NOT_LOADED" 40 | response.description = "Classifier is None | Training mandatory" 41 | response.status = "KO" 42 | log.error("predict_image | Seems that the classifier is not loaded :/") 43 | 44 | elif isinstance(prediction, int): 45 | response.status = "KO" 46 | if prediction == -1: 47 | response.error = "FACE_NOT_RECOGNIZED" 48 | response.description = "Seems that this face is related to nobody that i've seen before ..." 49 | response.status = "KO" 50 | log.error("predict_image | Face not recognized ...") 51 | 52 | # Saving unknown faces for future clustering 53 | now = str(datetime.now())[:23] 54 | now = now.replace(":", "_") 55 | now = now.replace(".", "_") 56 | _, tail = os.path.split(img_path) 57 | filename, file_extension = os.path.splitext(tail) 58 | filename = filename + "__" + now + file_extension 59 | filename = os.path.join(TMP_UNKNOWN, filename) 60 | log.info("Image not recognized, saving it in: {}".format(filename)) 61 | shutil.copy(img_path, filename) 62 | 63 | elif prediction == -2: 64 | response.error = "FILE_NOT_VALID" 65 | response.description = "Seems that the file that you have tried to upload is not valid ..." 66 | log.error( 67 | "predict_image |Seems that the file that you have tried to upload is not valid ...") 68 | 69 | # Manage no face found 70 | elif prediction == -3: 71 | log.error( 72 | "predict_image | Seems that this face is related to nobody that i've seen before ...") 73 | response.error = "FACE_NOT_FOUND" 74 | response.description = "No face found in the given image ..." 75 | 76 | # Manage success 77 | elif "predictions" in prediction and isinstance(prediction['predictions'], list): 78 | # Be sure to don't overwrite an existing image 79 | exists = True 80 | while exists: 81 | # img_name = os.path.join(PREDICTION_PATH, random_string() + ".png" 82 | img_name = random_string() + ".png" 83 | img_file_name = os.path.join(PREDICTION_PATH, img_name) 84 | if not os.path.exists(img_file_name): 85 | exists = False 86 | 87 | log.debug("predict_image | Generated a random name: {}".format(img_name)) 88 | log.debug("predict_image | Printing prediction on image ...") 89 | print_prediction_on_image( 90 | img_path, prediction["predictions"], img_file_name) 91 | 92 | return Response(status="OK", description="/uploads/" + img_name, data=prediction).__dict__ 93 | 94 | return response.__dict__ 95 | 96 | 97 | def train_network(folder_uncompress, zip_file, clf, DETECTION_MODEL, JITTER, encoding_models): 98 | """ 99 | Train a new neural model with the zip file provided 100 | :param folder_uncompress: 101 | :param zip_file: 102 | :param clf: 103 | :return: 104 | """ 105 | 106 | log.debug("train_network | Starting training phase ...") 107 | dataset = retrieve_dataset(folder_uncompress, zip_file, clf, DETECTION_MODEL, JITTER, encoding_models) 108 | if dataset is None: 109 | return Response(error="ERROR DURING LOADING DAT", description="Seems that the dataset is not valid").__dict__ 110 | 111 | timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') 112 | neural_model_file, _ = clf.train(dataset["X"], dataset["Y"], timestamp) 113 | 114 | response = Response(status="OK", data=neural_model_file) 115 | response.description = "Model successfully trained!" 116 | log.debug("train_network | Tuning phase finished! | {}".format( 117 | response.description)) 118 | 119 | return response.__dict__ 120 | 121 | 122 | def tune_network(folder_uncompress, zip_file, clf, DETECTION_MODEL, JITTER, encoding_models): 123 | """ 124 | Train a new neural model with the zip file provided 125 | :param folder_uncompress: 126 | :param zip_file: 127 | :param clf: 128 | :return: 129 | """ 130 | log.debug("tune_network | Starting tuning phase ...") 131 | dataset = retrieve_dataset(folder_uncompress, zip_file, clf, DETECTION_MODEL, JITTER, encoding_models) 132 | 133 | if dataset is None: 134 | return Response(error="ERROR DURING LOADING DAT", description="Seems that the dataset is not valid").__dict__ 135 | 136 | timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') 137 | neural_model_file, elapsed_time = clf.tuning( 138 | dataset["X"], dataset["Y"], timestamp) 139 | 140 | response = Response(status="OK", data=neural_model_file) 141 | response.description = "Model successfully trained! | {}".format( 142 | time.strftime("%H:%M:%S.%f", time.gmtime(elapsed_time))) 143 | log.debug("train_network | Tuning phase finished! | {}".format( 144 | response.description)) 145 | 146 | return response.__dict__ 147 | -------------------------------------------------------------------------------- /api/templates/login.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | PyRecognizer Login Page! 7 | 8 | 9 | 10 |
11 | 12 | 13 | 14 | 15 |
16 | 17 | 18 | -------------------------------------------------------------------------------- /api/templates/train.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | PyRecognizer Training Page! 7 | 8 | 9 | 10 |

Upload a zip file with all person that you want to save!

11 |
12 | 13 | 14 | 15 |
16 | {% with messages = get_flashed_messages() %} 17 | {% if messages %} 18 | {% for message in messages %} 19 |

{{ message }}

20 | {% endfor %} 21 | {% endif %} 22 | {% endwith %} 23 | 24 | 25 | -------------------------------------------------------------------------------- /api/templates/upload.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | PyRecognizer image predict! 6 | 7 | 8 | 9 |

Face recognition tool

10 |
11 | 12 | 13 |

14 | 15 | 18 | 19 |
20 | {% with messages = get_flashed_messages() %} 21 | {% if messages %} 22 | {% for message in messages %} 23 |

{{ message }}

24 | {% endfor %} 25 | {% endif %} 26 | {% endwith %} 27 | 28 | -------------------------------------------------------------------------------- /conf/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "PyRecognizer": { 3 | "Version": "0.3.3", 4 | "temp_upload_training": "uploads/training/", 5 | "temp_upload_predict": "uploads/predict/", 6 | "temp_upload": "uploads/upload", 7 | "temp_unknown": "uploads/unknown", 8 | "detection_model": "hog", 9 | "jitter": 1, 10 | "encoding_models": "small", 11 | "enable_dashboard": false 12 | }, 13 | "logging": { 14 | "path": "log/", 15 | "prefix": "pyrecognizer.log", 16 | "level": "debug" 17 | }, 18 | "network": { 19 | "host": "0.0.0.0", 20 | "port": 8081, 21 | "templates": "api/templates/", 22 | "csrf_protection": false, 23 | "SSL": { 24 | "enabled": false, 25 | "cert.pub": "conf/ssl/localhost.crt", 26 | "cert.priv": "conf/ssl/localhost.key" 27 | } 28 | }, 29 | "classifier": { 30 | "training_dir": "dataset/images/", 31 | "model_path": "dataset/model/", 32 | "timestamp": "20191202_153034" 33 | }, 34 | "dashboard": { 35 | "config_file": "conf/dashboard.ini" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /conf/dashboard.ini: -------------------------------------------------------------------------------- 1 | [dashboard] 2 | APP_VERSION = 1.0 3 | GIT = .git/ 4 | CUSTOM_LINK = dashboard 5 | MONITOR_LEVEL = 3 6 | OUTLIER_DETECTION_CONSTANT = 2.5 7 | SAMPLING_PERIOD = 20 8 | ENABLE_LOGGING = True 9 | 10 | [authentication] 11 | USERNAME = admin 12 | PASSWORD = admin 13 | GUEST_USERNAME = guest 14 | GUEST_PASSWORD = ['guest', 'password'] 15 | 16 | [database] 17 | TABLE_PREFIX = fmd 18 | DATABASE = sqlite:///conf/flask_monitoringdashboard.db -------------------------------------------------------------------------------- /conf/ssl/localhost.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIC5TCCAc2gAwIBAgIJAJPNi4jjHSy3MA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV 3 | BAMMCWxvY2FsaG9zdDAeFw0xOTA1MjIxNjMxMDJaFw0xOTA2MjExNjMxMDJaMBQx 4 | EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 5 | ggEBANskAjz6LENzhnpGkyJHztmIf3Pno8h/k70fjEI13osonv7W5alA3vgQ9az3 6 | ivD7cp6YPXkv5lK+mTx6dKccrdAPQLWQDZBqaotasTX1hBxaqILqNvh25QY5gjbz 7 | jdfK27E+82QDZUzdYsFDyZQ4ORQ8qVUz0k42ulS4WMpluBEaLk8rHkDIyZSM4psv 8 | EK+IcI7mN8z1YI8mS3jOW2ouQQVwRb60ZOe4b9wcFPYR7+NdNQM7rCR9UQU9ymjC 9 | U4VmTUrIonmXML1gRPHs0Z694AsQe+Mr5O3OxeYhbsFb7d1Ry4WcZiPM+ugJJiNS 10 | Fkpf4SDT7nHAcHbqFzibpSJPP7cCAwEAAaM6MDgwFAYDVR0RBA0wC4IJbG9jYWxo 11 | b3N0MAsGA1UdDwQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0B 12 | AQsFAAOCAQEADz/YL1DOV8n/15/ApaWCQhzcFGOPSv1DcnI6sY46I4zRKyG9yuHE 13 | N11XqkCmQuKF9UnowhFFMLIfxzlqkUTWjKtaWKasnOdAd/LOqO9Eh4cnsyC4yEBB 14 | aMO00YdUAdFb0eV3bR/UY3srji6LjRy9215Ad3eXYxjdTTB/btIsN75XTTsZLnbR 15 | F0V3TRkZlxCQXcYh/lpfPHG9xWLxPZ8g8e+hrwJhsmW3a0BMzYNF8nJdzhZi7Dls 16 | ldR2V8IqVP/Ip6dpsygn/CzbDlZVcZVV4jqhec8bbijsXdSizwm8bfc57TssRA1C 17 | HlvLlwAsoiDj6PZ4PwRCvc5k6ydDbXNftw== 18 | -----END CERTIFICATE----- 19 | -------------------------------------------------------------------------------- /conf/ssl/localhost.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDbJAI8+ixDc4Z6 3 | RpMiR87ZiH9z56PIf5O9H4xCNd6LKJ7+1uWpQN74EPWs94rw+3KemD15L+ZSvpk8 4 | enSnHK3QD0C1kA2QamqLWrE19YQcWqiC6jb4duUGOYI2843XytuxPvNkA2VM3WLB 5 | Q8mUODkUPKlVM9JONrpUuFjKZbgRGi5PKx5AyMmUjOKbLxCviHCO5jfM9WCPJkt4 6 | zltqLkEFcEW+tGTnuG/cHBT2Ee/jXTUDO6wkfVEFPcpowlOFZk1KyKJ5lzC9YETx 7 | 7NGeveALEHvjK+TtzsXmIW7BW+3dUcuFnGYjzProCSYjUhZKX+Eg0+5xwHB26hc4 8 | m6UiTz+3AgMBAAECggEBAIMpqFVK/9dXfDQPrd0k0cAOHQsIqFVHVuwpx8+RYqQ0 9 | KgYqJcgKVepwbDuc5oKaXd5jDNhOPTNldV5nhQ7I8ZfIqViC4juAFklWfR7o1qwJ 10 | 7zZ8bW6F60qwfSna2RlCCACsxw0joyxAje1TX4HhrPhZ3phqrgO2agxvUmXCQEur 11 | HmZXEXP2grR0XdWiXazWI5jlG0MsX6J+qsMHFCApGR/9KcsB8Lwe8RAiszc1SPPp 12 | TNGZopojkH1GK8DAXMFvODmTdwlStpDh1g711cX5KoINKlX5ppJjsoqcGOLhbEee 13 | uCsfckXGrHJm51GbJePPZ16x7Op/BUdyKjYvSL31fuECgYEA7mumpBMDq4NQ1gju 14 | n7kmU75k2ddrXSycvFJ5yxKCCec+hdJBtKm6WrGGD+uchjxFhZP37JRTimV/F5RL 15 | Ps6xVwgwX3DtSLpwyOelLR8Zo2wT1cDFKp6EfD4ltDVbTsOW2X8yyKeJHac23/wT 16 | HIRyv+8DUUo0GU4JMl4VAW9PwWkCgYEA60xv/8c0AfjOZIGlxdk2RCKWnZas6Rdk 17 | STChPXoIOj5T75B7OfxJukY4R8d7jzXOwX5WX3wS/rtEuom5tFW5+fLl16HWUyz5 18 | pXa7/QW5dQa7GLB3K6HBKhfTm7/fDkaFKDu/c+sF46RWoP7vxqct1ir0L0Z1BFnk 19 | /qSpSbhBtB8CgYA1/ajR9QBawbT3kzQ+dVYplq8N6cuFYQnpV5//DaTnCzfMZC2+ 20 | 9MSfrx3V0xwyBcoUksqNB5XXfF6If2t+wJ3GQLN7mX4Sfy31QQfVrPpIWLwxJqM/ 21 | oIAOBqDRK1gPARnTDQv6Bn51eZ1ioZnOVmwJ7N1KdkxQAqzwe/+zwHpGKQKBgQCH 22 | e/Pha2pe2Ey/QoeZbID6qo/fHatia72rBv1Q0Lt8Dfd2sdLCiKpLP7OYYRycUXdD 23 | ouNJB8BIPLxOTI9JbzMu4NXHW8B1FCiLRdrozisDX2TLypBT50e6XQ3TWJ+vMJvr 24 | lruem21ArpfTC/g0gn66GvGPZxpp7vkURuvTLu1mMQKBgQDI0yvH+FqxiXmnZjY6 25 | 4rqoq7shenmrHxbywHOCJbXMVlFMhFovZUCKZtJ0G14e3yGystA3wkNj8CJtBYj4 26 | /R1ucQIXBeiGJHKY9lVuRuJI258jUrIQ8z6hNv8zXVW/2oM0R58dJXL2UJVFHDpU 27 | ETwkYWrY5QeX4J4mxX2AfsrZ8Q== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /dataset/face_training_dataset.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/dataset/face_training_dataset.zip -------------------------------------------------------------------------------- /dataset/face_training_dataset_little.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/dataset/face_training_dataset_little.zip -------------------------------------------------------------------------------- /dataset/model/20191202_153034/model.clf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/dataset/model/20191202_153034/model.clf -------------------------------------------------------------------------------- /dataset/model/20191202_153034/model.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/dataset/model/20191202_153034/model.dat -------------------------------------------------------------------------------- /dataset/model/20191202_153034/model.json: -------------------------------------------------------------------------------- 1 | { 2 | "classifier_file": "20191202_153034/model.clf", 3 | "params": { 4 | "activation": "identity", 5 | "hidden_layer_sizes": [ 6 | 128 7 | ], 8 | "learning_rate": "constant", 9 | "solver": "adam" 10 | } 11 | } -------------------------------------------------------------------------------- /dataset/model/README.md: -------------------------------------------------------------------------------- 1 | ### Neural Network model folder 2 | 3 | This directory will contains the model generated by the neural network training among the give images 4 | 5 | #### image_dataset-DATE_TIME.**dat** 6 | 7 | Contains the dataset parsed from the image 8 | 9 | #### model-DATE_TIME.**clf** 10 | 11 | Is the neural model able to classify a given face 12 | 13 | #### model-DATE_TIME.**json** 14 | 15 | Contains the json configuration for training the neural network with the best parameters 16 | -------------------------------------------------------------------------------- /datastructure/Administrator.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | 4 | import bcrypt 5 | import redis 6 | from flask_login import UserMixin 7 | 8 | log = logging.getLogger() 9 | log.setLevel(logging.DEBUG) 10 | 11 | 12 | class Administrator(UserMixin): 13 | 14 | def __init__(self, name: str, mail: str, password: str): 15 | # Identifier will be "name:mail" 16 | self.name = name 17 | self.mail = mail 18 | self.password = password 19 | self.id = None 20 | self.redis_client = None 21 | if len(name) < 3 or len(mail) < 3 or len(password) < 5: 22 | print("Value not allowed! -> {}".format(vars(self))) 23 | return 24 | 25 | def init_redis_connection(self, host: str = "0.0.0.0", port: str = "6379", db: int = 0) -> bool: 26 | log.warning("Initializing a new redis connection") 27 | self.redis_client = redis.Redis(host=host, port=port, db=db) 28 | try: 29 | health_check = self.redis_client.ping() 30 | except redis.exceptions.ConnectionError: 31 | health_check = False 32 | log.warning("Connection not established!") 33 | return health_check 34 | 35 | @staticmethod 36 | def validate_password(password) -> bool: 37 | if len(password) < 5: 38 | log.warning( 39 | "Password not valid | Password have to be more than 5 characters long") 40 | return False 41 | return True 42 | 43 | def retrieve_password(self) -> str: 44 | if self.redis_client is None: 45 | log.warning("Redis connection is not initialized") 46 | return "" 47 | return self.redis_client.get(self.get_name()) 48 | 49 | def verify_user_exist(self) -> bool: 50 | """ 51 | Verify that an user is already registered. 52 | True -> Already exist 53 | False -> New user 54 | """ 55 | already_exists = self.retrieve_password() 56 | # Be sure that user does not exists 57 | return already_exists is not None 58 | 59 | def remove_user(self) -> bool: 60 | if not self.verify_user_exist(): 61 | log.warning("User {} does not exists!".format(vars(self))) 62 | return False 63 | log.warning("Removing user {} from redis!".format(vars(self))) 64 | self.redis_client.delete(self.get_name()) 65 | return True 66 | 67 | def add_user(self) -> bool: 68 | if self.verify_user_exist(): 69 | log.warning("User {} already exist or db is not reachable ...".format(vars(self))) 70 | return False 71 | if not self.validate_password(self.password): 72 | log.warning("Password not valid") 73 | return False 74 | 75 | log.warning("Encrypting password -> {}".format(self.password)) 76 | self.password = self.encrypt_password(str(self.password)) 77 | log.warning("Password encrypted {}".format(self.password)) 78 | self.redis_client.set(self.get_name(), self.password) 79 | log.warning("User {} registered!".format(self.get_name())) 80 | return True 81 | 82 | def verify_login(self, password: str) -> bool: 83 | if not self.verify_user_exist(): 84 | log.warning("User {} does not exists!".format(vars(self))) 85 | return False 86 | if not self.validate_password(password): 87 | log.warning("Password {} is not valid!".format(password)) 88 | return False 89 | 90 | # Retrieve password from DB 91 | psw = self.retrieve_password() 92 | return self.check_password(password, psw) 93 | 94 | def get_name(self) -> str: 95 | # return self.name+":"+self.mail 96 | return self.mail 97 | 98 | @staticmethod 99 | def encrypt_password(plain_text_password: str) -> str: 100 | # Hash a password for the first time 101 | # (Using bcrypt, the salt is saved into the hash itself) 102 | return bcrypt.hashpw(plain_text_password, bcrypt.gensalt()) 103 | 104 | @staticmethod 105 | def check_password(plain_text_password: str, hashed_password: str) -> bool: 106 | # Check hashed password. Using bcrypt, the salt is saved into the hash itself 107 | log.warning("Comparing plain: {} with hashed {}".format( 108 | plain_text_password, hashed_password)) 109 | check = bcrypt.checkpw(plain_text_password, hashed_password) 110 | if check: 111 | log.debug("Password match, user logged in!") 112 | else: 113 | log.debug("Password mismatch, user NOT logged in!") 114 | return check 115 | -------------------------------------------------------------------------------- /datastructure/Classifier.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Core utils for manage face recognition process 4 | """ 5 | import json 6 | import logging 7 | import os 8 | import pickle 9 | import time 10 | from pprint import pformat 11 | 12 | import face_recognition 13 | from sklearn.metrics import accuracy_score, balanced_accuracy_score, classification_report, \ 14 | precision_score 15 | from sklearn.model_selection import GridSearchCV, train_test_split 16 | from sklearn.neural_network import MLPClassifier 17 | from tqdm import tqdm 18 | 19 | from datastructure.Person import Person 20 | from utils.util import dump_dataset, load_image_file 21 | 22 | log = logging.getLogger() 23 | 24 | 25 | class Classifier(): 26 | """ 27 | Store the knowledge related to the people faces 28 | """ 29 | 30 | def __init__(self): 31 | self.training_dir = None 32 | self.model_path = None 33 | self.peoples_list = [] 34 | self.classifier = None 35 | self.parameters = {} 36 | 37 | def init_classifier(self): 38 | """ 39 | Initialize a new classifier after be sure that necessary data are initialized 40 | """ 41 | if self.classifier is None: 42 | log.debug("init_classifier | START!") 43 | if len(self.parameters) > 0: 44 | log.debug("init_classifier | Initializing a new classifier ... | {0}".format( 45 | pformat(self.__dict__))) 46 | self.classifier = MLPClassifier(**self.parameters) 47 | else: 48 | log.error( 49 | "init_classifier | Mandatory parameter not provided | Init a new KNN Classifier") 50 | self.classifier = MLPClassifier() 51 | 52 | def load_classifier_from_file(self, timestamp): 53 | """ 54 | Initialize the classifier from file. 55 | The classifier file represent the name of the directory related to the classifier that we want to load. 56 | 57 | The tree structure of the the model folder will be something like this 58 | 59 | Structure: 60 | model/ 61 | ├── <20190520_095119>/ --> Timestamp in which the model was created 62 | │ ├── model.dat --> Dataset generated by encoding the faces and pickelizing them 63 | │ ├── model.clf --> Classifier delegated to recognize a given face 64 | │ ├── model.json --> Hyperparameters related to the current classifier 65 | ├── <20190519_210950>/ 66 | │ ├── model.dat 67 | │ ├── model.clf 68 | │ ├── model.json 69 | └── ... 70 | 71 | :param timestamp: 72 | :return: 73 | """ 74 | log.debug( 75 | "load_classifier_from_file | Loading classifier from file ... | File: {}".format(timestamp)) 76 | 77 | # Load a trained KNN model (if one was passed in) 78 | err = None 79 | if self.classifier is None: 80 | if self.model_path is None or not os.path.isdir(self.model_path): 81 | raise Exception("Model folder not provided!") 82 | # Adding the conventional name used for the classifier -> 'model.clf' 83 | filename = os.path.join(self.model_path, timestamp, "model.clf") 84 | log.debug( 85 | "load_classifier_from_file | Loading classifier from file: {}".format(filename)) 86 | if os.path.isfile(filename): 87 | log.debug( 88 | "load_classifier_from_file | File {} exist! Loading classifier ...".format(filename)) 89 | with open(filename, 'rb') as f: 90 | self.classifier = pickle.load(f) 91 | log.debug("load_classifier_from_file | Classifier loaded!") 92 | else: 93 | err = "load_classifier_from_file | FATAL | File {} DOES NOT EXIST ...".format(filename) 94 | if err is not None: 95 | log.error("load_classifier_from_file | ERROR: {} | Seems that the model is gone :/ |" 96 | " Loading an empty classifier for training purpose ...".format(err)) 97 | self.classifier = None 98 | 99 | def train(self, X, Y, timestamp): 100 | """ 101 | Train a new model by the given data [X] related to the given target [Y] 102 | :param X: 103 | :param Y: 104 | :param timestamp: 105 | """ 106 | log.debug("train | START") 107 | if self.classifier is None: 108 | self.init_classifier() 109 | 110 | dump_dataset(X, Y, os.path.join(self.model_path, timestamp)) 111 | 112 | start_time = time.time() 113 | 114 | X_train, x_test, Y_train, y_test = train_test_split( 115 | X, Y, test_size=0.25) 116 | 117 | log.debug("train | Training ...") 118 | self.classifier.fit(X_train, Y_train) 119 | log.debug("train | Model Trained!") 120 | log.debug("train | Checking performance ...") 121 | y_pred = self.classifier.predict(x_test) 122 | # Static method 123 | self.verify_performance(y_test, y_pred) 124 | 125 | return self.dump_model(timestamp=timestamp, classifier=self.classifier), time.time() - start_time 126 | 127 | def tuning(self, X, Y, timestamp): 128 | """ 129 | Tune the hyperparameter of a new model by the given data [X] related to the given target [Y] 130 | 131 | :param X: 132 | :param Y: 133 | :param timestamp: 134 | :return: 135 | """ 136 | start_time = time.time() 137 | dump_dataset(X, Y, os.path.join(self.model_path, timestamp)) 138 | 139 | X_train, x_test, Y_train, y_test = train_test_split( 140 | X, Y, test_size=0.25) 141 | self.classifier = MLPClassifier(max_iter=250) 142 | # Hyperparameter of the neural network (MLP) to tune 143 | # Faces are encoded using 128 points 144 | parameter_space = { 145 | 'hidden_layer_sizes': [(128,), (200,), (200, 128,), ], 146 | 'activation': ['identity', 'tanh', 'relu'], 147 | 'solver': ['adam'], 148 | 'learning_rate': ['constant', 'adaptive'], 149 | } 150 | log.debug("tuning | Parameter -> {}".format(pformat(parameter_space))) 151 | grid = GridSearchCV(self.classifier, parameter_space, 152 | cv=2, scoring='accuracy', verbose=20, n_jobs=8) 153 | grid.fit(X_train, Y_train) 154 | log.info("TUNING COMPLETE | DUMPING DATA!") 155 | # log.info("tuning | Grid Scores: {}".format(pformat(grid.grid_scores_))) 156 | log.info('Best parameters found: {}'.format(grid.best_params_)) 157 | 158 | y_pred = grid.predict(x_test) 159 | 160 | log.info('Results on the test set: {}'.format( 161 | pformat(grid.score(x_test, y_test)))) 162 | 163 | self.verify_performance(y_test, y_pred) 164 | 165 | return self.dump_model(timestamp=timestamp, params=grid.best_params_, 166 | classifier=grid.best_estimator_), time.time() - start_time 167 | 168 | @staticmethod 169 | def verify_performance(y_test, y_pred): 170 | """ 171 | Verify the performance of the result analyzing the known-predict result 172 | :param y_test: 173 | :param y_pred: 174 | :return: 175 | """ 176 | 177 | log.debug("verify_performance | Analyzing performance ...") 178 | log.info("\nClassification Report: {}".format( 179 | pformat(classification_report(y_test, y_pred)))) 180 | log.info("balanced_accuracy_score: {}".format( 181 | pformat(balanced_accuracy_score(y_test, y_pred)))) 182 | log.info("accuracy_score: {}".format( 183 | pformat(accuracy_score(y_test, y_pred)))) 184 | log.info("precision_score: {}".format( 185 | pformat(precision_score(y_test, y_pred, average='weighted')))) 186 | 187 | def dump_model(self, timestamp, classifier, params=None, path=None): 188 | """ 189 | Dump the model to the given path, file 190 | :param params: 191 | :param timestamp: 192 | :param classifier: 193 | :param path: 194 | 195 | """ 196 | log.debug("dump_model | Dumping model ...") 197 | if path is None: 198 | if self.model_path is not None: 199 | if os.path.exists(self.model_path) and os.path.isdir(self.model_path): 200 | path = self.model_path 201 | config = {'classifier_file': os.path.join(timestamp, "model.clf"), 202 | 'params': params 203 | } 204 | if not os.path.isdir(path): 205 | os.makedirs(timestamp) 206 | classifier_folder = os.path.join(path, timestamp) 207 | classifier_file = os.path.join(classifier_folder, "model") 208 | 209 | log.debug("dump_model | Dumping model ... | Path: {} | Model folder: {}".format( 210 | path, timestamp)) 211 | if not os.path.exists(classifier_folder): 212 | os.makedirs(classifier_folder) 213 | 214 | with open(classifier_file + ".clf", 'wb') as f: 215 | pickle.dump(classifier, f) 216 | log.info('dump_model | Model saved to {0}.clf'.format( 217 | classifier_file)) 218 | 219 | with open(classifier_file + ".json", 'w') as f: 220 | json.dump(config, f) 221 | log.info('dump_model | Configuration saved to {0}.json'.format( 222 | classifier_file)) 223 | 224 | return config 225 | 226 | def init_peoples_list(self, detection_model, jitters, encoding_models, peoples_path=None): 227 | """ 228 | This method is delegated to iterate among the folder that contains the peoples's face in order to 229 | initialize the array of peoples 230 | :return: 231 | """ 232 | 233 | log.debug("init_peoples_list | Initializing people ...") 234 | if peoples_path is not None and os.path.isdir(peoples_path): 235 | self.training_dir = peoples_path 236 | else: 237 | raise Exception("Dataset (peoples faces) path not provided :/") 238 | 239 | # The init process can be parallelized, but BATCH method will perform better 240 | # pool = ThreadPool(3) 241 | # self.peoples_list = pool.map(self.init_peoples_list_core, os.listdir(self.training_dir)) 242 | 243 | files_list = os.listdir(self.training_dir) 244 | for people_name in tqdm(files_list, total=len(files_list), desc="Init people list ..."): 245 | self.peoples_list.append( 246 | self.init_peoples_list_core(detection_model, jitters, encoding_models, people_name)) 247 | 248 | self.peoples_list = list( 249 | filter(None.__ne__, self.peoples_list)) # Remove None 250 | 251 | def init_peoples_list_core(self, detection_model, jitters, encoding_models, people_name): 252 | """ 253 | Delegated core method for parallelize operation 254 | :param detection_model 255 | :param jitters 256 | :param people_name 257 | :param encoding_models 258 | :return: 259 | """ 260 | if os.path.isdir(os.path.join(self.training_dir, people_name)): 261 | log.debug("Initializing people {0}".format( 262 | os.path.join(self.training_dir, people_name))) 263 | person = Person() 264 | person.name = people_name 265 | person.path = os.path.join(self.training_dir, people_name) 266 | person.init_dataset(detection_model, jitters, encoding_models) 267 | return person 268 | 269 | log.debug("People {0} invalid folder!".format( 270 | os.path.join(self.training_dir, people_name))) 271 | return None 272 | 273 | def init_dataset(self): 274 | """ 275 | Initialize a new dataset joining all the data related to the peoples list 276 | :return: 277 | """ 278 | DATASET = { 279 | # Image data (numpy array) 280 | "X": [], 281 | # Person name 282 | "Y": [] 283 | } 284 | 285 | for people in self.peoples_list: 286 | log.debug(people.name) 287 | DATASET["X"] = [data for data in people.dataset["X"]] 288 | DATASET["Y"] = [data for data in people.dataset["Y"]] 289 | return DATASET 290 | 291 | # The method is delegated to try to retrieve the face from the given image. 292 | # In case of cuda_malloc error (out of memory), the image will be resized 293 | @staticmethod 294 | def extract_face_from_image(X_img_path, detection_model, jitters, encoding_models): 295 | # Load image data in a numpy array 296 | try: 297 | log.debug("extract_face_from_image | Loading image {}".format(X_img_path)) 298 | X_img, ratio = load_image_file(X_img_path) 299 | except OSError: 300 | log.error("extract_face_from_image | What have you uploaded ???") 301 | return -2, -2, -1 302 | log.debug("extract_face_from_image | Extracting faces locations ...") 303 | try: 304 | # TODO: Reduce size of the image at every iteration 305 | X_face_locations = face_recognition.face_locations( 306 | X_img, model=detection_model) # model="cnn") 307 | except RuntimeError: 308 | log.error( 309 | "extract_face_from_image | GPU does not have enough memory: FIXME unload data and retry") 310 | return None, None, ratio 311 | 312 | log.debug("extract_face_from_image | Found {} face(s) for the given image".format( 313 | len(X_face_locations))) 314 | 315 | # If no faces are found in the image, return an empty result. 316 | if len(X_face_locations) == 0: 317 | log.warning("extract_face_from_image | Seems that no faces was found :( ") 318 | return -3, -3, ratio 319 | 320 | # Find encodings for faces in the test image 321 | log.debug("extract_face_from_image | Encoding faces using [{}] jitters ...".format(jitters)) 322 | # num_jitters increase the distortion check 323 | faces_encodings = face_recognition.face_encodings( 324 | X_img, known_face_locations=X_face_locations, num_jitters=jitters, model=encoding_models) 325 | log.debug("extract_face_from_image | Face encoded! | Let's ask to the neural network ...") 326 | return faces_encodings, X_face_locations, ratio 327 | 328 | def predict(self, X_img_path: str, detection_model: str, jitters: int, encoding_models: str, 329 | distance_threshold: int = 0.45): 330 | """ 331 | Recognizes faces in given image using a trained KNN classifier 332 | 333 | :param detection_model: can be 'hog' (CPU) or 'cnn' (GPU) 334 | :param jitters: augmentation data (jitters=20 -> 20x time) 335 | :param X_img_path: path of the image to be recognized 336 | :param distance_threshold: (optional) distance threshold for face classification. the larger it is, 337 | the more chance of mis-classifying an unknown person as a known one. 338 | :return: a list of names and face locations for the recognized faces in the image: [(name, bounding box), ...]. 339 | For faces of unrecognized persons, the name 'unknown' will be returned. 340 | """ 341 | 342 | if self.classifier is None: 343 | log.error( 344 | "predict | Be sure that you have loaded/trained the neural network model") 345 | return None 346 | 347 | faces_encodings, X_face_locations = None, None 348 | # Resize image if necessary for avoid cuda-malloc error (important for low gpu) 349 | # In case of error, will be returned back an integer. 350 | # FIXME: manage gpu memory unload in case of None 351 | ratio = 2 352 | while faces_encodings is None or X_face_locations is None: 353 | faces_encodings, X_face_locations, ratio = Classifier.extract_face_from_image( 354 | X_img_path, detection_model, jitters, encoding_models) 355 | # In this case return back the error to the caller 356 | if isinstance(faces_encodings, int): 357 | return faces_encodings 358 | 359 | # Use the MLP model to find the best matches for the face(s) 360 | log.debug("predict | Understanding peoples recognized from NN ...") 361 | closest_distances = self.classifier.predict(faces_encodings) 362 | log.debug("predict | Persons recognized: [{}]".format( 363 | closest_distances)) 364 | 365 | log.debug("predict | Asking to the neural network for probability ...") 366 | predictions = self.classifier.predict_proba(faces_encodings) 367 | pred = [] 368 | for prediction in predictions: 369 | pred.append(dict([v for v in sorted(zip(self.classifier.classes_, prediction), 370 | key=lambda c: c[1], reverse=True)[:len(closest_distances)]])) 371 | log.debug("predict | Predict proba -> {}".format(pred)) 372 | face_prediction = [] 373 | for i in range(len(pred)): 374 | element = list(pred[i].items())[0] 375 | log.debug("pred in cycle: {}".format(element)) 376 | face_prediction.append(element) 377 | # log.debug("predict | *****MIN****| {}".format(min(closest_distances[0][i]))) 378 | log.debug("Scores -> {}".format(face_prediction)) 379 | 380 | _predictions = [] 381 | scores = [] 382 | if len(face_prediction) > 0: 383 | for person_score, loc in zip(face_prediction, X_face_locations): 384 | if person_score[1] < distance_threshold: 385 | log.warning("predict | Person {} does not outbounds threshold {}<{}".format( 386 | pred, person_score[1], distance_threshold)) 387 | else: 388 | log.debug("predict | Pred: {} | Loc: {} | Score: {}".format( 389 | person_score[0], loc, person_score[1])) 390 | if ratio > 0: 391 | log.debug( 392 | "predict | Fixing face location using ratio: {}".format(ratio)) 393 | 394 | x1, y1, x2, y2 = loc 395 | # 1200 < size < 1600 396 | if ratio < 1: 397 | ratio = pow(ratio, -1) 398 | x1 *= ratio 399 | x2 *= ratio 400 | y1 *= ratio 401 | y2 *= ratio 402 | loc = x1, y1, x2, y2 403 | 404 | _predictions.append((person_score[0], loc)) 405 | scores.append(person_score[1]) 406 | log.debug("predict | Prediction: {}".format(_predictions)) 407 | log.debug("predict | Score: {}".format(scores)) 408 | 409 | if len(_predictions) == 0 or len(face_prediction) == 0: 410 | log.debug("predict | Face not recognized :/") 411 | return -1 412 | 413 | return {"predictions": _predictions, "scores": scores} 414 | -------------------------------------------------------------------------------- /datastructure/Person.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Common structure for define how to manage a person 4 | """ 5 | from logging import getLogger 6 | from os.path import isdir 7 | 8 | from face_recognition import face_encodings, face_locations, load_image_file 9 | from face_recognition.face_recognition_cli import image_files_in_folder 10 | 11 | log = getLogger() 12 | 13 | 14 | class Person(): 15 | """ 16 | Represent the necessary information for classify a person's face 17 | """ 18 | 19 | def __init__(self): 20 | # Name of the user 21 | self.name = "" 22 | # Filesystem folder where images are stored 23 | self.path = "" 24 | # Image list used for train the model 25 | self.dataset = { 26 | # Image data (numpy array) 27 | "X": [], 28 | # Person name 29 | "Y": [] 30 | } 31 | 32 | def init_dataset(self, detection_model, jitters, encoding_models): 33 | """ 34 | This method is delegated to load the images related to a person and verify if the ones 35 | are suitable for training the neural network. 36 | 37 | The image will be discarded if: More than one face if found | No face is found 38 | :return: 39 | """ 40 | 41 | if self.path != "" and isdir(self.path): 42 | log.debug("initDataset | Parameter provided, iterating images ..") 43 | # Iterating the images in parallel 44 | # pool = ThreadPool(2) 45 | # self.dataset["X"] = pool.map(self.init_dataset_core, image_files_in_folder(self.path)) 46 | for image_path in image_files_in_folder(self.path): 47 | self.dataset["X"].append(self.init_dataset_core(detection_model, jitters, encoding_models, image_path)) 48 | self.dataset["X"] = list( 49 | filter(None.__ne__, self.dataset["X"])) # Remove None 50 | # Loading the Y [target] 51 | for _ in range(len(self.dataset["X"])): 52 | self.dataset["Y"].append(self.name) 53 | log.debug("Adding {} entries for {}".format( 54 | len(self.dataset["X"]), self.name)) 55 | 56 | @staticmethod 57 | def init_dataset_core(detection_model, jitters, encoding_models, img_path=None): 58 | """ 59 | Delegated core method for parallelize work 60 | :detection_model 61 | :jitters 62 | :param img_path: 63 | :return: 64 | """ 65 | try: 66 | image = load_image_file(img_path) 67 | except OSError: 68 | log.error( 69 | "init_dataset | === FATAL === | Image {} is corrupted!!".format(img_path)) 70 | return None 71 | # log.debug("initDataset | Image loaded! | Searching for face ...") 72 | # Array of w,x,y,z coordinates 73 | # NOTE: Can be used batch_face_locations in order to parallelize the image init, but unfortunately 74 | # it's the only GPU that i have right now. And, of course, i'll try to don't burn it 75 | face_bounding_boxes = face_locations(image, model=detection_model) 76 | face_data = None 77 | if len(face_bounding_boxes) == 1: 78 | log.info( 79 | "initDataset | Image {0} have only 1 face, loading for future training ...".format(img_path)) 80 | # Loading the X [data] using 300 different distortion 81 | face_data = face_encodings(image, known_face_locations=face_bounding_boxes, num_jitters=jitters, 82 | model=encoding_models)[0] 83 | else: 84 | log.error( 85 | "initDataset | Image {0} not suitable for training!".format(img_path)) 86 | if len(face_bounding_boxes) == 0: 87 | log.error("initDataset | I've not found any face :/ ") 88 | else: 89 | log.error( 90 | "initDataset | Found more than one face, too much for me Sir :&") 91 | return face_data 92 | -------------------------------------------------------------------------------- /datastructure/Response.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Define the standard response to return to the client 4 | """ 5 | 6 | import logging 7 | from datetime import datetime 8 | 9 | 10 | class Response(): 11 | """ 12 | Response class is delegated to standardize the response in order to better manage the interaction with other 13 | external tools 14 | """ 15 | 16 | def __init__(self, status="KO", description=None, error=None, data=None): 17 | self.status = status 18 | self.description = description 19 | self.error = error 20 | self.data = self.parse_data(data) 21 | self.date = str(datetime.now()) 22 | 23 | @staticmethod 24 | def parse_data(data): 25 | """ 26 | 27 | :param data: 28 | :return: 29 | """ 30 | log = logging.getLogger() 31 | log.debug("parse_data | Parsing {}".format(data)) 32 | 33 | t = {} 34 | if data is not None: 35 | log.debug("parse_data | Data not None ...") 36 | if isinstance(data, dict): 37 | log.debug("parse_data | Data is a dict") 38 | if "predictions" in data and "scores" in data: 39 | # if predictions data["predictions"] and data["scores"]: 40 | log.debug("parse_data | Predictions and scores provided") 41 | if isinstance(data["predictions"], list) and isinstance(data["scores"], list): 42 | predictions = data["predictions"] 43 | scores = data["scores"] 44 | if len(predictions) == len(scores): 45 | log.debug( 46 | "parse_data | Predictions and scores same length") 47 | for i in range(len(predictions)): 48 | t[predictions[i][0]] = scores[i] 49 | log.debug("parse_data | Dict initialized -> {}".format(t)) 50 | return t 51 | return None 52 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | 4 | pyrecognizer: 5 | image: pyrecognizer:v0.3.3 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | # NOTE: Be sure that the conf/conf.json reflect these port 10 | ports: 11 | - "0.0.0.0:8081:8081" 12 | restart: always 13 | depends_on: 14 | - redisdb 15 | network_mode: "host" 16 | 17 | # Necessary only in case of train/tune of the network 18 | redisdb: 19 | container_name: redis 20 | image: redis 21 | restart: always 22 | ports: 23 | - "0.0.0.0:6379:6379" 24 | expose: 25 | - '6379' 26 | volumes: 27 | - redis-db:/var/lib/redis 28 | entrypoint: redis-server --appendonly yes 29 | network_mode: "host" 30 | 31 | volumes: 32 | redis-db: 33 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyRecognizer loader 4 | """ 5 | import base64 6 | import os 7 | import signal 8 | import sys 9 | 10 | import flask_monitoringdashboard as dashboard 11 | from flask import Flask, flash, jsonify, render_template, request, send_from_directory, session 12 | from flask_login import LoginManager, UserMixin, login_required, login_user, current_user 13 | from werkzeug.exceptions import abort 14 | from werkzeug.utils import redirect, secure_filename 15 | 16 | from api.Api import predict_image, train_network, tune_network 17 | from datastructure.Administrator import Administrator 18 | from datastructure.Classifier import Classifier 19 | from datastructure.Response import Response 20 | from utils.util import init_main_data, random_string, secure_request, find_duplicates 21 | 22 | # ===== LOAD CONFIGURATION FILE ===== 23 | CONFIG_FILE = "conf/conf.json" 24 | 25 | CFG, log, TMP_UPLOAD_PREDICTION, TMP_UPLOAD_TRAINING, TMP_UPLOAD, TMP_UNKNOWN, DETECTION_MODEL, JITTER, \ 26 | ENCODING_MODELS, enable_dashboard = init_main_data(CONFIG_FILE) 27 | 28 | SSL_ENABLED = CFG["network"]["SSL"]["enabled"] 29 | # Disable CSRF protection for if you need to use as REST server instead of use the GUI 30 | ENABLE_CSRF = CFG["network"]["csrf_protection"] 31 | # $(base64 /dev/urandom | head -n 1 | md5sum | awk '{print $1}') 32 | SECRET_KEY = str(base64.b64encode(bytes(os.urandom(24)))).encode() 33 | 34 | login_manager = LoginManager() 35 | 36 | # ===== FLASK CONFIGURATION ===== 37 | app = Flask(__name__, template_folder=CFG["network"]["templates"]) 38 | app.secret_key = SECRET_KEY 39 | # Used by flask when a upload is made 40 | app.config['UPLOAD_FOLDER'] = TMP_UPLOAD 41 | PUB_KEY = CFG["network"]["SSL"]["cert.pub"] 42 | PRIV_KEY = CFG["network"]["SSL"]["cert.priv"] 43 | 44 | if not os.path.isfile(PUB_KEY) or not os.path.isfile(PRIV_KEY): 45 | log.error( 46 | "Unable to find certs file, be sure that the following certs exists, disabling SSL") 47 | log.warning("Public key: {}".format(PUB_KEY)) 48 | log.warning("Private key: {}".format(PRIV_KEY)) 49 | SSL_ENABLED = False 50 | 51 | # =====FLASK DASHBOARD CONFIGURATION ===== 52 | if enable_dashboard: 53 | dashboard.config.init_from(file=CFG["dashboard"]["config_file"]) 54 | dashboard.bind(app, SECRET_KEY) 55 | 56 | # flask-login 57 | login_manager.init_app(app) 58 | login_manager.login_view = "login" 59 | 60 | # ===== CLASSIFIER CONFIGURATION ===== 61 | 62 | log.debug("Init classifier ...") 63 | 64 | clf = Classifier() 65 | clf.model_path = CFG["classifier"]["model_path"] 66 | clf.load_classifier_from_file(CFG["classifier"]["timestamp"]) 67 | 68 | 69 | @app.route('/', methods=['GET']) 70 | def home(): 71 | """ 72 | Show the html template for upload the image 73 | """ 74 | return render_template("upload.html") 75 | 76 | 77 | @app.route('/', methods=["POST"]) 78 | def predict(): 79 | """ 80 | Load the image using the HTML page and predict who is 81 | :return: 82 | """ 83 | # check if the post request has the file part 84 | if 'file' not in request.files or request.files['file'].filename == '': 85 | # flash('No file choose :/', category="error") 86 | log.warning("predict_api | No file choose!") 87 | response = Response(status="KO", error="NO_FILE_IN_REQUEST") 88 | response.description = "You have sent a request without the photo to predict :/" 89 | return jsonify(response=response.__dict__) 90 | # return redirect(request.url) # Return to HTML page [GET] 91 | file = request.files['file'] 92 | threshold = request.form.get('threshold') 93 | log.debug("Received file [{}] and threshold [{}]".format(file, threshold)) 94 | if threshold is None or len(threshold) == 0: 95 | log.warning("Threshold not provided") 96 | response = Response(status="KO", error="THRESHOLD_NOT_PROVIDED") 97 | response.description = "You have sent a request without the `threshold` parameter :/" 98 | return jsonify(response=response.__dict__) 99 | 100 | try: 101 | threshold = int(threshold) 102 | except ValueError: 103 | log.error("Unable to convert threshold") 104 | response = Response(status="KO", error="UNABLE_CAST_INT") 105 | response.description = "Threshold is not an integer!" 106 | return jsonify(response=response.__dict__) 107 | if not 0 <= threshold <= 100: 108 | log.error("Threshold wrong value") 109 | response = Response(status="KO", error="THRESHOLD_ERROR_VALUE") 110 | response.description = "Threshold have to be greater than 0 and lesser than 100!" 111 | return jsonify(response=response.__dict__) 112 | 113 | threshold /= 100 114 | 115 | log.debug("Received file {} and threshold {}".format(file, threshold)) 116 | filename = secure_filename(file.filename) 117 | img_path = os.path.join(app.config['UPLOAD_FOLDER'], filename) 118 | file.save(img_path) 119 | return jsonify( 120 | response=predict_image(img_path, clf, TMP_UPLOAD_PREDICTION, TMP_UNKNOWN, DETECTION_MODEL, JITTER, 121 | ENCODING_MODELS, threshold)) 122 | 123 | 124 | @app.route('/train', methods=['GET']) 125 | @login_required 126 | def train(): 127 | """ 128 | Show the html template for training the neural network 129 | """ 130 | return render_template("train.html") 131 | 132 | 133 | @app.route('/train', methods=['POST']) 134 | @login_required 135 | def train_http(): 136 | """ 137 | 138 | :return: 139 | """ 140 | # check if the post request has the file part 141 | if 'file' not in request.files or request.files['file'].filename == '': 142 | flash('No file choose :/', category="error") 143 | return redirect(request.url) # Return to HTML page [GET] 144 | file = request.files['file'] 145 | file.save(os.path.join(TMP_UPLOAD_TRAINING, file.filename)) 146 | return jsonify(train_network(TMP_UPLOAD_TRAINING, file, clf, DETECTION_MODEL, JITTER, ENCODING_MODELS)) 147 | 148 | 149 | @app.route('/tune', methods=['GET']) 150 | @login_required 151 | def tune(): 152 | """ 153 | Show the html template for training the neural network 154 | """ 155 | return render_template("train.html") 156 | 157 | 158 | @app.route('/tune', methods=['POST']) 159 | @login_required 160 | def tune_http(): 161 | """ 162 | 163 | :return: 164 | """ 165 | # check if the post request has the file part 166 | if 'file' not in request.files or request.files['file'].filename == '': 167 | flash('No file choose :/', category="error") 168 | return redirect(request.url) # Return to HTML page [GET] 169 | file = request.files['file'] 170 | file.save(os.path.join(TMP_UPLOAD_TRAINING, file.filename)) 171 | return jsonify(tune_network(TMP_UPLOAD_TRAINING, file, clf, DETECTION_MODEL, JITTER, ENCODING_MODELS)) 172 | 173 | 174 | @app.route('/uploads/') 175 | def uploaded_file(filename): 176 | """ 177 | Expose images only to the one that know the image name in a secure method 178 | :param filename: 179 | :return: 180 | """ 181 | if os.path.exists(os.path.join(TMP_UPLOAD_PREDICTION, filename)): 182 | return send_from_directory(TMP_UPLOAD_PREDICTION, filename) 183 | return "PHOTOS_NOT_FOUND" 184 | 185 | 186 | class User(UserMixin): 187 | pass 188 | 189 | 190 | @login_manager.user_loader 191 | def user_loader(email): 192 | user = Administrator("administrator", email, "dummypassword") 193 | user.init_redis_connection() 194 | user_exists = user.verify_user_exist() 195 | user.redis_client.close() 196 | if not user_exists: 197 | return None 198 | user.id = email 199 | return user 200 | 201 | 202 | @app.route('/login', methods=['GET', 'POST']) 203 | def login(): 204 | if request.method == 'GET': 205 | return render_template("login.html") 206 | 207 | email = request.form['email'] 208 | password = request.form['password'] 209 | log.debug("Password in input -> {}".format(password)) 210 | # name (administrator) is not managed, only mail and psw will be used for login validation 211 | admin = Administrator("administrator", email, password) 212 | if not admin.init_redis_connection(): 213 | log.error("Unable to connect to redis-db!") 214 | response = Response(status="KO", error="UNABLE_CONNECT_REDIS_DB") 215 | response.description = "Seems that the DB is not reachable!" 216 | return jsonify(response=response.__dict__) 217 | authenticated = admin.verify_login(password) 218 | admin.redis_client.close() 219 | if not authenticated: 220 | log.error("Password is not valid!") 221 | response = Response(status="KO", error="PASSWORD_NOT_VALID") 222 | response.description = "The password inserted is not valid!" 223 | return jsonify(response=response.__dict__) 224 | user = User() 225 | user.id = email 226 | login_user(user) 227 | log.debug("Logged in!") 228 | return redirect('/train') 229 | 230 | 231 | @app.route('/protected') 232 | @login_required 233 | def protected(): 234 | return 'Logged in as: ' + current_user.id 235 | 236 | 237 | @app.before_request 238 | def csrf_protect(): 239 | """ 240 | Validate csrf token against the one in session 241 | :return: 242 | """ 243 | if ENABLE_CSRF: 244 | if "dashboard" not in str(request.url_rule): 245 | if request.method == "POST": 246 | token = session.pop('_csrf_token', None) 247 | if not token or token != request.form.get('_csrf_token'): 248 | abort(403) 249 | 250 | 251 | @app.after_request 252 | def secure_headers(response): 253 | """ 254 | Apply security headers to the response call 255 | :return: 256 | """ 257 | return secure_request(response, SSL_ENABLED) 258 | 259 | 260 | def generate_csrf_token(): 261 | """ 262 | Generate a random string and set the data into session 263 | :return: 264 | """ 265 | if '_csrf_token' not in session: 266 | session['_csrf_token'] = random_string() 267 | return session['_csrf_token'] 268 | 269 | 270 | def signal_handler(signal, frame): 271 | find_duplicates(TMP_UPLOAD) 272 | find_duplicates(TMP_UPLOAD_PREDICTION) 273 | find_duplicates(TMP_UNKNOWN) 274 | sys.exit(0) 275 | 276 | 277 | signal.signal(signal.SIGINT, signal_handler) 278 | 279 | app.jinja_env.globals['csrf_token'] = generate_csrf_token 280 | app.jinja_env.autoescape = True 281 | 282 | if __name__ == '__main__': 283 | if SSL_ENABLED: 284 | log.debug("main | RUNNING OVER SSL") 285 | app.run(host=CFG["network"]["host"], port=CFG["network"]["port"], threaded=False, debug=False, 286 | use_reloader=False, ssl_context=( 287 | PUB_KEY, PRIV_KEY)) 288 | else: 289 | log.debug("main | HTTPS DISABLED | RUNNING OVER HTTP") 290 | app.run(host=CFG["network"]["host"], port=CFG["network"]["port"], threaded=False, debug=False) 291 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Cython==0.29.27 2 | numpy==1.19.4 3 | requests 4 | flask-login==0.5.0 5 | face_recognition==1.3.0 6 | redis==4.1.2 7 | Flask_MonitoringDashboard==3.1.1 8 | Flask==1.1.2 9 | Werkzeug==2.0.3 10 | tqdm==4.62.3 11 | olefile==0.46 12 | Pillow==8.3.2 13 | py-bcrypt==0.4 14 | scikit_learn==0.22.2.post1 -------------------------------------------------------------------------------- /test/conf_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "PyRecognizer": { 3 | "Version": "0.3.3", 4 | "temp_upload_training": "../uploads/training/", 5 | "temp_upload_predict": "../uploads/predict/", 6 | "temp_upload": "../uploads/upload", 7 | "temp_unknown": "../uploads/unknown", 8 | "detection_model": "hog", 9 | "jitter": 1, 10 | "encoding_models": "small", 11 | "enable_dashboard": false 12 | }, 13 | "logging": { 14 | "path": "test_log/", 15 | "prefix": "pyrecognizer.log", 16 | "level": "debug" 17 | }, 18 | "network": { 19 | "host": "0.0.0.0", 20 | "port": 8081, 21 | "templates": "../api/templates/", 22 | "csrf_protection": false, 23 | "SSL": { 24 | "enabled": false, 25 | "cert.pub": "../conf/ssl/localhost.crt", 26 | "cert.priv": "../conf/ssl/localhost.key" 27 | } 28 | }, 29 | "classifier": { 30 | "training_dir": "../dataset/images/", 31 | "model_path": "../dataset/model/", 32 | "timestamp": "20191202_153034" 33 | }, 34 | "dashboard": { 35 | "config_file": "../conf/dashboard.ini" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test/test_admin.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | import unittest 4 | 5 | sys.path.insert(0, "../") 6 | from datastructure.Administrator import Administrator 7 | from utils.util import load_logger 8 | 9 | password = "mysecretpassword" 10 | mail = "alessiosavibtc@gmail.com" 11 | log = load_logger("debug", "test_log", "pyrecognizer.log") 12 | a = Administrator("alessio", mail, password) 13 | a.init_redis_connection() 14 | 15 | 16 | class TestAdmin(unittest.TestCase): 17 | 18 | def test_a_connection(self): 19 | self.assertTrue(a.init_redis_connection()) 20 | 21 | def test_b_invalid_password(self): 22 | self.assertFalse(a.validate_password("a")) 23 | self.assertFalse(a.validate_password("1")) 24 | self.assertFalse(a.validate_password("aaaa")) 25 | 26 | def test_c_valid_password(self): 27 | self.assertTrue(a.validate_password("aaaaa")) 28 | self.assertTrue(a.validate_password("11111")) 29 | self.assertTrue(a.validate_password("!!!!!")) 30 | self.assertTrue(a.validate_password(password)) 31 | 32 | def test_d_register_user(self): 33 | self.assertTrue(a.add_user()) 34 | 35 | def test_e_login_user(self): 36 | self.assertTrue(a.verify_login(password)) 37 | 38 | def test_f_remove_user(self): 39 | self.assertTrue(a.remove_user()) 40 | -------------------------------------------------------------------------------- /test/test_classifier.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Basic test case for classifier 4 | """ 5 | import json 6 | import sys 7 | import unittest 8 | from typing import Dict 9 | 10 | import requests 11 | 12 | sys.path.insert(0, "../") 13 | from datastructure.Classifier import Classifier 14 | from utils.util import init_main_data 15 | 16 | config_file: str = "conf_test.json" 17 | CFG, log, TMP_UPLOAD_PREDICTION, TMP_UPLOAD_TRAINING, TMP_UPLOAD, TMP_UNKNOWN, detection_model, jitter, encoding_models, _ = init_main_data( 18 | config_file) 19 | 20 | url = "http://0.0.0.0:8081/" 21 | 22 | 23 | def load_classifier(config: Dict) -> Classifier: 24 | clf = Classifier() 25 | clf.model_path = config["classifier"]["model_path"] 26 | clf.load_classifier_from_file(config["classifier"]["timestamp"]) 27 | return clf 28 | 29 | 30 | class TestPredict(unittest.TestCase): 31 | clf = load_classifier(CFG) 32 | 33 | def test_predict_without_file(self): 34 | values = {"threshold": "45"} 35 | r = requests.post(url, data=values) 36 | log.debug(r.content) 37 | response = json.loads(r.content)["response"]["error"] 38 | self.assertEqual("NO_FILE_IN_REQUEST", response) 39 | 40 | def test_predict_without_threshold(self): 41 | image = open('test_images/bush_test.jpg', 'rb') 42 | files = {"file": image} 43 | r = requests.post(url, files=files) 44 | image.close() 45 | log.debug(r.content) 46 | response = json.loads(r.content)["response"]["error"] 47 | self.assertEqual("THRESHOLD_NOT_PROVIDED", response) 48 | 49 | def test_predict_without_threshold_not_number(self): 50 | image = open('test_images/bush_test.jpg', 'rb') 51 | files = {"file": image} 52 | data = {"threshold": "a"} 53 | r = requests.post(url, files=files, data=data) 54 | image.close() 55 | log.debug(r.content) 56 | response = json.loads(r.content)["response"]["error"] 57 | self.assertEqual("UNABLE_CAST_INT", response) 58 | 59 | def test_predict_without_threshold_invalid_number(self): 60 | image = open('test_images/bush_test.jpg', 'rb') 61 | files = {"file": image} 62 | data = {"threshold": "-1"} 63 | r = requests.post(url, files=files, data=data) 64 | image.close() 65 | log.debug(r.content) 66 | response = json.loads(r.content)["response"]["error"] 67 | self.assertEqual("THRESHOLD_ERROR_VALUE", response) 68 | 69 | def test_predict_with_no_face(self): 70 | image = open('test_images/no_face_test.png', 'rb') 71 | files = {"file": image} 72 | data = {"threshold": "1"} 73 | r = requests.post(url, files=files, data=data) 74 | image.close() 75 | log.debug(r.content) 76 | response = json.loads(r.content)["response"] 77 | self.assertEqual("FACE_NOT_FOUND", response["error"]) 78 | 79 | def test_predict_one_face(self): 80 | image = open('test_images/bush_test.jpg', 'rb') 81 | files = {"file": image} 82 | data = {"threshold": "1"} 83 | r = requests.post(url, files=files, data=data) 84 | image.close() 85 | log.debug(r.content) 86 | response = json.loads(r.content)["response"] 87 | self.assertIsNone(response["error"]) 88 | data = response["data"] 89 | self.assertEqual(list(data.keys())[0], "George_W_Bush") 90 | self.assertGreater(list(data.values())[0], 0.99) 91 | 92 | def test_predict_multiple_face(self): 93 | image = open('test_images/multi_face_test.jpg', 'rb') 94 | files = {"file": image} 95 | data = {"threshold": "1"} 96 | r = requests.post(url, files=files, data=data) 97 | image.close() 98 | log.debug(r.content) 99 | response = json.loads(r.content)["response"] 100 | self.assertIsNone(response["error"]) 101 | data = response["data"] 102 | self.assertEqual(list(data.keys())[0], "Angelina_Jolie") 103 | self.assertEqual(list(data.keys())[1], "Clint_Eastwood") 104 | self.assertGreater(list(data.values())[0], 0.85) 105 | self.assertGreater(list(data.values())[1], 0.93) 106 | 107 | def test_predict_unknown_face(self): 108 | image = open('test_images/unknown_face.jpg', 'rb') 109 | files = {"file": image} 110 | data = {"threshold": "90"} 111 | r = requests.post(url, files=files, data=data) 112 | image.close() 113 | log.debug(r.content) 114 | response = json.loads(r.content)["response"] 115 | self.assertEqual("FACE_NOT_RECOGNIZED", response["error"]) 116 | 117 | 118 | if __name__ == '__main__': 119 | # begin the unittest.main() 120 | unittest.main() 121 | -------------------------------------------------------------------------------- /test/test_images/bush_test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/test/test_images/bush_test.jpg -------------------------------------------------------------------------------- /test/test_images/multi_face_test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/test/test_images/multi_face_test.jpg -------------------------------------------------------------------------------- /test/test_images/no_face_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/test/test_images/no_face_test.png -------------------------------------------------------------------------------- /test/test_images/unknown_face.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alessiosavi/PyRecognizer/bf8edc19247cf6aac267cf7892ca9cc1175b6bf2/test/test_images/unknown_face.jpg -------------------------------------------------------------------------------- /utils/add_users.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Generate the administrator of the neural network, delegated to train/tune the model 4 | 5 | NOTE: The login will be made using the `mail` and `password` fields, username is not necessary 6 | """ 7 | import sys 8 | 9 | sys.path.insert(0, "../") 10 | 11 | from datastructure.Administrator import Administrator 12 | 13 | # Creating a new administrator with the following credentials 14 | a = Administrator("username", "mail", "password") 15 | a.init_redis_connection() 16 | print("Remove user -> {}".format(a.remove_user())) 17 | print("Add user -> {}".format(a.add_user())) 18 | print("Verify login {}".format(a.verify_login("password"))) 19 | -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Common method for reuse code 4 | 5 | Generate certificate 6 | 7 | openssl req -x509 -out localhost.crt -keyout localhost.key \ 8 | -newkey rsa:2048 -nodes -sha256 \ 9 | -subj '/CN=localhost' -extensions EXT -config <( \ 10 | printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth") 11 | """ 12 | import filecmp 13 | import json 14 | import logging 15 | import os 16 | import pickle 17 | import random 18 | import shutil 19 | import string 20 | import zipfile 21 | from logging.handlers import TimedRotatingFileHandler 22 | from typing import Dict, Union 23 | 24 | import PIL 25 | import numpy as np 26 | from PIL import Image, ImageDraw 27 | 28 | levels = { 29 | 'debug': logging.DEBUG, 30 | 'info': logging.INFO, 31 | 'warning': logging.WARNING, 32 | 'error': logging.ERROR, 33 | 'critical': logging.CRITICAL 34 | } 35 | 36 | 37 | def print_prediction_on_image(img_path, predictions, file_to_save): 38 | """ 39 | Shows the face recognition results visually. 40 | 41 | :param file_to_save: 42 | :param img_path: path to image to be recognized 43 | :param predictions: results of the predict function 44 | :return: 45 | """ 46 | pil_image = Image.open(img_path).convert("RGB") 47 | draw = ImageDraw.Draw(pil_image) 48 | 49 | for name, (top, right, bottom, left) in predictions: 50 | # Draw a box around the face using the Pillow module 51 | draw.rectangle(((left, top), (right, bottom)), outline=(0, 0, 255)) 52 | 53 | # There's a bug in Pillow where it blows up with non-UTF-8 text 54 | # when using the default bitmap font 55 | name = name.encode("UTF-8") 56 | 57 | # Draw a label with a name below the face 58 | _, text_height = draw.textsize(name) 59 | draw.rectangle(((left, bottom - text_height - 10), 60 | (right, bottom)), fill=(0, 0, 255), outline=(0, 0, 255)) 61 | draw.text((left + 6, bottom - text_height - 5), 62 | name, fill=(255, 255, 255, 255)) 63 | # Remove the drawing library from memory as per the Pillow docs 64 | del draw 65 | 66 | # Display the resulting image 67 | # pil_image.show() 68 | pil_image.save(file_to_save, "PNG") 69 | 70 | 71 | def init_main_data(config_file): 72 | """ 73 | Parse the configuration file and return the necessary data for initalize the tool 74 | :param config_file: 75 | :return: 76 | """ 77 | try: 78 | with open(config_file) as f: 79 | CFG = json.load(f) 80 | except json.decoder.JSONDecodeError: 81 | raise Exception("Unable to load JSON File: {}".format(config_file)) 82 | 83 | log = load_logger(CFG["logging"]["level"], CFG["logging"]["path"], CFG["logging"]["prefix"]) 84 | 85 | # Store the image predicted drawing a boc in the face of the person 86 | TMP_UPLOAD_PREDICTION = CFG["PyRecognizer"]["temp_upload_predict"] 87 | # Uncompress the images in this folder 88 | TMP_UPLOAD_TRAINING = CFG["PyRecognizer"]["temp_upload_training"] 89 | # Save the images sent by the customer 90 | TMP_UPLOAD = CFG["PyRecognizer"]["temp_upload"] 91 | # Save the images of unknown people for future clustering/labeling 92 | TMP_UNKNOWN = CFG["PyRecognizer"]["temp_unknown"] 93 | # Use CNN if you have a GPU, use HOG if you have CPU only 94 | detection_model = CFG["PyRecognizer"]["detection_model"].lower() 95 | # Use data augmentation when retrieving face encodings 96 | jitter = CFG["PyRecognizer"]["jitter"] 97 | # Use 68 or 5 points 98 | encoding_models = CFG["PyRecognizer"]["encoding_models"].lower() 99 | 100 | enable_dashboard = CFG["PyRecognizer"]["enable_dashboard"] 101 | 102 | if detection_model not in ('hog', 'cnn'): 103 | log.warning("detection_model selected is not valid! [{}], using 'hog' as default!".format(detection_model)) 104 | detection_model = "hog" 105 | 106 | if jitter < 0: 107 | log.warning("jitter can be lower than 0, using 1 as default") 108 | jitter = 1 109 | 110 | if encoding_models not in ('small', 'large'): 111 | log.warning("encoding_models have to be or 'large' or 'small', using 'small' as fallback") 112 | encoding_models = "small" 113 | 114 | if not os.path.exists(TMP_UPLOAD_PREDICTION): 115 | os.makedirs(TMP_UPLOAD_PREDICTION) 116 | if not os.path.exists(TMP_UPLOAD_TRAINING): 117 | os.makedirs(TMP_UPLOAD_TRAINING) 118 | if not os.path.exists(TMP_UPLOAD): 119 | os.makedirs(TMP_UPLOAD) 120 | if not os.path.exists(TMP_UNKNOWN): 121 | os.makedirs(TMP_UNKNOWN) 122 | 123 | return CFG, log, TMP_UPLOAD_PREDICTION, TMP_UPLOAD_TRAINING, TMP_UPLOAD, TMP_UNKNOWN, detection_model, jitter, encoding_models, enable_dashboard 124 | 125 | 126 | def load_logger(level, path, name): 127 | """ 128 | 129 | :param level: 130 | :param path: 131 | :param name: 132 | """ 133 | logger = logging.getLogger() # set up root logger 134 | if not os.path.exists(path): 135 | logger.warning("Folder {} not found, creating a new one ...".format(path)) 136 | os.makedirs(path) 137 | filename = os.path.join(path, name) 138 | handler = TimedRotatingFileHandler(filename, when='H') 139 | handler.suffix = "%Y-%m-%d.log" 140 | handler.extMatch = r"^\d{4}-\d{2}-\d{2}\.log$" 141 | 142 | level = levels[level] 143 | handler.setLevel(level) # set level for handler 144 | formatter = '%(asctime)s - %(name)s - %(levelname)s | [%(filename)s:%(lineno)d] | %(message)s' 145 | handler.setFormatter(logging.Formatter(formatter)) 146 | logger.addHandler(handler) 147 | logger.setLevel(level) 148 | logger.warning("Logger initialized, dumping log in {}".format(filename)) 149 | return logger 150 | 151 | 152 | def random_string(string_length=13): 153 | """ 154 | Generate a random string of fixed length 155 | :param string_length: 156 | :return: 157 | """ 158 | letters = string.ascii_lowercase 159 | data = "" 160 | for _ in range(string_length): 161 | data += random.choice(letters) 162 | return data 163 | 164 | 165 | def zip_data(file_to_zip, path): 166 | """ 167 | 168 | :param file_to_zip: 169 | :param path: 170 | :return: 171 | """ 172 | 173 | shutil.make_archive(file_to_zip, 'zip', path) 174 | 175 | 176 | def unzip_data(unzipped_folder, zip_file): 177 | """ 178 | Unzip the zip file in input in the given 'unzipped_folder' 179 | :param unzipped_folder: 180 | :param zip_file: 181 | :return: The name of the folder in which find the unzipped data 182 | """ 183 | log = logging.getLogger() 184 | folder_name = os.path.join(unzipped_folder, random_string()) 185 | log.debug("unzip_data | Unzipping {} into {}".format(zip_file, folder_name)) 186 | zip_ref = zipfile.ZipFile(zip_file) 187 | zip_ref.extractall(folder_name) 188 | zip_ref.close() 189 | log.debug("unzip_data | File uncompressed!") 190 | return folder_name 191 | 192 | 193 | def dump_dataset(X, Y, path): 194 | """ 195 | 196 | :param X: 197 | :param Y: 198 | :param path: 199 | :return: 200 | """ 201 | log = logging.getLogger() 202 | dataset = { 203 | 'X': X, 204 | 'Y': Y 205 | } 206 | log.debug("dump_dataset | Dumping dataset int {}".format(path)) 207 | if not os.path.exists(path): 208 | os.makedirs(path) 209 | log.debug("dump_dataset | Path {} exist".format(path)) 210 | dataset_name = os.path.join(path, "model.dat") 211 | with open(dataset_name, 'wb') as f: 212 | pickle.dump(dataset, f) 213 | else: 214 | log.error( 215 | "dump_dataset | Path {} ALREADY EXIST exist, avoiding to overwrite".format(path)) 216 | 217 | 218 | def remove_dir(directory: str): 219 | """ 220 | Wrapper for remove a directory 221 | :param directory: 222 | :return: 223 | """ 224 | log = logging.getLogger() 225 | log.debug("remove_dir | Removing directory {}".format(directory)) 226 | if not os.path.exists(directory): 227 | log.warning("remove_dir | Folder {} does not exists!".format(directory)) 228 | return 229 | if not os.path.isdir(directory): 230 | log.warning("remove_dir | File {} is not a folder".format(directory)) 231 | return 232 | shutil.rmtree(directory) 233 | 234 | 235 | def verify_extension(folder, file): 236 | """ 237 | Wrapper for validate file 238 | :param folder: 239 | :param file: 240 | :return: 241 | """ 242 | log = logging.getLogger() 243 | extension = os.path.splitext(file)[1] 244 | log.debug("verify_extension | File: {} | Ext: {}".format(file, extension)) 245 | filepath = os.path.join(folder, file) 246 | file_ext = None 247 | if os.path.exists(filepath): 248 | if extension == ".zip": 249 | log.debug("verify_extension | Verifying zip bomb ...") 250 | zp = zipfile.ZipFile(filepath) 251 | size = sum(zinfo.file_size for zinfo in zp.filelist) 252 | zip_kb = float(size) / (1000 * 1000) # MB 253 | if zip_kb > 250: 254 | log.error("verify_extension | ZIP BOMB DETECTED! | Zip file size is to much {} MB ...".format(size)) 255 | return "ZIP_BOMB!" 256 | file_ext = "zip" 257 | 258 | elif extension == ".dat": 259 | # Photos have been already analyzed, dataset is ready! 260 | file_ext = "dat" 261 | 262 | log.error("verify_extension | Filename {} does not exist!".format(filepath)) 263 | return file_ext 264 | 265 | 266 | def retrieve_dataset(folder_uncompress, zip_file, clf, detection_model, jitter, encoding_models): 267 | """ 268 | 269 | :param folder_uncompress: 270 | :param zip_file: 271 | :param clf: 272 | :return: 273 | """ 274 | log = logging.getLogger() 275 | log.debug("retrieve_dataset | Parsing dataset ...") 276 | check = verify_extension(folder_uncompress, zip_file.filename) 277 | if check == "zip": # Image provided 278 | log.debug("retrieve_dataset | Zip file uploaded") 279 | folder_name = unzip_data(folder_uncompress, zip_file) 280 | log.debug("retrieve_dataset | zip file uncompressed!") 281 | clf.init_peoples_list(detection_model, jitter, encoding_models, peoples_path=folder_name) 282 | dataset = clf.init_dataset() 283 | log.debug("retrieve_dataset | Removing [{}]".format(folder_name)) 284 | remove_dir(folder_name) 285 | elif check == "dat": 286 | log.debug("retrieve_dataset | Pickle data uploaded") 287 | dataset = pickle.load(zip_file) 288 | else: 289 | log.warning("retrieve_dataset | Unable to understand the file type: {}".format(check)) 290 | dataset = None 291 | log.debug("retrieve_dataset | Dataset parsed!") 292 | return dataset 293 | 294 | 295 | def secure_request(request, ssl: bool): 296 | """ 297 | 298 | :param ssl: 299 | :param request: 300 | :return: 301 | """ 302 | # request.headers['Content-Security-Policy'] = "script-src 'self' cdnjs.cloudflare.com ; " 303 | request.headers['Feature-Policy'] = "geolocation 'none'; microphone 'none'; camera 'self'" 304 | request.headers['Referrer-Policy'] = 'no-referrer' 305 | request.headers['x-frame-options'] = 'SAMEORIGIN' 306 | request.headers['X-Content-Type-Options'] = 'nosniff' 307 | request.headers['X-Permitted-Cross-Domain-Policies'] = 'none' 308 | request.headers['X-XSS-Protection'] = '1; mode=block' 309 | if ssl: 310 | request.headers['expect-ct'] = 'max-age=60, enforce' 311 | request.headers["Content-Security-Policy"] = "upgrade-insecure-requests" 312 | request.headers['Strict-Transport-Security'] = "max-age=60; includeSubDomains; preload" 313 | 314 | return request 315 | 316 | 317 | def concatenate_dataset(*dictionary) -> Union[Dict, None]: 318 | """ 319 | Concatenate multiple dataset into a single one 320 | """ 321 | dataset = {} 322 | for d in dictionary: 323 | if type(d) is dict: 324 | dataset.update(d) 325 | 326 | if len(dataset) == 2: 327 | if "X" in dataset and "Y" in dataset: 328 | if len(dataset["X"]) != len(dataset["Y"]): 329 | return None 330 | return dataset 331 | 332 | 333 | def load_image_file(file, mode='RGB'): 334 | """ 335 | Loads an image file (.jpg, .png, etc) into a numpy array 336 | 337 | :param file: image file name or file object to load 338 | :param mode: format to convert the image to. Only 'RGB' (8-bit RGB, 3 channels) and 'L' (black and white) are supported. 339 | :return: image contents as numpy array 340 | """ 341 | 342 | im = PIL.Image.open(file) 343 | width, height = im.size 344 | w, h = width, height 345 | log = logging.getLogger() 346 | 347 | ratio = -1 348 | # Ratio for resize the image 349 | log.debug("load_image_file | Image dimension: ({}:{})".format(w, h)) 350 | # Resize in case of to bigger dimension 351 | # In first instance manage the HIGH-Dimension photos 352 | if width > 3600 or height > 3600: 353 | if width > height: 354 | ratio = width / 800 355 | else: 356 | ratio = height / 800 357 | 358 | elif 1200 <= width <= 1600 or 1200 <= height <= 1600: 359 | ratio = 1 / 2 360 | elif 1600 <= width <= 3600 or 1600 <= height <= 3600: 361 | ratio = 1 / 3 362 | 363 | log.debug("load_image_file | Dimension: w: {} | h: {}".format(w, h)) 364 | log.debug("load_image_file | New ratio -> {}".format(ratio)) 365 | 366 | if 0 < ratio < 1: 367 | # Scale image in case of width > 1600 368 | w = width * ratio 369 | h = height * ratio 370 | elif ratio > 1: 371 | # Scale image in case of width > 3600 372 | w = width / ratio 373 | h = height / ratio 374 | if w != width: 375 | # Check if scaling was applied 376 | maxsize = (w, h) 377 | log.debug("load_image_file | Image have to high dimension, avoiding memory error. Resizing to {}" 378 | .format(maxsize)) 379 | im.thumbnail(maxsize, PIL.Image.ANTIALIAS) 380 | 381 | if mode: 382 | im = im.convert(mode) 383 | return np.array(im), ratio 384 | 385 | 386 | def find_duplicates(directory: str): 387 | log = logging.getLogger() 388 | # List files in the directory 389 | files = [] 390 | for _file in os.listdir(directory): 391 | files.append(_file) 392 | 393 | equals = [] 394 | # Searching duplicates files 395 | for i in range(len(files)): 396 | for j in range(i + 1, len(files)): 397 | if filecmp.cmp(os.path.join(directory, files[i]), os.path.join(directory, files[j])): 398 | equals.append(os.path.join(directory, files[i])) 399 | break 400 | log.info("find_duplicates | Removing the following duplicates files: {}".format(equals)) 401 | for _file in equals: 402 | log.debug("find_duplicates | Removing file: {}".format(_file)) 403 | os.remove(_file) 404 | -------------------------------------------------------------------------------- /wsgi.py: -------------------------------------------------------------------------------- 1 | from main import app 2 | 3 | # gunicorn --certfile="conf/ssl/localhost.crt" --keyfile="conf/ssl/localhost.key" --bind 0.0.0.0:8081 wsgi:app 4 | if __name__ == "__main__": 5 | app.run() 6 | --------------------------------------------------------------------------------