├── .gitignore ├── LICENSE ├── README.md ├── cnn_filter_visualization.ipynb ├── dataset ├── faceforensics_download_v4.py ├── frame_extraction.py ├── images_tiny │ ├── original │ │ ├── 001 │ │ │ ├── 0000.png │ │ │ ├── 0001.png │ │ │ ├── 0002.png │ │ │ ├── 0003.png │ │ │ ├── 0004.png │ │ │ ├── 0005.png │ │ │ ├── 0006.png │ │ │ ├── 0007.png │ │ │ ├── 0008.png │ │ │ ├── 0009.png │ │ │ └── 0010.png │ │ └── 034 │ │ │ ├── 034_0000.png │ │ │ ├── 034_0001.png │ │ │ └── 034_0002.png │ └── tampered │ │ ├── 002_003 │ │ ├── 0001.png │ │ ├── 0002.png │ │ ├── 0003.png │ │ ├── 0004.png │ │ ├── 0005.png │ │ ├── 0006.png │ │ ├── 0007.png │ │ ├── 0008.png │ │ ├── 0009.png │ │ └── 0010.png │ │ └── 259_345 │ │ ├── 259_345_0000.png │ │ ├── 259_345_0001.png │ │ ├── 259_345_0002.png │ │ ├── 259_345_0003.png │ │ ├── 259_345_0004.png │ │ ├── 259_345_0005.png │ │ ├── 259_345_0006.png │ │ ├── 259_345_0007.png │ │ ├── 259_345_0008.png │ │ ├── 259_345_0009.png │ │ ├── 259_345_0010.png │ │ └── 259_345_0011.png ├── prepare_full_dataset.sh ├── requirements.txt └── splits │ ├── test.json │ ├── train.json │ └── val.json ├── diagrams ├── 3d_conv.png ├── 3dconv.drawio ├── CNN_LSTM.drawio ├── CNN_LSTM.png ├── Inception-resnet.drawio ├── ff_benchmark.png └── incep_resnet_v1.png ├── model_evalution.ipynb ├── requirements.txt ├── src ├── FaceCrop.py ├── benchmark.py ├── data_loader.py ├── evaluate.py ├── model.py ├── resnet3d.py ├── train.py └── utils.py └── training_documentation /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/venv,python,pycharm,jetbrains,virtualenv,jupyternotebooks 3 | # Edit at https://www.gitignore.io/?templates=venv,python,pycharm,jetbrains,virtualenv,jupyternotebooks 4 | 5 | ### JetBrains ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | # *.iml 40 | # *.ipr 41 | 42 | # CMake 43 | cmake-build-*/ 44 | 45 | # Mongo Explorer plugin 46 | .idea/**/mongoSettings.xml 47 | 48 | # File-based project format 49 | *.iws 50 | 51 | # IntelliJ 52 | out/ 53 | 54 | # mpeltonen/sbt-idea plugin 55 | .idea_modules/ 56 | 57 | # JIRA plugin 58 | atlassian-ide-plugin.xml 59 | 60 | # Cursive Clojure plugin 61 | .idea/replstate.xml 62 | 63 | # Crashlytics plugin (for Android Studio and IntelliJ) 64 | com_crashlytics_export_strings.xml 65 | crashlytics.properties 66 | crashlytics-build.properties 67 | fabric.properties 68 | 69 | # Editor-based Rest Client 70 | .idea/httpRequests 71 | 72 | # Android studio 3.1+ serialized cache file 73 | .idea/caches/build_file_checksums.ser 74 | 75 | ### JetBrains Patch ### 76 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 77 | 78 | # *.iml 79 | # modules.xml 80 | # .idea/misc.xml 81 | # *.ipr 82 | 83 | # Sonarlint plugin 84 | .idea/**/sonarlint/ 85 | 86 | # SonarQube Plugin 87 | .idea/**/sonarIssues.xml 88 | 89 | # Markdown Navigator plugin 90 | .idea/**/markdown-navigator.xml 91 | .idea/**/markdown-navigator/ 92 | 93 | ### JupyterNotebooks ### 94 | # gitignore template for Jupyter Notebooks 95 | # website: http://jupyter.org/ 96 | 97 | .ipynb_checkpoints 98 | */.ipynb_checkpoints/* 99 | 100 | # IPython 101 | profile_default/ 102 | ipython_config.py 103 | 104 | # Remove previous ipynb_checkpoints 105 | # git rm -r .ipynb_checkpoints/ 106 | 107 | ### PyCharm ### 108 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 109 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 110 | 111 | # User-specific stuff 112 | 113 | # Generated files 114 | 115 | # Sensitive or high-churn files 116 | 117 | # Gradle 118 | 119 | # Gradle and Maven with auto-import 120 | # When using Gradle or Maven with auto-import, you should exclude module files, 121 | # since they will be recreated, and may cause churn. Uncomment if using 122 | # auto-import. 123 | # .idea/modules.xml 124 | # .idea/*.iml 125 | # .idea/modules 126 | # *.iml 127 | # *.ipr 128 | 129 | # CMake 130 | 131 | # Mongo Explorer plugin 132 | 133 | # File-based project format 134 | 135 | # IntelliJ 136 | 137 | # mpeltonen/sbt-idea plugin 138 | 139 | # JIRA plugin 140 | 141 | # Cursive Clojure plugin 142 | 143 | # Crashlytics plugin (for Android Studio and IntelliJ) 144 | 145 | # Editor-based Rest Client 146 | 147 | # Android studio 3.1+ serialized cache file 148 | 149 | ### PyCharm Patch ### 150 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 151 | 152 | # *.iml 153 | # modules.xml 154 | # .idea/misc.xml 155 | # *.ipr 156 | 157 | # Sonarlint plugin 158 | 159 | # SonarQube Plugin 160 | 161 | # Markdown Navigator plugin 162 | 163 | ### Python ### 164 | # Byte-compiled / optimized / DLL files 165 | __pycache__/ 166 | *.py[cod] 167 | *$py.class 168 | 169 | # C extensions 170 | *.so 171 | 172 | # Distribution / packaging 173 | .Python 174 | build/ 175 | develop-eggs/ 176 | dist/ 177 | downloads/ 178 | eggs/ 179 | .eggs/ 180 | lib/ 181 | lib64/ 182 | parts/ 183 | sdist/ 184 | var/ 185 | wheels/ 186 | pip-wheel-metadata/ 187 | share/python-wheels/ 188 | *.egg-info/ 189 | .installed.cfg 190 | *.egg 191 | MANIFEST 192 | 193 | # PyInstaller 194 | # Usually these files are written by a python script from a template 195 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 196 | *.manifest 197 | *.spec 198 | 199 | # Installer logs 200 | pip-log.txt 201 | pip-delete-this-directory.txt 202 | 203 | # Unit test / coverage reports 204 | htmlcov/ 205 | .tox/ 206 | .nox/ 207 | .coverage 208 | .coverage.* 209 | .cache 210 | nosetests.xml 211 | coverage.xml 212 | *.cover 213 | .hypothesis/ 214 | .pytest_cache/ 215 | 216 | # Translations 217 | *.mo 218 | *.pot 219 | 220 | # Scrapy stuff: 221 | .scrapy 222 | 223 | # Sphinx documentation 224 | docs/_build/ 225 | 226 | # PyBuilder 227 | target/ 228 | 229 | # pyenv 230 | .python-version 231 | 232 | # pipenv 233 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 234 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 235 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 236 | # install all needed dependencies. 237 | #Pipfile.lock 238 | 239 | # celery beat schedule file 240 | celerybeat-schedule 241 | 242 | # SageMath parsed files 243 | *.sage.py 244 | 245 | # Spyder project settings 246 | .spyderproject 247 | .spyproject 248 | 249 | # Rope project settings 250 | .ropeproject 251 | 252 | # Mr Developer 253 | .mr.developer.cfg 254 | .project 255 | .pydevproject 256 | 257 | # mkdocs documentation 258 | /site 259 | 260 | # mypy 261 | .mypy_cache/ 262 | .dmypy.json 263 | dmypy.json 264 | 265 | # Pyre type checker 266 | .pyre/ 267 | 268 | ### venv ### 269 | # Virtualenv 270 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 271 | [Bb]in 272 | [Ii]nclude 273 | [Ll]ib 274 | [Ll]ib64 275 | [Ll]ocal 276 | [Ss]cripts 277 | pyvenv.cfg 278 | .env 279 | .venv 280 | env/ 281 | venv/ 282 | ENV/ 283 | env.bak/ 284 | venv.bak/ 285 | pip-selfcheck.json 286 | 287 | ### VirtualEnv ### 288 | # Virtualenv 289 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 290 | 291 | # End of https://www.gitignore.io/api/venv,python,pycharm,jetbrains,virtualenv,jupyternotebooks 292 | .idea 293 | /src/runs/ 294 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Nika Dogonadze, Jana Obernosterer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## DeepFaceForgery Detection 2 | This repository contains code for deep face forgery detection in video frames. This is a student project from [Advanced Deep Learning for Computer Vision](https://dvl.in.tum.de/teaching/adl4cv-ws18/) course at [TUM](https://www.tum.de/). Publication available on [Arxiv](https://arxiv.org/abs/2004.11804) 3 | 4 | ### FaceForensics Benchmark 5 | Using transfer learning we were able to achieve a new state of the art performance on [faceforenics benchmark](http://kaldir.vc.in.tum.de/faceforensics_benchmark/) 6 | ![State of the art results on public benchmark](diagrams/ff_benchmark.png) 7 | 8 | 9 | ### Dataset and technologies 10 | For detecting video frame forgeries we use [FaceForensics++](http://www.niessnerlab.org/projects/roessler2019faceforensicspp.html) dataset of pristine and manipulated videos. As a preprocessing step we extract faces from all the frames using [MTCNN](https://github.com/ipazc/mtcnn). 11 | Total dataset is ~507GB and contains 7 million frames. Dataset downloading and frame extraction code is located in [dataset](dataset) directory. 12 | For model training, we use the split from [FaceForensics++](http://www.niessnerlab.org/projects/roessler2019faceforensicspp.html) repository. 13 | 14 | Main technologies: 15 | 1. `Python` as main programming language 16 | 2. `Pytorch` as deep learning library 17 | 3. `pip` for dependency management 18 | 19 | ### Training/Evaluation 20 | All the training and evaluation code together with various models are stored in [src](src) directory. All scripts are self-documenting. Just run them with `--help` option. 21 | They automatically use gpu when available, but will also work on cpu, but very slowly, unfortunately. 22 | 23 | ### Single frame model 24 | We got the best single-frame classification accuracy using a version of [Inception Resnet V1 model](src/model.py#L109) pretrained on VGGFace2 face recognition dataset. 25 | ![Inception Resnet V1 diagram](diagrams/incep_resnet_v1.png) 26 | 27 | 28 | ### Window frame models 29 | We also evaluated how performance improves when incorporating temporal data. The task in this case changes from single frame classification to frame sequence classification. 30 | We used 2 different models for such an approach 3D convolutional and Bi-LSTM. 31 | 32 | #### 3D convolutional model 33 | ![3D convolutional model diagram](diagrams/3d_conv.png) 34 | Temporal feature locality assumption that 3D convolutional model has, seems reasonable in this case, but it is very slow to train for large window sizes. 35 | 36 | #### LSTM with 2D CNN encoder 37 | ![LSTM with 2D CNN encoder diagram](diagrams/CNN_LSTM.png) 38 | 39 | ## Citation 40 | ``` 41 | @misc{dogonadze2020deep, 42 | title={Deep Face Forgery Detection}, 43 | author={Nika Dogonadze and Jana Obernosterer and Ji Hou}, 44 | year={2020}, 45 | eprint={2004.11804}, 46 | archivePrefix={arXiv}, 47 | primaryClass={cs.CV} 48 | } 49 | ``` 50 | 51 | ### Model Weights 52 | Various model weights are available here - [models](https://drive.google.com/file/d/18-ki7vY0Yt4fzq-5A5A5w11kxy9xxQaW/view?usp=sharing) 53 | -------------------------------------------------------------------------------- /dataset/faceforensics_download_v4.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ Downloads FaceForensics++ and Deep Fake Detection public data release 3 | Example usage: 4 | see -h or https://github.com/ondyari/FaceForensics 5 | """ 6 | # -*- coding: utf-8 -*- 7 | import argparse 8 | import json 9 | import os 10 | import sys 11 | import tempfile 12 | import time 13 | import urllib 14 | import urllib.request 15 | from os.path import join 16 | 17 | from tqdm import tqdm 18 | 19 | # URLs and filenames 20 | FILELIST_URL = 'misc/filelist.json' 21 | DEEPFEAKES_DETECTION_URL = 'misc/deepfake_detection_filenames.json' 22 | DEEPFAKES_MODEL_NAMES = ['decoder_A.h5', 'decoder_B.h5', 'encoder.h5',] 23 | 24 | # Parameters 25 | DATASETS = { 26 | 'original_youtube_videos': 'misc/downloaded_youtube_videos.zip', 27 | 'original_youtube_videos_info': 'misc/downloaded_youtube_videos_info.zip', 28 | 'original': 'original_sequences/youtube', 29 | 'DeepFakeDetection_original': 'original_sequences/actors', 30 | 'Deepfakes': 'manipulated_sequences/Deepfakes', 31 | 'DeepFakeDetection': 'manipulated_sequences/DeepFakeDetection', 32 | 'Face2Face': 'manipulated_sequences/Face2Face', 33 | 'FaceSwap': 'manipulated_sequences/FaceSwap', 34 | 'NeuralTextures': 'manipulated_sequences/NeuralTextures' 35 | } 36 | ALL_DATASETS = ['original', 'DeepFakeDetection_original', 'Deepfakes', 37 | 'DeepFakeDetection', 'Face2Face', 'FaceSwap', 38 | 'NeuralTextures'] 39 | COMPRESSION = ['raw', 'c23', 'c40'] 40 | TYPE = ['videos', 'masks', 'models'] 41 | SERVERS = ['EU', 'EU2', 'CA'] 42 | 43 | 44 | def parse_args(): 45 | parser = argparse.ArgumentParser( 46 | description='Downloads FaceForensics v2 public data release.', 47 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 48 | ) 49 | parser.add_argument('output_path', type=str, help='Output directory.') 50 | parser.add_argument('-d', '--dataset', type=str, default='all', 51 | help='Which dataset to download, either pristine or ' 52 | 'manipulated data or the downloaded youtube ' 53 | 'videos.', 54 | choices=list(DATASETS.keys()) + ['all'] 55 | ) 56 | parser.add_argument('-c', '--compression', type=str, default='raw', 57 | help='Which compression degree. All videos ' 58 | 'have been generated with h264 with a varying ' 59 | 'codec. Raw (c0) videos are lossless compressed.', 60 | choices=COMPRESSION 61 | ) 62 | parser.add_argument('-t', '--type', type=str, default='videos', 63 | help='Which file type, i.e. videos, masks, for our ' 64 | 'manipulation methods, models, for Deepfakes.', 65 | choices=TYPE 66 | ) 67 | parser.add_argument('-n', '--num_videos', type=int, default=None, 68 | help='Select a number of videos number to ' 69 | "download if you don't want to download the full" 70 | ' dataset.') 71 | parser.add_argument('--server', type=str, default='EU', 72 | help='Server to download the data from. If you ' 73 | 'encounter a slow download speed, consider ' 74 | 'changing the server.', 75 | choices=SERVERS 76 | ) 77 | args = parser.parse_args() 78 | 79 | # URLs 80 | server = args.server 81 | if server == 'EU': 82 | server_url = 'http://canis.vc.in.tum.de:8100/' 83 | elif server == 'EU2': 84 | server_url = 'http://kaldir.vc.in.tum.de/faceforensics/' 85 | elif server == 'CA': 86 | server_url = 'http://falas.cmpt.sfu.ca:8100/' 87 | else: 88 | raise Exception('Wrong server name. Choices: {}'.format(str(SERVERS))) 89 | args.tos_url = server_url + 'webpage/FaceForensics_TOS.pdf' 90 | args.base_url = server_url + 'v3/' 91 | args.deepfakes_model_url = server_url + 'v3/manipulated_sequences/' + \ 92 | 'Deepfakes/models/' 93 | 94 | return args 95 | 96 | 97 | 98 | def download_files(filenames, base_url, output_path, report_progress=True): 99 | os.makedirs(output_path, exist_ok=True) 100 | if report_progress: 101 | filenames = tqdm(filenames) 102 | for filename in filenames: 103 | download_file(base_url + filename, join(output_path, filename)) 104 | 105 | 106 | def reporthook(count, block_size, total_size): 107 | global start_time 108 | if count == 0: 109 | start_time = time.time() 110 | return 111 | duration = time.time() - start_time 112 | progress_size = int(count * block_size) 113 | speed = int(progress_size / (1024 * duration)) 114 | percent = int(count * block_size * 100 / total_size) 115 | sys.stdout.write("\rProgress: %d%%, %d MB, %d KB/s, %d seconds passed" % 116 | (percent, progress_size / (1024 * 1024), speed, duration)) 117 | sys.stdout.flush() 118 | 119 | 120 | def download_file(url, out_file, report_progress=False): 121 | out_dir = os.path.dirname(out_file) 122 | if not os.path.isfile(out_file): 123 | fh, out_file_tmp = tempfile.mkstemp(dir=out_dir) 124 | f = os.fdopen(fh, 'w') 125 | f.close() 126 | if report_progress: 127 | urllib.request.urlretrieve(url, out_file_tmp, 128 | reporthook=reporthook) 129 | else: 130 | urllib.request.urlretrieve(url, out_file_tmp) 131 | os.rename(out_file_tmp, out_file) 132 | else: 133 | tqdm.write('WARNING: skipping download of existing file ' + out_file) 134 | 135 | 136 | def main(args): 137 | # TOS 138 | print('By pressing any key to continue you confirm that you have agreed ' \ 139 | 'to the FaceForensics terms of use as described at:') 140 | print(args.tos_url) 141 | print('***') 142 | print('Press any key to continue, or CTRL-C to exit.') 143 | _ = input('') 144 | 145 | # Extract arguments 146 | c_datasets = [args.dataset] if args.dataset != 'all' else ALL_DATASETS 147 | c_type = args.type 148 | c_compression = args.compression 149 | num_videos = args.num_videos 150 | output_path = args.output_path 151 | os.makedirs(output_path, exist_ok=True) 152 | 153 | # Check for special dataset cases 154 | for dataset in c_datasets: 155 | dataset_path = DATASETS[dataset] 156 | # Special cases 157 | if 'original_youtube_videos' in dataset: 158 | # Here we download the original youtube videos zip file 159 | print('Downloading original youtube videos.') 160 | if not 'info' in dataset_path: 161 | print('Please be patient, this may take a while (~40gb)') 162 | suffix = '' 163 | else: 164 | suffix = 'info' 165 | download_file(args.base_url + '/' + dataset_path, 166 | out_file=join(output_path, 167 | 'downloaded_videos{}.zip'.format( 168 | suffix)), 169 | report_progress=True) 170 | return 171 | 172 | # Else: regular datasets 173 | print('Downloading {} of dataset "{}"'.format( 174 | c_type, dataset_path 175 | )) 176 | 177 | # Get filelists and video lenghts list from server 178 | if 'DeepFakeDetection' in dataset_path or 'actors' in dataset_path: 179 | filepaths = json.loads(urllib.request.urlopen(args.base_url + '/' + 180 | DEEPFEAKES_DETECTION_URL).read().decode("utf-8")) 181 | if 'actors' in dataset_path: 182 | filelist = filepaths['actors'] 183 | else: 184 | filelist = filepaths['DeepFakesDetection'] 185 | elif 'original' in dataset_path: 186 | # Load filelist from server 187 | file_pairs = json.loads(urllib.request.urlopen(args.base_url + '/' + 188 | FILELIST_URL).read().decode("utf-8")) 189 | filelist = [] 190 | for pair in file_pairs: 191 | filelist += pair 192 | else: 193 | # Load filelist from server 194 | file_pairs = json.loads(urllib.request.urlopen(args.base_url + '/' + 195 | FILELIST_URL).read().decode("utf-8")) 196 | # Get filelist 197 | filelist = [] 198 | for pair in file_pairs: 199 | filelist.append('_'.join(pair)) 200 | if c_type != 'models': 201 | filelist.append('_'.join(pair[::-1])) 202 | # Maybe limit number of videos for download 203 | if num_videos is not None and num_videos > 0: 204 | print('Downloading the first {} videos'.format(num_videos)) 205 | filelist = filelist[:num_videos] 206 | 207 | # Server and local paths 208 | dataset_videos_url = args.base_url + '{}/{}/{}/'.format( 209 | dataset_path, c_compression, c_type) 210 | dataset_mask_url = args.base_url + '{}/{}/videos/'.format( 211 | dataset_path, 'masks', c_type) 212 | 213 | if c_type == 'videos': 214 | dataset_output_path = join(output_path, dataset_path, c_compression, 215 | c_type) 216 | print('Output path: {}'.format(dataset_output_path)) 217 | filelist = [filename + '.mp4' for filename in filelist] 218 | download_files(filelist, dataset_videos_url, dataset_output_path) 219 | elif c_type == 'masks': 220 | dataset_output_path = join(output_path, dataset_path, c_type, 221 | 'videos') 222 | print('Output path: {}'.format(dataset_output_path)) 223 | if 'original' in dataset: 224 | if args.dataset != 'all': 225 | print('Only videos available for original data. Aborting.') 226 | return 227 | else: 228 | print('Only videos available for original data. ' 229 | 'Skipping original.\n') 230 | continue 231 | filelist = [filename + '.mp4' for filename in filelist] 232 | download_files(filelist, dataset_mask_url, dataset_output_path) 233 | 234 | # Else: models for deepfakes 235 | else: 236 | if dataset != 'Deepfakes' and c_type == 'models': 237 | print('Models only available for Deepfakes. Aborting') 238 | return 239 | dataset_output_path = join(output_path, dataset_path, c_type) 240 | print('Output path: {}'.format(dataset_output_path)) 241 | 242 | # Get Deepfakes models 243 | for folder in tqdm(filelist): 244 | folder_filelist = DEEPFAKES_MODEL_NAMES 245 | 246 | # Folder paths 247 | folder_base_url = args.deepfakes_model_url + folder + '/' 248 | folder_dataset_output_path = join(dataset_output_path, 249 | folder) 250 | download_files(folder_filelist, folder_base_url, 251 | folder_dataset_output_path, 252 | report_progress=False) # already done 253 | 254 | 255 | if __name__ == "__main__": 256 | args = parse_args() 257 | main(args) 258 | -------------------------------------------------------------------------------- /dataset/frame_extraction.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from os.path import join 4 | 5 | import cv2 6 | import mmcv 7 | import torch 8 | from PIL import Image 9 | from facenet_pytorch import MTCNN 10 | from tqdm import tqdm 11 | 12 | 13 | def extract_frames(face_detector, data_path, output_path, file_prefix): 14 | os.makedirs(output_path, exist_ok=True) 15 | video = mmcv.VideoReader(data_path) 16 | length = video.frame_cnt 17 | for frame_num, frame in enumerate(video): 18 | image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) 19 | out_file_path = join(output_path, '{}{:04d}.png'.format(file_prefix, frame_num)) 20 | if not os.path.exists(out_file_path): 21 | face_detector(image, save_path=out_file_path) 22 | return length 23 | 24 | 25 | def extract_images(device, videos_path, out_path, num_videos): 26 | print('extracting video frames from {} to {}'.format(videos_path, out_path)) 27 | 28 | video_files = os.listdir(videos_path) 29 | print('total videos found - {}, extracting from - {}'.format(len(video_files), min(len(video_files), num_videos))) 30 | video_files = video_files[:num_videos] 31 | 32 | def get_video_input_output_pairs(): 33 | for index, video_file in enumerate(video_files): 34 | video_file_name = video_file.split('.')[0] 35 | v_out_path = os.path.join(out_path, video_file_name) 36 | v_path = os.path.join(videos_path, video_file) 37 | f_prefix = '{}_'.format(video_file_name) 38 | yield v_path, v_out_path, f_prefix 39 | 40 | face_detector = MTCNN(device=device, margin=16) 41 | face_detector.eval() 42 | 43 | for data_path, output_path, file_prefix in tqdm(get_video_input_output_pairs(), total=len(video_files)): 44 | extract_frames(face_detector, data_path, output_path, file_prefix) 45 | 46 | 47 | def parse_args(): 48 | args_parser = argparse.ArgumentParser() 49 | args_parser.add_argument('path_to_videos', type=str, help='path for input videos') 50 | args_parser.add_argument('output_path', type=str, help='output images path') 51 | args_parser.add_argument('--num_videos', type=int, default=10, help='number of videos to extract images from') 52 | args = args_parser.parse_args() 53 | return args 54 | 55 | 56 | def main(): 57 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 58 | print('Running on device: {}'.format(device)) 59 | 60 | args = parse_args() 61 | videos_path = args.path_to_videos 62 | out_path = args.output_path 63 | num_videos = args.num_videos 64 | extract_images(device, videos_path, out_path, num_videos) 65 | 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0000.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0001.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0002.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0003.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0004.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0005.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0006.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0007.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0008.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0009.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/001/0010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/001/0010.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/034/034_0000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/034/034_0000.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/034/034_0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/034/034_0001.png -------------------------------------------------------------------------------- /dataset/images_tiny/original/034/034_0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/original/034/034_0002.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0001.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0002.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0003.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0004.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0005.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0006.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0007.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0008.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0009.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/002_003/0010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/002_003/0010.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0000.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0001.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0002.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0003.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0004.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0005.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0006.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0007.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0008.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0009.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0010.png -------------------------------------------------------------------------------- /dataset/images_tiny/tampered/259_345/259_345_0011.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/dataset/images_tiny/tampered/259_345/259_345_0011.png -------------------------------------------------------------------------------- /dataset/prepare_full_dataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | echo "YES" | python3 faceforensics_download_v4.py -d original -c c40 $1/videos/original_c40 && \ 5 | python3 frame_extraction.py $1/videos/original_c40/original_sequences/youtube/c40/videos/ $1/mtcnn/original_faces_c40 --num_videos 999999 6 | 7 | echo "YES" | python3 faceforensics_download_v4.py -d original -c c23 $1/videos/original_c23 && \ 8 | python3 frame_extraction.py $1/videos/original_c23/original_sequences/youtube/c23/videos/ $1/mtcnn/original_faces_c23 --num_videos 999999 9 | 10 | echo "YES" | python3 faceforensics_download_v4.py -d original -c raw $1/videos/original_raw && \ 11 | python3 frame_extraction.py $1/videos/original_raw/original_sequences/youtube/raw/videos/ $1/mtcnn/original_faces_raw --num_videos 999999 12 | 13 | 14 | echo "YES" | python3 faceforensics_download_v4.py -d NeuralTextures -c c40 $1/videos/neural_textures_c40 && \ 15 | python3 frame_extraction.py $1/videos/neural_textures_c40/manipulated_sequences/NeuralTextures/c40/videos/ $1/mtcnn/neural_textures_faces_c40 --num_videos 999999 16 | 17 | echo "YES" | python3 faceforensics_download_v4.py -d NeuralTextures -c c23 $1/videos/neural_textures_c23 && \ 18 | python3 frame_extraction.py $1/videos/neural_textures_c23/manipulated_sequences/NeuralTextures/c23/videos/ $1/mtcnn/neural_textures_faces_c23 --num_videos 999999 19 | 20 | echo "YES" | python3 faceforensics_download_v4.py -d NeuralTextures -c raw $1/videos/neural_textures_raw && \ 21 | python3 frame_extraction.py $1/videos/neural_textures_raw/manipulated_sequences/NeuralTextures/raw/videos/ $1/mtcnn/neural_textures_faces_raw --num_videos 999999 22 | 23 | 24 | echo "YES" | python3 faceforensics_download_v4.py -d Deepfakes -c c40 $1/videos/deepfakes_c40 && \ 25 | python3 frame_extraction.py $1/videos/deepfakes_c40/manipulated_sequences/Deepfakes/c40/videos/ $1/mtcnn/deepfakes_faces_c40 --num_videos 999999 26 | 27 | echo "YES" | python3 faceforensics_download_v4.py -d Deepfakes -c c23 $1/videos/deepfakes_c23 && \ 28 | python3 frame_extraction.py $1/videos/deepfakes_c23/manipulated_sequences/Deepfakes/c23/videos/ $1/mtcnn/deepfakes_faces_c23 --num_videos 999999 29 | 30 | echo "YES" | python3 faceforensics_download_v4.py -d Deepfakes -c raw $1/videos/deepfakes_raw && \ 31 | python3 frame_extraction.py $1/videos/deepfakes_raw/manipulated_sequences/Deepfakes/raw/videos/ $1/mtcnn/deepfakes_faces_raw --num_videos 999999 32 | 33 | 34 | echo "YES" | python3 faceforensics_download_v4.py -d Face2Face -c c40 $1/videos/face2face_c40 && \ 35 | python3 frame_extraction.py $1/videos/face2face_c40/manipulated_sequences/Face2Face/c40/videos/ $1/mtcnn/face2face_faces_c40 --num_videos 999999 36 | 37 | echo "YES" | python3 faceforensics_download_v4.py -d Face2Face -c c23 $1/videos/face2face_c23 && \ 38 | python3 frame_extraction.py $1/videos/face2face_c23/manipulated_sequences/Face2Face/c23/videos/ $1/mtcnn/face2face_faces_c23 --num_videos 999999 39 | 40 | echo "YES" | python3 faceforensics_download_v4.py -d Face2Face -c raw $1/videos/face2face_raw && \ 41 | python3 frame_extraction.py $1/videos/face2face_raw/manipulated_sequences/Face2Face/raw/videos/ $1/mtcnn/face2face_faces_raw --num_videos 999999 42 | 43 | 44 | echo "YES" | python3 faceforensics_download_v4.py -d FaceSwap -c c40 $1/videos/faceswap_c40 && \ 45 | python3 frame_extraction.py $1/videos/faceswap_c40/manipulated_sequences/FaceSwap/c40/videos/ $1/mtcnn/faceswap_faces_c40 --num_videos 999999 46 | 47 | echo "YES" | python3 faceforensics_download_v4.py -d FaceSwap -c c23 $1/videos/faceswap_c23 && \ 48 | python3 frame_extraction.py $1/videos/faceswap_c23/manipulated_sequences/FaceSwap/c23/videos/ $1/mtcnn/faceswap_faces_c23 --num_videos 999999 49 | 50 | echo "YES" | python3 faceforensics_download_v4.py -d FaceSwap -c raw $1/videos/faceswap_raw && \ 51 | python3 frame_extraction.py $1/videos/faceswap_raw/manipulated_sequences/FaceSwap/raw/videos/ $1/mtcnn/faceswap_faces_raw --num_videos 999999 52 | -------------------------------------------------------------------------------- /dataset/requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm==4.66.3 2 | opencv-python==4.8.1.78 3 | facenet-pytorch==2.2.9 4 | mmcv==0.5.7 5 | requests==2.32.0 6 | requests-oauthlib==1.3.0 7 | -------------------------------------------------------------------------------- /dataset/splits/test.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "953", 4 | "974" 5 | ], 6 | [ 7 | "012", 8 | "026" 9 | ], 10 | [ 11 | "078", 12 | "955" 13 | ], 14 | [ 15 | "623", 16 | "630" 17 | ], 18 | [ 19 | "919", 20 | "015" 21 | ], 22 | [ 23 | "367", 24 | "371" 25 | ], 26 | [ 27 | "847", 28 | "906" 29 | ], 30 | [ 31 | "529", 32 | "633" 33 | ], 34 | [ 35 | "418", 36 | "507" 37 | ], 38 | [ 39 | "227", 40 | "169" 41 | ], 42 | [ 43 | "389", 44 | "480" 45 | ], 46 | [ 47 | "821", 48 | "812" 49 | ], 50 | [ 51 | "670", 52 | "661" 53 | ], 54 | [ 55 | "158", 56 | "379" 57 | ], 58 | [ 59 | "423", 60 | "421" 61 | ], 62 | [ 63 | "352", 64 | "319" 65 | ], 66 | [ 67 | "579", 68 | "701" 69 | ], 70 | [ 71 | "488", 72 | "399" 73 | ], 74 | [ 75 | "695", 76 | "422" 77 | ], 78 | [ 79 | "288", 80 | "321" 81 | ], 82 | [ 83 | "705", 84 | "707" 85 | ], 86 | [ 87 | "306", 88 | "278" 89 | ], 90 | [ 91 | "865", 92 | "739" 93 | ], 94 | [ 95 | "995", 96 | "233" 97 | ], 98 | [ 99 | "755", 100 | "759" 101 | ], 102 | [ 103 | "467", 104 | "462" 105 | ], 106 | [ 107 | "314", 108 | "347" 109 | ], 110 | [ 111 | "741", 112 | "731" 113 | ], 114 | [ 115 | "970", 116 | "973" 117 | ], 118 | [ 119 | "634", 120 | "660" 121 | ], 122 | [ 123 | "494", 124 | "445" 125 | ], 126 | [ 127 | "706", 128 | "479" 129 | ], 130 | [ 131 | "186", 132 | "170" 133 | ], 134 | [ 135 | "176", 136 | "190" 137 | ], 138 | [ 139 | "380", 140 | "358" 141 | ], 142 | [ 143 | "214", 144 | "255" 145 | ], 146 | [ 147 | "454", 148 | "527" 149 | ], 150 | [ 151 | "425", 152 | "485" 153 | ], 154 | [ 155 | "388", 156 | "308" 157 | ], 158 | [ 159 | "384", 160 | "932" 161 | ], 162 | [ 163 | "035", 164 | "036" 165 | ], 166 | [ 167 | "257", 168 | "420" 169 | ], 170 | [ 171 | "924", 172 | "917" 173 | ], 174 | [ 175 | "114", 176 | "102" 177 | ], 178 | [ 179 | "732", 180 | "691" 181 | ], 182 | [ 183 | "550", 184 | "452" 185 | ], 186 | [ 187 | "280", 188 | "249" 189 | ], 190 | [ 191 | "842", 192 | "714" 193 | ], 194 | [ 195 | "625", 196 | "650" 197 | ], 198 | [ 199 | "024", 200 | "073" 201 | ], 202 | [ 203 | "044", 204 | "945" 205 | ], 206 | [ 207 | "896", 208 | "128" 209 | ], 210 | [ 211 | "862", 212 | "047" 213 | ], 214 | [ 215 | "607", 216 | "683" 217 | ], 218 | [ 219 | "517", 220 | "521" 221 | ], 222 | [ 223 | "682", 224 | "669" 225 | ], 226 | [ 227 | "138", 228 | "142" 229 | ], 230 | [ 231 | "552", 232 | "851" 233 | ], 234 | [ 235 | "376", 236 | "381" 237 | ], 238 | [ 239 | "000", 240 | "003" 241 | ], 242 | [ 243 | "048", 244 | "029" 245 | ], 246 | [ 247 | "724", 248 | "725" 249 | ], 250 | [ 251 | "608", 252 | "675" 253 | ], 254 | [ 255 | "386", 256 | "154" 257 | ], 258 | [ 259 | "220", 260 | "219" 261 | ], 262 | [ 263 | "801", 264 | "855" 265 | ], 266 | [ 267 | "161", 268 | "141" 269 | ], 270 | [ 271 | "949", 272 | "868" 273 | ], 274 | [ 275 | "880", 276 | "135" 277 | ], 278 | [ 279 | "429", 280 | "404" 281 | ] 282 | ] -------------------------------------------------------------------------------- /dataset/splits/train.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "071", 4 | "054" 5 | ], 6 | [ 7 | "087", 8 | "081" 9 | ], 10 | [ 11 | "881", 12 | "856" 13 | ], 14 | [ 15 | "187", 16 | "234" 17 | ], 18 | [ 19 | "645", 20 | "688" 21 | ], 22 | [ 23 | "754", 24 | "758" 25 | ], 26 | [ 27 | "811", 28 | "920" 29 | ], 30 | [ 31 | "710", 32 | "788" 33 | ], 34 | [ 35 | "628", 36 | "568" 37 | ], 38 | [ 39 | "312", 40 | "021" 41 | ], 42 | [ 43 | "950", 44 | "836" 45 | ], 46 | [ 47 | "059", 48 | "050" 49 | ], 50 | [ 51 | "524", 52 | "580" 53 | ], 54 | [ 55 | "751", 56 | "752" 57 | ], 58 | [ 59 | "918", 60 | "934" 61 | ], 62 | [ 63 | "604", 64 | "703" 65 | ], 66 | [ 67 | "296", 68 | "293" 69 | ], 70 | [ 71 | "518", 72 | "131" 73 | ], 74 | [ 75 | "536", 76 | "540" 77 | ], 78 | [ 79 | "969", 80 | "897" 81 | ], 82 | [ 83 | "372", 84 | "413" 85 | ], 86 | [ 87 | "357", 88 | "432" 89 | ], 90 | [ 91 | "809", 92 | "799" 93 | ], 94 | [ 95 | "092", 96 | "098" 97 | ], 98 | [ 99 | "302", 100 | "323" 101 | ], 102 | [ 103 | "981", 104 | "985" 105 | ], 106 | [ 107 | "512", 108 | "495" 109 | ], 110 | [ 111 | "088", 112 | "060" 113 | ], 114 | [ 115 | "795", 116 | "907" 117 | ], 118 | [ 119 | "535", 120 | "587" 121 | ], 122 | [ 123 | "297", 124 | "270" 125 | ], 126 | [ 127 | "838", 128 | "810" 129 | ], 130 | [ 131 | "850", 132 | "764" 133 | ], 134 | [ 135 | "476", 136 | "400" 137 | ], 138 | [ 139 | "268", 140 | "269" 141 | ], 142 | [ 143 | "033", 144 | "097" 145 | ], 146 | [ 147 | "226", 148 | "491" 149 | ], 150 | [ 151 | "784", 152 | "769" 153 | ], 154 | [ 155 | "195", 156 | "442" 157 | ], 158 | [ 159 | "678", 160 | "460" 161 | ], 162 | [ 163 | "320", 164 | "328" 165 | ], 166 | [ 167 | "451", 168 | "449" 169 | ], 170 | [ 171 | "409", 172 | "382" 173 | ], 174 | [ 175 | "556", 176 | "588" 177 | ], 178 | [ 179 | "027", 180 | "009" 181 | ], 182 | [ 183 | "196", 184 | "310" 185 | ], 186 | [ 187 | "241", 188 | "210" 189 | ], 190 | [ 191 | "295", 192 | "099" 193 | ], 194 | [ 195 | "043", 196 | "110" 197 | ], 198 | [ 199 | "753", 200 | "789" 201 | ], 202 | [ 203 | "716", 204 | "712" 205 | ], 206 | [ 207 | "508", 208 | "831" 209 | ], 210 | [ 211 | "005", 212 | "010" 213 | ], 214 | [ 215 | "276", 216 | "185" 217 | ], 218 | [ 219 | "498", 220 | "433" 221 | ], 222 | [ 223 | "294", 224 | "292" 225 | ], 226 | [ 227 | "105", 228 | "180" 229 | ], 230 | [ 231 | "984", 232 | "967" 233 | ], 234 | [ 235 | "318", 236 | "334" 237 | ], 238 | [ 239 | "356", 240 | "324" 241 | ], 242 | [ 243 | "344", 244 | "020" 245 | ], 246 | [ 247 | "289", 248 | "228" 249 | ], 250 | [ 251 | "022", 252 | "489" 253 | ], 254 | [ 255 | "137", 256 | "165" 257 | ], 258 | [ 259 | "095", 260 | "053" 261 | ], 262 | [ 263 | "999", 264 | "960" 265 | ], 266 | [ 267 | "481", 268 | "469" 269 | ], 270 | [ 271 | "534", 272 | "490" 273 | ], 274 | [ 275 | "543", 276 | "559" 277 | ], 278 | [ 279 | "150", 280 | "153" 281 | ], 282 | [ 283 | "598", 284 | "178" 285 | ], 286 | [ 287 | "475", 288 | "265" 289 | ], 290 | [ 291 | "671", 292 | "677" 293 | ], 294 | [ 295 | "204", 296 | "230" 297 | ], 298 | [ 299 | "863", 300 | "853" 301 | ], 302 | [ 303 | "561", 304 | "998" 305 | ], 306 | [ 307 | "163", 308 | "031" 309 | ], 310 | [ 311 | "655", 312 | "444" 313 | ], 314 | [ 315 | "038", 316 | "125" 317 | ], 318 | [ 319 | "735", 320 | "774" 321 | ], 322 | [ 323 | "184", 324 | "205" 325 | ], 326 | [ 327 | "499", 328 | "539" 329 | ], 330 | [ 331 | "717", 332 | "684" 333 | ], 334 | [ 335 | "878", 336 | "866" 337 | ], 338 | [ 339 | "127", 340 | "129" 341 | ], 342 | [ 343 | "286", 344 | "267" 345 | ], 346 | [ 347 | "032", 348 | "944" 349 | ], 350 | [ 351 | "681", 352 | "711" 353 | ], 354 | [ 355 | "236", 356 | "237" 357 | ], 358 | [ 359 | "989", 360 | "993" 361 | ], 362 | [ 363 | "537", 364 | "563" 365 | ], 366 | [ 367 | "814", 368 | "871" 369 | ], 370 | [ 371 | "509", 372 | "525" 373 | ], 374 | [ 375 | "221", 376 | "206" 377 | ], 378 | [ 379 | "808", 380 | "829" 381 | ], 382 | [ 383 | "696", 384 | "686" 385 | ], 386 | [ 387 | "431", 388 | "447" 389 | ], 390 | [ 391 | "737", 392 | "719" 393 | ], 394 | [ 395 | "609", 396 | "596" 397 | ], 398 | [ 399 | "408", 400 | "424" 401 | ], 402 | [ 403 | "976", 404 | "954" 405 | ], 406 | [ 407 | "156", 408 | "243" 409 | ], 410 | [ 411 | "434", 412 | "438" 413 | ], 414 | [ 415 | "627", 416 | "658" 417 | ], 418 | [ 419 | "025", 420 | "067" 421 | ], 422 | [ 423 | "635", 424 | "642" 425 | ], 426 | [ 427 | "523", 428 | "541" 429 | ], 430 | [ 431 | "572", 432 | "554" 433 | ], 434 | [ 435 | "215", 436 | "208" 437 | ], 438 | [ 439 | "651", 440 | "835" 441 | ], 442 | [ 443 | "975", 444 | "978" 445 | ], 446 | [ 447 | "792", 448 | "903" 449 | ], 450 | [ 451 | "931", 452 | "936" 453 | ], 454 | [ 455 | "846", 456 | "845" 457 | ], 458 | [ 459 | "899", 460 | "914" 461 | ], 462 | [ 463 | "209", 464 | "016" 465 | ], 466 | [ 467 | "398", 468 | "457" 469 | ], 470 | [ 471 | "797", 472 | "844" 473 | ], 474 | [ 475 | "360", 476 | "437" 477 | ], 478 | [ 479 | "738", 480 | "804" 481 | ], 482 | [ 483 | "694", 484 | "767" 485 | ], 486 | [ 487 | "790", 488 | "014" 489 | ], 490 | [ 491 | "657", 492 | "644" 493 | ], 494 | [ 495 | "374", 496 | "407" 497 | ], 498 | [ 499 | "728", 500 | "673" 501 | ], 502 | [ 503 | "193", 504 | "030" 505 | ], 506 | [ 507 | "876", 508 | "891" 509 | ], 510 | [ 511 | "553", 512 | "545" 513 | ], 514 | [ 515 | "331", 516 | "260" 517 | ], 518 | [ 519 | "873", 520 | "872" 521 | ], 522 | [ 523 | "109", 524 | "107" 525 | ], 526 | [ 527 | "121", 528 | "093" 529 | ], 530 | [ 531 | "143", 532 | "140" 533 | ], 534 | [ 535 | "778", 536 | "798" 537 | ], 538 | [ 539 | "983", 540 | "113" 541 | ], 542 | [ 543 | "504", 544 | "502" 545 | ], 546 | [ 547 | "709", 548 | "390" 549 | ], 550 | [ 551 | "940", 552 | "941" 553 | ], 554 | [ 555 | "894", 556 | "848" 557 | ], 558 | [ 559 | "311", 560 | "387" 561 | ], 562 | [ 563 | "562", 564 | "626" 565 | ], 566 | [ 567 | "330", 568 | "162" 569 | ], 570 | [ 571 | "112", 572 | "892" 573 | ], 574 | [ 575 | "765", 576 | "867" 577 | ], 578 | [ 579 | "124", 580 | "085" 581 | ], 582 | [ 583 | "665", 584 | "679" 585 | ], 586 | [ 587 | "414", 588 | "385" 589 | ], 590 | [ 591 | "555", 592 | "516" 593 | ], 594 | [ 595 | "072", 596 | "037" 597 | ], 598 | [ 599 | "086", 600 | "090" 601 | ], 602 | [ 603 | "202", 604 | "348" 605 | ], 606 | [ 607 | "341", 608 | "340" 609 | ], 610 | [ 611 | "333", 612 | "377" 613 | ], 614 | [ 615 | "082", 616 | "103" 617 | ], 618 | [ 619 | "569", 620 | "921" 621 | ], 622 | [ 623 | "750", 624 | "743" 625 | ], 626 | [ 627 | "211", 628 | "177" 629 | ], 630 | [ 631 | "770", 632 | "791" 633 | ], 634 | [ 635 | "329", 636 | "327" 637 | ], 638 | [ 639 | "613", 640 | "685" 641 | ], 642 | [ 643 | "007", 644 | "132" 645 | ], 646 | [ 647 | "304", 648 | "300" 649 | ], 650 | [ 651 | "860", 652 | "905" 653 | ], 654 | [ 655 | "986", 656 | "994" 657 | ], 658 | [ 659 | "378", 660 | "368" 661 | ], 662 | [ 663 | "761", 664 | "766" 665 | ], 666 | [ 667 | "232", 668 | "248" 669 | ], 670 | [ 671 | "136", 672 | "285" 673 | ], 674 | [ 675 | "601", 676 | "653" 677 | ], 678 | [ 679 | "693", 680 | "698" 681 | ], 682 | [ 683 | "359", 684 | "317" 685 | ], 686 | [ 687 | "246", 688 | "258" 689 | ], 690 | [ 691 | "500", 692 | "592" 693 | ], 694 | [ 695 | "776", 696 | "676" 697 | ], 698 | [ 699 | "262", 700 | "301" 701 | ], 702 | [ 703 | "307", 704 | "365" 705 | ], 706 | [ 707 | "600", 708 | "505" 709 | ], 710 | [ 711 | "833", 712 | "826" 713 | ], 714 | [ 715 | "361", 716 | "448" 717 | ], 718 | [ 719 | "473", 720 | "366" 721 | ], 722 | [ 723 | "885", 724 | "802" 725 | ], 726 | [ 727 | "277", 728 | "335" 729 | ], 730 | [ 731 | "667", 732 | "446" 733 | ], 734 | [ 735 | "522", 736 | "337" 737 | ], 738 | [ 739 | "018", 740 | "019" 741 | ], 742 | [ 743 | "430", 744 | "459" 745 | ], 746 | [ 747 | "886", 748 | "877" 749 | ], 750 | [ 751 | "456", 752 | "435" 753 | ], 754 | [ 755 | "239", 756 | "218" 757 | ], 758 | [ 759 | "771", 760 | "849" 761 | ], 762 | [ 763 | "065", 764 | "089" 765 | ], 766 | [ 767 | "654", 768 | "648" 769 | ], 770 | [ 771 | "151", 772 | "225" 773 | ], 774 | [ 775 | "152", 776 | "149" 777 | ], 778 | [ 779 | "229", 780 | "247" 781 | ], 782 | [ 783 | "624", 784 | "570" 785 | ], 786 | [ 787 | "290", 788 | "240" 789 | ], 790 | [ 791 | "011", 792 | "805" 793 | ], 794 | [ 795 | "461", 796 | "250" 797 | ], 798 | [ 799 | "251", 800 | "375" 801 | ], 802 | [ 803 | "639", 804 | "841" 805 | ], 806 | [ 807 | "602", 808 | "397" 809 | ], 810 | [ 811 | "028", 812 | "068" 813 | ], 814 | [ 815 | "338", 816 | "336" 817 | ], 818 | [ 819 | "964", 820 | "174" 821 | ], 822 | [ 823 | "782", 824 | "787" 825 | ], 826 | [ 827 | "478", 828 | "506" 829 | ], 830 | [ 831 | "313", 832 | "283" 833 | ], 834 | [ 835 | "659", 836 | "749" 837 | ], 838 | [ 839 | "690", 840 | "689" 841 | ], 842 | [ 843 | "893", 844 | "913" 845 | ], 846 | [ 847 | "197", 848 | "224" 849 | ], 850 | [ 851 | "253", 852 | "183" 853 | ], 854 | [ 855 | "373", 856 | "394" 857 | ], 858 | [ 859 | "803", 860 | "017" 861 | ], 862 | [ 863 | "305", 864 | "513" 865 | ], 866 | [ 867 | "051", 868 | "332" 869 | ], 870 | [ 871 | "238", 872 | "282" 873 | ], 874 | [ 875 | "621", 876 | "546" 877 | ], 878 | [ 879 | "401", 880 | "395" 881 | ], 882 | [ 883 | "510", 884 | "528" 885 | ], 886 | [ 887 | "410", 888 | "411" 889 | ], 890 | [ 891 | "049", 892 | "946" 893 | ], 894 | [ 895 | "663", 896 | "231" 897 | ], 898 | [ 899 | "477", 900 | "487" 901 | ], 902 | [ 903 | "252", 904 | "266" 905 | ], 906 | [ 907 | "952", 908 | "882" 909 | ], 910 | [ 911 | "315", 912 | "322" 913 | ], 914 | [ 915 | "216", 916 | "164" 917 | ], 918 | [ 919 | "061", 920 | "080" 921 | ], 922 | [ 923 | "603", 924 | "575" 925 | ], 926 | [ 927 | "828", 928 | "830" 929 | ], 930 | [ 931 | "723", 932 | "704" 933 | ], 934 | [ 935 | "870", 936 | "001" 937 | ], 938 | [ 939 | "201", 940 | "203" 941 | ], 942 | [ 943 | "652", 944 | "773" 945 | ], 946 | [ 947 | "108", 948 | "052" 949 | ], 950 | [ 951 | "272", 952 | "396" 953 | ], 954 | [ 955 | "040", 956 | "997" 957 | ], 958 | [ 959 | "988", 960 | "966" 961 | ], 962 | [ 963 | "281", 964 | "474" 965 | ], 966 | [ 967 | "077", 968 | "100" 969 | ], 970 | [ 971 | "146", 972 | "256" 973 | ], 974 | [ 975 | "972", 976 | "718" 977 | ], 978 | [ 979 | "303", 980 | "309" 981 | ], 982 | [ 983 | "582", 984 | "172" 985 | ], 986 | [ 987 | "222", 988 | "168" 989 | ], 990 | [ 991 | "884", 992 | "968" 993 | ], 994 | [ 995 | "217", 996 | "117" 997 | ], 998 | [ 999 | "118", 1000 | "120" 1001 | ], 1002 | [ 1003 | "242", 1004 | "182" 1005 | ], 1006 | [ 1007 | "858", 1008 | "861" 1009 | ], 1010 | [ 1011 | "101", 1012 | "096" 1013 | ], 1014 | [ 1015 | "697", 1016 | "581" 1017 | ], 1018 | [ 1019 | "763", 1020 | "930" 1021 | ], 1022 | [ 1023 | "839", 1024 | "864" 1025 | ], 1026 | [ 1027 | "542", 1028 | "520" 1029 | ], 1030 | [ 1031 | "122", 1032 | "144" 1033 | ], 1034 | [ 1035 | "687", 1036 | "615" 1037 | ], 1038 | [ 1039 | "544", 1040 | "532" 1041 | ], 1042 | [ 1043 | "721", 1044 | "715" 1045 | ], 1046 | [ 1047 | "179", 1048 | "212" 1049 | ], 1050 | [ 1051 | "591", 1052 | "605" 1053 | ], 1054 | [ 1055 | "275", 1056 | "887" 1057 | ], 1058 | [ 1059 | "996", 1060 | "056" 1061 | ], 1062 | [ 1063 | "825", 1064 | "074" 1065 | ], 1066 | [ 1067 | "530", 1068 | "594" 1069 | ], 1070 | [ 1071 | "757", 1072 | "573" 1073 | ], 1074 | [ 1075 | "611", 1076 | "760" 1077 | ], 1078 | [ 1079 | "189", 1080 | "200" 1081 | ], 1082 | [ 1083 | "392", 1084 | "339" 1085 | ], 1086 | [ 1087 | "734", 1088 | "699" 1089 | ], 1090 | [ 1091 | "977", 1092 | "075" 1093 | ], 1094 | [ 1095 | "879", 1096 | "963" 1097 | ], 1098 | [ 1099 | "910", 1100 | "911" 1101 | ], 1102 | [ 1103 | "889", 1104 | "045" 1105 | ], 1106 | [ 1107 | "962", 1108 | "929" 1109 | ], 1110 | [ 1111 | "515", 1112 | "519" 1113 | ], 1114 | [ 1115 | "062", 1116 | "066" 1117 | ], 1118 | [ 1119 | "937", 1120 | "888" 1121 | ], 1122 | [ 1123 | "199", 1124 | "181" 1125 | ], 1126 | [ 1127 | "785", 1128 | "736" 1129 | ], 1130 | [ 1131 | "079", 1132 | "076" 1133 | ], 1134 | [ 1135 | "155", 1136 | "576" 1137 | ], 1138 | [ 1139 | "748", 1140 | "355" 1141 | ], 1142 | [ 1143 | "819", 1144 | "786" 1145 | ], 1146 | [ 1147 | "577", 1148 | "593" 1149 | ], 1150 | [ 1151 | "464", 1152 | "463" 1153 | ], 1154 | [ 1155 | "439", 1156 | "441" 1157 | ], 1158 | [ 1159 | "574", 1160 | "547" 1161 | ], 1162 | [ 1163 | "747", 1164 | "854" 1165 | ], 1166 | [ 1167 | "403", 1168 | "497" 1169 | ], 1170 | [ 1171 | "965", 1172 | "948" 1173 | ], 1174 | [ 1175 | "726", 1176 | "713" 1177 | ], 1178 | [ 1179 | "943", 1180 | "942" 1181 | ], 1182 | [ 1183 | "160", 1184 | "928" 1185 | ], 1186 | [ 1187 | "496", 1188 | "417" 1189 | ], 1190 | [ 1191 | "700", 1192 | "813" 1193 | ], 1194 | [ 1195 | "756", 1196 | "503" 1197 | ], 1198 | [ 1199 | "213", 1200 | "083" 1201 | ], 1202 | [ 1203 | "039", 1204 | "058" 1205 | ], 1206 | [ 1207 | "781", 1208 | "806" 1209 | ], 1210 | [ 1211 | "620", 1212 | "619" 1213 | ], 1214 | [ 1215 | "351", 1216 | "346" 1217 | ], 1218 | [ 1219 | "959", 1220 | "957" 1221 | ], 1222 | [ 1223 | "264", 1224 | "271" 1225 | ], 1226 | [ 1227 | "006", 1228 | "002" 1229 | ], 1230 | [ 1231 | "391", 1232 | "406" 1233 | ], 1234 | [ 1235 | "631", 1236 | "551" 1237 | ], 1238 | [ 1239 | "501", 1240 | "326" 1241 | ], 1242 | [ 1243 | "412", 1244 | "274" 1245 | ], 1246 | [ 1247 | "641", 1248 | "662" 1249 | ], 1250 | [ 1251 | "111", 1252 | "094" 1253 | ], 1254 | [ 1255 | "166", 1256 | "167" 1257 | ], 1258 | [ 1259 | "130", 1260 | "139" 1261 | ], 1262 | [ 1263 | "938", 1264 | "987" 1265 | ], 1266 | [ 1267 | "055", 1268 | "147" 1269 | ], 1270 | [ 1271 | "990", 1272 | "008" 1273 | ], 1274 | [ 1275 | "013", 1276 | "883" 1277 | ], 1278 | [ 1279 | "614", 1280 | "616" 1281 | ], 1282 | [ 1283 | "772", 1284 | "708" 1285 | ], 1286 | [ 1287 | "840", 1288 | "800" 1289 | ], 1290 | [ 1291 | "415", 1292 | "484" 1293 | ], 1294 | [ 1295 | "287", 1296 | "426" 1297 | ], 1298 | [ 1299 | "680", 1300 | "486" 1301 | ], 1302 | [ 1303 | "057", 1304 | "070" 1305 | ], 1306 | [ 1307 | "590", 1308 | "034" 1309 | ], 1310 | [ 1311 | "194", 1312 | "235" 1313 | ], 1314 | [ 1315 | "291", 1316 | "874" 1317 | ], 1318 | [ 1319 | "902", 1320 | "901" 1321 | ], 1322 | [ 1323 | "343", 1324 | "363" 1325 | ], 1326 | [ 1327 | "279", 1328 | "298" 1329 | ], 1330 | [ 1331 | "393", 1332 | "405" 1333 | ], 1334 | [ 1335 | "674", 1336 | "744" 1337 | ], 1338 | [ 1339 | "244", 1340 | "822" 1341 | ], 1342 | [ 1343 | "133", 1344 | "148" 1345 | ], 1346 | [ 1347 | "636", 1348 | "578" 1349 | ], 1350 | [ 1351 | "637", 1352 | "427" 1353 | ], 1354 | [ 1355 | "041", 1356 | "063" 1357 | ], 1358 | [ 1359 | "869", 1360 | "780" 1361 | ], 1362 | [ 1363 | "733", 1364 | "935" 1365 | ], 1366 | [ 1367 | "259", 1368 | "345" 1369 | ], 1370 | [ 1371 | "069", 1372 | "961" 1373 | ], 1374 | [ 1375 | "783", 1376 | "916" 1377 | ], 1378 | [ 1379 | "191", 1380 | "188" 1381 | ], 1382 | [ 1383 | "526", 1384 | "436" 1385 | ], 1386 | [ 1387 | "123", 1388 | "119" 1389 | ], 1390 | [ 1391 | "207", 1392 | "908" 1393 | ], 1394 | [ 1395 | "796", 1396 | "740" 1397 | ], 1398 | [ 1399 | "815", 1400 | "730" 1401 | ], 1402 | [ 1403 | "173", 1404 | "171" 1405 | ], 1406 | [ 1407 | "383", 1408 | "353" 1409 | ], 1410 | [ 1411 | "458", 1412 | "722" 1413 | ], 1414 | [ 1415 | "533", 1416 | "450" 1417 | ], 1418 | [ 1419 | "618", 1420 | "629" 1421 | ], 1422 | [ 1423 | "646", 1424 | "643" 1425 | ], 1426 | [ 1427 | "531", 1428 | "549" 1429 | ], 1430 | [ 1431 | "428", 1432 | "466" 1433 | ], 1434 | [ 1435 | "859", 1436 | "843" 1437 | ], 1438 | [ 1439 | "692", 1440 | "610" 1441 | ] 1442 | ] -------------------------------------------------------------------------------- /dataset/splits/val.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "720", 4 | "672" 5 | ], 6 | [ 7 | "939", 8 | "115" 9 | ], 10 | [ 11 | "284", 12 | "263" 13 | ], 14 | [ 15 | "402", 16 | "453" 17 | ], 18 | [ 19 | "820", 20 | "818" 21 | ], 22 | [ 23 | "762", 24 | "832" 25 | ], 26 | [ 27 | "834", 28 | "852" 29 | ], 30 | [ 31 | "922", 32 | "898" 33 | ], 34 | [ 35 | "104", 36 | "126" 37 | ], 38 | [ 39 | "106", 40 | "198" 41 | ], 42 | [ 43 | "159", 44 | "175" 45 | ], 46 | [ 47 | "416", 48 | "342" 49 | ], 50 | [ 51 | "857", 52 | "909" 53 | ], 54 | [ 55 | "599", 56 | "585" 57 | ], 58 | [ 59 | "443", 60 | "514" 61 | ], 62 | [ 63 | "566", 64 | "617" 65 | ], 66 | [ 67 | "472", 68 | "511" 69 | ], 70 | [ 71 | "325", 72 | "492" 73 | ], 74 | [ 75 | "816", 76 | "649" 77 | ], 78 | [ 79 | "583", 80 | "558" 81 | ], 82 | [ 83 | "933", 84 | "925" 85 | ], 86 | [ 87 | "419", 88 | "824" 89 | ], 90 | [ 91 | "465", 92 | "482" 93 | ], 94 | [ 95 | "565", 96 | "589" 97 | ], 98 | [ 99 | "261", 100 | "254" 101 | ], 102 | [ 103 | "992", 104 | "980" 105 | ], 106 | [ 107 | "157", 108 | "245" 109 | ], 110 | [ 111 | "571", 112 | "746" 113 | ], 114 | [ 115 | "947", 116 | "951" 117 | ], 118 | [ 119 | "926", 120 | "900" 121 | ], 122 | [ 123 | "493", 124 | "538" 125 | ], 126 | [ 127 | "468", 128 | "470" 129 | ], 130 | [ 131 | "915", 132 | "895" 133 | ], 134 | [ 135 | "362", 136 | "354" 137 | ], 138 | [ 139 | "440", 140 | "364" 141 | ], 142 | [ 143 | "640", 144 | "638" 145 | ], 146 | [ 147 | "827", 148 | "817" 149 | ], 150 | [ 151 | "793", 152 | "768" 153 | ], 154 | [ 155 | "837", 156 | "890" 157 | ], 158 | [ 159 | "004", 160 | "982" 161 | ], 162 | [ 163 | "192", 164 | "134" 165 | ], 166 | [ 167 | "745", 168 | "777" 169 | ], 170 | [ 171 | "299", 172 | "145" 173 | ], 174 | [ 175 | "742", 176 | "775" 177 | ], 178 | [ 179 | "586", 180 | "223" 181 | ], 182 | [ 183 | "483", 184 | "370" 185 | ], 186 | [ 187 | "779", 188 | "794" 189 | ], 190 | [ 191 | "971", 192 | "564" 193 | ], 194 | [ 195 | "273", 196 | "807" 197 | ], 198 | [ 199 | "991", 200 | "064" 201 | ], 202 | [ 203 | "664", 204 | "668" 205 | ], 206 | [ 207 | "823", 208 | "584" 209 | ], 210 | [ 211 | "656", 212 | "666" 213 | ], 214 | [ 215 | "557", 216 | "560" 217 | ], 218 | [ 219 | "471", 220 | "455" 221 | ], 222 | [ 223 | "042", 224 | "084" 225 | ], 226 | [ 227 | "979", 228 | "875" 229 | ], 230 | [ 231 | "316", 232 | "369" 233 | ], 234 | [ 235 | "091", 236 | "116" 237 | ], 238 | [ 239 | "023", 240 | "923" 241 | ], 242 | [ 243 | "702", 244 | "612" 245 | ], 246 | [ 247 | "904", 248 | "046" 249 | ], 250 | [ 251 | "647", 252 | "622" 253 | ], 254 | [ 255 | "958", 256 | "956" 257 | ], 258 | [ 259 | "606", 260 | "567" 261 | ], 262 | [ 263 | "632", 264 | "548" 265 | ], 266 | [ 267 | "927", 268 | "912" 269 | ], 270 | [ 271 | "350", 272 | "349" 273 | ], 274 | [ 275 | "595", 276 | "597" 277 | ], 278 | [ 279 | "727", 280 | "729" 281 | ] 282 | ] -------------------------------------------------------------------------------- /diagrams/3d_conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/diagrams/3d_conv.png -------------------------------------------------------------------------------- /diagrams/3dconv.drawio: -------------------------------------------------------------------------------- 1 | 7V1tc5s4EP41nmk/xAMCBP6Y117uek3azF2unzoyyDYtRhTLsdNffxIvBixhkxgMTnGmCVqBkLXPPtpdJDrQLufrDyEKZn8TB3sDoDjrgXY1AGBkWew3FzzHAmgpsWAauk4sUjPBg/sLJ8L0tKXr4EXhREqIR92gKLSJ72ObFmQoDMmqeNqEeMW7BmiKBcGDjTxR+ug6dBZLLUPJ5H9gdzpL76wqSc0cpScngsUMOWSVE2nXA+0yJITGR/P1Jfb42KXjEl93U1K76ViIfVrlgm9f/rLnxrX36fPd5/PV9/uL+3+0M6jFzTwhb5l84wGAHmvwYswOpvwgFUwIuxH7HvQ5GRz4c0nSirNFpLpzdoIKg3VWmWsFzQN24I8XQVRWHmas886A9zu5Zbjj7MdomBfSc/dffevbOKAu8Sud/QUvfEzLb8VGOR4NQZwbNVAYLBCSpe9grgyFVa9mLsUPAbJ57YqZDpPN6NxjJTUZ0sQYVMDLruddEo+EUVuag7A1sZl8QUPyA+dqoG3h8YTVIM+d+kzm4UnWnTxaEgA94ZDidU6UoOcDJnNMw2d2SlKbYj6xZHOUlFeZXZgp2Gc5mwBaIkSJLU43TWdwZQcJYuXoXV2HX1V9fns5uvnzm+k4U//x55lqlqJ3ESBfCtYxsn9MI12c2fGwccyG0/E7oFoRFllfFKCB7NgA70U8AwOuVXPNu7BRfnzTEv1jhxFLUiQhnZEp8ZF3nUkvigjJzvlISJDg4jum9DkBBlpSUkQNXrv0P3750EhKX5PG+PHVOl94Tgs+00TuIl78mrbHC9llUSm7LmVryEocRS5jzfMEdXPXcaLvtA3kUhzy4dmJwhB7iLpPRV6WQSq59J64EV+tUxSCITQLCB5xJedbWZBlaOPkwjyVStrSNLj5mFqxXYVXZx+jeBeKwimmwl0iC9h839cbBSi1icoErssI/IHieSn/HcR1RW7DqmNgU8ZtIzbQiKPNnbNp93wRxBO+Wg+9qcaoFCA5ilN1CcU1x3BQos23TSydYgizIYaAbTKE6PPd+sGS9/6dtlahwv+9r89/0ZuzT/OI9in1nzXz9Mwxb4w523ypOcamsANmenxeDOZdYwjqtu+DKFcXrOMmRHPMRNe+TRzXn9Y63bHJznJ02XRngTFjEOl0dxzzstTWpz/r5Q7+tjMDuDMTI1npnf965mgPjbF3sRnMFLQ+8XHHZnCroRnc0tucwUWv0FhvMhxncfbi7LxWnppYNralKYexZeiG0iZPte4GiHNGn4c4EhXVnYfIuzVSXRsV3Zo0g318t2ZXt3MQjQP6tiP3BihCg+WpntYoog/kq5uYWdHEjE5ZmJiMPoWwWmYsrc+nmt4bR9nYWBWNo72wele3f7ewWmJeRw2rpboYvdxd7aPqLnNC6mvuJQWrU5yQdvsQIEqfVcmj4WqgOnqQ3BDLtD6Jq+rLtdtTSTej4qYzd7DGp/NwO3NXbPZYD+flNgEEm3izYbjR8gN1uQJkC97eNhd0yqhfm47fY9THep4ux5SY/j2FyF9mn607DanqTsgc1bwx1vRAXY6zqqnnjoX+qph7/i1if4mBtR/7S1eU9cF/P0vLYsd6Z+mjPTOXA/8Vi8X7ZENFWmvdb5CsFMoxytumjlML9k2FAQiWsYRhKS2yBJClJN82fDqKA6i3ioPyjXEvc5N6P+ck0GaANtGmj1pVe5nS+WbWbbWzwj0OXfaNcShAoRAJ7521WGm7qfqWaYHKwfKobvwd5MgAMVjmv5RL4j/lN6RW9FYj4tmioKi9C0TtGfv7iYTzmtr9gr2lgOPFDAX80F6O8X4Xdxxj+uN4I9iEl3dL6rk+TuQOCn/csWZcGuFpqBhFIYikr0sONJAKgMW9s0a6Cz7vL8v2zlqN5dnK0wCHbRPk85smzHA7dwt2foYrpzWzyGvWEYgtTe7tJTbQreVxab9ziGOcZiPKRgDFO/PrDI4N/iMz83j2ZzXTEDkuu3k+cI5+mqEADepDTVeyT3FDwWbr/D5KgE1RQrq/v3dC6rPVUUVbTXMlXbFVcYVY74ScvBMClK55IbDd3NxbpBy96lJA2C33QBcTbz3lnDzlaHrXKMcsX3NaOfAxZIEP01O05uOkY54aeQjUTS8laT1VH5Zn9bafNzWdxxNjnHqi6o+MAVDYzOt3dr9qrMJTxhp4YvNawDT5r4o8AS2RJlJHtf5gqHxL72Ga/Pf26vruKIrcegPctl4nGMr16pijsVKTXlWoFvSqKlBUrGFIJgDQWJjbbq59CJp/xlLjZmcoepO73inUEWcSijzMTfaMW6WgfQZlWlRWajk2Gy/uopc/Uw8xs3c0jpriIx/wmSX6NsbFwLgalL4qI7JA/qnkd+0EsmCNm7ehJh0b5F84KrNS5kbqwCoYarq39rWzc3oKmUwWuJG5FoqOHD0VN6sOw97pob/KsHftau6KYYt7ijjaBuBCatpHVb/5JvWfLm7vCAAM8bnV4+2nq7tH8S2+dflSusx5moAS5wmOoQFrCp5H+pbzlFt5l/OfrGO6T7B88/Bh8fMNgw/FfhcD6DK7VkHBso0XJgr3bhWoz4+DVVdD1B+2H4a2ktUQ508o5G9uZ0cOCqJ+AOWeEE/ASZ+H47W6pQ61Ips0mYpjxezF9rHDl/3vANr1/w== -------------------------------------------------------------------------------- /diagrams/CNN_LSTM.drawio: -------------------------------------------------------------------------------- 1 | 7V1bd5s4EP41Pmf3IT6AkIDHxHG62U3TpNlt2n3pwUa2aTFyMUmc/fUrcQfJLra5OYWcONYAgpFmvrloIAMwWm7eeeZq8Z5Y2BkokrUZgMuBohi6Tj8Z4TUkICCHhLlnWyEpQ3iw/8MRUYqoT7aF17kDfUIc317liVPiunjq52im55GX/GEz4uSvujLnmCM8TE2Hpz7alr8IqTqUUvof2J4v4ivLUrRnacYHR4T1wrTIS4YExgMw8gjxw2/LzQg7bOzicQnPu9qyN7kxD7t+mRO+fvxruoRj5/b+w/35y7e7i7t/wBkCYTfPpvMUcTxQkEM7vFjQK6A5+xZTJowL/zUaGvTjid36xYy4/tk6mLhzeoCsrDbpzmIX7OBSvaBdvUy8IoWyHXbNkScCGs9aN5k1lyv6xZ2sV0FbYjN1Y75ij/19+Pv9AdwLh47n3rKfuzgglcy+gLee3Z7dnt2e3Z7dnt2e3Z7dnt3j2FVy96N45Mm1MItDJLr7ZWH7+GFlTtneFxo1UtrCXzq0JUd3HcWBssLatuOMiEO8oC9gmVifTSl97XvkO87sQVMdT2Z0j+nYc5fSHDxLbycbKEWx0zP2fLzJkKLA6R0mS+x7r/SQeG8c78VRrB61X9KYEKoRbZGJB5M41ozi0HnSdxqq0S9RtCaO3F7G3hdZXV6PjKs/v2qWNXcff5zJmiByK4w7tmgsGzWJ5y/InLimM06pF/mZSY+5IWQVzcc37Puv0YSYTz7Jzxbe2P5ndvpQBlHzS9CU4r2Xm6j3oPEaN1w6COGJMG5+iftkjfS0oJWeFycJEG2xGbRpsH4ezfjStqyAr6IQsWvfYc+mY4+9qKutcrEmT94U7xj7aKR905vj6FT4+fHfG309lq/uH9wrSb1bGv4ZDI9j07BTyjzsmL79nE85iCQmOvWO2IHKR9KpAGWItJyAGkDJ9xLyFJ2YzRII+gIAJZsG8v1KbHe6wfxVwhHhrhIIeMLv4TKvcCKv0A9pdHtLP8fulFhBaHztTvHKt4lLv3/Eaxcz/j7JnHrsB0t5GMKyBbEmgiGDjpnJhNNemnN8vl6FaakitMGKkAnlkclQBMikCJAJVABMQqFP8l0t4RDMwlBZDArBqzwI0VZ1cAJ4ONkl/tXByVEGiM8cXnnmElPSb2BDxZL9/l6dK1CVvkCDw0pN4VRGViWKc/VojTARq3bMmh+kRU1Y8vqtttGM1VaZ1UZGsumwGguuAs5k12WkhcMnG29dckuJoFDLta4j/a67PjGkB0gtA/MNYrwhwPgwfl6vTPfwRIAC0UbWNiwiTMLxsMct8fivoo/1Q3geanW1YajVT3BqdahlJ1caGppem78t1ENZPUkYlnkf8RRwOMmLdQaHY7XJ5hFSCD05hWrV664bYjWJxmuZLR+6QV1qJh0llKM40/ELBW0dlQOktioH24uoesfuDUobVNqUNsi7AZ+uL8cfqjX8hTW0QvZ7NsNoKlyEszRjIu2cu31y23Ju3GUJCZLbcb4ht+xWl+sAW05uKPUr3UEJN/FgoZJ+NuiUn40UTsEYNJwxmOZmn4qyn5+sWHOmdLxYsLLdyfIwNQDmJOiKjfyKQUbADbwYwEtKccwJdi7M6fd5IEM5DWRbKY99pyBz2piUUkc3NshWK4u0VBpKqqLnFDVe+DwUheNDyGy2xrWAKOJdN/+tW9PSsd1Bir0rc9gVxZZFk04l4UKo2o1Ov/Ym5z9OWXREAOL7zgjA4/Xt5YfHAVdrX6UvxTlPM2WL84QmCKJqnCdg5NPfspRZ+cz4T7qoaqmu2gC5VS2TsyqWAO7PlCyrYhmNO2ZtaFexUPPLk0fNKOQ0au+yREhD3yDmLUS/JZ5wYUWLR1yhslsdjW9utt7qUVBSKH6EWLdUEXToygQgtA/Kl4cSRdWLNRO6xEdiqiwsmdBrA5NWs23dBhOjJJjE5akdQRNZlFH9iY4GGnS2DlWIaalLvKXpBMMn1Y81lVy+GSbfNkqhwjqTEKIEzk5t+KSIKrR7Ye6FuUzqky9T1ASVvVuqFGsTacAHUG3a4NjQ7mWDkzC5bhuslDXCSmtpEuHtoBMs4N5vTsO0KvEs7MX67BKXscFAyXbnlMBS36u45OQhIZdOnBwnEkDvlEjE992bst6U7WvKQMEvE9uxJo2YKGfRG7EtgyWVNWLdykuBPpLsEetAxIKoY4iFWqnzzY6sehyI6CVBBHXLEwZ8oeiIuFOTXyo+RshnkP2IhDwsPoqmIkMPt5rMtW4MdSClW34ZCUFeF1AaqzayiqSpJxeg7Gmvq1O9Ew04+nijt94HApiqdSwPDNt5qKDqNaeuvO2i+MYDXS1MXOkHZandoicnm5x/ADd5x2dDD3NB3tloVW6qCjF/bnoqX4MUz3fy0p7YrUcHCo4C2Xp4RnBalZv4MdeM3NzYLja9ap1UfYrFRUsTHapwr/qyvZ7hLS6DJI+TZR8fa/QFJ6iVGvDqXEIkeAp9p2R1xCdE/HrqpUdW5KnicKwrko4Ejkuzkq6JkmhtRl46ypbdnLG3jPWvGauhiGeL3QPaEGUyA3I+NWAAaQgOtKiaOpS3W1SjWYuq8XXirXpiLT/i2x0HTs47cIZ0oAOHNDjMyHH8gpy2xE3pxW0fcat+pWlLfKhUJG6qru4MNJuWN9Hjyr28bZe3yhcltpnXiuStKWtKm+m/LwkPT/8HDBj/Dw== -------------------------------------------------------------------------------- /diagrams/CNN_LSTM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/diagrams/CNN_LSTM.png -------------------------------------------------------------------------------- /diagrams/Inception-resnet.drawio: -------------------------------------------------------------------------------- 1 | 7Zphc5s2GMc/je+6F86BZAF+aTtp1+uWukl7W1/tMAibDiMmixj3008CYSAiDplNFO/sxAl6JIHQ//lJj0ADOFtnH6ibrH4nPo4GwPCzAbweADB2HP5XGHaFwYJmYVjS0C9MNcN9+BNLoyGtaejjTaMgIyRiYdI0eiSOsccaNpdSsm0WC0jUvGriLrFiuPfcSLX+EfpsVVgdZFT2X3G4XJVXNg2Zs3bLwtKwWbk+2dZM8GYAZ5QQVhytsxmORN+V/VLUe/9E7r5hFMesS4W/7j55a3QT3X75/GWy/TGfzr/BoSnleXCjVN4xQFZm2vxXNpvtyr7APu8amSSUrciSxG50U1mnlKSxj8UFDZ6qyvxGSMKNJjf+wIztpM5uygg3rdg6krk4C9mfovoVkqnv8mTi+DqrJ3ZlImZ0V6skkt/L84lEVS1PVfVKf7N46gFTFnLdJ1G4jLltHfp+fk8BiZksZ3LRpmq/SylE99QMUoUPmKwxvywvQHHksvCh6VmudNDlvty+6pyE/BLAkCzZxvjKqn2kQBIs5BhXsPZBzdMzly4xk2esvIQf1JpYmXLfeYEflU5e86P/u/u8UT+wRlr9ALb4gRXxDppuEjduOIT1TyrGvhyw4SaXdMILmCDJqkx+tBT/a6NScTrevOKMRf7F2/SMOuCVvO3zV3BrfUIjmxnf4q3lB9drb2hqGWQ2JKUenmMa8vZjep+4XhgveaYQVN5xe2ZzxtkUeV/zy4q2bBglf+/DDHOv+iOJW92gaNShvpLBU96+A+VAL14ETdQcpcoJ47U8RRmWPsZJKhr4DmamZYjvL4ozNV1luwoZFoKK3C2PdptuEYRRNCMRoXldGCDxs1e1llNwJAe+mr347FUXAQnODuuu6lTG3Uaju+0ynN5WMawJpW1Vi1+h8bSyDU1eLIA6MWgNNA9C3OT0INK9Mgs6Mjs6ktmjlAWKsPcMr0/Kku9iJ/BaWfIcvAh6YYYHUpqZaQmqnbGVOZmjJdr5j5HLWZAGO5Jm6SRNHUJNI/sYezhhIYmHd3gTYzZUQ+GjpjHHw14regsHjZDRC3oj/dPV6DJdvRiiUUeIkE6IVGFRk6FbztDk/BlCtnaG0Fuavs6CINSRIKiTIFXWO+ynXs7PacHxEXb8URs4DljAfA3Vw+Qz1g6OrU709hhkMIMXcg4FZh3IsXWSYx0g57Rh2+uQA+y39pgBqA96Lug8g47dER1HJzrqkIjUpc/sDMO2xwy9gccOaogsGLrwc4iLDvyMdfKjvuefcLcVmx+AMSckEn11fo/sHsOjf9ED1Oj4As9zUHSAxzTaHeF16Bkrol5TkpCUvTOu0GnfG+kBR/+iB6jB8QWcZ4HoQs6x+wKOW8uq7zCmLvNW3HRL6GlfEwVBANoDNt9aWKinRY/+h9VADY750JS/3r7A0w8TT+xygM1dDtB4JHnfuxzU1e/7NIrEnc2K/bFc2dMSh60niLPHC6OnJRLscYnEk9WW3EKXal8zvPkX -------------------------------------------------------------------------------- /diagrams/ff_benchmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/diagrams/ff_benchmark.png -------------------------------------------------------------------------------- /diagrams/incep_resnet_v1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Megatvini/DeepFaceForgeryDetection/d547936c1702336d089192ffae534b36d26bf889/diagrams/incep_resnet_v1.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.22.0 2 | Pillow>=8.1.1 3 | six==1.13.0 4 | torch==2.2.0 5 | torchvision==0.6.0 6 | tqdm==4.66.3 7 | tensorboard==2.2.1 8 | facenet-pytorch==2.2.9 9 | -------------------------------------------------------------------------------- /src/FaceCrop.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import cv2 4 | import dlib 5 | from tqdm import tqdm 6 | 7 | """ 8 | Function facecrop automatically extracts face region for all frames and saves the cropped image 9 | - Assumes current file structure of one folder per video containing all extracted frames 10 | - Drastically reduces the size of the data we need to store 11 | """ 12 | 13 | 14 | # generates a quadratic bounding box 15 | # source: https://github.com/ondyari/FaceForensics/blob/master/classification/detect_from_video.py 16 | def get_boundingbox(face, width, height, scale=1.3, minsize=None): 17 | """ 18 | Expects a dlib face to generate a quadratic bounding box. 19 | :param face: dlib face class 20 | :param width: frame width 21 | :param height: frame height 22 | :param scale: bounding box size multiplier to get a bigger face region 23 | :param minsize: set minimum bounding box size 24 | :return: x, y, bounding_box_size in opencv form 25 | """ 26 | x1 = face.left() 27 | y1 = face.top() 28 | x2 = face.right() 29 | y2 = face.bottom() 30 | size_bb = int(max(x2 - x1, y2 - y1) * scale) 31 | if minsize: 32 | if size_bb < minsize: 33 | size_bb = minsize 34 | center_x, center_y = (x1 + x2) // 2, (y1 + y2) // 2 35 | 36 | # Check for out of bounds, x-y top left corner 37 | x1 = max(int(center_x - size_bb // 2), 0) 38 | y1 = max(int(center_y - size_bb // 2), 0) 39 | # Check for too big bb size for given x, y 40 | size_bb = min(width - x1, size_bb) 41 | size_bb = min(height - y1, size_bb) 42 | 43 | return x1, y1, size_bb 44 | 45 | 46 | # generates cropped image containing only the face region for all frames 47 | # available in input path and saves resulting image to output path 48 | 49 | def facecrop(input_path, output_path): 50 | os.makedirs(output_path, exist_ok=True) 51 | face_detector = dlib.get_frontal_face_detector() 52 | # iterate overall folders in input path containing drames 53 | for foldername in os.listdir(input_path): 54 | framepath = os.path.join(input_path, foldername) 55 | # save cropped image in folderof same name in ouput path 56 | output_folder = os.path.join(output_path, foldername) 57 | os.makedirs(output_folder, exist_ok=True) 58 | print(framepath) 59 | length = len(os.listdir(framepath)) 60 | with tqdm(total=length) as p_bar: 61 | # iterate over all files in the current folder 62 | for filename in os.listdir(framepath): 63 | # go to the next file if the current file is not a png-image 64 | if filename[-3:] == 'png': 65 | filepath = os.path.join(framepath, filename) 66 | # read current frame 67 | im = cv2.imread(filepath) 68 | height, width = im.shape[:2] 69 | gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) 70 | faces = face_detector(gray, 1) 71 | if len(faces): 72 | # For now only take biggest face 73 | face = faces[0] 74 | x, y, size = get_boundingbox(face, width, height) 75 | # generate cropped image 76 | cropped_face = im[y:y + size, x:x + size] 77 | # save cropped image in the desired location 78 | im_out_path = os.path.join(output_folder, filename) 79 | cv2.imwrite(im_out_path, cropped_face) 80 | p_bar.update(1) 81 | pass 82 | 83 | 84 | # assert that a cropped image has been generated for each frame of every available video 85 | def checkfacecropping(input_path, ouput_path): 86 | nr_input_videos = len(os.listdir(input_path)) 87 | nr_output_videos = len(os.listdir(output_path)) 88 | oklist = [] 89 | notoklist = [] 90 | # check whether the number of video folders in the input path matches the number of folders in the ouput path 91 | if nr_input_videos == nr_output_videos: 92 | print('All videos have been cropped.') 93 | else: 94 | print(nr_input_videos - nr_output_videos, 'videos have not been cropped.') 95 | # for each video folder in the output path check whether all frames have been cropped 96 | for out_folder in os.listdir(output_path): 97 | out_frame_path = os.path.join(output_path, out_folder) 98 | in_frame_path = os.path.join(input_path, out_folder) 99 | nr_input_frames = len(os.listdir(in_frame_path)) 100 | nr_output_frames = len(os.listdir(out_frame_path)) 101 | # if all frames have been cropped append the number of the current video to oklist 102 | if nr_input_frames == nr_output_frames: 103 | oklist.append(out_folder) 104 | # if not all frames have been cropped append the number of the current video to notoklist 105 | else: 106 | notoklist.append(out_folder) 107 | # return list of all videos for which all frames have been cropped and list 108 | # of videos for which only a subset of frames have been cropped 109 | return oklist, notoklist 110 | 111 | 112 | if __name__ == '__main__': 113 | p = argparse.ArgumentParser( 114 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 115 | p.add_argument('--input_path', '-i', type=str, 116 | default='/home/jober/Documents/GR/original_sequences/c0/videos/images') 117 | p.add_argument('--output_path', '-o', type=str, 118 | default='/home/jober/Documents/GR/original_sequences/c0/images') 119 | args = p.parse_args() 120 | 121 | input_path = args.input_path 122 | output_path = args.output_path 123 | 124 | # facecrop(input_path, output_path) 125 | 126 | oklist, notoklist = checkfacecropping(input_path, output_path) -------------------------------------------------------------------------------- /src/benchmark.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | import torch 6 | from PIL import Image 7 | from facenet_pytorch import fixed_image_standardization 8 | from torch.utils.data import Dataset 9 | from torchvision import transforms 10 | from tqdm import tqdm 11 | 12 | from model import FaceRecognitionCNN 13 | from utils import write_json 14 | from facenet_pytorch import MTCNN 15 | 16 | FACES_DATA_DIR = '../dataset/faceforensics_benchmark_faces' 17 | 18 | # Device configuration 19 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 20 | print('running on', device) 21 | 22 | 23 | class ImagesDataset(Dataset): 24 | def __init__(self, images_dir, transform) -> None: 25 | super().__init__() 26 | self._images = [] 27 | self.read_images(images_dir) 28 | self.transform = transform 29 | 30 | def __getitem__(self, index: int): 31 | image_name, image_path = self._images[index] 32 | image = Image.open(image_path) 33 | if self.transform is not None: 34 | image = self.transform(image) 35 | return image_name, image 36 | 37 | def __len__(self) -> int: 38 | return len(self._images) 39 | 40 | def read_images(self, images_dir): 41 | for image_name in os.listdir(images_dir): 42 | image_path = os.path.join(images_dir, image_name) 43 | self._images.append((image_name, image_path)) 44 | 45 | 46 | def run_evaluate(model_path): 47 | # Image preprocessing, normalization for the pretrained resnet 48 | transform = transforms.Compose([ 49 | transforms.Resize((160, 160)), 50 | np.float32, 51 | transforms.ToTensor(), 52 | fixed_image_standardization 53 | ]) 54 | 55 | full_dataset = ImagesDataset(FACES_DATA_DIR, transform) 56 | 57 | # Build the models 58 | model = FaceRecognitionCNN().to(device) 59 | state_dict = torch.load(model_path, map_location=device) 60 | model.load_state_dict(state_dict) 61 | model.eval() 62 | 63 | res = {} 64 | with torch.no_grad(): 65 | for image_name, image in tqdm(full_dataset, desc='Evaluating frames'): 66 | image = image.to(device) 67 | output = model(image.unsqueeze(0)).item() 68 | prediction = 'fake' if output > 0.0 else 'real' 69 | res[image_name] = prediction 70 | 71 | write_json(res, 'benchmark.json') 72 | 73 | 74 | def extract_faces(data_dir): 75 | face_detector = MTCNN(device=device, margin=16) 76 | face_detector.eval() 77 | for image_name in tqdm(os.listdir(data_dir), desc='Extracting faces'): 78 | inp_img_path = os.path.join(data_dir, image_name) 79 | out_img_path = os.path.join(FACES_DATA_DIR, image_name) 80 | if not os.path.exists(out_img_path): 81 | image = Image.open(inp_img_path) 82 | face_detector(image, save_path=out_img_path) 83 | 84 | 85 | def main(): 86 | parser = argparse.ArgumentParser() 87 | parser.add_argument('model_path', type=str, help='path for the model to evaluate') 88 | parser.add_argument('data_dir', type=str, help='path to images to classify') 89 | args = parser.parse_args() 90 | extract_faces(args.data_dir) 91 | run_evaluate(args.model_path) 92 | 93 | 94 | if __name__ == '__main__': 95 | main() 96 | -------------------------------------------------------------------------------- /src/data_loader.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import random 4 | 5 | import torch 6 | from PIL import Image, ImageFile 7 | from torch import tensor 8 | from torch.utils.data import Dataset 9 | 10 | ImageFile.LOAD_TRUNCATED_IMAGES = True 11 | 12 | 13 | class CompositeDataset(Dataset): 14 | def __init__(self, *datasets) -> None: 15 | super().__init__() 16 | self.datasets = datasets 17 | 18 | def __getitem__(self, index: int): 19 | for d in self.datasets: 20 | if index < len(d): 21 | return d[index] 22 | index -= len(d) 23 | 24 | def __len__(self) -> int: 25 | return sum(map(len, self.datasets)) 26 | 27 | 28 | class ImagesDataset(Dataset): 29 | def __init__(self, video_dirs, name, target, max_images_per_video, max_videos, transform, window_size): 30 | self.name = name 31 | self.target = torch.tensor(target).float() 32 | self.max_images_per_video = max_images_per_video 33 | self.max_videos = max_videos 34 | self.image_paths = [] 35 | self.transform = transform 36 | self.window_size = window_size 37 | self._read_images(video_dirs, name) 38 | self.image_paths = sorted(self.image_paths, key=lambda x: x['img_path']) 39 | 40 | def _read_images(self, video_dirs, class_name): 41 | for video_dir in video_dirs[:self.max_videos]: 42 | self._read_class_images(class_name, video_dir) 43 | 44 | def _read_class_images(self, class_name, video_dir): 45 | video_id = get_file_name(video_dir) 46 | sorted_images_names = sorted(os.listdir(video_dir))[:self.max_images_per_video] 47 | for image_name in sorted_images_names: 48 | frame_id = image_name.split('_')[-1].split('.')[0] 49 | self.image_paths.append({ 50 | 'video_id': video_id, 51 | 'frame_id': frame_id, 52 | 'class': class_name, 53 | 'img_path': os.path.join(video_dir, image_name) 54 | }) 55 | 56 | def __getitem__(self, index): 57 | data = [self._get_item(index + i) for i in range(-self.window_size//2 + 1, self.window_size//2 + 1)] 58 | mid_video_id, mid_frame_id, mid_image, target = data[len(data)//2] 59 | images = [x[2] if x[0] == mid_video_id else mid_image for x in data] 60 | if self.window_size > 1: 61 | return mid_video_id, mid_frame_id, torch.stack(images).permute(1, 0, 2, 3), target 62 | else: 63 | image_tensor = images[0] 64 | return mid_video_id, mid_frame_id, image_tensor, target 65 | 66 | def _get_item(self, index): 67 | img = self.image_paths[index] 68 | target = self.target 69 | image = Image.open(img['img_path']) 70 | if self.transform is not None: 71 | image = self.transform(image) 72 | return img['video_id'], img['frame_id'], image, target 73 | 74 | def __len__(self): 75 | return len(self.image_paths) - self.window_size // 2 76 | 77 | 78 | def listdir_with_full_paths(dir_path): 79 | return [os.path.join(dir_path, x) for x in os.listdir(dir_path)] 80 | 81 | 82 | def random_split(data, split): 83 | size = int(len(data)*split) 84 | random.shuffle(data) 85 | return data[:size], data[size:] 86 | 87 | 88 | def get_file_name(file_path): 89 | return file_path.split('/')[-1] 90 | 91 | 92 | def read_json(file_path): 93 | with open(file_path) as inp: 94 | return json.load(inp) 95 | 96 | 97 | def get_sets(data): 98 | return {x[0] for x in data} | {x[1] for x in data} | {'_'.join(x) for x in data} | {'_'.join(x[::-1]) for x in data} 99 | 100 | 101 | def get_video_ids(spl, splits_path): 102 | return get_sets(read_json(os.path.join(splits_path, f'{spl}.json'))) 103 | 104 | 105 | def read_train_test_val_dataset( 106 | dataset_dir, name, target, splits_path, **dataset_kwargs 107 | ): 108 | for spl in ['train', 'val', 'test']: 109 | video_ids = get_video_ids(spl, splits_path) 110 | video_paths = listdir_with_full_paths(dataset_dir) 111 | videos = [x for x in video_paths if get_file_name(x) in video_ids] 112 | dataset = ImagesDataset(videos, name, target, **dataset_kwargs) 113 | yield dataset 114 | 115 | 116 | def read_dataset( 117 | data_dir, transform, max_videos, window_size, 118 | max_images_per_video, splits_path='../dataset/splits/' 119 | ): 120 | data_class_dirs = os.listdir(data_dir) 121 | data_sets = {} 122 | for data_class_dir in data_class_dirs: 123 | data_class_dir_path = os.path.join(data_dir, data_class_dir) 124 | target = 0 if 'original' in data_class_dir.lower() else 1 125 | data_sets[data_class_dir] = read_train_test_val_dataset( 126 | data_class_dir_path, data_class_dir, target, splits_path, transform=transform, 127 | max_videos=max_videos, max_images_per_video=max_images_per_video, window_size=window_size 128 | ) 129 | return data_sets 130 | 131 | 132 | def get_loader(dataset, batch_size, shuffle, num_workers, drop_last=True): 133 | data_loader = torch.utils.data.DataLoader(dataset=dataset, 134 | batch_size=batch_size, 135 | shuffle=shuffle, 136 | num_workers=num_workers, 137 | drop_last=drop_last, 138 | pin_memory=True) 139 | return data_loader 140 | -------------------------------------------------------------------------------- /src/evaluate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import numpy as np 4 | import torch 5 | import torch.nn as nn 6 | from facenet_pytorch import fixed_image_standardization 7 | from torchvision import transforms 8 | from tqdm import tqdm 9 | 10 | from data_loader import get_loader, read_dataset 11 | from model import FaceRecognitionCNN 12 | 13 | 14 | def read_testing_dataset(args, transform): 15 | datasets = read_dataset( 16 | args.data_dir, transform=transform, 17 | max_images_per_video=args.max_images_per_video, max_videos=args.max_videos, 18 | window_size=args.window_size, splits_path=args.splits_path 19 | ) 20 | return datasets 21 | 22 | 23 | def run_evaluate(args): 24 | # Image preprocessing, normalization for the pretrained resnet 25 | transform = transforms.Compose([ 26 | transforms.Resize((160, 160)), 27 | np.float32, 28 | transforms.ToTensor(), 29 | fixed_image_standardization 30 | ]) 31 | 32 | # transform = transforms.Compose([ 33 | # transforms.Resize((224, 224)), 34 | # transforms.ToTensor(), 35 | # transforms.Normalize((0.485, 0.456, 0.406), 36 | # (0.229, 0.224, 0.225)) 37 | # ]) 38 | 39 | full_dataset = read_testing_dataset(args, transform) 40 | 41 | # Device configuration 42 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 43 | print('evaluating on', device) 44 | 45 | # Build the models 46 | model = FaceRecognitionCNN().to(device) 47 | state_dict = torch.load(args.model_path, map_location=device) 48 | model.load_state_dict(state_dict) 49 | model.eval() 50 | 51 | for test_dataset_name, dt in full_dataset.items(): 52 | if 'c40' in test_dataset_name and ('original' in test_dataset_name or 'neural' in test_dataset_name): 53 | _, _, test_dataset = dt 54 | evaluate(args, device, model, test_dataset, test_dataset_name) 55 | 56 | 57 | def evaluate(args, device, model, test_dataset, test_dataset_name): 58 | tqdm.write(f'evaluating for {test_dataset_name}') 59 | tqdm.write('test data size: {}'.format(len(test_dataset))) 60 | 61 | # Build data loader 62 | test_loader = get_loader( 63 | test_dataset, args.batch_size, shuffle=False, num_workers=args.num_workers, drop_last=False 64 | ) 65 | 66 | criterion = nn.BCEWithLogitsLoss() 67 | 68 | with torch.no_grad(): 69 | loss_values = [] 70 | all_predictions = [] 71 | all_targets = [] 72 | for video_ids, frame_ids, images, targets in tqdm(test_loader, desc=test_dataset_name): 73 | images = images.to(device) 74 | targets = targets.to(device) 75 | 76 | outputs = model(images) 77 | loss = criterion(outputs, targets) 78 | loss_values.append(loss.item()) 79 | 80 | predictions = outputs > 0.0 81 | all_predictions.append(predictions) 82 | all_targets.append(targets) 83 | 84 | val_loss = sum(loss_values) / len(loss_values) 85 | 86 | all_predictions = torch.cat(all_predictions).int() 87 | all_targets = torch.cat(all_targets).int() 88 | test_accuracy = (all_predictions == all_targets).sum().float().item() / all_targets.shape[0] 89 | 90 | tqdm.write('Testing results - Loss: {:.3f}, Acc: {:.3f}'.format(val_loss, test_accuracy)) 91 | 92 | 93 | def main(): 94 | parser = argparse.ArgumentParser() 95 | parser.add_argument('model_path', type=str, help='path for the model to evaluate') 96 | parser.add_argument('--data_dir', type=str, default='../dataset/images_tiny', help='directory for data with images') 97 | parser.add_argument('--max_images_per_video', type=int, default=999999, help='maximum images to use from one video') 98 | parser.add_argument('--batch_size', type=int, default=2) 99 | parser.add_argument('--num_workers', type=int, default=2) 100 | parser.add_argument('--window_size', type=int, default=1) 101 | parser.add_argument('--max_videos', type=int, default=1000) 102 | parser.add_argument('--splits_path', type=str, default='../dataset/splits/') 103 | parser.add_argument('--comment', type=str, default='') 104 | args = parser.parse_args() 105 | run_evaluate(args) 106 | 107 | 108 | if __name__ == '__main__': 109 | main() 110 | -------------------------------------------------------------------------------- /src/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | from facenet_pytorch import InceptionResnetV1 4 | from torch import nn 5 | 6 | import resnet3d 7 | 8 | 9 | class NNLambda(nn.Module): 10 | def __init__(self, fn) -> None: 11 | super().__init__() 12 | self.fn = fn 13 | 14 | def forward(self, x): 15 | return self.fn(x) 16 | 17 | 18 | class CNN_LSTM(nn.Module): 19 | def __init__(self, face_recognition_cnn_path, hidden_size=64): 20 | super(CNN_LSTM, self).__init__() 21 | image_encoding_size = 64 22 | 23 | face_cnn = FaceRecognitionCNN() 24 | state_dict = torch.load(face_recognition_cnn_path, map_location='cpu') 25 | face_cnn.load_state_dict(state_dict) 26 | face_cnn = nn.Sequential(*list(face_cnn.resnet.children()))[:-12] 27 | 28 | for p in face_cnn.parameters(): 29 | p.requires_grad_(False) 30 | 31 | self.cnn_encoder = nn.Sequential( 32 | face_cnn, 33 | 34 | nn.Conv2d(192, 128, 5, bias=False), 35 | nn.BatchNorm2d(128), 36 | nn.ReLU(), 37 | 38 | nn.Conv2d(128, image_encoding_size, 5, bias=False), 39 | nn.BatchNorm2d(image_encoding_size), 40 | nn.ReLU(), 41 | 42 | nn.AdaptiveAvgPool2d(1) 43 | ) 44 | 45 | self.relu1 = nn.ReLU() 46 | self.dropout1 = nn.Dropout(0.5) 47 | self.lstm = nn.LSTM( 48 | image_encoding_size, hidden_size, num_layers=2, bias=True, 49 | batch_first=True, bidirectional=True, dropout=0.5 50 | ) 51 | self.fc = nn.Linear(2*hidden_size, 1) 52 | 53 | def forward(self, images): 54 | batch_size, num_channels, depth, height, width = images.shape 55 | inp = images.permute(0, 2, 1, 3, 4).reshape(batch_size * depth, num_channels, height, width) 56 | 57 | inp = self.cnn_encoder(inp).reshape(batch_size, depth, -1) 58 | inp = self.relu1(inp) 59 | inp = self.dropout1(inp) 60 | 61 | out, _ = self.lstm(inp) 62 | 63 | mid_out = out[:, depth // 2, :] 64 | 65 | res = self.fc(mid_out) 66 | return res.squeeze() 67 | 68 | 69 | class ResNet3d(nn.Module): 70 | def __init__(self): 71 | super(ResNet3d, self).__init__() 72 | self.model = resnet3d.resnet10(num_classes=1) 73 | 74 | def forward(self, images): 75 | return self.model(images).squeeze() 76 | 77 | 78 | class ResNet2d(nn.Module): 79 | def __init__(self, final_hidden_dim=256, dropout=0.5, pretrained=True): 80 | super(ResNet2d, self).__init__() 81 | resnet = torchvision.models.resnet18(pretrained=pretrained) 82 | resnet.fc = nn.Linear(512, final_hidden_dim) 83 | self.model = nn.Sequential( 84 | resnet, 85 | nn.ReLU(), 86 | nn.Dropout(dropout), 87 | nn.Linear(final_hidden_dim, 1) 88 | ) 89 | 90 | def forward(self, images): 91 | return self.model(images.squeeze()).squeeze() 92 | 93 | 94 | class SqueezeNet2d(nn.Module): 95 | def __init__(self, dropout=0.5, pretrained=True): 96 | super(SqueezeNet2d, self).__init__() 97 | squeeze_net = torchvision.models.squeezenet1_1(pretrained=pretrained) 98 | self.model = nn.Sequential( 99 | squeeze_net, 100 | nn.ReLU(), 101 | nn.Dropout(dropout), 102 | nn.Linear(1000, 1), 103 | ) 104 | 105 | def forward(self, images): 106 | return self.model(images.squeeze()).squeeze() 107 | 108 | 109 | class FaceRecognitionCNN(nn.Module): 110 | def __init__(self): 111 | super(FaceRecognitionCNN, self).__init__() 112 | self.resnet = InceptionResnetV1(pretrained='vggface2') 113 | self.relu = nn.ReLU() 114 | self.dropout = nn.Dropout(0.5) 115 | self.fc = nn.Linear(512, 1) 116 | 117 | def forward(self, images): 118 | out = self.resnet(images) 119 | out = self.relu(out) 120 | out = self.dropout(out) 121 | out = self.fc(out) 122 | return out.squeeze() 123 | 124 | 125 | class Encoder2DConv3D(nn.Module): 126 | def __init__(self, face_recognition_cnn_path=None): 127 | super(Encoder2DConv3D, self).__init__() 128 | 129 | face_cnn = FaceRecognitionCNN() 130 | if face_recognition_cnn_path is not None: 131 | state_dict = torch.load(face_recognition_cnn_path, map_location='cpu') 132 | face_cnn.load_state_dict(state_dict) 133 | 134 | self.encoder2d = nn.Sequential(*list(face_cnn.resnet.children()))[:-10] 135 | self.encoder3d = nn.Sequential( 136 | nn.Conv3d(256, 256, 3, padding=1, bias=False), 137 | nn.BatchNorm3d(256), 138 | nn.ReLU(), 139 | 140 | nn.Conv3d(256, 512, 3, padding=1, bias=False), 141 | nn.BatchNorm3d(512), 142 | nn.ReLU(), 143 | 144 | nn.AdaptiveAvgPool3d(1), 145 | nn.Flatten(), 146 | nn.Dropout(0.5), 147 | nn.Linear(512, 1) 148 | ) 149 | 150 | def forward(self, images): 151 | batch_size, num_channels, depth, height, width = images.shape 152 | images = images.permute(0, 2, 1, 3, 4) 153 | images = images.reshape(batch_size * depth, num_channels, height, width) 154 | out = self.encoder2d(images) 155 | out = out.reshape(batch_size, depth, 256, 17, 17) 156 | out = out.permute(0, 2, 1, 3, 4) 157 | out = self.encoder3d(out) 158 | return out.squeeze() 159 | 160 | 161 | class MajorityVoteModel(nn.Module): 162 | def __init__(self, face_recognition_cnn_path): 163 | super(MajorityVoteModel, self).__init__() 164 | 165 | face_cnn = FaceRecognitionCNN() 166 | state_dict = torch.load(face_recognition_cnn_path, map_location='cpu') 167 | face_cnn.load_state_dict(state_dict) 168 | 169 | self.cnn_encoder = face_cnn 170 | 171 | def forward(self, images): 172 | batch_size, num_channels, depth, height, width = images.shape 173 | images = images.permute(0, 2, 1, 3, 4) 174 | images = images.reshape(batch_size * depth, num_channels, height, width) 175 | out = self.cnn_encoder(images) 176 | out = out.reshape(batch_size, depth) 177 | out = ((out > 0.0).sum(axis=1) > depth // 2).float() 178 | # out = out[:, depth//2] 179 | return out 180 | -------------------------------------------------------------------------------- /src/resnet3d.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from torch.autograd import Variable 7 | 8 | 9 | """ 10 | Initial 3d resnet code taken from https://github.com/kenshohara/3D-ResNets-PyTorch 11 | """ 12 | 13 | 14 | def resnet10(**kwargs): 15 | """Constructs a ResNet-18 model. 16 | """ 17 | model = ResNet(BasicBlock, [1, 1, 1, 1], **kwargs) 18 | return model 19 | 20 | 21 | def resnet18(**kwargs): 22 | """Constructs a ResNet-18 model. 23 | """ 24 | model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) 25 | return model 26 | 27 | 28 | def resnet34(**kwargs): 29 | """Constructs a ResNet-34 model. 30 | """ 31 | model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) 32 | return model 33 | 34 | 35 | def resnet50(**kwargs): 36 | """Constructs a ResNet-50 model. 37 | """ 38 | model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) 39 | return model 40 | 41 | 42 | def resnet101(**kwargs): 43 | """Constructs a ResNet-101 model. 44 | """ 45 | model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) 46 | return model 47 | 48 | 49 | def resnet152(**kwargs): 50 | """Constructs a ResNet-101 model. 51 | """ 52 | model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) 53 | return model 54 | 55 | 56 | def resnet200(**kwargs): 57 | """Constructs a ResNet-101 model. 58 | """ 59 | model = ResNet(Bottleneck, [3, 24, 36, 3], **kwargs) 60 | return model 61 | 62 | 63 | def conv3x3x3(in_planes, out_planes, stride=1): 64 | # 3x3x3 convolution with padding 65 | return nn.Conv3d( 66 | in_planes, 67 | out_planes, 68 | kernel_size=3, 69 | stride=stride, 70 | padding=1, 71 | bias=False 72 | ) 73 | 74 | 75 | def downsample_basic_block(x, planes, stride): 76 | out = F.avg_pool3d(x, kernel_size=1, stride=stride) 77 | zero_pads = torch.Tensor( 78 | out.size(0), planes - out.size(1), out.size(2), out.size(3), 79 | out.size(4) 80 | ).zero_() 81 | if isinstance(out.data, torch.cuda.FloatTensor): 82 | zero_pads = zero_pads.cuda() 83 | 84 | out = Variable(torch.cat([out.data, zero_pads], dim=1)) 85 | 86 | return out 87 | 88 | 89 | class BasicBlock(nn.Module): 90 | expansion = 1 91 | 92 | def __init__(self, inplanes, planes, stride=1, downsample=None): 93 | super(BasicBlock, self).__init__() 94 | self.conv1 = conv3x3x3(inplanes, planes, stride) 95 | self.bn1 = nn.BatchNorm3d(planes) 96 | self.relu = nn.ReLU(inplace=True) 97 | self.conv2 = conv3x3x3(planes, planes) 98 | self.bn2 = nn.BatchNorm3d(planes) 99 | self.downsample = downsample 100 | self.stride = stride 101 | 102 | def forward(self, x): 103 | residual = x 104 | 105 | out = self.conv1(x) 106 | out = self.bn1(out) 107 | out = self.relu(out) 108 | 109 | out = self.conv2(out) 110 | out = self.bn2(out) 111 | 112 | if self.downsample is not None: 113 | residual = self.downsample(x) 114 | 115 | out += residual 116 | out = self.relu(out) 117 | 118 | return out 119 | 120 | 121 | class Bottleneck(nn.Module): 122 | expansion = 4 123 | 124 | def __init__(self, inplanes, planes, stride=1, downsample=None): 125 | super(Bottleneck, self).__init__() 126 | self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False) 127 | self.bn1 = nn.BatchNorm3d(planes) 128 | self.conv2 = nn.Conv3d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) 129 | self.bn2 = nn.BatchNorm3d(planes) 130 | self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False) 131 | self.bn3 = nn.BatchNorm3d(planes * 4) 132 | self.relu = nn.ReLU(inplace=True) 133 | self.downsample = downsample 134 | self.stride = stride 135 | 136 | def forward(self, x): 137 | residual = x 138 | 139 | out = self.conv1(x) 140 | out = self.bn1(out) 141 | out = self.relu(out) 142 | 143 | out = self.conv2(out) 144 | out = self.bn2(out) 145 | out = self.relu(out) 146 | 147 | out = self.conv3(out) 148 | out = self.bn3(out) 149 | 150 | if self.downsample is not None: 151 | residual = self.downsample(x) 152 | 153 | out += residual 154 | out = self.relu(out) 155 | 156 | return out 157 | 158 | 159 | class ResNet(nn.Module): 160 | def __init__(self, block, layers, shortcut_type='B', num_classes=1): 161 | self.inplanes = 64 162 | super(ResNet, self).__init__() 163 | self.conv1 = nn.Conv3d(3, 64, kernel_size=7, stride=(1, 2, 2), padding=(3, 3, 3), bias=False) 164 | self.bn1 = nn.BatchNorm3d(64) 165 | self.relu = nn.ReLU(inplace=True) 166 | self.maxpool = nn.MaxPool3d(kernel_size=(3, 3, 3), stride=2, padding=1) 167 | self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type) 168 | self.layer2 = self._make_layer(block, 128, layers[1], shortcut_type, stride=2) 169 | self.layer3 = self._make_layer(block, 256, layers[2], shortcut_type, stride=2) 170 | self.layer4 = self._make_layer(block, 512, layers[3], shortcut_type, stride=2) 171 | self.avgpool = nn.AdaptiveAvgPool3d((1, 1, 1)) 172 | self.fc = nn.Linear(512 * block.expansion, num_classes) 173 | 174 | for m in self.modules(): 175 | if isinstance(m, nn.Conv3d): 176 | nn.init.kaiming_normal_(m.weight, mode='fan_out') 177 | elif isinstance(m, nn.BatchNorm3d): 178 | m.weight.data.fill_(1) 179 | m.bias.data.zero_() 180 | 181 | def _make_layer(self, block, planes, blocks, shortcut_type, stride=1): 182 | downsample = None 183 | if stride != 1 or self.inplanes != planes * block.expansion: 184 | if shortcut_type == 'A': 185 | downsample = partial( 186 | downsample_basic_block, 187 | planes=planes * block.expansion, 188 | stride=stride 189 | ) 190 | else: 191 | downsample = nn.Sequential( 192 | nn.Conv3d( 193 | self.inplanes, 194 | planes * block.expansion, 195 | kernel_size=1, 196 | stride=stride, 197 | bias=False 198 | ), 199 | nn.BatchNorm3d(planes * block.expansion) 200 | ) 201 | layers = [block(self.inplanes, planes, stride, downsample)] 202 | self.inplanes = planes * block.expansion 203 | for i in range(1, blocks): 204 | layers.append(block(self.inplanes, planes)) 205 | 206 | return nn.Sequential(*layers) 207 | 208 | def forward(self, x): 209 | x = self.conv1(x) 210 | x = self.bn1(x) 211 | x = self.relu(x) 212 | x = self.maxpool(x) 213 | 214 | x = self.layer1(x) 215 | x = self.layer2(x) 216 | x = self.layer3(x) 217 | x = self.layer4(x) 218 | 219 | x = self.avgpool(x) 220 | 221 | x = x.view(x.size(0), -1) 222 | x = self.fc(x) 223 | 224 | return x 225 | -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | import torch 6 | import torch.nn as nn 7 | from facenet_pytorch import fixed_image_standardization 8 | from torch.utils.tensorboard import SummaryWriter 9 | from torchvision import transforms 10 | from tqdm import tqdm 11 | 12 | from data_loader import get_loader, read_dataset, CompositeDataset 13 | from model import FaceRecognitionCNN 14 | from utils import write_json, copy_file, count_parameters 15 | 16 | 17 | def read_training_dataset(args, transform): 18 | datasets = read_dataset( 19 | args.data_dir, transform=transform, 20 | max_images_per_video=args.max_images_per_video, max_videos=args.max_videos, 21 | window_size=args.window_size, splits_path=args.splits_path 22 | ) 23 | # only neural textures c40 and original c40 24 | datasets = { 25 | k: v for k, v in datasets.items() 26 | if ('original' in k or 'neural' in k) and 'c40' in k 27 | } 28 | print('Using training data: ') 29 | print('\n'.join(sorted(datasets.keys()))) 30 | 31 | trains, vals, tests = [], [], [] 32 | for data_dir_name, dataset in datasets.items(): 33 | train, val, test = dataset 34 | # repeat original data multiple times to balance out training data 35 | compression = data_dir_name.split('_')[-1] 36 | num_tampered_with_same_compression = len({x for x in datasets.keys() if compression in x}) - 1 37 | count = 1 if 'original' not in data_dir_name else num_tampered_with_same_compression 38 | for _ in range(count): 39 | trains.append(train) 40 | vals.append(val) 41 | tests.append(test) 42 | return CompositeDataset(*trains), CompositeDataset(*vals), CompositeDataset(*tests) 43 | 44 | 45 | def run_train(args): 46 | # show tensorboard graphs with following command: tensorboard --logdir=src/runs 47 | writer = SummaryWriter(comment=args.comment) 48 | 49 | # Image preprocessing, normalization for the pretrained resnet 50 | transform = transforms.Compose([ 51 | transforms.Resize((160, 160)), 52 | np.float32, 53 | transforms.ToTensor(), 54 | fixed_image_standardization 55 | ]) 56 | 57 | # transform = transforms.Compose([ 58 | # transforms.Resize((224, 224)), 59 | # transforms.ToTensor(), 60 | # transforms.Normalize((0.485, 0.456, 0.406), 61 | # (0.229, 0.224, 0.225)) 62 | # ]) 63 | 64 | train_dataset, val_dataset, test_dataset = read_training_dataset(args, transform) 65 | 66 | tqdm.write('train data size: {}, validation data size: {}'.format(len(train_dataset), len(val_dataset))) 67 | 68 | # Build data loader 69 | train_loader = get_loader( 70 | train_dataset, args.batch_size, shuffle=True, num_workers=args.num_workers 71 | ) 72 | val_loader = get_loader( 73 | val_dataset, args.batch_size, shuffle=False, num_workers=args.num_workers 74 | ) 75 | 76 | # Device configuration 77 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 78 | print('training on', device) 79 | 80 | # Build the models 81 | model = FaceRecognitionCNN().to(device) 82 | if args.freeze_first_epoch: 83 | for m in model.resnet.parameters(): 84 | m.requires_grad_(False) 85 | 86 | input_shape = next(iter(train_loader))[2].shape 87 | print('input shape', input_shape) 88 | # need to call this before summary!!! 89 | model.eval() 90 | # summary(model, input_shape[1:], batch_size=input_shape[0], device=device) 91 | print('model params (trainable, total):', count_parameters(model)) 92 | 93 | # Loss and optimizer 94 | criterion = nn.BCEWithLogitsLoss() 95 | optimizer = torch.optim.Adam( 96 | model.parameters(), lr=args.learning_rate, weight_decay=args.regularization 97 | ) 98 | 99 | # decrease learning rate if validation accuracy has not increased 100 | lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( 101 | optimizer, mode='max', factor=1/4, patience=args.patience, verbose=True, 102 | ) 103 | 104 | writer.add_hparams(args.__dict__, {}) 105 | writer.add_text('model', str(model)) 106 | 107 | # Train the models 108 | total_step = len(train_loader) 109 | step = 1 110 | best_val_acc = 0.5 111 | for epoch in range(args.num_epochs): 112 | for i, (video_ids, frame_ids, images, targets) in \ 113 | tqdm(enumerate(train_loader), desc=f'training epoch {epoch}', total=len(train_loader)): 114 | model.train() 115 | # Set mini-batch dataset 116 | images = images.to(device) 117 | targets = targets.to(device) 118 | 119 | # Forward, backward and optimize 120 | outputs = model(images) 121 | loss = criterion(outputs, targets) 122 | model.zero_grad() 123 | loss.backward() 124 | optimizer.step() 125 | 126 | batch_accuracy = float((outputs > 0.0).eq(targets).sum()) / len(targets) 127 | 128 | # Print log info 129 | step += 1 130 | 131 | if (i + 1) % args.log_step == 0: 132 | print_training_info(batch_accuracy, loss, step, writer) 133 | 134 | if (i + 1) % args.val_step == 0: 135 | val_acc, pr_acc, tmp_acc = print_validation_info( 136 | args, criterion, device, model, val_loader, writer, epoch, step 137 | ) 138 | if val_acc > best_val_acc: 139 | save_model_checkpoint(args, epoch, model, (val_acc, pr_acc, tmp_acc), writer.get_logdir()) 140 | best_val_acc = val_acc 141 | 142 | # validation step after full epoch 143 | val_acc, pr_acc, tmp_acc = print_validation_info( 144 | args, criterion, device, model, val_loader, writer, epoch, step 145 | ) 146 | lr_scheduler.step(val_acc) 147 | if val_acc > best_val_acc: 148 | save_model_checkpoint(args, epoch, model, (val_acc, pr_acc, tmp_acc), writer.get_logdir()) 149 | best_val_acc = val_acc 150 | 151 | if args.freeze_first_epoch and epoch == 0: 152 | for m in model.resnet.parameters(): 153 | m.requires_grad_(True) 154 | tqdm.write('Fine tuning on') 155 | 156 | writer.close() 157 | 158 | 159 | def save_model_checkpoint(args, epoch, model, val_acc, writer_log_dir): 160 | run_id = writer_log_dir.split('/')[-1] 161 | model_dir = os.path.join(args.model_path, run_id) 162 | os.makedirs(model_dir, exist_ok=True) 163 | 164 | model_path = os.path.join(model_dir, f'model.pt') 165 | torch.save(model.state_dict(), model_path) 166 | 167 | model_info = { 168 | 'epoch': epoch, 169 | 'val_acc': val_acc, 170 | 'model_str': str(model) 171 | } 172 | json_path = os.path.join(model_dir, 'info.json') 173 | write_json(model_info, json_path) 174 | 175 | src_model_file = os.path.join(os.path.dirname(__file__), 'model.py') 176 | dest_model_file = os.path.join(model_dir, 'model.py') 177 | copy_file(src_model_file, dest_model_file) 178 | 179 | tqdm.write(f'New checkpoint saved at {model_path}') 180 | 181 | 182 | def print_training_info(batch_accuracy, loss, step, writer): 183 | log_info = 'Training - Loss: {:.4f}, Accuracy: {:.4f}'.format(loss.item(), batch_accuracy) 184 | tqdm.write(log_info) 185 | 186 | writer.add_scalar('training loss', loss.item(), step) 187 | writer.add_scalar('training acc', batch_accuracy, step) 188 | 189 | 190 | def print_validation_info(args, criterion, device, model, val_loader, writer, epoch, step): 191 | model.eval() 192 | with torch.no_grad(): 193 | loss_values = [] 194 | all_predictions = [] 195 | all_targets = [] 196 | for video_ids, frame_ids, images, targets in tqdm(val_loader, desc=f'validation ep. {epoch}'): 197 | images = images.to(device) 198 | targets = targets.to(device) 199 | 200 | outputs = model(images) 201 | loss = criterion(outputs, targets) 202 | loss_values.append(loss.item()) 203 | 204 | predictions = outputs > 0.0 205 | all_predictions.append(predictions) 206 | all_targets.append(targets) 207 | if args.debug: 208 | tqdm.write(outputs) 209 | tqdm.write(predictions) 210 | tqdm.write(targets) 211 | 212 | val_loss = sum(loss_values) / len(loss_values) 213 | 214 | all_predictions = torch.cat(all_predictions).int() 215 | all_targets = torch.cat(all_targets).int() 216 | val_accuracy = (all_predictions == all_targets).sum().float().item() / all_targets.shape[0] 217 | 218 | total_target_tampered = all_targets.sum().float().item() 219 | tampered_accuracy = (all_predictions * all_targets).sum().item() / total_target_tampered 220 | 221 | total_target_pristine = (1 - all_targets).sum().float().item() 222 | pristine_accuracy = (1 - (all_predictions | all_targets)).sum().item() / total_target_pristine 223 | 224 | tqdm.write( 225 | 'Validation - Loss: {:.3f}, Acc: {:.3f}, Prs: {:.3f}, Tmp: {:.3f}'.format( 226 | val_loss, val_accuracy, pristine_accuracy, tampered_accuracy 227 | ) 228 | ) 229 | writer.add_scalar('validation loss', val_loss, step) 230 | writer.add_scalar('validation acc', val_accuracy, step) 231 | writer.add_scalar('pristine acc', pristine_accuracy, step) 232 | writer.add_scalar('tampered acc', tampered_accuracy, step) 233 | return val_accuracy, pristine_accuracy, tampered_accuracy 234 | 235 | 236 | def main(): 237 | parser = argparse.ArgumentParser() 238 | parser.add_argument('--model_path', type=str, default='models/', help='path for saving trained models') 239 | parser.add_argument('--data_dir', type=str, default='../dataset/images_tiny', help='directory for data with images') 240 | parser.add_argument('--log_step', type=int, default=10, help='step size for printing training log info') 241 | parser.add_argument('--val_step', type=int, default=100, help='step size for validation during epoch') 242 | parser.add_argument('--max_images_per_video', type=int, default=10, help='maximum images to use from one video') 243 | parser.add_argument('--debug', type=bool, default=False, help='include additional debugging ifo') 244 | 245 | parser.add_argument('--num_epochs', type=int, default=15) 246 | parser.add_argument('--regularization', type=float, default=0.001) 247 | parser.add_argument('--batch_size', type=int, default=22) 248 | parser.add_argument('--num_workers', type=int, default=2) 249 | parser.add_argument('--learning_rate', type=float, default=0.00001) 250 | parser.add_argument('--patience', type=int, default=2) 251 | parser.add_argument('--window_size', type=int, default=1) 252 | parser.add_argument('--max_videos', type=int, default=1000) 253 | parser.add_argument('--splits_path', type=str, default='../dataset/splits/') 254 | parser.add_argument('--encoder_model_path', type=str, default='models/Jan12_10-57-19_gpu-training/model.pt') 255 | parser.add_argument('--freeze_first_epoch', type=bool, default=False) 256 | parser.add_argument('--comment', type=str, default='') 257 | args = parser.parse_args() 258 | run_train(args) 259 | 260 | 261 | if __name__ == '__main__': 262 | main() 263 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from collections import OrderedDict 3 | 4 | import numpy as np 5 | import torch 6 | from torch import nn as nn 7 | 8 | 9 | def write_json(data, file_path): 10 | with open(file_path, 'w') as out: 11 | json.dump(data, out, indent=4) 12 | 13 | 14 | def copy_file(src_path, dest_path): 15 | with open(src_path) as src: 16 | with open(dest_path, 'w') as dest: 17 | dest.write(src.read()) 18 | 19 | 20 | def summary(model, input_size, batch_size=-1, device=torch.device('cuda:0'), dtypes=None): 21 | def get_shape(output): 22 | if isinstance(output, (list, tuple)): 23 | shape = [get_shape(o) for o in output] 24 | else: 25 | shape = list(output.size()) 26 | 27 | for i in range(len(shape)): 28 | if shape[i] == batch_size: 29 | shape[i] = -1 30 | return shape 31 | return shape 32 | 33 | if dtypes == None: 34 | dtypes = [torch.FloatTensor]*len(input_size) 35 | 36 | def register_hook(module): 37 | 38 | def hook(module, input, output): 39 | class_name = str(module.__class__).split(".")[-1].split("'")[0] 40 | module_idx = len(summary) 41 | 42 | if class_name in ('InceptionResnetV1', 'CNN_LSTM'): 43 | return 44 | 45 | m_key = "%s-%i" % (class_name, module_idx + 1) 46 | summary[m_key] = OrderedDict() 47 | summary[m_key]["input_shape"] = list(input[0].size()) 48 | summary[m_key]["input_shape"][0] = batch_size 49 | if isinstance(output, (list, tuple)): 50 | summary[m_key]["output_shape"] = get_shape(output) 51 | else: 52 | summary[m_key]["output_shape"] = list(output.size()) 53 | summary[m_key]["output_shape"][0] = batch_size 54 | 55 | params = sum([m.numel() for m in module.parameters()]) 56 | summary[m_key]["nb_params"] = params 57 | 58 | if ( 59 | not isinstance(module, nn.Sequential) 60 | and not isinstance(module, nn.ModuleList) 61 | ): 62 | hooks.append(module.register_forward_hook(hook)) 63 | 64 | # multiple inputs to the network 65 | if isinstance(input_size, tuple): 66 | input_size = [input_size] 67 | 68 | # batch_size of 2 for batchnorm 69 | x = [ torch.rand(2, *in_size).type(dtype).to(device=device) for in_size, dtype in zip(input_size, dtypes)] 70 | 71 | # create properties 72 | summary = OrderedDict() 73 | hooks = [] 74 | 75 | # register hook 76 | model.apply(register_hook) 77 | 78 | # make a forward pass 79 | # print(x.shape) 80 | model(*x) 81 | 82 | # remove these hooks 83 | for h in hooks: 84 | h.remove() 85 | 86 | print("----------------------------------------------------------------") 87 | line_new = "{:>20} {:>25} {:>15}".format("Layer (type)", "Output Shape", "Param #") 88 | print(line_new) 89 | print("================================================================") 90 | trainable_params, total_params = count_parameters(model) 91 | total_output = 0 92 | for layer in summary: 93 | # input_shape, output_shape, trainable, nb_params 94 | output_shape = summary[layer]["output_shape"] 95 | line_new = "{:>20} {:>25} {:>15}".format( 96 | layer, 97 | str(output_shape), 98 | "{0:,}".format(summary[layer]["nb_params"]), 99 | ) 100 | 101 | total_output += np.prod(output_shape[0] if isinstance(output_shape[0], list) else output_shape) 102 | print(line_new) 103 | 104 | # assume 4 bytes/number (float on cuda). 105 | total_input_size = abs(np.prod(sum(input_size, ())) * batch_size * 4. / (1024 ** 2.)) 106 | total_output_size = abs(2. * total_output * 4. / (1024 ** 2.)) # x2 for gradients 107 | total_params_size = abs(total_params * 4. / (1024 ** 2.)) 108 | total_size = total_params_size + total_output_size + total_input_size 109 | 110 | print("================================================================") 111 | print("Total params: {0:,}".format(total_params)) 112 | print("Trainable params: {0:,}".format(trainable_params)) 113 | print("Non-trainable params: {0:,}".format(total_params - trainable_params)) 114 | print("----------------------------------------------------------------") 115 | print("Input size (MB): %0.2f" % total_input_size) 116 | print("Forward/backward pass size (MB): %0.2f" % total_output_size) 117 | print("Params size (MB): %0.2f" % total_params_size) 118 | print("Estimated Total Size (MB): %0.2f" % total_size) 119 | print("----------------------------------------------------------------") 120 | # return summary 121 | return total_params, trainable_params 122 | 123 | 124 | def count_parameters(model): 125 | trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) 126 | total = sum(p.numel() for p in model.parameters()) 127 | return trainable, total 128 | -------------------------------------------------------------------------------- /training_documentation: -------------------------------------------------------------------------------- 1 | Dec04_11-57-43_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 15 epochs, regularization 0.001, batch size 22, learning rate 0.00001, training set size 540, validations set size 60 2 | Dec04_12-33-59_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 15 epochs, regularization 0.001, batch size 22, learning rate 0.0001, training set size 540, validations set size 60 3 | Dec04_13-01-49_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 15 epochs, regularization 0.001, batch size 22, learning rate 0.000001, training set size 540, validations set size 60 4 | Dec04_18-00-28_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 25 epochs, regularization 0.001, batch size 22, learning rate 0.000001, training set size 540, validations set size 60 5 | Dec04_18-39-28_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 25 epochs, regularization 0.01, batch size 22, learning rate 0.000001, training set size 540, validations set size 60 6 | Dec04_19-19-48_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 25 epochs, regularization 0.005, batch size 22, learning rate 0.000001, training set size 540, validations set size 60 7 | Dec04_20-00-47_Horst: resnet 50, pretrained false, neurAL TEXTURES, C40, 10 images per video, 25 epochs, regularization 0.0025, batch size 22, learning rate 0.000001, training set size 540, validations set size 60 8 | 9 | --------------------------------------------------------------------------------