├── README.md ├── backend ├── face_swap_py.so ├── image_engine.py ├── requirements.txt ├── server.py └── swapper.py ├── config.json ├── face_swap ├── .vscode │ ├── launch.json │ └── settings.json ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake │ ├── FindEigen3.cmake │ ├── face_swap-config-version.cmake.in │ └── face_swap-config.cmake.in ├── data │ └── images │ │ ├── brad_pitt_01.jpg │ │ └── bruce_willis_01.jpg ├── doc │ ├── CMakeLists.txt │ └── Doxyfile.in ├── face_swap │ ├── CMakeLists.txt │ ├── basel_3dmm.cpp │ ├── cnn_3dmm.cpp │ ├── cnn_3dmm_expr.cpp │ ├── face_data.proto │ ├── face_detection_landmarks.cpp │ ├── face_seg.cpp │ ├── face_swap │ │ ├── basel_3dmm.h │ │ ├── cnn_3dmm.h │ │ ├── cnn_3dmm_expr.h │ │ ├── face_detection_landmarks.h │ │ ├── face_seg.h │ │ ├── face_swap_c_interface.h │ │ ├── face_swap_engine.h │ │ ├── face_swap_engine_impl.h │ │ ├── landmarks_utilities.h │ │ ├── render_utilities.h │ │ ├── segmentation_utilities.h │ │ └── utilities.h │ ├── face_swap_c_interface.cpp │ ├── face_swap_engine_impl.cpp │ ├── landmarks_utilities.cpp │ ├── render_utilities.cpp │ ├── segmentation_utilities.cpp │ └── utilities.cpp ├── face_swap_batch │ ├── CMakeLists.txt │ ├── face_swap_batch.cfg │ └── face_swap_batch.cpp ├── face_swap_image │ ├── CMakeLists.txt │ ├── face_swap_image.cfg │ └── face_swap_image.cpp ├── face_swap_image2video │ ├── CMakeLists.txt │ ├── face_swap_image2video.cfg │ └── face_swap_image2video.cpp ├── face_swap_single2many │ ├── CMakeLists.txt │ ├── face_swap_single2many.cfg │ └── face_swap_single2many.cpp ├── interfaces │ └── python │ │ ├── CMakeLists.txt │ │ └── face_swap_py.cpp ├── iris_sfs │ ├── BaselFace.cpp │ ├── BaselFace.h │ ├── BaselFaceEstimator.cpp │ ├── BaselFaceEstimator.h │ ├── CMakeLists.txt │ ├── FImRenderer.cpp │ ├── FImRenderer.h │ ├── FTModel.cpp │ ├── FTModel.h │ ├── FaceServices2.cpp │ ├── FaceServices2.h │ ├── MeshModel.cpp │ ├── MeshModel.h │ ├── RenderModel.cpp │ ├── RenderModel.h │ ├── epnp.cpp │ ├── epnp.h │ ├── utility.cpp │ └── utility.h └── tests │ ├── CMakeLists.txt │ ├── test_flip.cfg │ ├── test_flip.cpp │ ├── test_landmarks.cfg │ ├── test_landmarks.cpp │ ├── test_memory.cfg │ ├── test_memory.cpp │ ├── test_resolution.cfg │ ├── test_resolution.cpp │ ├── test_resolution_batch.cfg │ └── test_resolution_batch.cpp ├── frontend └── faceswap │ ├── .buckconfig │ ├── .flowconfig │ ├── .gitattributes │ ├── .gitignore │ ├── .watchmanconfig │ ├── App.js │ ├── __tests__ │ └── App-test.js │ ├── android │ ├── app │ │ ├── BUCK │ │ ├── build.gradle │ │ ├── build_defs.bzl │ │ ├── proguard-rules.pro │ │ └── src │ │ │ ├── debug │ │ │ └── AndroidManifest.xml │ │ │ └── main │ │ │ ├── AndroidManifest.xml │ │ │ ├── java │ │ │ └── com │ │ │ │ └── faceswap │ │ │ │ ├── MainActivity.java │ │ │ │ └── MainApplication.java │ │ │ └── res │ │ │ ├── mipmap-hdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-mdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xxhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ │ ├── mipmap-xxxhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ │ └── values │ │ │ ├── strings.xml │ │ │ └── styles.xml │ ├── build.gradle │ ├── gradle.properties │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── keystores │ │ ├── BUCK │ │ └── debug.keystore.properties │ └── settings.gradle │ ├── app.json │ ├── babel.config.js │ ├── build.sh │ ├── index.js │ ├── ios │ ├── faceswap-tvOS │ │ └── Info.plist │ ├── faceswap-tvOSTests │ │ └── Info.plist │ ├── faceswap.xcodeproj │ │ ├── project.pbxproj │ │ └── xcshareddata │ │ │ └── xcschemes │ │ │ ├── faceswap-tvOS.xcscheme │ │ │ └── faceswap.xcscheme │ ├── faceswap │ │ ├── AppDelegate.h │ │ ├── AppDelegate.m │ │ ├── Base.lproj │ │ │ └── LaunchScreen.xib │ │ ├── Images.xcassets │ │ │ ├── AppIcon.appiconset │ │ │ │ └── Contents.json │ │ │ └── Contents.json │ │ ├── Info.plist │ │ └── main.m │ └── faceswapTests │ │ ├── Info.plist │ │ └── faceswapTests.m │ ├── metro.config.js │ ├── package.json │ ├── release.sh │ ├── run.sh │ └── src │ ├── index.js │ ├── submit.js │ ├── swap.js │ ├── upload.js │ ├── waiting.js │ └── welcome.js └── images ├── android-release.png ├── overview.png ├── result-0.png └── result-1.png /README.md: -------------------------------------------------------------------------------- 1 | # Transfiguring Portraits 2 | This is a face swapping system introduced in the paper[1]. Users could type a portrait style and take a selfie, then the system will return some own portraits with the input style by using image search engine and face swapping method. 3 | 4 | Note: The detailed swapping method of this repo is not exactly the same as the original paper. 5 | 6 | ![Overview](images/overview.png) 7 | 8 | ## Sample Results 9 | 10 | - (Portrait - Barack Obama, Style - Blonde Hair) && (Portrait - Kobe Bryant, Style - Curl Hair) 11 | ![Result-0](images/result-0.png) 12 | 13 | - (Portrait - Donald Trump, Style - Kim Jong-un) && (Portrait - Leo Messi, Style - Cristiano Ronaldo) 14 | ![Result-1](images/result-1.png) 15 | 16 | ## Installation and Usage 17 | 18 | ### Faceswap 19 | Based on [face_swap](https://github.com/YuvalNirkin/face_swap/)[2,3] with minor changes. Please refer to [face_swap](https://github.com/YuvalNirkin/face_swap/) for more information. Remember to build it with python interface and copy built pyd/so module into backend folder, and also download the data including 3dmm face model, cnn models, etc into backend folder. 20 | 21 | ### Backend Server 22 | Written by python3 with flask. Currently Bing image search is used with limited free account. Use your own account if you want to have a try. To run on local machine, please configure local ip and azure key in config.json file. 23 | 24 | Install dependences by 25 | > pip -r install requirements.txt 26 | 27 | Run server by 28 | > python server.py 29 | 30 | ### Frontend App 31 | Written by javascript with react-native. 32 | 33 | Install dependences by 34 | > npm install 35 | 36 | Build and debug with build.sh and run.sh. Please refer to react-native docs for more details. 37 | 38 | 39 | 40 | ## Demo App 41 | A simple client app for Android is also provided. Download and try! 42 | 43 | Note: may not work well due to limited server resource, trying it yourself with local machine is recommended. 44 | 45 | ![Android-release](images/android-release.png) 46 | 47 | ## References 48 | >1. Kemelmacher-Shlizerman, Ira. "Transfiguring portraits." ACM Transactions on Graphics (TOG) 35.4 (2016): 94. 49 | >2. Tuan Tran, Anh, et al. "Regressing robust and discriminative 3D morphable models with a very deep neural network." Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2017. 50 | >3. Nirkin, Yuval, et al. "On face segmentation, face swapping, and face perception." 2018 13th IEEE International Conference on Automatic Face & Gesture Recognition (FG 2018). IEEE, 2018. -------------------------------------------------------------------------------- /backend/face_swap_py.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/backend/face_swap_py.so -------------------------------------------------------------------------------- /backend/image_engine.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import threading 4 | import numpy 5 | from azure.cognitiveservices.search.imagesearch import ImageSearchAPI 6 | from msrest.authentication import CognitiveServicesCredentials 7 | from urllib.request import urlopen 8 | from PIL import Image 9 | 10 | class ImageEngine: 11 | def __init__(self): 12 | self.msg = None 13 | 14 | def request(self, item): 15 | raise NotImplementedError 16 | 17 | class BingImageEngine(ImageEngine): 18 | def __init__(self, key): 19 | super(BingImageEngine, self).__init__() 20 | self.key = key 21 | self.client = ImageSearchAPI(CognitiveServicesCredentials(self.key)) 22 | 23 | def request(self, item): 24 | image_results = self.client.images.search(query=item) 25 | images = list() 26 | images = None 27 | if image_results.value: 28 | print("Total number of images returned: {}".format(len(image_results.value))) 29 | self.msg = "Successfully requested!" 30 | 31 | images = [None]* len(image_results.value) 32 | def get_image(idx, list, results): 33 | image_result = results[idx] 34 | # print("image thumbnail url: {}".format(image_result.thumbnail_url)) 35 | # print("image content url: {}".format(image_result.content_url)) 36 | img = Image.open(urlopen(image_result.thumbnail_url)) 37 | #images.append(cv2.cvtColor(numpy.array(img), cv2.COLOR_RGB2BGR)) 38 | list[idx] = cv2.cvtColor(numpy.array(img), cv2.COLOR_RGB2BGR) 39 | #print(idx) 40 | 41 | threads = list() 42 | #for i in range(len(image_results.value)): 43 | for i in range(int(len(image_results.value))): 44 | x = threading.Thread(target=get_image, args=(i, images, image_results.value)) 45 | threads.append(x) 46 | x.start() 47 | 48 | for t in threads: 49 | t.join() 50 | else: 51 | self.msg = "No image results returned!" 52 | print("No image results returned!") 53 | return images 54 | 55 | 56 | if __name__ == "__main__": 57 | key = "3e49862cde1443c3b88b0538bb42b6cd" 58 | bie = BingImageEngine(key) 59 | bie.request("obama") 60 | -------------------------------------------------------------------------------- /backend/requirements.txt: -------------------------------------------------------------------------------- 1 | azure-cognitiveservices-search-imagesearch==1.0.0 2 | Flask==1.0.2 3 | numpy==1.16.3 4 | opencv-python==4.1.0.25 5 | Pillow==6.0.0 6 | -------------------------------------------------------------------------------- /backend/server.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | import json 4 | import os 5 | from flask import Flask, request, jsonify, send_file, url_for, Response 6 | import numpy as np 7 | import cv2 8 | from image_engine import BingImageEngine 9 | from swapper import * 10 | 11 | 12 | app = Flask(__name__) 13 | 14 | user_cache = 'user.json' 15 | if not os.path.exists(user_cache): 16 | with open(user_cache,'w') as f: 17 | json.dump(dict(),f) 18 | 19 | with open(user_cache,"r") as f: 20 | uuid_list = json.load(f) 21 | 22 | 23 | static_cache = 'static' 24 | if not os.path.exists(static_cache): 25 | os.mkdir(static_cache) 26 | 27 | image_cache = os.path.join(static_cache, 'image') 28 | if not os.path.exists(image_cache): 29 | os.mkdir(image_cache) 30 | 31 | swapper = Swapper() 32 | swapper.start_img_engine() 33 | swapper.start_fs_engine() 34 | 35 | @app.route("/swapped", methods=["GET"]) 36 | def has_swapped(): 37 | # send image back 38 | uuid = request.args.get('uuid') 39 | item = request.args.get('item') 40 | if uuid in uuid_list and item in uuid_list[uuid] and len(uuid_list[uuid][item]) > 0: 41 | return jsonify({ 42 | "status": "Accepted", 43 | }) 44 | else: 45 | return jsonify({ 46 | "status": "WA", 47 | }) 48 | 49 | @app.route("/result", methods=["GET"]) 50 | def swapped_images(): 51 | # send image back 52 | uuid = request.args.get('uuid') 53 | item = request.args.get('item') 54 | img_idx = int(request.args.get('idx')) 55 | if img_idx < len(uuid_list[uuid][item]): 56 | img_url = os.path.join(image_cache, uuid, uuid_list[uuid][item][img_idx]) 57 | print(img_url) 58 | img_url = os.path.join(request.url_root, img_url) 59 | return jsonify({ 60 | "status": "Success", 61 | "url": img_url 62 | }) 63 | else: 64 | return jsonify({ 65 | "status": "WI", 66 | }) 67 | 68 | @app.route("/", methods=["POST"]) 69 | def swap(): 70 | img = request.files["image"].read() 71 | item = request.form['item'] 72 | uuid = request.form['uuid'] 73 | 74 | def swapping(uuid, img, item): 75 | # need set gpu mode in each thread 76 | swapper.set_mode_gpu() 77 | 78 | uuid_cache = os.path.join(image_cache, uuid) 79 | if not uuid in uuid_list: 80 | os.mkdir(uuid_cache) 81 | uuid_list[uuid] = dict() 82 | # I need better way to save 83 | with open(user_cache, "w") as f: 84 | json.dump(uuid_list, f) 85 | 86 | if not item in uuid_list[uuid]: 87 | uuid_list[uuid][item] = list() 88 | # I need better way to save 89 | with open(user_cache, "w") as f: 90 | json.dump(uuid_list, f) 91 | 92 | npimg = np.fromstring(img, np.int8) 93 | cvimg = cv2.imdecode(npimg, 1) 94 | # scale in client is better 95 | img_h, img_w, _ = cvimg.shape 96 | scale = int(img_w / 1000) + 4 97 | resized = (int(img_w / scale), int(img_h /scale)) 98 | print(resized) 99 | scaled_img = cv2.resize(cvimg, resized) 100 | 101 | time_start = time.time() 102 | src_img, suc = swapper.set_image(scaled_img) 103 | # no face in source image 104 | if not suc: 105 | return False 106 | time_end = time.time() 107 | print('set source image time:', time_end-time_start) 108 | 109 | time_start = time.time() 110 | tgt_imgs = swapper.request_images(item) 111 | time_end = time.time() 112 | print('request images time:', time_end-time_start) 113 | 114 | for idx in range(len(tgt_imgs)): 115 | time_start = time.time() 116 | if tgt_imgs[idx] is None: 117 | continue 118 | tgt_img, suc = swapper.set_image(tgt_imgs[idx]) 119 | if not suc: 120 | continue 121 | time_end = time.time() 122 | print('set target image time:', time_end-time_start) 123 | 124 | time_start = time.time() 125 | swapped_img = swapper.process_one(src_img, tgt_img) 126 | time_end = time.time() 127 | print('process_one time:', time_end-time_start) 128 | if swapped_img is not None and swapped_img.size != 0: 129 | img_name = '%s-%d.jpg' % (item, len(uuid_list[uuid][item])) 130 | img_file = os.path.join(uuid_cache, img_name) 131 | cv2.imwrite(img_file, swapped_img) 132 | uuid_list[uuid][item].append(img_name) 133 | # I need better way to save 134 | with open(user_cache, "w") as f: 135 | json.dump(uuid_list, f) 136 | return True 137 | 138 | if not swapping(uuid, img, item): 139 | return jsonify({ 140 | "status": "Invalid", 141 | }) 142 | else: 143 | return jsonify({ 144 | "status": "OK", 145 | }) 146 | 147 | 148 | 149 | if __name__ == "__main__": 150 | with open(config) as json_file: 151 | conf = json.load(json_file) 152 | app.run(host=conf['local_ip'], port=conf['local_port']) 153 | #app.run(host='127.0.0.1',port=9080, threaded=True) 154 | -------------------------------------------------------------------------------- /backend/swapper.py: -------------------------------------------------------------------------------- 1 | import json 2 | from image_engine import BingImageEngine 3 | import face_swap_py as fspy 4 | 5 | landmarks_path = 'data/shape_predictor_68_face_landmarks.dat' 6 | model_3dmm_h5_path = 'data/BaselFaceModel_mod_wForehead_noEars.h5' 7 | model_3dmm_dat_path = 'data/BaselFace.dat' 8 | reg_model_path = 'data/3dmm_cnn_resnet_101.caffemodel' 9 | reg_deploy_path = 'data/3dmm_cnn_resnet_101_deploy.prototxt' 10 | reg_mean_path = 'data/3dmm_cnn_resnet_101_mean.binaryproto' 11 | seg_model_path = 'data/face_seg_fcn8s_300.caffemodel' # or 'data/face_seg_fcn8s_300.caffemodel' for lower resolution 12 | seg_deploy_path = 'data/face_seg_fcn8s_300_deploy.prototxt' # or 'data/face_seg_fcn8s_300_deploy.prototxt' for lower resolution 13 | generic = False 14 | with_expr = False 15 | with_gpu = True 16 | gpu_device_id = 0 17 | 18 | config = "../config.json" 19 | 20 | 21 | class Swapper: 22 | def __init__(self): 23 | self.src_img = None 24 | self.cur_tgt = None 25 | self.fs_engine = None 26 | self.img_engine = None 27 | 28 | def set_mode_gpu(self): 29 | self.fs_engine.set_mode_gpu() 30 | 31 | def start_fs_engine(self): 32 | self.fs_engine = fspy.FaceSwap(landmarks_path, model_3dmm_h5_path, 33 | model_3dmm_dat_path, reg_model_path, 34 | reg_deploy_path, reg_mean_path, 35 | seg_model_path, seg_deploy_path, 36 | generic, with_expr, with_gpu, gpu_device_id) 37 | 38 | def start_img_engine(self): 39 | with open(config) as json_file: 40 | conf = json.load(json_file) 41 | key = conf["azure_key"] 42 | self.img_engine = BingImageEngine(key) 43 | 44 | def set_image(self, img): 45 | img = fspy.FaceData(img) 46 | success = self.fs_engine.estimate(img) 47 | return img, success 48 | 49 | def request_images(self, item): 50 | return self.img_engine.request(item) 51 | 52 | def swap(self, src_img, cur_tgt): 53 | swapped = src_img 54 | # invoke faceswapping (self.src_img, tgt_img) 55 | swapped = self.fs_engine.transfer(src_img, cur_tgt) 56 | return swapped 57 | 58 | def compare(self, src_img, tgt_img): 59 | # cur_tgt = fspy.FaceData(tgt_img) 60 | # self.fs_engine.estimate(cur_tgt) 61 | # invoke faceswapping (self.src_img, tgt_img) 62 | ret = self.fs_engine.compare(src_img, tgt_img) 63 | #return True 64 | return ret 65 | 66 | def process_one(self, src_img, tgt_img): 67 | output = None 68 | if self.compare(src_img, tgt_img): 69 | output = self.swap(src_img, tgt_img) 70 | return output 71 | 72 | def process(self, img, item): 73 | src_img = self.set_image(img) 74 | tgt_imgs = self.request_images(item) 75 | if not tgt_imgs: 76 | print("Cannot search for images!") 77 | 78 | for idx in range(len(tgt_imgs)): 79 | self.process_one(src_img, tgt_imgs[idx]) 80 | 81 | 82 | 83 | if __name__ == "__main__": 84 | 85 | fs_engine = fspy.FaceSwap(landmarks_path, model_3dmm_h5_path, 86 | model_3dmm_dat_path, reg_model_path, 87 | reg_deploy_path, reg_mean_path, 88 | seg_model_path, seg_deploy_path, 89 | generic, with_expr, with_gpu, gpu_device_id) 90 | 91 | import cv2 92 | img1 = fspy.FaceData(cv2.imread('trump1.jpg')) 93 | img2 = fspy.FaceData(cv2.imread('xi1.jpg')) 94 | fs_engine.estimate(img1) 95 | fs_engine.estimate(img2) 96 | 97 | 98 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "local_ip": "192.168.31.216", 3 | "local_port": 5000, 4 | "azure_key": "3e49862cde1443c3b88b0538bb42b6cd" 5 | } -------------------------------------------------------------------------------- /face_swap/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "(gdb) Launch", 9 | "type": "cppdbg", 10 | "request": "launch", 11 | "program": "enter program name, for example ${workspaceFolder}/a.out", 12 | "args": [], 13 | "stopAtEntry": false, 14 | "cwd": "${workspaceFolder}", 15 | "environment": [], 16 | "externalConsole": true, 17 | "MIMode": "gdb", 18 | "setupCommands": [ 19 | { 20 | "description": "Enable pretty-printing for gdb", 21 | "text": "-enable-pretty-printing", 22 | "ignoreFailures": true 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /face_swap/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.associations": { 3 | "array": "cpp", 4 | "*.tcc": "cpp", 5 | "unordered_map": "cpp", 6 | "fstream": "cpp", 7 | "istream": "cpp", 8 | "optional": "cpp", 9 | "ostream": "cpp", 10 | "vector": "cpp", 11 | "sstream": "cpp", 12 | "streambuf": "cpp", 13 | "string_view": "cpp", 14 | "thread": "cpp", 15 | "tuple": "cpp", 16 | "chrono": "cpp", 17 | "future": "cpp", 18 | "memory": "cpp" 19 | } 20 | } -------------------------------------------------------------------------------- /face_swap/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4) 2 | project(face_swap) 3 | 4 | # Set version 5 | set(FACE_SWAP_MAJOR_VERSION 1) 6 | set(FACE_SWAP_MINOR_VERSION 0) 7 | set(FACE_SWAP_PATCH_VERSION 2) 8 | set(FACE_SWAP_VERSION ${FACE_SWAP_MAJOR_VERSION}.${FACE_SWAP_MINOR_VERSION}.${FACE_SWAP_PATCH_VERSION}) 9 | 10 | # Global configurations 11 | # =================================================== 12 | if(WIN32) 13 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNOMINMAX") 14 | set(DEF_INSTALL_CMAKE_DIR cmake) 15 | else() 16 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") 17 | set(DEF_INSTALL_CMAKE_DIR lib/cmake/${PROJECT_NAME}) 18 | endif() 19 | set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH 20 | "Installation directory for CMake files") 21 | set(CMAKE_CXX_STANDARD 14) 22 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") 23 | set(CMAKE_DEBUG_POSTFIX "_d" CACHE STRING 24 | "Add a custom \"postfix\" to static and shared libraries when in Debug build") 25 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) 26 | 27 | # Set build output directories 28 | set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) 29 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) 30 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) 31 | 32 | # Optional 3rd party components 33 | # =================================================== 34 | option(WITH_BOOST_STATIC "Boost static libraries" ON) 35 | option(WITH_PROTOBUF "Protocol Buffers - Google's data interchange format" ON) 36 | 37 | # Build components 38 | # =================================================== 39 | option(BUILD_APPS "Build applications" ON) 40 | option(BUILD_SHARED_LIBS "Build Shared Libraries" OFF) 41 | option(BUILD_DOCS "Build documentation using Doxygen" ON) 42 | option(BUILD_INTERFACE_PYTHON "Build interface for Python" ON) 43 | option(BUILD_TESTS "Build tests" ON) 44 | 45 | # Set library type 46 | set(LIB_TYPE STATIC) 47 | if(BUILD_SHARED_LIBS) 48 | set(LIB_TYPE SHARED) 49 | endif(BUILD_SHARED_LIBS) 50 | 51 | # Find dependencies 52 | # =================================================== 53 | 54 | # Boost 55 | set(Boost_USE_STATIC_LIBS ${WITH_BOOST_STATIC}) 56 | set(BOOST_ALL_DYN_LINK NOT ${WITH_BOOST_STATIC}) 57 | if(WIN32) 58 | if(${WITH_BOOST_STATIC}) 59 | add_definitions(-DBOOST_ALL_NO_LIB) 60 | else() 61 | add_definitions(-DBOOST_ALL_DYN_LINK) 62 | endif() 63 | endif() 64 | find_package(Boost REQUIRED filesystem program_options regex timer thread chrono) 65 | 66 | # OpenCV 67 | find_package(OpenCV REQUIRED highgui imgproc imgcodecs calib3d photo) 68 | 69 | # dlib 70 | find_package(dlib REQUIRED) 71 | 72 | # Caffe 73 | find_package(Caffe REQUIRED) 74 | 75 | # HDF5 76 | if(MSVC) 77 | # Find HDF5 using it's hdf5-config.cmake file with MSVC 78 | if(DEFINED HDF5_DIR) 79 | list(APPEND CMAKE_MODULE_PATH ${HDF5_DIR}) 80 | endif() 81 | find_package(HDF5 COMPONENTS C HL CXX REQUIRED) 82 | set(HDF5_LIBRARIES hdf5-shared hdf5_cpp-shared) 83 | set(HDF5_HL_LIBRARIES hdf5_hl-shared) 84 | else() 85 | #find_package(HDF5 COMPONENTS HL REQUIRED) 86 | find_package(HDF5 COMPONENTS C CXX HL REQUIRED) 87 | endif() 88 | #find_package(HDF5 REQUIRED CXX) 89 | 90 | # Eigen 91 | find_package(Eigen3 REQUIRED) 92 | 93 | # protobuf 94 | if(WITH_PROTOBUF) 95 | find_package(Protobuf) 96 | endif() 97 | 98 | # Doxygen 99 | find_package(Doxygen) 100 | 101 | # SWIG 102 | find_package(SWIG) 103 | 104 | # Add sub-directories 105 | # =================================================== 106 | add_subdirectory(iris_sfs) 107 | add_subdirectory(face_swap) 108 | if(BUILD_APPS) 109 | add_subdirectory(face_swap_image) 110 | add_subdirectory(face_swap_batch) 111 | add_subdirectory(face_swap_single2many) 112 | add_subdirectory(face_swap_image2video) 113 | endif(BUILD_APPS) 114 | if(BUILD_TESTS) 115 | add_subdirectory(tests) 116 | endif(BUILD_TESTS) 117 | 118 | # Documentation 119 | if(BUILD_DOCS) 120 | add_subdirectory(doc) 121 | endif() 122 | 123 | # Interfaces 124 | if(BUILD_INTERFACE_PYTHON) 125 | add_subdirectory(interfaces/python) 126 | endif() 127 | 128 | # Export configuration 129 | # =================================================== 130 | 131 | # Add all targets to the build-tree export set 132 | set(FACE_SWAP_TARGETS face_swap iris_sfs) 133 | export(TARGETS ${FACE_SWAP_TARGETS} 134 | FILE "${PROJECT_BINARY_DIR}/face_swap-targets.cmake") 135 | 136 | # Export the package for use from the build-tree 137 | # (this registers the build-tree with a global CMake-registry) 138 | export(PACKAGE face_swap) 139 | 140 | # Create config files 141 | configure_file(cmake/face_swap-config.cmake.in 142 | "${PROJECT_BINARY_DIR}/face_swap-config.cmake" @ONLY) 143 | configure_file(cmake/face_swap-config-version.cmake.in 144 | "${PROJECT_BINARY_DIR}/face_swap-config-version.cmake" @ONLY) 145 | 146 | # Install config files 147 | install(FILES 148 | "${PROJECT_BINARY_DIR}/face_swap-config.cmake" 149 | "${PROJECT_BINARY_DIR}/face_swap-config-version.cmake" 150 | DESTINATION "cmake" COMPONENT dev) 151 | 152 | # Install the export set for use with the install-tree 153 | install(EXPORT face_swap-targets DESTINATION cmake COMPONENT dev) 154 | 155 | # Install sample data 156 | file(GLOB SAMPLE_IMAGES "data/images/*.jpg") 157 | install(FILES ${SAMPLE_IMAGES} DESTINATION data/images) 158 | -------------------------------------------------------------------------------- /face_swap/README.md: -------------------------------------------------------------------------------- 1 | # End-to-end, automatic face swapping pipeline 2 | ![alt text](https://yuvalnirkin.github.io/assets/img/projects/face_swap/joker_teaser.jpg "Samples") 3 | The Joker (Heath Ledger) swapped using our method onto very different subjects and images. 4 | 5 | [Yuval Nirkin](http://www.nirkin.com/), [Iacopo Masi](http://www-bcf.usc.edu/~iacopoma/), [Anh Tuan Tran](https://sites.google.com/site/anhttranusc/), [Tal Hassner](http://www.openu.ac.il/home/hassner/), and [Gerard Medioni](http://iris.usc.edu/people/medioni/index.html). 6 | 7 | # Overview 8 | Code for the automatic, image-to-image face swapping method described in the paper: 9 | 10 | Yuval Nirkin, Iacopo Masi, Anh Tuan Tran, Tal Hassner, Gerard Medioni, "[On Face Segmentation, Face Swapping, and Face Perception](https://arxiv.org/abs/1704.06729)", IEEE Conference on Automatic Face and Gesture Recognition (FG), Xi'an, China on May 2018 11 | 12 | Please see [project page](http://www.openu.ac.il/home/hassner/projects/faceswap/) for more details, more resources and updates on this project. 13 | 14 | If you find this code useful, please make sure to cite our paper in your work. 15 | 16 | ## News 17 | Version 1.0 is released: 18 | - Better performance and results. 19 | - Python binding. 20 | - Easier to build: removed many of the dependencies. 21 | - Software renderer: OpenGL is no longer required, the project can now be built on gui-less servers without any problems. 22 | - Added low resolution segmentation model for GPUs with low memory. 23 | 24 | ## Installation 25 | Both Linux and Windows are supported. 26 | - [Ubuntu installation guide](https://github.com/YuvalNirkin/face_swap/wiki/Ubuntu-Installation-Guide) 27 | - [Windows installation guide](https://github.com/YuvalNirkin/face_swap/wiki/Windows-Installation-Guide) 28 | 29 | ## Usage 30 | - For Python follow the [Python guide](https://github.com/YuvalNirkin/face_swap/wiki/Python-Guide) 31 | - For running the applications follow the [applications guide](https://github.com/YuvalNirkin/face_swap/wiki/Applications-Guide). 32 | - For using the library's C++ interface, please take a look at the [Doxygen generated documentation](https://yuvalnirkin.github.io/docs/face_swap/). 33 | 34 | ## Citation 35 | Please cite our paper with the following bibtex if you use our face renderer: 36 | 37 | ``` latex 38 | @inproceedings{nirkin2018_faceswap, 39 | title={On Face Segmentation, Face Swapping, and Face Perception}, 40 | booktitle = {IEEE Conference on Automatic Face and Gesture Recognition}, 41 | author={Nirkin, Yuval and Masi, Iacopo and Tran, Anh Tuan and Hassner, Tal and Medioni, G\'{e}rard}, 42 | year={2018}, 43 | } 44 | ``` 45 | 46 | ## Related projects 47 | - [Deep face segmentation](https://github.com/YuvalNirkin/face_segmentation), used to segment face regions in the face swapping pipeline. 48 | - [Interactive system for fast face segmentation ground truth labeling](https://github.com/YuvalNirkin/face_video_segment), used to produce the training set for our deep face segmentation. 49 | - [CNN3DMM](http://www.openu.ac.il/home/hassner/projects/CNN3DMM/), used to estimate 3D face shapes from single images. 50 | - [ResFace101](http://www.openu.ac.il/home/hassner/projects/augmented_faces/), deep face recognition used in the paper to test face swapping capabilities. 51 | - [Hand augmentations](https://github.com/YuvalNirkin/egohands_augmentations), used to generate the hand augmentations for training the face segmentation. 52 | 53 | ## Copyright 54 | Copyright 2018, Yuval Nirkin, Iacopo Masi, Anh Tuan Tran, Tal Hassner, and Gerard Medioni 55 | 56 | The SOFTWARE provided in this page is provided "as is", without any guarantee made as to its suitability or fitness for any particular use. It may contain bugs, so use of this tool is at your own risk. We take no responsibility for any damage of any sort that may unintentionally be caused through its use. 57 | 58 | ## License 59 | This project is released under the GPLv3 license. 60 | 61 | For a more permitting license please contact the [author](mailto:yuval.nirkin@gmail.com). 62 | -------------------------------------------------------------------------------- /face_swap/cmake/FindEigen3.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Eigen3 lib 2 | # 3 | # This module supports requiring a minimum version, e.g. you can do 4 | # find_package(Eigen3 3.1.2) 5 | # to require version 3.1.2 or newer of Eigen3. 6 | # 7 | # Once done this will define 8 | # 9 | # EIGEN3_FOUND - system has eigen lib with correct version 10 | # EIGEN3_INCLUDE_DIR - the eigen include directory 11 | # EIGEN3_VERSION - eigen version 12 | 13 | # Copyright (c) 2006, 2007 Montel Laurent, 14 | # Copyright (c) 2008, 2009 Gael Guennebaud, 15 | # Copyright (c) 2009 Benoit Jacob 16 | # Redistribution and use is allowed according to the terms of the 2-clause BSD license. 17 | 18 | if(NOT Eigen3_FIND_VERSION) 19 | if(NOT Eigen3_FIND_VERSION_MAJOR) 20 | set(Eigen3_FIND_VERSION_MAJOR 2) 21 | endif(NOT Eigen3_FIND_VERSION_MAJOR) 22 | if(NOT Eigen3_FIND_VERSION_MINOR) 23 | set(Eigen3_FIND_VERSION_MINOR 91) 24 | endif(NOT Eigen3_FIND_VERSION_MINOR) 25 | if(NOT Eigen3_FIND_VERSION_PATCH) 26 | set(Eigen3_FIND_VERSION_PATCH 0) 27 | endif(NOT Eigen3_FIND_VERSION_PATCH) 28 | 29 | set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}") 30 | endif(NOT Eigen3_FIND_VERSION) 31 | 32 | macro(_eigen3_check_version) 33 | file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) 34 | 35 | string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}") 36 | set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}") 37 | string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}") 38 | set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}") 39 | string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}") 40 | set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}") 41 | 42 | set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION}) 43 | if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 44 | set(EIGEN3_VERSION_OK FALSE) 45 | else(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 46 | set(EIGEN3_VERSION_OK TRUE) 47 | endif(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 48 | 49 | if(NOT EIGEN3_VERSION_OK) 50 | 51 | message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, " 52 | "but at least version ${Eigen3_FIND_VERSION} is required") 53 | endif(NOT EIGEN3_VERSION_OK) 54 | endmacro(_eigen3_check_version) 55 | 56 | if (EIGEN3_INCLUDE_DIR) 57 | 58 | # in cache already 59 | _eigen3_check_version() 60 | set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) 61 | 62 | else (EIGEN3_INCLUDE_DIR) 63 | 64 | find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library 65 | PATHS 66 | ${CMAKE_INSTALL_PREFIX}/include 67 | ${KDE4_INCLUDE_DIR} 68 | PATH_SUFFIXES eigen3 eigen 69 | ) 70 | 71 | if(EIGEN3_INCLUDE_DIR) 72 | _eigen3_check_version() 73 | endif(EIGEN3_INCLUDE_DIR) 74 | 75 | include(FindPackageHandleStandardArgs) 76 | find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK) 77 | 78 | mark_as_advanced(EIGEN3_INCLUDE_DIR) 79 | 80 | endif(EIGEN3_INCLUDE_DIR) 81 | 82 | -------------------------------------------------------------------------------- /face_swap/cmake/face_swap-config-version.cmake.in: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@FACE_SWAP_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() -------------------------------------------------------------------------------- /face_swap/cmake/face_swap-config.cmake.in: -------------------------------------------------------------------------------- 1 | # - Config file for the face_swap package 2 | # It defines the following variables 3 | # face_swap_INCLUDE_DIRS - include directories for face_swap 4 | # face_swap_LIBRARIES - libraries to link against 5 | # face_swap_EXECUTABLE - the face_swap executable 6 | 7 | # Calculated paths 8 | get_filename_component(face_swap_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" DIRECTORY) 9 | set(face_swap_INCLUDE_DIRS "@CMAKE_INSTALL_PREFIX@/include") 10 | 11 | # Our library dependencies (contains definitions for IMPORTED targets) 12 | if(NOT TARGET face_swap AND NOT face_swap_BINARY_DIR) 13 | include("${face_swap_CMAKE_DIR}/face_swap-targets.cmake") 14 | endif() 15 | 16 | # These are IMPORTED targets created by face_swap-targets.cmake 17 | set(face_swap_LIBRARIES face_swap) 18 | if(TARGET face_swap_images) 19 | set(face_swap_EXECUTABLE face_swap_images) 20 | endif() -------------------------------------------------------------------------------- /face_swap/data/images/brad_pitt_01.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/face_swap/data/images/brad_pitt_01.jpg -------------------------------------------------------------------------------- /face_swap/data/images/bruce_willis_01.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/face_swap/data/images/bruce_willis_01.jpg -------------------------------------------------------------------------------- /face_swap/doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # additional config 2 | set(doxyfile "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile") 3 | set(DOXYGEN_INPUT_PATH "\"${CMAKE_SOURCE_DIR}/face_swap/face_swap\"") 4 | set(DOXYGEN_OUTPUT_PATH "\"${CMAKE_CURRENT_BINARY_DIR}/doxygen\"") 5 | set(DOXYGEN_PROJECT_NAME "\"Nirkin Face Swap\"") 6 | set(DOXYGEN_PROJECT_NUMBER ${FACE_SWAP_VERSION}) 7 | set(DOXYGEN_PROJECT_BRIEF "\"Face swap.\"") 8 | 9 | # Write doxygen configuration file 10 | configure_file(Doxyfile.in ${doxyfile} @ONLY) 11 | 12 | add_custom_target(doxygen 13 | COMMAND ${CMAKE_COMMAND} -E echo "Building API Documentation..." 14 | COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} 15 | COMMAND ${CMAKE_COMMAND} -E echo "Done." 16 | DEPENDS ${doxyfile} 17 | ) 18 | 19 | install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html 20 | DESTINATION doc 21 | COMPONENT "docs" OPTIONAL 22 | ) -------------------------------------------------------------------------------- /face_swap/face_swap/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Source 2 | set(SRC 3 | basel_3dmm.cpp 4 | cnn_3dmm.cpp 5 | cnn_3dmm_expr.cpp 6 | face_seg.cpp 7 | face_swap_engine_impl.cpp 8 | face_swap_c_interface.cpp 9 | render_utilities.cpp 10 | utilities.cpp 11 | face_detection_landmarks.cpp 12 | landmarks_utilities.cpp 13 | segmentation_utilities.cpp 14 | ) 15 | set(HDR 16 | face_swap/basel_3dmm.h 17 | face_swap/cnn_3dmm.h 18 | face_swap/cnn_3dmm_expr.h 19 | face_swap/face_seg.h 20 | face_swap/face_swap_engine.h 21 | face_swap/face_swap_engine_impl.h 22 | face_swap/face_swap_c_interface.h 23 | face_swap/render_utilities.h 24 | face_swap/utilities.h 25 | face_swap/face_detection_landmarks.h 26 | face_swap/landmarks_utilities.h 27 | face_swap/segmentation_utilities.h 28 | ) 29 | 30 | if(PROTOBUF_FOUND) 31 | set(PROTO_FILES face_data.proto) 32 | protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES}) 33 | set(SRC ${SRC} ${PROTO_SRCS} ${PROTO_HDRS} ${PROTO_FILES}) 34 | add_definitions(-DWITH_PROTOBUF) 35 | endif() 36 | 37 | # Target 38 | add_library(face_swap ${LIB_TYPE} ${SRC} ${HDR}) 39 | target_include_directories(face_swap PUBLIC 40 | $ 41 | $ 42 | ${OpenCV_INCLUDE_DIRS} 43 | ${Caffe_INCLUDE_DIRS} 44 | ${HDF5_INCLUDE_DIRS} 45 | ) 46 | target_link_libraries(face_swap PUBLIC 47 | iris_sfs 48 | ${OpenCV_LIBS} 49 | dlib::dlib 50 | ${Caffe_LIBRARIES} 51 | ${HDF5_LIBRARIES} 52 | ) 53 | if(PROTOBUF_FOUND) 54 | target_include_directories(face_swap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) 55 | target_include_directories(face_swap PUBLIC ${PROTOBUF_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS}) 56 | target_link_libraries(face_swap PUBLIC ${PROTOBUF_LIBRARIES} ${Boost_LIBRARIES}) 57 | endif() 58 | 59 | # Add export header 60 | include (GenerateExportHeader) 61 | if(UNIX) 62 | set(USE_COMPILER_HIDDEN_VISIBILITY OFF) 63 | endif(UNIX) 64 | generate_export_header(face_swap EXPORT_FILE_NAME ${PROJECT_BINARY_DIR}/face_swap/face_swap_export.h) 65 | install(FILES ${PROJECT_BINARY_DIR}/face_swap/face_swap_export.h DESTINATION include/face_swap) 66 | 67 | 68 | # Installations 69 | install(TARGETS face_swap 70 | EXPORT face_swap-targets 71 | RUNTIME DESTINATION bin COMPONENT dev 72 | LIBRARY DESTINATION lib COMPONENT dev 73 | ARCHIVE DESTINATION lib COMPONENT dev) 74 | install(FILES ${HDR} DESTINATION include/face_swap) 75 | -------------------------------------------------------------------------------- /face_swap/face_swap/basel_3dmm.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/basel_3dmm.h" 2 | #include "face_swap/utilities.h" 3 | #include 4 | #include 5 | #include // debug 6 | 7 | // Boost 8 | #include 9 | 10 | // HDF5 11 | #include 12 | 13 | using namespace boost::filesystem; 14 | 15 | namespace face_swap 16 | { 17 | void Mesh::save_ply(const Mesh& mesh, const std::string & ply_file) 18 | { 19 | std::ofstream ply(ply_file); 20 | ply.precision(6); 21 | 22 | // Write header 23 | ply << "ply" << std::endl; 24 | ply << "format ascii 1.0" << std::endl; 25 | ply << "element vertex " << mesh.vertices.rows << std::endl; 26 | ply << "property float x" << std::endl; 27 | ply << "property float y" << std::endl; 28 | ply << "property float z" << std::endl; 29 | ply << "property uchar red" << std::endl; 30 | ply << "property uchar green" << std::endl; 31 | ply << "property uchar blue" << std::endl; 32 | ply << "element face " << mesh.faces.rows << std::endl; 33 | ply << "property list uchar int vertex_indices" << std::endl; 34 | ply << "end_header" << std::endl; 35 | 36 | // Write vertices 37 | float* vert_data = (float*)mesh.vertices.data; 38 | unsigned char* color_data = mesh.colors.data; 39 | for (int i = 0; i < mesh.vertices.rows; ++i) 40 | { 41 | ply << *vert_data++ << " "; 42 | ply << *vert_data++ << " "; 43 | ply << *vert_data++ << " "; 44 | ply << (int)*color_data++ << " "; 45 | ply << (int)*color_data++ << " "; 46 | ply << (int)*color_data++ << std::endl; 47 | } 48 | 49 | // Write faces 50 | unsigned short* faces_data = (unsigned short*)mesh.faces.data; 51 | for (int i = 0; i < mesh.faces.rows; ++i) 52 | { 53 | ply << "3 " << (int)*faces_data++ << " "; 54 | ply << (int)*faces_data++ << " "; 55 | ply << (int)*faces_data++ << std::endl; 56 | } 57 | } 58 | 59 | cv::Mat readH5Dataset(const H5::H5File& file, const std::string& datasetName) 60 | { 61 | cv::Mat out; 62 | 63 | // Open the specified dataset in the file 64 | H5::DataSet dataset = file.openDataSet(datasetName); 65 | 66 | // Get dataset info 67 | H5T_class_t type_class = dataset.getTypeClass(); 68 | H5::DataSpace filespace = dataset.getSpace(); 69 | hsize_t dims[2]; // dataset dimensions 70 | int rank = filespace.getSimpleExtentDims(dims); 71 | 72 | // Read dataset 73 | int sizes[2] = { (int)dims[0], (int)dims[1] }; 74 | out.create(rank, sizes, CV_32FC1); 75 | dataset.read(out.data, H5::PredType::NATIVE_FLOAT, H5::DataSpace(rank, dims), filespace); 76 | 77 | return out; 78 | /* 79 | DSetCreatPropList cparms = dataset.getCreatePlist(); 80 | if (H5D_CHUNKED == cparms.getLayout()) 81 | { 82 | // Get chunking information: rank and dimensions 83 | rank = cparms.getChunk(2, dims); 84 | 85 | // Define the memory space to read a chunk. 86 | H5::DataSpace cspace(rank, dims); 87 | 88 | // Define chunk in the file (hyperslab) to read. 89 | } 90 | else 91 | { 92 | } 93 | */ 94 | } 95 | 96 | Mesh Basel3DMM::sample(const cv::Mat & shape_coefficients, 97 | const cv::Mat & tex_coefficients) 98 | { 99 | Mesh mesh; 100 | mesh.faces = faces; 101 | 102 | cv::Mat s = shape_coefficients.mul(shapeEV); 103 | cv::Mat t = tex_coefficients.mul(texEV); 104 | 105 | mesh.vertices = shapePC * s + shapeMU; 106 | mesh.colors = texPC * t + texMU; 107 | 108 | int total_vertices = mesh.vertices.rows / 3; 109 | mesh.vertices = mesh.vertices.reshape(0, total_vertices); 110 | mesh.colors = mesh.colors.reshape(0, total_vertices); 111 | mesh.colors.convertTo(mesh.colors, CV_8U); 112 | 113 | return mesh; 114 | } 115 | 116 | Mesh Basel3DMM::sample(const cv::Mat& shape_coefficients, 117 | const cv::Mat& tex_coefficients, const cv::Mat& expr_coefficients) 118 | { 119 | Mesh mesh; 120 | mesh.faces = faces; 121 | 122 | cv::Mat s = shape_coefficients.mul(shapeEV); 123 | cv::Mat t = tex_coefficients.mul(texEV); 124 | cv::Mat e = expr_coefficients.mul(exprEV); 125 | 126 | mesh.vertices = shapePC * s + shapeMU + exprPC * e + exprMU; 127 | mesh.colors = texPC * t + texMU; 128 | 129 | int total_vertices = mesh.vertices.rows / 3; 130 | mesh.vertices = mesh.vertices.reshape(0, total_vertices); 131 | mesh.colors = mesh.colors.reshape(0, total_vertices); 132 | mesh.colors.convertTo(mesh.colors, CV_8U); 133 | 134 | return mesh; 135 | } 136 | 137 | Basel3DMM Basel3DMM::load(const std::string & model_file) 138 | { 139 | Basel3DMM basel_3dmm; 140 | 141 | try 142 | { 143 | // Turn off the auto-printing when failure occurs so that we can 144 | // handle the errors appropriately 145 | H5::Exception::dontPrint(); 146 | 147 | // Open the specified file and the specified dataset in the file 148 | H5::H5File file(model_file.c_str(), H5F_ACC_RDONLY); 149 | cv::Mat faces = readH5Dataset(file, "/faces"); 150 | basel_3dmm.shapeMU = readH5Dataset(file, "/shapeMU"); 151 | basel_3dmm.shapePC = readH5Dataset(file, "/shapePC"); 152 | basel_3dmm.shapeEV = readH5Dataset(file, "/shapeEV"); 153 | basel_3dmm.texMU = readH5Dataset(file, "/texMU"); 154 | basel_3dmm.texPC = readH5Dataset(file, "/texPC"); 155 | basel_3dmm.texEV = readH5Dataset(file, "/texEV"); 156 | basel_3dmm.exprMU = readH5Dataset(file, "/expMU"); 157 | basel_3dmm.exprPC = readH5Dataset(file, "/expPC"); 158 | basel_3dmm.exprEV = readH5Dataset(file, "/expEV"); 159 | 160 | // Convert faces to unsigned int 161 | float* faces_data = (float*)faces.data; 162 | int faces_size = faces.total(); 163 | basel_3dmm.faces.create(faces.size(), CV_16U); 164 | unsigned short* out_faces_data = (unsigned short*)basel_3dmm.faces.data; 165 | for (int i = 0; i < faces_size; ++i) 166 | *out_faces_data++ = (unsigned short)(*faces_data++); 167 | } 168 | catch (H5::DataSetIException error) 169 | { 170 | throw std::runtime_error(error.getDetailMsg()); 171 | } 172 | 173 | return basel_3dmm; 174 | } 175 | 176 | } // namespace face_swap 177 | -------------------------------------------------------------------------------- /face_swap/face_swap/cnn_3dmm.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/cnn_3dmm.h" 2 | #include 3 | #include // debug 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | using std::string; 11 | using namespace caffe; 12 | 13 | namespace face_swap 14 | { 15 | CNN3DMM::CNN3DMM(const string& deploy_file, const string& caffe_model_file, 16 | const std::string& mean_file, bool init_cnn, bool with_gpu, int gpu_device_id) : 17 | m_num_channels(0), m_with_gpu(with_gpu) 18 | { 19 | if (!init_cnn) return; 20 | 21 | // Initialize device mode 22 | if (with_gpu) 23 | { 24 | Caffe::SetDevice(gpu_device_id); 25 | Caffe::set_mode(Caffe::GPU); 26 | } 27 | else Caffe::set_mode(Caffe::CPU); 28 | 29 | // Set test mode 30 | //Caffe::set_phase(Caffe::TEST); 31 | 32 | // Load the network 33 | m_net.reset(new Net(deploy_file, caffe::TEST)); 34 | m_net->CopyTrainedLayersFrom(caffe_model_file); 35 | 36 | CHECK_EQ(m_net->num_inputs(), 1) << "Network should have exactly one input."; 37 | CHECK_EQ(m_net->num_outputs(), 1) << "Network should have exactly one output."; 38 | 39 | Blob* input_layer = m_net->input_blobs()[0]; 40 | m_num_channels = input_layer->channels(); 41 | CHECK(m_num_channels == 3 || m_num_channels == 1) 42 | << "Input layer should have 1 or 3 channels."; 43 | m_input_size = cv::Size(input_layer->width(), input_layer->height()); 44 | 45 | // Load mean file 46 | m_mean = readMean(mean_file); 47 | } 48 | 49 | void CNN3DMM::process(const cv::Mat& img, 50 | cv::Mat& shape_coefficients, cv::Mat& tex_coefficients) 51 | { 52 | // Prepare input data 53 | cv::Mat img_processed = preprocess(img); 54 | copyInputData(img_processed); 55 | const vector*>& output_blobs = m_net->Forward(); 56 | 57 | // Output results 58 | const std::vector& shape = m_net->blob_by_name("fc_ftnew")->shape(); 59 | float* featues = m_net->blob_by_name("fc_ftnew")->mutable_cpu_data(); 60 | shape_coefficients = cv::Mat_(99, 1, featues).clone(); 61 | tex_coefficients = cv::Mat_(99, 1, featues + 99).clone(); 62 | } 63 | 64 | CNN3DMM::~CNN3DMM() 65 | { 66 | } 67 | 68 | void CNN3DMM::wrapInputLayer(std::vector& input_channels) 69 | { 70 | Blob* input_layer = m_net->input_blobs()[0]; 71 | 72 | int width = input_layer->width(); 73 | int height = input_layer->height(); 74 | 75 | #ifdef CPU_ONLY 76 | float* input_data = input_layer->mutable_cpu_data(); 77 | #else 78 | float* input_data = input_layer->mutable_gpu_data(); 79 | #endif 80 | 81 | for (int i = 0; i < input_layer->channels(); ++i) { 82 | cv::Mat channel(height, width, CV_32FC1, input_data); 83 | input_channels.push_back(channel); 84 | input_data += width * height; 85 | } 86 | } 87 | 88 | cv::Mat CNN3DMM::readMean(const std::string & mean_file) const 89 | { 90 | // Read binary proto file 91 | caffe::BlobProto blob_proto; 92 | caffe::ReadProtoFromBinaryFileOrDie(mean_file, &blob_proto); 93 | 94 | // Convert from BlobProto to Blob 95 | Blob mean_blob; 96 | mean_blob.FromProto(blob_proto); 97 | CHECK_EQ(mean_blob.channels(), m_num_channels) 98 | << "Number of channels of mean file doesn't match input layer."; 99 | 100 | // The format of the mean file is planar 32-bit float BGR or grayscale 101 | std::vector channels; 102 | float* data = mean_blob.mutable_cpu_data(); 103 | for (int i = 0; i < m_num_channels; ++i) { 104 | /* Extract an individual channel. */ 105 | cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_32FC1, data); 106 | channels.push_back(channel); 107 | data += mean_blob.height() * mean_blob.width(); 108 | } 109 | 110 | // Merge the separate channels into a single image 111 | cv::Mat mean; 112 | cv::merge(channels, mean); 113 | 114 | return mean; 115 | } 116 | 117 | cv::Mat CNN3DMM::preprocess(const cv::Mat& img) 118 | { 119 | // Convert the input image to the input image format of the network. 120 | cv::Mat sample; 121 | if (img.channels() == 3 && m_num_channels == 1) 122 | cv::cvtColor(img, sample, cv::COLOR_BGR2GRAY); 123 | else if (img.channels() == 4 && m_num_channels == 1) 124 | cv::cvtColor(img, sample, cv::COLOR_BGRA2GRAY); 125 | else if (img.channels() == 4 && m_num_channels == 3) 126 | cv::cvtColor(img, sample, cv::COLOR_BGRA2BGR); 127 | else if (img.channels() == 1 && m_num_channels == 3) 128 | cv::cvtColor(img, sample, cv::COLOR_GRAY2BGR); 129 | else 130 | sample = img; 131 | 132 | cv::Mat sample_resized; 133 | if (sample.size() != m_input_size) 134 | cv::resize(sample, sample_resized, m_input_size, 0, 0, cv::INTER_CUBIC); 135 | else 136 | sample_resized = sample; 137 | 138 | cv::Mat sample_float; 139 | if (m_num_channels == 3) 140 | sample_resized.convertTo(sample_float, CV_32FC3); 141 | else 142 | sample_resized.convertTo(sample_float, CV_32FC1); 143 | 144 | //cv::Mat sample_normalized; 145 | //cv::subtract(sample_float, mean_, sample_normalized); 146 | 147 | // Normalize the image by substracting the mean 148 | //subtractMean_c3(sample_float); 149 | //cv::Mat sample_normalized = sample_float; 150 | sample_float -= m_mean; 151 | 152 | return sample_float; 153 | } 154 | 155 | void CNN3DMM::copyInputData(cv::Mat& img) 156 | { 157 | Blob* input_layer = m_net->input_blobs()[0]; 158 | float* buf = new float[input_layer->count()]; 159 | 160 | int r, c, cnl, channels = img.channels(); 161 | float* img_data = nullptr; 162 | float* buf_data = buf; 163 | for(cnl = 0; cnl < channels; ++cnl) 164 | { 165 | img_data = ((float*)img.data) + cnl; 166 | for(r = 0; r < img.rows; ++r) 167 | { 168 | for(c = 0; c < img.cols; ++c) 169 | { 170 | *buf_data++ = *img_data; 171 | img_data += channels; 172 | } 173 | } 174 | } 175 | 176 | if(m_with_gpu) 177 | caffe_copy(input_layer->count(), buf, input_layer->mutable_gpu_data()); 178 | else 179 | caffe_copy(input_layer->count(), buf, input_layer->mutable_cpu_data()); 180 | 181 | delete[] buf; 182 | } 183 | 184 | } // namespace face_swap 185 | -------------------------------------------------------------------------------- /face_swap/face_swap/cnn_3dmm_expr.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/cnn_3dmm.h" 2 | #include "face_swap/cnn_3dmm_expr.h" 3 | 4 | // iris_sfs 5 | #include 6 | 7 | // std 8 | #include 9 | #include 10 | 11 | // OpenCV 12 | #include 13 | #include // debug 14 | 15 | namespace face_swap 16 | { 17 | CNN3DMMExpr::CNN3DMMExpr(const std::string& deploy_file, 18 | const std::string& caffe_model_file, const std::string& mean_file, 19 | const std::string& model_file, bool generic, bool with_expr, 20 | bool with_gpu, int gpu_device_id) : 21 | CNN3DMM(deploy_file, caffe_model_file, mean_file, 22 | !generic, with_gpu, gpu_device_id), 23 | m_generic(generic), m_with_expr(with_expr) 24 | { 25 | // Load Basel 3DMM 26 | BaselFace::load_BaselFace_data(model_file.c_str()); 27 | 28 | // Initialize face service 29 | fservice = std::make_unique(); 30 | } 31 | 32 | CNN3DMMExpr::~CNN3DMMExpr() 33 | { 34 | } 35 | 36 | void CNN3DMMExpr::process(const cv::Mat& img, 37 | const std::vector& landmarks, cv::Mat& shape_coefficients, 38 | cv::Mat& tex_coefficients, cv::Mat& expr_coefficients, 39 | cv::Mat& vecR, cv::Mat& vecT, cv::Mat& K) 40 | { 41 | // Calculate shape and texture coefficients 42 | if (m_generic) 43 | { 44 | shape_coefficients = cv::Mat::zeros(99, 1, CV_32F); 45 | tex_coefficients = cv::Mat::zeros(99, 1, CV_32F); 46 | } 47 | else CNN3DMM::process(img, shape_coefficients, tex_coefficients); 48 | 49 | // Set up face service 50 | //fservice->setUp(img.cols, img.rows, 1000.0f); 51 | fservice->init(img.cols, img.rows, 1000.0f); 52 | 53 | // Convert landmarks format 54 | //cv::Mat_ LMs(68 * 2, 1); 55 | //for (int i = 0; i < 68; ++i) LMs.at(i) = landmarks[i].x; 56 | //for (int i = 0; i < 68; ++i) LMs.at(i + 68) = landmarks[i].y; 57 | cv::Mat LMs(68, 2, CV_32F); 58 | float* lms_data = (float*)LMs.data; 59 | for (int i = 0; i < 68; ++i) 60 | { 61 | *lms_data++ = (float)landmarks[i].x; 62 | *lms_data++ = (float)landmarks[i].y; 63 | } 64 | 65 | // Calculate pose and expression 66 | fservice->estimatePoseExpr(img, LMs, shape_coefficients, vecR, vecT, K, 67 | expr_coefficients, "", m_with_expr); 68 | } 69 | } // namespace face_swap 70 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_data.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package face_swap.io; 4 | 5 | message FaceData { 6 | optional bytes cropped_seg = 1; 7 | repeated int32 scaled_landmarks = 2 [packed = true]; 8 | repeated int32 bbox = 3 [packed = true]; 9 | repeated float shape_coefficients = 4 [packed = true]; 10 | repeated float tex_coefficients = 5 [packed = true]; 11 | repeated float expr_coefficients = 6 [packed = true]; 12 | repeated float vecR = 7 [packed = true]; 13 | repeated float vecT = 8 [packed = true]; 14 | repeated float K = 9 [packed = true]; 15 | repeated float shape_coefficients_flipped = 10 [packed = true]; 16 | repeated float tex_coefficients_flipped = 11 [packed = true]; 17 | repeated float expr_coefficients_flipped = 12 [packed = true]; 18 | repeated float vecR_flipped = 13 [packed = true]; 19 | repeated float vecT_flipped = 14 [packed = true]; 20 | required bool enable_seg = 15; 21 | required int32 max_bbox_res = 16; 22 | } -------------------------------------------------------------------------------- /face_swap/face_swap/face_detection_landmarks.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/face_detection_landmarks.h" 2 | 3 | // std 4 | #include 5 | 6 | // OpenCV 7 | #include 8 | 9 | // dlib 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | using std::string; 16 | using std::runtime_error; 17 | 18 | namespace face_swap 19 | { 20 | void dlib_obj_to_points(const dlib::full_object_detection& obj, 21 | std::vector& points) 22 | { 23 | points.resize(obj.num_parts()); 24 | for (unsigned long i = 0; i < obj.num_parts(); ++i) 25 | { 26 | cv::Point& p = points[i]; 27 | const dlib::point& obj_p = obj.part(i); 28 | p.x = (float)obj_p.x(); 29 | p.y = (float)obj_p.y(); 30 | } 31 | } 32 | 33 | class FaceDetectionLandmarksImpl : public FaceDetectionLandmarks 34 | { 35 | private: 36 | // dlib 37 | dlib::frontal_face_detector m_detector; 38 | dlib::shape_predictor m_landmarks_model; 39 | 40 | public: 41 | FaceDetectionLandmarksImpl(const std::string& landmarks_path) 42 | { 43 | // Face detector for finding bounding boxes for each face in an image 44 | m_detector = dlib::get_frontal_face_detector(); 45 | 46 | // Shape predictor for finding landmark positions given an image and face bounding box. 47 | if (landmarks_path.empty()) return; 48 | dlib::deserialize(landmarks_path) >> m_landmarks_model; 49 | } 50 | 51 | void process(const cv::Mat& frame, std::vector& faces) 52 | { 53 | // Convert OpenCV's mat to dlib format 54 | dlib::cv_image dlib_frame(frame); 55 | 56 | // Detect bounding boxes around all the faces in the image. 57 | std::vector dlib_rects = m_detector(dlib_frame); 58 | 59 | // Extract landmarks for each face we detected. 60 | std::vector shapes; 61 | for (size_t i = 0; i < dlib_rects.size(); ++i) 62 | { 63 | faces.push_back(Face()); 64 | Face& curr_face = faces.back(); 65 | dlib::rectangle& dlib_rect = dlib_rects[i]; 66 | 67 | // Set landmarks 68 | dlib::full_object_detection shape = m_landmarks_model(dlib_frame, dlib_rect); 69 | dlib_obj_to_points(shape, curr_face.landmarks); 70 | 71 | // Set face bounding box 72 | curr_face.bbox.x = (int)dlib_rect.left(); 73 | curr_face.bbox.y = (int)dlib_rect.top(); 74 | curr_face.bbox.width = (int)dlib_rect.width(); 75 | curr_face.bbox.height = (int)dlib_rect.height(); 76 | } 77 | } 78 | }; 79 | 80 | std::shared_ptr FaceDetectionLandmarks::create( 81 | const std::string& landmarks_path) 82 | { 83 | return std::make_shared(landmarks_path); 84 | } 85 | 86 | } // namespace face_swap -------------------------------------------------------------------------------- /face_swap/face_swap/face_seg.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/face_seg.h" 2 | #include "face_swap/segmentation_utilities.h" 3 | #include 4 | #include 5 | #include // debug 6 | 7 | using namespace caffe; 8 | 9 | namespace face_swap 10 | { 11 | 12 | FaceSeg::FaceSeg(const string& deploy_file, const string& model_file, 13 | bool with_gpu, int gpu_device_id, bool scale, bool postprocess_seg) : 14 | m_num_channels(0), m_with_gpu(with_gpu), m_scale(scale), m_postprocess_seg(postprocess_seg), 15 | m_gpu_device_id(gpu_device_id) 16 | { 17 | if (with_gpu) 18 | { 19 | Caffe::SetDevice(gpu_device_id); 20 | Caffe::set_mode(Caffe::GPU); 21 | } 22 | else Caffe::set_mode(Caffe::CPU); 23 | 24 | std::cout << "Caffe gpu in segment:" << m_with_gpu << std::endl; 25 | 26 | // Load the network 27 | m_net.reset(new Net(deploy_file, caffe::TEST)); 28 | m_net->CopyTrainedLayersFrom(model_file); 29 | 30 | CHECK_EQ(m_net->num_inputs(), 1) << "Network should have exactly one input."; 31 | CHECK_EQ(m_net->num_outputs(), 1) << "Network should have exactly one output."; 32 | 33 | // Get suggested input size 34 | Blob* input_layer = m_net->input_blobs()[0]; 35 | m_num_channels = input_layer->channels(); 36 | CHECK(m_num_channels == 3 || m_num_channels == 1) 37 | << "Input layer should have 1 or 3 channels."; 38 | m_input_size = cv::Size(input_layer->width(), input_layer->height()); 39 | 40 | // Check number of output channels 41 | Blob* output_layer = m_net->output_blobs()[0]; 42 | if (output_layer->channels() == 21) 43 | m_foreground_channel = 15; 44 | } 45 | 46 | FaceSeg::~FaceSeg() 47 | { 48 | } 49 | 50 | void FaceSeg::set_mode_gpu() 51 | { 52 | Caffe::SetDevice(m_gpu_device_id); 53 | Caffe::set_mode(Caffe::GPU); 54 | } 55 | 56 | cv::Mat FaceSeg::process(const cv::Mat& img) 57 | { 58 | cv::Mat img_scaled; 59 | if (!m_scale) 60 | { 61 | // Enforce network maximum size 62 | if (img.cols > m_input_size.width) 63 | { 64 | float scale = (float)m_input_size.width / (float)img.cols; 65 | cv::resize(img, img_scaled, cv::Size(), scale, scale, cv::INTER_CUBIC); 66 | } 67 | else img_scaled = img; 68 | 69 | // Reshape net 70 | Blob* input_layer = m_net->input_blobs()[0]; 71 | std::vector shape = { 1, img_scaled.channels(), img_scaled.rows, img_scaled.cols }; 72 | input_layer->Reshape(shape); 73 | 74 | // Forward dimension change to all layers 75 | m_net->Reshape(); 76 | } 77 | else img_scaled = img; 78 | 79 | // Prepare input data 80 | std::vector input_channels; 81 | wrapInputLayer(input_channels); 82 | preprocess(img_scaled, input_channels); 83 | 84 | // Forward pass 85 | m_net->Forward(); 86 | 87 | // Extract background and foreground from output layer 88 | Blob* output_layer = m_net->output_blobs()[0]; 89 | cv::Mat background(output_layer->height(), output_layer->width(), CV_32F, 90 | (void*)output_layer->cpu_data()); 91 | cv::Mat foreground(output_layer->height(), output_layer->width(), CV_32F, 92 | (void*)(output_layer->cpu_data() + m_foreground_channel * (output_layer->height() * output_layer->width()))); 93 | 94 | // Calculate argmax 95 | cv::Mat seg(output_layer->height(), output_layer->width(), CV_8U); 96 | unsigned char* seg_data = seg.data; 97 | float* back_data = (float*)background.data; 98 | float* fore_data = (float*)foreground.data; 99 | for (int i = 0; i < seg.total(); ++i) 100 | *seg_data++ = (*back_data++ < *fore_data++) ? 255 : 0; 101 | 102 | // Refine segmentation 103 | //cv::Mat kernel = cv::getStructuringElement(cv::MorphShapes::MORPH_ELLIPSE, cv::Size(3, 3)); 104 | //cv::erode(seg, seg, kernel, cv::Point(-1, -1), 5); 105 | //cv::Mat kernel = cv::getStructuringElement(cv::MorphShapes::MORPH_ELLIPSE, cv::Size(5, 5)); 106 | //cv::erode(seg, seg, kernel, cv::Point(-1, -1), 1); 107 | if(m_postprocess_seg) smoothFlaws(seg, 1, 2); 108 | 109 | // Resize to original image size 110 | if (seg.size() != img.size()) 111 | cv::resize(seg, seg, img.size(), 0, 0, cv::INTER_NEAREST); 112 | 113 | // Output results 114 | return seg; 115 | } 116 | 117 | void FaceSeg::wrapInputLayer(std::vector& input_channels) 118 | { 119 | Blob* input_layer = m_net->input_blobs()[0]; 120 | 121 | int width = input_layer->width(); 122 | int height = input_layer->height(); 123 | 124 | float* input_data = input_layer->mutable_cpu_data(); 125 | 126 | for (int i = 0; i < input_layer->channels(); ++i) { 127 | cv::Mat channel(height, width, CV_32FC1, input_data); 128 | input_channels.push_back(channel); 129 | input_data += width * height; 130 | } 131 | } 132 | 133 | void FaceSeg::preprocess(const cv::Mat& img, std::vector& input_channels) 134 | { 135 | // Convert the input image to the input image format of the network. 136 | cv::Mat sample; 137 | if (img.channels() == 3 && m_num_channels == 1) 138 | cv::cvtColor(img, sample, cv::COLOR_BGR2GRAY); 139 | else if (img.channels() == 4 && m_num_channels == 1) 140 | cv::cvtColor(img, sample, cv::COLOR_BGRA2GRAY); 141 | else if (img.channels() == 4 && m_num_channels == 3) 142 | cv::cvtColor(img, sample, cv::COLOR_BGRA2BGR); 143 | else if (img.channels() == 1 && m_num_channels == 3) 144 | cv::cvtColor(img, sample, cv::COLOR_GRAY2BGR); 145 | else 146 | sample = img; 147 | 148 | cv::Mat sample_resized; 149 | if (m_scale && sample.size() != m_input_size) 150 | cv::resize(sample, sample_resized, m_input_size, 0, 0, cv::INTER_CUBIC); 151 | else 152 | sample_resized = sample; 153 | 154 | cv::Mat sample_float; 155 | if (m_num_channels == 3) 156 | sample_resized.convertTo(sample_float, CV_32FC3); 157 | else 158 | sample_resized.convertTo(sample_float, CV_32FC1); 159 | 160 | // Normalize the image by substracting the mean 161 | subtractMean_c3(sample_float); 162 | 163 | // This operation will write the separate BGR planes directly to the 164 | // input layer of the network because it is wrapped by the cv::Mat 165 | // objects in input_channels. 166 | cv::split(sample_float, input_channels); 167 | } 168 | 169 | void FaceSeg::subtractMean_c3(cv::Mat& img) 170 | { 171 | int r, c; 172 | float* img_data = (float*)img.data; 173 | for (r = 0; r < img.rows; ++r) 174 | { 175 | for (c = 0; c < img.cols; ++c) 176 | { 177 | *img_data++ -= MB; 178 | *img_data++ -= MG; 179 | *img_data++ -= MR; 180 | } 181 | } 182 | } 183 | 184 | } // namespace face_swap 185 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/basel_3dmm.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_BASEL_3DMM_H 2 | #define FACE_SWAP_BASEL_3DMM_H 3 | 4 | #include "face_swap/face_swap_export.h" 5 | 6 | // Includes 7 | #include 8 | 9 | #include 10 | 11 | namespace face_swap 12 | { 13 | /** Represents a 3D renderable shape. 14 | */ 15 | struct FACE_SWAP_EXPORT Mesh 16 | { 17 | /** Save mesh to a ply file. 18 | @param mesh The mesh to write to file. 19 | @param ply_file The name of the file (.ply) 20 | */ 21 | static void save_ply(const Mesh& mesh, const std::string& ply_file); 22 | 23 | cv::Mat vertices; 24 | cv::Mat colors; 25 | cv::Mat faces; 26 | cv::Mat uv; 27 | cv::Mat tex; 28 | cv::Mat normals; 29 | }; 30 | 31 | /** Represents Basel's 3D Morphable Model. 32 | This is a PCA model of 3D faces that includes shape, texture and expressions. 33 | Based on the paper: 34 | A 3D Face Model for Pose and Illumination Invariant Face Recognition, 35 | P. Paysan and R. Knothe and B. Amberg and S. Romdhani and T. Vetter. 36 | */ 37 | struct Basel3DMM 38 | { 39 | /** Sample a mesh from the PCA model. 40 | @param[in] shape_coefficients PCA shape coefficients. 41 | @param[in] tex_coefficients PCA texture coefficients. 42 | */ 43 | Mesh sample(const cv::Mat& shape_coefficients, 44 | const cv::Mat& tex_coefficients); 45 | 46 | /** Sample a mesh from the PCA model. 47 | @param[in] shape_coefficients PCA shape coefficients. 48 | @param[in] tex_coefficients PCA texture coefficients. 49 | @param[in] expr_coefficients PCA expression coefficients. 50 | */ 51 | Mesh sample(const cv::Mat& shape_coefficients, 52 | const cv::Mat& tex_coefficients, const cv::Mat& expr_coefficients); 53 | 54 | /** Load a Basel's 3DMM from file. 55 | @param model_file Path to 3DMM file (.h5). 56 | */ 57 | static Basel3DMM load(const std::string& model_file); 58 | 59 | cv::Mat faces; 60 | cv::Mat shapeMU, shapePC, shapeEV; 61 | cv::Mat texMU, texPC, texEV; 62 | cv::Mat exprMU, exprPC, exprEV; 63 | }; 64 | 65 | } // namespace face_swap 66 | 67 | #endif // FACE_SWAP_BASEL_3DMM_H 68 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/cnn_3dmm.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_CNN_3DMM_H 2 | #define FACE_SWAP_CNN_3DMM_H 3 | 4 | // Includes 5 | #include 6 | 7 | // Caffe 8 | #include 9 | 10 | #include 11 | 12 | namespace face_swap 13 | { 14 | /** This class provided face shape and texture estimation using Caffe with 15 | a convolutional neural network. 16 | The CNN estimates shape and texture coefficients for a PCA model that is based on 17 | Basel's 3D Morphable Model. 18 | This is an implementation of the following papers: 19 | -# Regressing Robust and Discriminative 3D Morphable Models with a very Deep 20 | Neural Network, Anh Tuan Tran, Tal Hassner, Iacopo Masi and Gerard Medioni. 21 | -# A 3D Face Model for Pose and Illumination Invariant Face Recognition, 22 | P. Paysan and R. Knothe and B. Amberg and S. Romdhani and T. Vetter. 23 | */ 24 | class CNN3DMM 25 | { 26 | public: 27 | 28 | /** Creates an instance of CNN3DMM. 29 | @param deploy_file Path to 3DMM regression CNN deploy file (.prototxt). 30 | @param caffe_model_file Path to 3DMM regression CNN model file (.caffemodel). 31 | @param mean_file Path to 3DMM regression CNN mean file (.binaryproto). 32 | @param init_cnn if true the CNN will be initialized. 33 | @param with_gpu Toggle GPU\CPU execution. 34 | @param gpu_device_id Set the GPU's device id. 35 | */ 36 | CNN3DMM(const std::string& deploy_file, const std::string& caffe_model_file, 37 | const std::string& mean_file, bool init_cnn = true, 38 | bool with_gpu = true, int gpu_device_id = 0); 39 | 40 | /** Destructor. 41 | */ 42 | ~CNN3DMM(); 43 | 44 | /** Estimate face shape and texture from image. 45 | @param[in] img The image to process. 46 | @param[out] shape_coefficients PCA shape coefficients. 47 | @param[out] tex_coefficients PCA texture coefficients. 48 | */ 49 | void process(const cv::Mat& img, 50 | cv::Mat& shape_coefficients, cv::Mat& tex_coefficients); 51 | 52 | private: 53 | void wrapInputLayer(std::vector& input_channels); 54 | 55 | cv::Mat readMean(const std::string& mean_file) const; 56 | 57 | cv::Mat preprocess(const cv::Mat& img); 58 | 59 | void copyInputData(cv::Mat& img); 60 | 61 | protected: 62 | std::shared_ptr > m_net; 63 | int m_num_channels; 64 | cv::Size m_input_size; 65 | cv::Mat m_mean; 66 | bool m_with_gpu; 67 | }; 68 | 69 | } // namespace face_swap 70 | 71 | #endif // FACE_SWAP_CNN_3DMM_H 72 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/cnn_3dmm_expr.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_CNN_3DMM_EXPR_H 2 | #define FACE_SWAP_CNN_3DMM_EXPR_H 3 | 4 | 5 | #include "cnn_3dmm.h" 6 | 7 | class FaceServices2; 8 | 9 | namespace face_swap 10 | { 11 | /** This class provided face shape, texture, expression and pose estimations using 12 | Caffe with a convolutional neural network. 13 | The CNN estimates shape and texture coefficients for a PCA model that is based on 14 | Basel's 3D Morphable Model. The pose and expression are then estimated using epnp 15 | optimization. This is an implementation of the following papers: 16 | -# Regressing Robust and Discriminative 3D Morphable Models with a very Deep 17 | Neural Network, Anh Tuan Tran, Tal Hassner, Iacopo Masi and Gerard Medioni. 18 | -# A 3D Face Model for Pose and Illumination Invariant Face Recognition, 19 | P. Paysan and R. Knothe and B. Amberg and S. Romdhani and T. Vetter. 20 | */ 21 | class CNN3DMMExpr : public CNN3DMM 22 | { 23 | public: 24 | 25 | /** Creates an instance of CNN3DMMExpr. 26 | @param deploy_file Path to 3DMM regression CNN deploy file (.prototxt). 27 | @param caffe_model_file Path to 3DMM regression CNN model file (.caffemodel). 28 | @param mean_file Path to 3DMM regression CNN mean file (.binaryproto). 29 | @param model_file Path to 3DMM file (.dat). 30 | @param generic Use generic model without shape regression. 31 | @param with_expr Toggle fitting face expressions. 32 | @param with_gpu Toggle GPU\CPU execution. 33 | @param gpu_device_id Set the GPU's device id. 34 | */ 35 | CNN3DMMExpr(const std::string& deploy_file, const std::string& caffe_model_file, 36 | const std::string& mean_file, const std::string& model_file, 37 | bool generic = false, bool with_expr = true, 38 | bool with_gpu = true, int gpu_device_id = 0); 39 | 40 | /** Destructor. 41 | */ 42 | ~CNN3DMMExpr(); 43 | 44 | /** Estimate face pose and shape, texture, expression coefficients from image. 45 | @param[in] img The image to process. 46 | @param[in] landmarks The face landmarks detected on the specified image. 47 | @param[out] shape_coefficients PCA shape coefficients. 48 | @param[out] tex_coefficients PCA texture coefficients. 49 | @param[out] expr_coefficients PCA expression coefficients. 50 | @param[out] vecR Face's rotation vector [Euler angles]. 51 | @param[out] vecT Face's translation vector. 52 | @param[out] K Camera intrinsic parameters. 53 | */ 54 | void process(const cv::Mat& img, const std::vector& landmarks, 55 | cv::Mat& shape_coefficients, cv::Mat& tex_coefficients, 56 | cv::Mat& expr_coefficients, cv::Mat& vecR, cv::Mat& vecT, cv::Mat& K); 57 | private: 58 | std::unique_ptr fservice; 59 | bool m_generic, m_with_expr; 60 | }; 61 | 62 | } // namespace face_swap 63 | 64 | #endif // FACE_SWAP_CNN_3DMM_EXPR_H 65 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/face_detection_landmarks.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_FACE_DETECTION_LANDMARKS_H 2 | #define FACE_SWAP_FACE_DETECTION_LANDMARKS_H 3 | 4 | #include "face_swap/face_swap_export.h" 5 | 6 | // std 7 | #include 8 | #include 9 | #include 10 | 11 | // OpenCV 12 | #include 13 | 14 | 15 | namespace face_swap 16 | { 17 | /** @brief Represents a face detected in a frame. 18 | */ 19 | struct Face 20 | { 21 | cv::Rect bbox; ///< Bounding box. 22 | std::vector landmarks; ///< Face landmarks. 23 | }; 24 | 25 | /** @brief This class provide face detection and landmarks extraction 26 | functionality for single images. 27 | */ 28 | class FACE_SWAP_EXPORT FaceDetectionLandmarks 29 | { 30 | public: 31 | /** @brief Process a frame. 32 | @param frame The frame to process [BGR]. 33 | @param faces The output faces detected in the frame. 34 | */ 35 | virtual void process(const cv::Mat& frame, std::vector& faces) = 0; 36 | 37 | /** @brief Create an instance initialized with a landmarks model file. 38 | @param landmarks_path Path to the landmarks model file (.dat). 39 | */ 40 | static std::shared_ptr create(const std::string& landmarks_path); 41 | }; 42 | 43 | } // namespace face_swap 44 | 45 | #endif // FACE_SWAP_FACE_DETECTION_LANDMARKS_H 46 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/face_seg.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_FACE_SEG_H 2 | #define FACE_SWAP_FACE_SEG_H 3 | 4 | #include "face_swap/face_swap_export.h" 5 | 6 | // std 7 | #include 8 | 9 | // OpenCV 10 | #include 11 | 12 | // Caffe 13 | #include 14 | 15 | namespace face_swap 16 | { 17 | /** This class provided deep face segmentation using Caffe with a fully connected 18 | convolutional neural network. 19 | */ 20 | class FACE_SWAP_EXPORT FaceSeg 21 | { 22 | public: 23 | /** Construct FaceSeg instance. 24 | @param deploy_file Network definition file for deployment (.prototxt). 25 | @param model_file Network weights model file (.caffemodel). 26 | @param with_gpu Toggle GPU\CPU. 27 | @param gpu_device_id Set the GPU's device id. 28 | @param scale Scale image to the network's maximum size (depicted by the prototxt file). 29 | @param postprocess_seg Toggle postprocessing of the segmentation. 30 | */ 31 | FaceSeg(const std::string& deploy_file, const std::string& model_file, 32 | bool with_gpu = true, int gpu_device_id = 0, 33 | bool scale = true, bool postprocess_seg = false); 34 | 35 | ~FaceSeg(); 36 | 37 | /** Do face segmentation. 38 | @param img BGR color image. 39 | @return 8-bit segmentation mask, 255 for face pixels and 0 for 40 | background pixels. 41 | */ 42 | cv::Mat process(const cv::Mat& img); 43 | 44 | void set_mode_gpu(); 45 | 46 | private: 47 | 48 | /** Wrap the input layer of the network in separate cv::Mat objects 49 | (one per channel). This way we save one memcpy operation and we 50 | don't need to rely on cudaMemcpy2D. The last preprocessing operation 51 | will write the separate channels directly to the input layer. 52 | @param input_channels Input image channels. 53 | */ 54 | void wrapInputLayer(std::vector& input_channels); 55 | 56 | /** Preprocess image for network. 57 | @param img BGR color image. 58 | @param input_channels Input image channels. 59 | */ 60 | void preprocess(const cv::Mat& img, std::vector& input_channels); 61 | 62 | /** Substract mean color from image. 63 | @param img BGR float image. 64 | */ 65 | void subtractMean_c3(cv::Mat& img); 66 | 67 | protected: 68 | std::shared_ptr> m_net; 69 | int m_num_channels; 70 | cv::Size m_input_size; 71 | bool m_with_gpu; 72 | bool m_scale; 73 | bool m_postprocess_seg; 74 | int m_foreground_channel = 1; 75 | 76 | int m_gpu_device_id; 77 | 78 | // Mean pixel color 79 | const float MB = 104.00699f, MG = 116.66877f, MR = 122.67892f; 80 | }; 81 | 82 | } // namespace face_swap 83 | 84 | #endif // FACE_SWAP_FACE_SEG_H 85 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/face_swap_c_interface.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_C_INTERFACE_H 2 | #define FACE_SWAP_C_INTERFACE_H 3 | 4 | #include "face_swap/face_swap_export.h" 5 | 6 | extern "C" { 7 | 8 | struct FaceDataInterface 9 | { 10 | // Input 11 | unsigned char* img; 12 | unsigned char* seg; 13 | int w, h; 14 | 15 | // Intermediate pipeline data 16 | unsigned char* scaled_img; 17 | unsigned char* scaled_seg; 18 | int scaled_w, scaled_h; 19 | unsigned char* cropped_img; 20 | unsigned char* cropped_seg; 21 | int cropped_w, cropped_h; 22 | int* scaled_landmarks; 23 | int* cropped_landmarks; 24 | int* bbox; 25 | int* scaled_bbox; 26 | float *shape_coefficients, *tex_coefficients, *expr_coefficients; 27 | float *vecR, *vecT, *K; 28 | 29 | // Flipped image data 30 | float *shape_coefficients_flipped, *tex_coefficients_flipped, *expr_coefficients_flipped; 31 | float *vecR_flipped, *vecT_flipped; 32 | 33 | // Processing parameters 34 | bool enable_seg = true; 35 | int max_bbox_res = 0; 36 | }; 37 | 38 | FACE_SWAP_EXPORT int init(const char* landmarks_path, const char* model_3dmm_h5_path, 39 | const char* model_3dmm_dat_path, const char* reg_model_path, 40 | const char* reg_deploy_path, const char* reg_mean_path, 41 | const char* seg_model_path, const char* seg_deploy_path, 42 | bool generic = false, bool with_expr = true, bool with_gpu = true, 43 | int gpu_device_id = 0); 44 | 45 | FACE_SWAP_EXPORT int process(FaceDataInterface* face_data); 46 | 47 | FACE_SWAP_EXPORT int swap(FaceDataInterface* src_data, FaceDataInterface* tgt_data, 48 | unsigned char* out); 49 | } 50 | 51 | 52 | #endif // FACE_SWAP_C_INTERFACE_H 53 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/face_swap_engine.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_FACE_SWAP_ENGINE_H 2 | #define FACE_SWAP_FACE_SWAP_ENGINE_H 3 | 4 | #include "face_swap/face_swap_export.h" 5 | #include "face_swap/basel_3dmm.h" 6 | 7 | // std 8 | #include 9 | 10 | // OpenCV 11 | #include 12 | 13 | 14 | namespace face_swap 15 | { 16 | struct FaceData 17 | { 18 | // Input 19 | cv::Mat img; 20 | cv::Mat seg; 21 | 22 | // Intermediate pipeline data 23 | float scale= 1; 24 | cv::Mat scaled_img; 25 | cv::Mat scaled_seg; 26 | cv::Mat cropped_img; 27 | cv::Mat cropped_seg; 28 | std::vector scaled_landmarks; 29 | std::vector cropped_landmarks; 30 | cv::Rect bbox; 31 | cv::Rect scaled_bbox; 32 | cv::Mat shape_coefficients, tex_coefficients, expr_coefficients; 33 | cv::Mat vecR, vecT, K; 34 | 35 | // Flipped image data 36 | cv::Mat shape_coefficients_flipped, tex_coefficients_flipped, expr_coefficients_flipped; 37 | cv::Mat vecR_flipped, vecT_flipped; 38 | 39 | // Processing parameters 40 | bool enable_seg = true; 41 | int max_bbox_res = 0; 42 | }; 43 | 44 | /** Face swap interface. 45 | */ 46 | class FACE_SWAP_EXPORT FaceSwapEngine 47 | { 48 | public: 49 | 50 | /** Transfer the face in the source image onto the face in the target image. 51 | @param[in] src_data Includes all the images and intermediate data for the specific face. 52 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 53 | @return The output face swapped image. 54 | */ 55 | virtual cv::Mat swap(FaceData& src_data, FaceData& tgt_data) = 0; 56 | 57 | /** Process a single image and save the intermediate face data. 58 | @param[in] face_data Includes all the images and intermediate data for the specific face. 59 | @param[in] process_flipped Toggle processing of flipped image. 60 | @return true for success and false for failure. 61 | */ 62 | virtual bool process(FaceData& face_data, bool process_flipped = false) = 0; 63 | 64 | virtual cv::Mat renderFaceData(const FaceData& face_data, float scale = 1.0f) = 0; 65 | 66 | /** Construct FaceSwapEngine instance. 67 | @param landmarks_path Path to the landmarks model file. 68 | @param model_3dmm_h5_path Path to 3DMM file (.h5). 69 | @param model_3dmm_dat_path Path to 3DMM file (.dat). 70 | @param reg_model_path Path to 3DMM regression CNN model file (.caffemodel). 71 | @param reg_deploy_path Path to 3DMM regression CNN deploy file (.prototxt). 72 | @param reg_mean_path Path to 3DMM regression CNN mean file (.binaryproto). 73 | @param seg_model_path Path to face segmentation CNN model file (.caffemodel). 74 | @param seg_deploy_path Path to face segmentation CNN deploy file (.prototxt). 75 | @param generic Use generic model without shape regression. 76 | @param with_expr Toggle fitting face expressions. 77 | @param with_gpu Toggle GPU\CPU execution. 78 | @param gpu_device_id Set the GPU's device id. 79 | */ 80 | static std::shared_ptr createInstance( 81 | const std::string& landmarks_path, const std::string& model_3dmm_h5_path, 82 | const std::string& model_3dmm_dat_path, const std::string& reg_model_path, 83 | const std::string& reg_deploy_path, const std::string& reg_mean_path, 84 | const std::string& seg_model_path, const std::string& seg_deploy_path, 85 | bool generic = false, bool with_expr = true, bool with_gpu = true, 86 | int gpu_device_id = 0); 87 | 88 | 89 | /** Estimate single image and save the related face data for comparing. 90 | @param[in] face_data Includes all the images and intermediate data for the specific face. 91 | @return true for success and false for failure. 92 | */ 93 | virtual bool estimate(FaceData& face_data) = 0; 94 | 95 | /** Segment face mask of single image and save the face mask. 96 | @param[in] face_data Includes all the images and intermediate data for the specific face. 97 | @return true for success and false for failure. 98 | */ 99 | virtual bool segment(FaceData& face_data) = 0; 100 | 101 | /** get face landmarks of single image and save the landmarks. 102 | @param[in] face_data Includes all the images and intermediate data for the specific face. 103 | @return true for success and false for failure. 104 | */ 105 | virtual bool landmarks(FaceData& face_data) = 0; 106 | 107 | /** get face shape info of single image and save the info. 108 | @param[in] face_data Includes all the images and intermediate data for the specific face. 109 | @return true for success and false for failure. 110 | */ 111 | virtual bool shape(FaceData& face_data) = 0; 112 | 113 | /** Compare the face in the source image and the face in the target image. 114 | @param[in] src_data Includes all the images and intermediate data for the specific face. 115 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 116 | @return The result of comparison. 117 | */ 118 | virtual bool compare(FaceData& src_data, FaceData& tgt_data) = 0; 119 | 120 | /** swapping the face in the source image and the face in the target image. 121 | @param[in] src_data Includes all the images and intermediate data for the specific face. 122 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 123 | @return The output face swapped image. 124 | */ 125 | virtual cv::Mat transfer(FaceData& src_data, FaceData& tgt_data) = 0; 126 | 127 | /** align the face in the source image and the face in the target image. 128 | @param[in] src_data Includes all the images and intermediate data for the specific face. 129 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 130 | @return The aligned matrix. 131 | */ 132 | virtual cv::Mat align(FaceData& src_data, FaceData& tgt_data) = 0; 133 | 134 | /** fine-tuning the alignment the face in the source image and the face in the target image. 135 | @param[in] src_data Includes all the images and intermediate data for the specific face. 136 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 137 | @return The aligned matrix. 138 | */ 139 | virtual cv::Mat fine_tune(cv::Mat& src_data, cv::Mat& tgt_data) = 0; 140 | 141 | virtual void set_mode_gpu() = 0; 142 | }; 143 | 144 | } // namespace face_swap 145 | 146 | #endif // FACE_SWAP_FACE_SWAP_ENGINE_H 147 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/face_swap_engine_impl.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_FACE_SWAP_ENGINE_IMPL_H 2 | #define FACE_SWAP_FACE_SWAP_ENGINE_IMPL_H 3 | 4 | #include "face_swap_engine.h" 5 | #include "face_swap/cnn_3dmm_expr.h" 6 | #include "face_swap/basel_3dmm.h" 7 | #include "face_swap/face_detection_landmarks.h" 8 | #include "face_swap/face_seg.h" 9 | 10 | // std 11 | #include 12 | 13 | 14 | namespace face_swap 15 | { 16 | /** Face swap interface. 17 | */ 18 | class FaceSwapEngineImpl : public FaceSwapEngine 19 | { 20 | public: 21 | FaceSwapEngineImpl( 22 | const std::string& landmarks_path, const std::string& model_3dmm_h5_path, 23 | const std::string& model_3dmm_dat_path, const std::string& reg_model_path, 24 | const std::string& reg_deploy_path, const std::string& reg_mean_path, 25 | const std::string& seg_model_path, const std::string& seg_deploy_path, 26 | bool generic = false, bool with_expr = true, bool with_gpu = true, 27 | int gpu_device_id = 0); 28 | 29 | /** Transfer the face in the source image onto the face in the target image. 30 | @param[in] src_data Includes all the images and intermediate data for the specific face. 31 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 32 | @return The output face swapped image. 33 | */ 34 | cv::Mat swap(FaceData& src_data, FaceData& tgt_data); 35 | 36 | /** Process a single image and save the intermediate face data. 37 | @param[in] face_data Includes all the images and intermediate data for the specific face. 38 | @param[in] process_flipped Toggle processing of flipped image. 39 | @return true for success and false for failure. 40 | */ 41 | bool process(FaceData& face_data, bool process_flipped = false); 42 | 43 | cv::Mat renderFaceData(const FaceData& img_data, float scale = 1.0f); 44 | 45 | 46 | 47 | /** Estimate single image and save the related face data for comparing. 48 | @param[in] face_data Includes all the images and intermediate data for the specific face. 49 | @return true for success and false for failure. 50 | */ 51 | bool estimate(FaceData& face_data); 52 | 53 | /** Compare the face in the source image and the face in the target image. 54 | @param[in] src_data Includes all the images and intermediate data for the specific face. 55 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 56 | @return The result of comparison. 57 | */ 58 | bool compare(FaceData& src_data, FaceData& tgt_data); 59 | 60 | /** swapping the face in the source image and the face in the target image. 61 | @param[in] src_data Includes all the images and intermediate data for the specific face. 62 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 63 | @return The output face swapped image. 64 | */ 65 | cv::Mat transfer(FaceData& src_data, FaceData& tgt_data); 66 | 67 | /** Segment face mask of single image and save the face mask. 68 | @param[in] face_data Includes all the images and intermediate data for the specific face. 69 | @return true for success and false for failure. 70 | */ 71 | bool segment(FaceData& face_data); 72 | 73 | /** get face landmarks of single image and save the landmarks. 74 | @param[in] face_data Includes all the images and intermediate data for the specific face. 75 | @return true for success and false for failure. 76 | */ 77 | bool landmarks(FaceData& face_data); 78 | 79 | /** get face shape info of single image and save the info. 80 | @param[in] face_data Includes all the images and intermediate data for the specific face. 81 | @return true for success and false for failure. 82 | */ 83 | bool shape(FaceData& face_data); 84 | 85 | /** align the face in the source image and the face in the target image. 86 | @param[in] src_data Includes all the images and intermediate data for the specific face. 87 | @param[in] tgt_data Includes all the images and intermediate data for the specific face. 88 | @return The aligned matrix. 89 | */ 90 | cv::Mat align(FaceData& src_data, FaceData& tgt_data); 91 | 92 | /** fine-tuning the alignment the face in the source image and the face in the target image. 93 | @param[in] src_data Includes globally-aligned segment data for the specific face. 94 | @param[in] tgt_data Includes globally-aligned segment data for the specific face. 95 | @return The aligned matrix. 96 | */ 97 | cv::Mat fine_tune(cv::Mat& src_data, cv::Mat& tgt_data); 98 | 99 | void set_mode_gpu(); 100 | 101 | private: 102 | 103 | /** Crops the image and it's corresponding segmentation according 104 | to the detected face landmarks. Optinally scale all the images and segmetations. 105 | @param[in] face_data Includes all the images and intermediate data for the specific face. 106 | @return true for success and false for failure. 107 | */ 108 | bool preprocessImages(FaceData& face_data); 109 | 110 | /** Compute affine transform for two point sets 111 | @param[in] srcPoints Source set. 112 | @param[in] dstPoints Target set. 113 | @return the transform. 114 | */ 115 | bool computeRigid(const std::vector &srcPoints, const std::vector &dstPoints, cv::Mat &transf, bool xyExchange = true); 116 | 117 | /** icp algorithm 118 | @param[in] srcPoints Source set. 119 | @param[in] dstPoints Target set. 120 | @return the transform. 121 | */ 122 | bool icp(const std::vector &srcPoints, const std::vector &dstPoints, cv::Mat &transf); 123 | 124 | private: 125 | //std::shared_ptr m_sfl; 126 | std::shared_ptr m_lms; 127 | std::unique_ptr m_cnn_3dmm_expr; 128 | std::unique_ptr m_basel_3dmm; 129 | std::unique_ptr m_face_seg; 130 | 131 | bool m_with_gpu; 132 | int m_gpu_device_id; 133 | }; 134 | 135 | } // namespace face_swap 136 | 137 | #endif // FACE_SWAP_FACE_SWAP_ENGINE_IMPL_H 138 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/landmarks_utilities.h: -------------------------------------------------------------------------------- 1 | /** @file 2 | @brief Face landmarks utility functions. 3 | */ 4 | 5 | #ifndef FACE_SWAP_LANDMARKS_UTILITIES_H 6 | #define FACE_SWAP_LANDMARKS_UTILITIES_H 7 | 8 | #include "face_detection_landmarks.h" 9 | #include "face_swap/face_swap_export.h" 10 | 11 | // OpenCV 12 | #include 13 | 14 | namespace face_swap 15 | { 16 | /** @brief Render landmarks. 17 | @param img The image that the landmarks will be rendered on. 18 | @param landmarks The landmark points to render. 19 | @param drawLabels if true, for each landmark, it's 0 based index will be 20 | rendererd as a label. 21 | @param color Line/point and label color. 22 | @param thickness Line/point thickness. 23 | */ 24 | FACE_SWAP_EXPORT void render(cv::Mat& img, const std::vector& landmarks, 25 | bool drawLabels = false, const cv::Scalar& color = cv::Scalar(0, 255, 0), 26 | int thickness = 1); 27 | 28 | /** @brief Render bounding box. 29 | @param img The image that the bounding box will be rendered on. 30 | @param bbox The bounding box rectangle to render. 31 | @param color Line color. 32 | @param thickness Line thickness. 33 | */ 34 | FACE_SWAP_EXPORT void render(cv::Mat& img, const cv::Rect& bbox, 35 | const cv::Scalar& color = cv::Scalar(0, 255, 0), int thickness = 1); 36 | 37 | /** @brief Render face's bounding box and landmarks. 38 | @param img The image that the face will be rendered on. 39 | @param face The face to render. 40 | @param drawLabels if true, for each landmark, it's 0 based index will be 41 | rendererd as a label. 42 | @param bbox_color Bounding box line color. 43 | @param landmarks_color Landmarks line/point and label color. 44 | @param thickness Line/point thickness. 45 | @param fontScale The size of the font for the labels. 46 | */ 47 | FACE_SWAP_EXPORT void render(cv::Mat& img, const Face& face, bool drawLabels = false, 48 | const cv::Scalar& bbox_color = cv::Scalar(0, 0, 255), 49 | const cv::Scalar& landmarks_color = cv::Scalar(0, 255, 0), int thickness = 1, 50 | double fontScale = 1.0); 51 | 52 | /** @brief Render all frame faces including bounding boxs and landmarks. 53 | @param img The image that the faces will be rendered on. 54 | @param frame The frame to render. 55 | @param drawIDs if true, the 0 based id will be rendererd as a label. 56 | @param drawLabels if true, for each landmark, it's 0 based index will be 57 | rendererd as a label. 58 | @param bbox_color Bounding box line color. 59 | @param landmarks_color Landmarks line/point and label color. 60 | @param thickness Line/point thickness. 61 | @param fontScale The size of the font for the labels. 62 | */ 63 | FACE_SWAP_EXPORT void render(cv::Mat& img, const std::vector& faces, bool drawLabels = false, 64 | const cv::Scalar& bbox_color = cv::Scalar(0, 0, 255), 65 | const cv::Scalar& landmarks_color = cv::Scalar(0, 255, 0), int thickness = 1, 66 | double fontScale = 1.0); 67 | 68 | /** @brief Get the main face index in a frame. 69 | */ 70 | FACE_SWAP_EXPORT int getMainFaceID(const std::vector& faces, const cv::Size& frame_size); 71 | 72 | /** @brief Get the face's left eye center position (right eye in the image). 73 | @param landmarks 68 face points. 74 | */ 75 | FACE_SWAP_EXPORT cv::Point2f getFaceLeftEye(const std::vector& landmarks); 76 | 77 | /** @brief Get the face's right eye center position (left eye in the image). 78 | @param landmarks 68 face points. 79 | */ 80 | FACE_SWAP_EXPORT cv::Point2f getFaceRightEye(const std::vector& landmarks); 81 | 82 | /** @brief Get the face's vertical angle [radians]. 83 | The angles are in the range [-75/180*pi, 75/180*pi]. 84 | When the face is looking up the angle will be positive and when it is 85 | looking down it will be negative. 86 | @param landmarks 68 face points. 87 | */ 88 | FACE_SWAP_EXPORT float getFaceApproxVertAngle(const std::vector& landmarks); 89 | 90 | /** @brief Get the face's horizontal angle [radians]. 91 | The angles are in the range [-75/180*pi, 75/180*pi]. 92 | When the face is looking right (left in the image) the angle will be positive and 93 | when it is looking left (right in the image) it will be negative. 94 | @param landmarks 68 face points. 95 | */ 96 | FACE_SWAP_EXPORT float getFaceApproxHorAngle(const std::vector& landmarks); 97 | 98 | /** @brief Get the face's tilt angle [radians]. 99 | The angles are in the range [-75/180*pi, 75/180*pi]. 100 | When the face is tilting left (right in the image) the angle will be positive and 101 | when it is tilting right (left in the image) it will be negative. 102 | @param landmarks 68 face points. 103 | */ 104 | FACE_SWAP_EXPORT float getFaceApproxTiltAngle(const std::vector& landmarks); 105 | 106 | /** @brief Get the face's euler angles [radians]. 107 | The angles are in the range [-75/180*pi, 75/180*pi]. 108 | @param landmarks 68 face points. 109 | @return Return a vector with the 3 euler angles. 110 | The x axis represents vertical rotation angle, up is positive. 111 | The y axis represents horizontal rotation angle, right is positive. 112 | The z axis represents tilt rotation angle, left is positive. 113 | */ 114 | FACE_SWAP_EXPORT cv::Point3f getFaceApproxEulerAngles(const std::vector& landmarks); 115 | 116 | /** @brief Get face bounding box from landmarks. 117 | @param landmarks Face points. 118 | @param frameSize The size of the image. 119 | @param square Make the bounding box square (limited to frame boundaries). 120 | */ 121 | FACE_SWAP_EXPORT cv::Rect getFaceBBoxFromLandmarks(const std::vector& landmarks, 122 | const cv::Size& frameSize, bool square); 123 | 124 | } // namespace face_swap 125 | 126 | #endif // FACE_SWAP_LANDMARKS_UTILITIES_H 127 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/render_utilities.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_RENDER_UTILITIES_H 2 | #define FACE_SWAP_RENDER_UTILITIES_H 3 | 4 | #include "face_swap/face_swap_engine.h" 5 | #include 6 | 7 | namespace face_swap 8 | { 9 | FACE_SWAP_EXPORT void renderWireframe(cv::Mat& img, const Mesh& mesh, const cv::Mat& P, 10 | float scale = 1, const cv::Scalar& color = cv::Scalar(0, 255, 0)); 11 | 12 | FACE_SWAP_EXPORT void renderWireframe(cv::Mat& img, const Mesh& mesh, 13 | const cv::Mat& rvec, const cv::Mat& tvec, const cv::Mat& K, 14 | float scale = 1, const cv::Scalar& color = cv::Scalar(0, 255, 0)); 15 | 16 | FACE_SWAP_EXPORT void renderWireframeUV(cv::Mat& img, const Mesh& mesh, const cv::Mat& uv, 17 | const cv::Scalar& color = cv::Scalar(0, 255, 0)); 18 | 19 | FACE_SWAP_EXPORT void renderBoundary(cv::Mat& img, const Mesh& mesh, 20 | const cv::Mat& rvec, const cv::Mat& tvec, const cv::Mat& K, 21 | const cv::Scalar& color = cv::Scalar(255, 0, 0)); 22 | 23 | FACE_SWAP_EXPORT void renderSegmentation(cv::Mat& img, const cv::Mat& seg, float alpha = 0.5f, 24 | const cv::Scalar& color = cv::Scalar(255, 0, 0)); 25 | 26 | /** @brief Render a textured mesh. 27 | All the polygons must be triangles. The texture can be either in BGR or BGRA format. 28 | @param img The image to render the mesh to. 29 | @param mesh The mesh to render. 30 | @param rvec Euler angles (Pitch, Yaw, Roll) [3x1]. 31 | @param tvec translation vector [3x1]. 32 | @param K Intrinsic camera matrix [3x3]. 33 | @param depth Output depth map (same size as img, in floating point format). 34 | */ 35 | FACE_SWAP_EXPORT void renderMesh(cv::Mat& img, const Mesh& mesh, 36 | const cv::Mat& rvec, const cv::Mat& tvec, const cv::Mat& K, 37 | cv::Mat& depthbuf, int ss = 1); 38 | 39 | /** @brief Render depth map. 40 | The depth values are inversed and rendered as a heat map, hotter values correspond to closer pixels. 41 | Pixels of infinite values (MAX_FLOAT) are considered background. 42 | @param depth_map The depth map to render. 43 | @return Rendered depth map. 44 | */ 45 | FACE_SWAP_EXPORT cv::Mat renderDepthMap(const cv::Mat& depth_map); 46 | 47 | FACE_SWAP_EXPORT cv::Mat renderImagePipe(const std::vector& images, int padding = 4, 48 | const cv::Scalar& border_color = cv::Scalar(255, 255, 255)); 49 | 50 | FACE_SWAP_EXPORT void overlayImage(cv::Mat& img, const cv::Mat& overlay, 51 | const cv::Point& loc, const cv::Mat& mask = cv::Mat()); 52 | 53 | FACE_SWAP_EXPORT void renderImageOverlay(cv::Mat& img, const cv::Rect& bbox, 54 | const cv::Mat& src, const cv::Mat& tgt, 55 | const cv::Scalar border_color = cv::Scalar(255.0, 255.0, 255.0)); 56 | 57 | FACE_SWAP_EXPORT bool is_ccw(const cv::Point2f& p1, const cv::Point2f& p2, const cv::Point2f& p3); 58 | 59 | inline unsigned int nextPow2(unsigned int x) 60 | { 61 | --x; 62 | x |= x >> 1; 63 | x |= x >> 2; 64 | x |= x >> 4; 65 | x |= x >> 8; 66 | x |= x >> 16; 67 | return ++x; 68 | } 69 | 70 | FACE_SWAP_EXPORT cv::Mat computeFaceNormals(const Mesh& mesh); 71 | 72 | FACE_SWAP_EXPORT cv::Mat computeVertexNormals(const Mesh& mesh); 73 | 74 | } // namespace face_swap 75 | 76 | #endif // FACE_SWAP_RENDER_UTILITIES_H 77 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/segmentation_utilities.h: -------------------------------------------------------------------------------- 1 | /** @file 2 | @brief Face segmentation utility functions. 3 | */ 4 | 5 | #ifndef FACE_SWAP_SEGMENTATION_UTILITIES_H 6 | #define FACE_SWAP_SEGMENTATION_UTILITIES_H 7 | 8 | #include "face_swap/face_swap_export.h" 9 | 10 | // OpenCV 11 | #include 12 | 13 | namespace face_swap 14 | { 15 | FACE_SWAP_EXPORT void removeSmallerComponents(cv::Mat& seg); 16 | 17 | FACE_SWAP_EXPORT void smoothFlaws(cv::Mat& seg, int smooth_iterations = 1, int smooth_kernel_radius = 2); 18 | 19 | FACE_SWAP_EXPORT void fillHoles(cv::Mat& seg); 20 | 21 | FACE_SWAP_EXPORT void postprocessSegmentation(cv::Mat& seg, bool disconnected = true, 22 | bool holes = true, bool smooth = true, int smooth_iterations = 1, 23 | int smooth_kernel_radius = 2); 24 | 25 | } // namespace face_swap 26 | 27 | #endif // FACE_SWAP_SEGMENTATION_UTILITIES_H 28 | -------------------------------------------------------------------------------- /face_swap/face_swap/face_swap/utilities.h: -------------------------------------------------------------------------------- 1 | #ifndef FACE_SWAP_UTILITIES_H 2 | #define FACE_SWAP_UTILITIES_H 3 | 4 | #include "face_swap/render_utilities.h" 5 | #include "face_swap/basel_3dmm.h" 6 | #include 7 | 8 | namespace face_swap 9 | { 10 | /** @brief Create a rotation matrix from Euler angles. 11 | The rotation values are given in radians and estimated using the RPY convention. 12 | Yaw is applied first to the model, then pitch, then roll (R * P * Y * vertex). 13 | @param x Pitch. rotation around the X axis [Radians]. 14 | @param y Yaw. rotation around the Y axis [Radians]. 15 | @param z Roll. rotation around the Z axis [Radians]. 16 | */ 17 | FACE_SWAP_EXPORT cv::Mat euler2RotMat(float x, float y, float z); 18 | 19 | FACE_SWAP_EXPORT cv::Mat euler2RotMat(const cv::Mat& euler); 20 | 21 | FACE_SWAP_EXPORT cv::Mat createModelView(const cv::Mat& euler, const cv::Mat& translation); 22 | 23 | FACE_SWAP_EXPORT cv::Mat createOrthoProj4x4(const cv::Mat& euler, const cv::Mat& translation, 24 | int width, int height); 25 | 26 | FACE_SWAP_EXPORT cv::Mat createOrthoProj3x4(const cv::Mat& euler, const cv::Mat& translation, 27 | int width, int height); 28 | 29 | FACE_SWAP_EXPORT cv::Mat createPerspectiveProj3x4(const cv::Mat& euler, 30 | const cv::Mat& translation, const cv::Mat& K); 31 | 32 | FACE_SWAP_EXPORT cv::Mat refineMask(const cv::Mat& img, const cv::Mat& mask); 33 | 34 | FACE_SWAP_EXPORT void horFlipLandmarks(std::vector& landmarks, int width); 35 | 36 | /** Generate texture for the mesh based on the image size, intrinsic and 37 | extrinsic transformations. 38 | @param[in] mesh The mesh to generate the texture for. 39 | @param[in] img The image for the texture 40 | @param[in] seg The segmentation for the texture (will be used as the 41 | texture's alpha channel). 42 | @param[in] vecR Mesh's rotation vector [Euler angles]. 43 | @param[in] vecT Mesh's translation vector. 44 | @param[in] K Camera intrinsic parameters. 45 | @param[out] tex Generated texture image. 46 | @param[out] uv Generated texture coordinates. 47 | */ 48 | FACE_SWAP_EXPORT void generateTexture(const Mesh& mesh, const cv::Mat& img, const cv::Mat& seg, 49 | const cv::Mat& vecR, const cv::Mat& vecT, const cv::Mat& K, 50 | cv::Mat& tex, cv::Mat& uv); 51 | 52 | /** Generate texture coordinates for the mesh based on the image size, 53 | intrinsic and extrinsic transformations. 54 | @param[in] mesh The mesh to generate the texture coordinates for. 55 | @param[in] img_size The image size that the texture coordinates will be 56 | relative to. 57 | @param[in] vecR Mesh's rotation vector [Euler angles]. 58 | @param[in] vecT Mesh's translation vector. 59 | @param[in] K Camera intrinsic parameters. 60 | @return n X 2 matrix where n is the number of vertices. Each row contain the 61 | xy texture coordinate of the corresponding vertex. 62 | */ 63 | FACE_SWAP_EXPORT cv::Mat generateTextureCoordinates(const Mesh& mesh, const cv::Size& img_size, 64 | const cv::Mat& vecR, const cv::Mat& vecT, const cv::Mat& K); 65 | 66 | /** Blend the source and destination images based on a segmentation mask. 67 | @param[in] src The source image. 68 | @param[in] dst The destination image. 69 | @param[in] mask Object segmentation mask. 70 | @return The src and dst blended image. 71 | */ 72 | FACE_SWAP_EXPORT cv::Mat blend(const cv::Mat& src, const cv::Mat& dst, 73 | const cv::Mat& mask = cv::Mat()); 74 | 75 | /** Read saved face data. 76 | @param[in] path Path to an image or a directory. If the path is an image, 77 | a directory will be created with the name of the image without the extension. 78 | @param[in] face_data Includes all the images and intermediate data for the specific face. 79 | @return true if cache was loaded, false otherwise. 80 | */ 81 | FACE_SWAP_EXPORT bool readFaceData(const std::string& path, FaceData& face_data); 82 | 83 | /** Write face data to file. 84 | @param[in] path Path to an image or a directory. If the path is an image, 85 | a directory will be created with the name of the image without the extension. 86 | @param[in] face_data Includes all the images and intermediate data for the specific face. 87 | @param[in] overwrite Toggle whether to overwrite existing cache or leave as it is. 88 | @return true if the face data was written to file, else false. 89 | */ 90 | FACE_SWAP_EXPORT bool writeFaceData(const std::string& path, const FaceData& face_data, 91 | bool overwrite = false); 92 | 93 | /** Write img to file. 94 | @param[in] path Path to an image or a directory. 95 | @param[in] face_data Includes all the images and intermediate data for the specific face. 96 | @return true if the image was written to file, else false. 97 | */ 98 | FACE_SWAP_EXPORT bool writeImage(const std::string& path, const cv::Mat& image); 99 | 100 | 101 | } // namespace face_swap 102 | 103 | #endif // FACE_SWAP_UTILITIES_H 104 | -------------------------------------------------------------------------------- /face_swap/face_swap/segmentation_utilities.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/segmentation_utilities.h" 2 | 3 | // OpenCV 4 | #include 5 | 6 | namespace face_swap 7 | { 8 | void removeSmallerComponents(cv::Mat& seg) 9 | { 10 | cv::Mat labels; 11 | cv::Mat stats, centroids; 12 | cv::connectedComponentsWithStats(seg, labels, stats, centroids); 13 | if (stats.rows <= 2) return; 14 | 15 | // Find the label of the connected component with maximum area 16 | cv::Mat areas = stats.colRange(4, 5).clone(); 17 | int* areas_data = (int*)areas.data; 18 | int max_label = std::distance(areas_data, 19 | std::max_element(areas_data + 1, areas_data + stats.rows)); 20 | 21 | // Clear smaller components 22 | unsigned char* seg_data = seg.data; 23 | int* labels_data = (int*)labels.data; 24 | for (size_t i = 0; i < seg.total(); ++i, ++seg_data) 25 | if (*labels_data++ != max_label) *seg_data = 0; 26 | } 27 | 28 | void smoothFlaws(cv::Mat& seg, int smooth_iterations, int smooth_kernel_radius) 29 | { 30 | int kernel_size = smooth_kernel_radius * 2 + 1; 31 | cv::Mat kernel = cv::getStructuringElement( 32 | cv::MorphShapes::MORPH_ELLIPSE, cv::Size(kernel_size, kernel_size)); 33 | // for (int i = 0; i < smooth_iterations; ++i) 34 | { 35 | cv::morphologyEx(seg, seg, cv::MORPH_OPEN, kernel, cv::Point(-1, -1), smooth_iterations); 36 | cv::morphologyEx(seg, seg, cv::MORPH_CLOSE, kernel, cv::Point(-1, -1), smooth_iterations); 37 | } 38 | } 39 | 40 | void fillHoles(cv::Mat& seg) 41 | { 42 | double min_val, max_val; 43 | cv::minMaxLoc(seg, &min_val, &max_val); 44 | cv::Mat holes = seg.clone(); 45 | cv::floodFill(holes, cv::Point2i(0, 0), cv::Scalar(max_val)); 46 | for (size_t i = 0; i < seg.total(); ++i) 47 | { 48 | if (holes.data[i] == 0) 49 | seg.data[i] = (unsigned char)max_val; 50 | } 51 | } 52 | 53 | void postprocessSegmentation(cv::Mat & seg, bool disconnected, 54 | bool holes, bool smooth, int smooth_iterations, int smooth_kernel_radius) 55 | { 56 | if (disconnected) removeSmallerComponents(seg); 57 | if (holes) fillHoles(seg); 58 | if (smooth) smoothFlaws(seg, smooth_iterations, smooth_kernel_radius); 59 | if (disconnected) removeSmallerComponents(seg); 60 | if (holes) fillHoles(seg); 61 | } 62 | 63 | } // namespace face_swap 64 | 65 | -------------------------------------------------------------------------------- /face_swap/face_swap_batch/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Target 2 | add_executable(face_swap_batch face_swap_batch.cpp) 3 | target_include_directories(face_swap_batch PRIVATE 4 | ${Boost_INCLUDE_DIRS} 5 | ) 6 | target_link_libraries(face_swap_batch PRIVATE 7 | face_swap 8 | ${Boost_LIBRARIES} 9 | ) 10 | 11 | # Installations 12 | install(TARGETS face_swap_batch EXPORT face_swap-targets DESTINATION bin COMPONENT app) 13 | install(FILES face_swap_batch.cfg DESTINATION bin COMPONENT app) 14 | -------------------------------------------------------------------------------- /face_swap/face_swap_batch/face_swap_batch.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /face_swap/face_swap_image/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Target 2 | add_executable(face_swap_image face_swap_image.cpp) 3 | target_include_directories(face_swap_image PRIVATE 4 | ${Boost_INCLUDE_DIRS} 5 | ) 6 | target_link_libraries(face_swap_image PRIVATE 7 | face_swap 8 | ${Boost_LIBRARIES} 9 | ) 10 | 11 | # Installations 12 | install(TARGETS face_swap_image EXPORT face_swap-targets DESTINATION bin COMPONENT app) 13 | install(FILES face_swap_image.cfg DESTINATION bin COMPONENT app) 14 | -------------------------------------------------------------------------------- /face_swap/face_swap_image/face_swap_image.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /face_swap/face_swap_image2video/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Target 2 | add_executable(face_swap_image2video face_swap_image2video.cpp) 3 | target_include_directories(face_swap_image2video PRIVATE 4 | ${Boost_INCLUDE_DIRS} 5 | ) 6 | target_link_libraries(face_swap_image2video PRIVATE 7 | face_swap 8 | ${Boost_LIBRARIES} 9 | ) 10 | 11 | # Installations 12 | install(TARGETS face_swap_image2video EXPORT face_swap-targets DESTINATION bin COMPONENT app) 13 | install(FILES face_swap_image2video.cfg DESTINATION bin COMPONENT app) 14 | -------------------------------------------------------------------------------- /face_swap/face_swap_image2video/face_swap_image2video.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 13 | reverse = 0 14 | cache = 1 15 | source_seg = 1 16 | target_seg = 1 17 | source_max_res = 500 18 | target_max_res = 500 19 | 20 | -------------------------------------------------------------------------------- /face_swap/face_swap_single2many/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Target 2 | add_executable(face_swap_single2many face_swap_single2many.cpp) 3 | target_include_directories(face_swap_single2many PRIVATE 4 | ${Boost_INCLUDE_DIRS} 5 | ) 6 | target_link_libraries(face_swap_single2many PRIVATE 7 | face_swap 8 | ${Boost_LIBRARIES} 9 | ) 10 | 11 | # Installations 12 | install(TARGETS face_swap_single2many EXPORT face_swap-targets DESTINATION bin COMPONENT app) 13 | install(FILES face_swap_single2many.cfg DESTINATION bin COMPONENT app) 14 | -------------------------------------------------------------------------------- /face_swap/face_swap_single2many/face_swap_single2many.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 13 | reverse = 0 14 | cache = 1 15 | source_seg = 1 16 | target_seg = 1 17 | source_max_res = 500 18 | target_max_res = 500 19 | 20 | -------------------------------------------------------------------------------- /face_swap/interfaces/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Load the package that SWIG uses to generate Python 2 | find_package(PythonLibs REQUIRED) 3 | find_package(Boost REQUIRED python3 numpy3) 4 | 5 | # Target 6 | add_library(face_swap_py SHARED face_swap_py.cpp) 7 | target_include_directories(face_swap_py PUBLIC 8 | ${PYTHON_INCLUDE_DIRS} 9 | ${Boost_INCLUDE_DIRS} 10 | ) 11 | target_link_libraries(face_swap_py 12 | face_swap 13 | ${PYTHON_LIBRARIES} 14 | ${Boost_LIBRARIES} 15 | ) 16 | message(${Boost_LIBRARIES}) 17 | message(${PYTHON_LIBRARIES}) 18 | set_target_properties(face_swap_py PROPERTIES 19 | PREFIX "" 20 | ) 21 | if(WIN32) 22 | set_target_properties(face_swap_py PROPERTIES 23 | SUFFIX ".pyd" 24 | ) 25 | endif() 26 | if(MSVC) 27 | set_target_properties(face_swap_py 28 | PROPERTIES 29 | ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" 30 | LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" 31 | RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" 32 | ) 33 | else() 34 | set_target_properties(face_swap_py 35 | PROPERTIES 36 | ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" 37 | LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" 38 | RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" 39 | ) 40 | endif() 41 | set_target_properties(face_swap_py PROPERTIES DEBUG_POSTFIX "") 42 | 43 | # Installations 44 | install(TARGETS face_swap_py 45 | EXPORT face_swap-targets 46 | RUNTIME DESTINATION interfaces/python COMPONENT python 47 | LIBRARY DESTINATION interfaces/python COMPONENT python 48 | ARCHIVE DESTINATION interfaces/python COMPONENT python) 49 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/BaselFace.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | class BaselFace{ 3 | public: 4 | 5 | /// ---- faces ---- 6 | static int BaselFace_faces_w; 7 | static int BaselFace_faces_h; 8 | static int* BaselFace_faces; 9 | 10 | /// ---- shapeMU ---- 11 | static int BaselFace_shapeMU_w; 12 | static int BaselFace_shapeMU_h; 13 | static float* BaselFace_shapeMU; 14 | 15 | /// ---- shapePC ---- 16 | static int BaselFace_shapePC_w; 17 | static int BaselFace_shapePC_h; 18 | static float* BaselFace_shapePC; 19 | 20 | /// ---- shapeEV ---- 21 | static int BaselFace_shapeEV_w; 22 | static int BaselFace_shapeEV_h; 23 | static float* BaselFace_shapeEV; 24 | 25 | /// ---- texMU ---- 26 | static int BaselFace_texMU_w; 27 | static int BaselFace_texMU_h; 28 | static float* BaselFace_texMU; 29 | 30 | /// ---- texPC ---- 31 | static int BaselFace_texPC_w; 32 | static int BaselFace_texPC_h; 33 | static float* BaselFace_texPC; 34 | 35 | /// ---- texEV ---- 36 | static int BaselFace_texEV_w; 37 | static int BaselFace_texEV_h; 38 | static float* BaselFace_texEV; 39 | 40 | /// ---- segbin ---- 41 | static int BaselFace_segbin_w; 42 | static int BaselFace_segbin_h; 43 | static char* BaselFace_segbin; 44 | 45 | /// ---- wparts ---- 46 | static int BaselFace_wparts_w; 47 | static int BaselFace_wparts_h; 48 | static float* BaselFace_wparts; 49 | 50 | /// ---- lmInd ---- 51 | static int BaselFace_lmInd_w; 52 | static int BaselFace_lmInd_h; 53 | static int* BaselFace_lmInd; 54 | 55 | /// ---- lmInd2 ---- 56 | static int BaselFace_lmInd2_w; 57 | static int BaselFace_lmInd2_h; 58 | static int* BaselFace_lmInd2; 59 | 60 | /// ---- keepV ---- 61 | static int BaselFace_keepV_w; 62 | static int BaselFace_keepV_h; 63 | static char* BaselFace_keepV; 64 | 65 | /// ---- faces_extra ---- 66 | static int BaselFace_faces_extra_w; 67 | static int BaselFace_faces_extra_h; 68 | static int* BaselFace_faces_extra; 69 | 70 | /// ---- mid ---- 71 | static int BaselFace_mid_w; 72 | static int BaselFace_mid_h; 73 | static char* BaselFace_mid; 74 | 75 | /// ---- texEdges ---- 76 | static int BaselFace_texEdges_w; 77 | static int BaselFace_texEdges_h; 78 | static int* BaselFace_texEdges; 79 | 80 | /// ---- canContour ---- 81 | static int BaselFace_canContour_w; 82 | static int BaselFace_canContour_h; 83 | static char* BaselFace_canContour; 84 | 85 | /// ---- keepVT ---- 86 | static int BaselFace_keepVT_w; 87 | static int BaselFace_keepVT_h; 88 | static int* BaselFace_keepVT; 89 | 90 | /// ---- pair ---- 91 | static int BaselFace_pair_w; 92 | static int BaselFace_pair_h; 93 | static int* BaselFace_pair; 94 | 95 | /// ---- pairKeepVT ---- 96 | static int BaselFace_pairKeepVT_w; 97 | static int BaselFace_pairKeepVT_h; 98 | static int* BaselFace_pairKeepVT; 99 | 100 | /// ---- vseg_bin ---- 101 | static int BaselFace_vseg_bin_w; 102 | static int BaselFace_vseg_bin_h; 103 | static char* BaselFace_vseg_bin; 104 | 105 | /// ---- indPX ---- 106 | static int BaselFace_indPX_w; 107 | static int BaselFace_indPX_h; 108 | static int* BaselFace_indPX; 109 | 110 | /// ---- indNX ---- 111 | static int BaselFace_indNX_w; 112 | static int BaselFace_indNX_h; 113 | static int* BaselFace_indNX; 114 | 115 | /// ---- symSPC ---- 116 | static int BaselFace_symSPC_w; 117 | static int BaselFace_symSPC_h; 118 | static float* BaselFace_symSPC; 119 | 120 | /// ---- symTPC ---- 121 | static int BaselFace_symTPC_w; 122 | static int BaselFace_symTPC_h; 123 | static float* BaselFace_symTPC; 124 | 125 | /// ---- expMU ---- 126 | static int BaselFace_expMU_w; 127 | static int BaselFace_expMU_h; 128 | static float* BaselFace_expMU; 129 | 130 | /// ---- expEV ---- 131 | static int BaselFace_expEV_w; 132 | static int BaselFace_expEV_h; 133 | static float* BaselFace_expEV; 134 | 135 | /// ---- expPC ---- 136 | static int BaselFace_expPC_w; 137 | static int BaselFace_expPC_h; 138 | static float* BaselFace_expPC; 139 | 140 | /// ---- expPCFlip ---- 141 | static int BaselFace_expPCFlip_w; 142 | static int BaselFace_expPCFlip_h; 143 | static float* BaselFace_expPCFlip; 144 | static bool load_BaselFace_data(const char* fname); 145 | }; 146 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/BaselFaceEstimator.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | //#include "cv.h" 4 | #include "highgui.h" 5 | //#include "FTModel.h" 6 | #include "BaselFace.h" 7 | 8 | class BaselFaceEstimator 9 | { 10 | cv::Mat coef2object(cv::Mat weight, cv::Mat MU, cv::Mat PCs, cv::Mat EV); 11 | cv::Mat coef2objectParts(cv::Mat &weight, cv::Mat &MU, cv::Mat &PCs, cv::Mat &EV); 12 | 13 | public: 14 | BaselFaceEstimator(/*std::string baselFile = ""*/); 15 | cv::Mat getFaces(); 16 | cv::Mat getFaces_fill(); 17 | cv::Mat getShape(cv::Mat weight, cv::Mat exprWeight = cv::Mat()); 18 | cv::Mat getShape2(cv::Mat weight, cv::Mat exprWeight = cv::Mat()); 19 | cv::Mat getTexture(cv::Mat weight); 20 | cv::Mat getTexture2(cv::Mat weight); 21 | cv::Mat getShapeParts(cv::Mat weight, cv::Mat exprWeight = cv::Mat()); 22 | cv::Mat getTextureParts(cv::Mat weight); 23 | cv::Mat getLM(cv::Mat shape, float yaw); 24 | cv::Mat getLMByAlpha(cv::Mat alpha, float yaw, std::vector inds, cv::Mat exprWeight = cv::Mat()); 25 | cv::Mat getLMByAlphaParts(cv::Mat alpha, float yaw, std::vector inds, cv::Mat exprWeight = cv::Mat()); 26 | cv::Mat getTriByAlpha(cv::Mat alpha, std::vector inds, cv::Mat exprWeight = cv::Mat()); 27 | cv::Mat getTriByAlphaParts(cv::Mat alpha, std::vector inds, cv::Mat exprWeight = cv::Mat()); 28 | cv::Mat getTriByBeta(cv::Mat beta, std::vector inds); 29 | cv::Mat getTriByBetaParts(cv::Mat beta, std::vector inds); 30 | void estimatePose3D0(cv::Mat landModel, cv::Mat landImage, cv::Mat k_m, cv::Mat &r, cv::Mat &t); 31 | void estimatePose3D(cv::Mat landModel, cv::Mat landImage, cv::Mat k_m, cv::Mat &r, cv::Mat &t); 32 | int* getLMIndices(int &count); 33 | 34 | //void estimatePose(cv::Mat landModel, cv::Mat lm, cv::Mat &A, cv::Mat &S, cv::Mat &R, cv::Mat &t); 35 | //void estimateT3D(cv::Mat landModel, cv::Mat landImage, int w, int h, cv::Mat R, float f, cv::Mat &t3D); 36 | //void fixSign(cv::Mat &R, cv::Mat &t); 37 | //cv::Mat estimateShape(cv::Mat landModel, cv::Mat landImage, cv::Mat A, cv::Mat t, int M); 38 | cv::Mat estimateShape3D(cv::Mat landModel, cv::Mat lm, cv::Mat k_m, cv::Mat r, cv::Mat t); 39 | 40 | ~BaselFaceEstimator(void); 41 | }; 42 | 43 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Source 2 | SET(IRIS_SFS_SOURCE 3 | utility.cpp 4 | epnp.cpp 5 | BaselFace.cpp 6 | BaselFaceEstimator.cpp 7 | FaceServices2.cpp 8 | ) 9 | 10 | SET(IRIS_SFS_HEADERS 11 | utility.h 12 | epnp.h 13 | BaselFace.h 14 | BaselFaceEstimator.h 15 | FaceServices2.h 16 | ) 17 | 18 | # Target 19 | add_library(iris_sfs STATIC ${IRIS_SFS_SOURCE} ${IRIS_SFS_HEADERS}) 20 | target_include_directories(iris_sfs PUBLIC 21 | $ 22 | ${OpenCV_INCLUDE_DIRS} 23 | ${EIGEN3_INCLUDE_DIR} 24 | ) 25 | target_link_libraries(iris_sfs PUBLIC 26 | ${OpenCV_LIBS} 27 | ) 28 | 29 | # Installations 30 | install(TARGETS iris_sfs 31 | EXPORT face_swap-targets 32 | RUNTIME DESTINATION bin COMPONENT dev 33 | LIBRARY DESTINATION lib COMPONENT dev 34 | ARCHIVE DESTINATION lib COMPONENT dev) 35 | #install(FILES ${IRIS_SFS_HEADERS} DESTINATION include) 36 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/FImRenderer.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | //Glew needs to be first 10 | //#include 11 | //#include 12 | //#include 13 | //#include 14 | //#include 15 | #include "FTModel.h" 16 | #include "FBRender.h" 17 | 18 | class FImRenderer 19 | { 20 | public: 21 | FBRender *render_; 22 | Face* face_; 23 | cv::Mat img_; 24 | float zNear, zFar; 25 | FImRenderer(cv::Mat img); 26 | void init(const cv::Mat& img); // Yuval 27 | void loadPLYFile(char* filename, bool useTexture = false); 28 | void loadMesh(cv::Mat shape, cv::Mat tex, cv::Mat faces); 29 | void computeTexture(float *rot, float *t, float f); 30 | void computeTexture(cv::Mat rVec, cv::Mat tVec, float f); 31 | void computeNormals(); 32 | void loadModel(char* meshFile = 0); 33 | void setIm(cv::Mat inIm); 34 | void mapRendering(float* r, float* t, float f, cv::Mat *im, cv::Mat *mask); 35 | void render( float *r, float *t, float f, cv::Mat &color, cv::Mat &depth ); 36 | void unmapRendering(); 37 | void resetRenderer(Face* face2, bool resetTexture = false, float *rot = 0, float *t = 0, float f = 1000); 38 | 39 | void copyFaces(cv::Mat faces); 40 | void copyShape(cv::Mat shape); 41 | void copyColors(cv::Mat colors); 42 | void copyNormals(cv::Mat normals); 43 | ~FImRenderer(void); 44 | }; 45 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/FTModel.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | //---------------------------------------------------------------------- 3 | // File: FTModel.h 4 | // Authors: Yann Dumortier (yann.dumortier@gmail.com), 5 | // Jongmoo Choi (jongmooc@usc.edu), 6 | // Sang-il Choi (csichoisi@gmail.net) 7 | // Description: 8 | // This file is part of the "Real-time 3D Face Tracking and Modeling Using a Webcam" 9 | // developed at the University of Southern California by: 10 | // 11 | // Yann Dumortier (yann.dumortier@gmail.com), 12 | // Jongmoo Choi (jongmooc@usc.edu), 13 | // Gerard Medioni (medioni@usc.edu). 14 | //---------------------------------------------------------------------- 15 | // Copyright (c) 2011 University of Southern California. All Rights Reserved. 16 | // 17 | 18 | 19 | 20 | #ifndef FTMODEL_H 21 | #define FTMODEL_H 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | #include "SimpleMesh.h" 34 | #include "utility.h" 35 | //#include "ASM.h" 36 | 37 | #define MODEL_SCALE 15.f 38 | #define MODEL_TX .0f 39 | #define MODEL_TY .3f 40 | #define MODEL_TZ .75f 41 | 42 | #define MAX_ERR 5.f 43 | #define MAX_LDMKS 85 44 | class StatiCam 45 | { 46 | public: 47 | StatiCam( float f=0.f, float cx=0.f, float cy=.0f ): f_( f ), cx_( cx ), cy_( cy ){} 48 | ~StatiCam(){} 49 | 50 | int calibIntWoRot( float cx, float cy, unsigned n, float *pts2D, float *pts3D, float *t ); 51 | int calibIntWoRot2( float cx, float cy, unsigned n, float *pts2D, float *pts3D, float *t ); 52 | 53 | public: 54 | float f_; 55 | float cx_, cy_; 56 | }; 57 | 58 | class Face 59 | { 60 | public: 61 | Face( unsigned id=0 ); 62 | Face( Face &f, unsigned id=0 ); 63 | ~Face(); 64 | 65 | float& tx() { return t_[0]; } 66 | float& ty() { return t_[1]; } 67 | float& tz() { return t_[2]; } 68 | 69 | float& rx() { return R_[0]; } 70 | float& ry() { return R_[1]; } 71 | float& rz() { return R_[2]; } 72 | 73 | int loadPLYModel( const char* fileName ); 74 | int loadPLYModel2( const char* fileName ); 75 | int loadPLYLandmarks( const char* fileName ); 76 | void loadMesh( cv::Mat shape, cv::Mat tex, cv::Mat faces ); 77 | void savePLYModel( const char* fileName ); 78 | bool estimateNormals( ); 79 | private: 80 | void triangleNormalFromVertex(int face_id, int vertex_id, float &nx, float &ny, float &nz); 81 | int invalidPLYFile() 82 | { 83 | std::cerr << "Invalid PLY file.\n"; 84 | return -1; 85 | } 86 | 87 | public: 88 | unsigned id_; //Face ID, 0 means generic face 89 | 90 | SimpleMesh mesh_; //3D Face mesh 91 | 92 | int landmarks_[MAX_LDMKS]; //Face landmark vertices' index 93 | unsigned nLdmks_; //Number of landmark 94 | 95 | float *R_; 96 | float *t_; 97 | }; 98 | 99 | class CylCoord { 100 | public: 101 | uchar r, g, b; 102 | float depth, theta, y_cyl; 103 | 104 | CylCoord() { 105 | r= g= b= depth= theta= y_cyl = 0; 106 | } 107 | 108 | CylCoord( uchar r, uchar g, uchar b, float depth, float theta, float y_cyl ) { 109 | this->r = r; 110 | this->g = g; 111 | this->b = b; 112 | 113 | this->depth = depth; 114 | this->theta = theta; 115 | this->y_cyl = y_cyl; 116 | } 117 | 118 | }; 119 | 120 | #endif /*FACEMODEL_H*/ 121 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/FaceServices2.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | #include "cv.h" 4 | #include "highgui.h" 5 | #include "BaselFaceEstimator.h" 6 | #include 7 | #include 8 | 9 | #define NUM_EXTRA_FEATURES 1 10 | #define FEATURES_LANDMARK 0 11 | #define REG_FROM_PREV 100 12 | #define REG_FROM_CURR 0 13 | 14 | typedef Eigen::SparseMatrix SpMat; // declares a column-major sparse matrix type of double 15 | typedef Eigen::Triplet SpT; 16 | typedef std::vector > IndWeight; 17 | 18 | #define RENDER_PARAMS_COUNT 21 19 | 20 | #define RENDER_PARAMS_R 0 21 | #define RENDER_PARAMS_T 3 22 | #define RENDER_PARAMS_AMBIENT 6 23 | #define RENDER_PARAMS_DIFFUSE 9 24 | #define RENDER_PARAMS_LDIR 12 25 | #define RENDER_PARAMS_CONTRAST 14 26 | #define RENDER_PARAMS_GAIN 15 27 | #define RENDER_PARAMS_OFFSET 18 28 | #define RENDER_PARAMS_SPECULAR 21 29 | #define RENDER_PARAMS_SHINENESS 24 30 | 31 | #define RENDER_PARAMS_AMBIENT_DEFAULT 0.5f 32 | #define RENDER_PARAMS_DIFFUSE_DEFAULT 0.5f 33 | #define RENDER_PARAMS_CONTRAST_DEFAULT 1.0f 34 | #define RENDER_PARAMS_GAIN_DEFAULT 1.0f 35 | #define RENDER_PARAMS_OFFSET_DEFAULT 0.0f 36 | #define RENDER_PARAMS_SPECULAR_DEFAULT 80.0f 37 | #define RENDER_PARAMS_SHINENESS_DEFAULT 16.0f 38 | 39 | typedef struct BFMParams { 40 | float sI; 41 | float sF[NUM_EXTRA_FEATURES]; 42 | float sR[RENDER_PARAMS_COUNT]; 43 | float initR[RENDER_PARAMS_COUNT]; 44 | bool doOptimize[RENDER_PARAMS_COUNT]; 45 | bool optimizeAB[2]; 46 | bool optimizeExpr; 47 | bool computeEI; 48 | float sExpr; 49 | cv::Mat hessDiag; 50 | cv::Mat gradVec; 51 | 52 | void init(){ 53 | memset(doOptimize,false,sizeof(bool)*RENDER_PARAMS_COUNT); 54 | optimizeAB[0] = false; 55 | optimizeAB[1] = false; 56 | optimizeExpr = true; 57 | computeEI = false; 58 | sExpr = 1; 59 | for (int i=0;i<6;i++) doOptimize[i] = true; 60 | 61 | if (RENDER_PARAMS_AMBIENT < RENDER_PARAMS_COUNT){ 62 | for (int i=0;i<3;i++) initR[RENDER_PARAMS_AMBIENT+i] = RENDER_PARAMS_AMBIENT_DEFAULT; 63 | } 64 | if (RENDER_PARAMS_DIFFUSE < RENDER_PARAMS_COUNT){ 65 | for (int i=0;i<3;i++) initR[RENDER_PARAMS_DIFFUSE+i] = 0.0f; 66 | } 67 | if (RENDER_PARAMS_LDIR < RENDER_PARAMS_COUNT){ 68 | for (int i=0;i<2;i++) initR[RENDER_PARAMS_LDIR+i] = 0.0f; 69 | } 70 | if (RENDER_PARAMS_CONTRAST < RENDER_PARAMS_COUNT){ 71 | initR[RENDER_PARAMS_CONTRAST] = RENDER_PARAMS_CONTRAST_DEFAULT; 72 | } 73 | if (RENDER_PARAMS_GAIN < RENDER_PARAMS_COUNT){ 74 | for (int i=0;i<3;i++) initR[RENDER_PARAMS_GAIN+i] = RENDER_PARAMS_GAIN_DEFAULT; 75 | } 76 | if (RENDER_PARAMS_OFFSET < RENDER_PARAMS_COUNT){ 77 | for (int i=0;i<3;i++) initR[RENDER_PARAMS_OFFSET+i] = RENDER_PARAMS_OFFSET_DEFAULT; 78 | } 79 | if (RENDER_PARAMS_SPECULAR < RENDER_PARAMS_COUNT){ 80 | for (int i=0;i<3;i++) initR[RENDER_PARAMS_SPECULAR+i] = RENDER_PARAMS_SPECULAR_DEFAULT; 81 | } 82 | if (RENDER_PARAMS_SHINENESS < RENDER_PARAMS_COUNT){ 83 | initR[RENDER_PARAMS_SHINENESS] = RENDER_PARAMS_SHINENESS_DEFAULT; 84 | } 85 | 86 | // sR 87 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_R+i] = (M_PI/6)*(M_PI/6); 88 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_T+i] = 900.0f; 89 | if (RENDER_PARAMS_AMBIENT < RENDER_PARAMS_COUNT){ 90 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_AMBIENT+i] = 1; 91 | } 92 | if (RENDER_PARAMS_DIFFUSE < RENDER_PARAMS_COUNT){ 93 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_DIFFUSE+i] = 1; 94 | } 95 | if (RENDER_PARAMS_LDIR < RENDER_PARAMS_COUNT){ 96 | for (int i=0;i<2;i++) sR[RENDER_PARAMS_LDIR+i] = M_PI*M_PI; 97 | } 98 | if (RENDER_PARAMS_CONTRAST < RENDER_PARAMS_COUNT){ 99 | sR[RENDER_PARAMS_CONTRAST] = 1; 100 | } 101 | if (RENDER_PARAMS_GAIN < RENDER_PARAMS_COUNT){ 102 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_GAIN+i] = 4.0f; 103 | } 104 | if (RENDER_PARAMS_OFFSET < RENDER_PARAMS_COUNT){ 105 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_OFFSET+i] = 10000.0f; 106 | } 107 | if (RENDER_PARAMS_SPECULAR < RENDER_PARAMS_COUNT){ 108 | for (int i=0;i<3;i++) sR[RENDER_PARAMS_SPECULAR+i] = 10000.0f; 109 | } 110 | if (RENDER_PARAMS_SHINENESS < RENDER_PARAMS_COUNT){ 111 | sR[RENDER_PARAMS_SHINENESS] = 1000000.0f; 112 | } 113 | } 114 | } BFMParams; 115 | 116 | 117 | class FaceServices2 118 | { 119 | float _k[9]; 120 | cv::Mat faces, shape, tex; 121 | BaselFaceEstimator festimator; 122 | 123 | float prevEF; 124 | float cEF; 125 | float mstep; 126 | int countFail; 127 | float maxVal; 128 | float mlambda; 129 | float PREV_USE_THRESH; 130 | 131 | public: 132 | FaceServices2(void); 133 | void setUp(int w, int h, float f); 134 | void init(int w, int h, float f = 1000.0f); // Yuval 135 | float updateHessianMatrix(bool part, cv::Mat alpha, float* renderParams, cv::Mat faces, cv::Mat colorIm,std::vector lmInds, cv::Mat landIm, BFMParams ¶ms, cv::Mat &prevR, cv::Mat &prevT, cv::Mat exprW = cv::Mat() ); 136 | cv::Mat computeGradient(bool part, cv::Mat alpha, float* renderParams, cv::Mat faces,cv::Mat colorIm,std::vector lmInds, cv::Mat landIm, BFMParams ¶ms,std::vector &inds, cv::Mat exprW, cv::Mat &prevR, cv::Mat &prevT); 137 | void sno_step2(bool part, cv::Mat &alpha, float* renderParams, cv::Mat faces,cv::Mat colorIm,std::vector lmInds, cv::Mat landIm, BFMParams ¶ms, cv::Mat &exprW, cv::Mat &prevR, cv::Mat &prevT); 138 | float line_search(bool part, cv::Mat &alpha, float* renderParams, cv::Mat &dirMove,std::vector inds, cv::Mat faces,cv::Mat colorIm,std::vector lmInds, cv::Mat landIm, BFMParams ¶ms, cv::Mat &exprW, cv::Mat &prevR, cv::Mat &prevT, int maxIters = 4); 139 | float computeCost(float vEF, cv::Mat &alpha, float* renderParams, BFMParams ¶ms, cv::Mat &exprW, cv::Mat &prevR, cv::Mat &prevT ); 140 | 141 | float eF(bool part, cv::Mat alpha, std::vector inds, cv::Mat landIm, float* renderParams, cv::Mat exprW); 142 | bool loadReference(std::string refDir, std::string model_file, cv::Mat &alpha, cv::Mat &beta, float* renderParams, int &M, cv::Mat &exprW, int &EM); 143 | bool loadReference2(std::string refDir, std::string model_file, cv::Mat &alpha, cv::Mat &beta, int &M); 144 | 145 | void initRenderer(cv::Mat &colorIm); 146 | void mergeIm(cv::Mat* output,cv::Mat bg,cv::Mat depth); 147 | ~FaceServices2(void); 148 | 149 | bool estimatePoseExpr(cv::Mat colorIm, cv::Mat lms, cv::Mat alpha, cv::Mat &vecR, cv::Mat &vecT, cv::Mat& K, cv::Mat &exprWeightse, const char* outputDir, bool with_expr = true); 150 | bool updatePoseExpr(cv::Mat colorIm, cv::Mat lms, cv::Mat alpha, cv::Mat &vecR, cv::Mat &vecT, cv::Mat &exprWeightse, char* outputDir, cv::Mat &prevR, cv::Mat &prevT); 151 | 152 | void nextMotion(int &currFrame, cv::Mat &vecR, cv::Mat &vecT, cv::Mat &exprWeights); 153 | }; 154 | 155 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/MeshModel.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "utility.h" 11 | //#include 12 | //#include 13 | 14 | #define MESH_COLOR 1 15 | #define MESH_NORMAL 2 16 | 17 | #define PROP_X 0 18 | #define PROP_Y 1 19 | #define PROP_Z 2 20 | #define PROP_R 3 21 | #define PROP_G 4 22 | #define PROP_B 5 23 | #define PROP_NX 6 24 | #define PROP_NY 7 25 | #define PROP_NZ 8 26 | 27 | using Eigen::Matrix3Xd; 28 | 29 | class MeshModel 30 | { 31 | public: 32 | int nVertices; 33 | int nFaces; 34 | int type; 35 | 36 | float* vertices_; 37 | int* faces_; 38 | unsigned char* colors_; 39 | float* normals_; 40 | 41 | MeshModel(char* ply_file); 42 | MeshModel(Matrix3Xd m); 43 | MeshModel(); 44 | //MeshModel(pcl::PointCloud::Ptr cloud); 45 | //MeshModel(pcl::PointCloud::Ptr cloud); 46 | bool save2File(char* ply_file ); 47 | 48 | //void toPointCloud(pcl::PointCloud::Ptr cloud_ptr); 49 | Matrix3Xd* toMatrix3Xd(); 50 | //void toPointCloud(pcl::PointCloud::Ptr cloud_ptr); 51 | //void copyNormals(pcl::PointCloud::Ptr normals); 52 | //void copyMesh(pcl::PolygonMesh mesh); 53 | //pcl::PointXYZ centerPoint(); 54 | void translate(double tx, double ty, double tz); 55 | ~MeshModel(void); 56 | }; 57 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/RenderModel.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | #include "cv.h" 4 | #include "highgui.h" 5 | 6 | #define RENDER_PARAMS_COUNT 21 7 | 8 | #define RENDER_PARAMS_R 0 9 | #define RENDER_PARAMS_T 3 10 | #define RENDER_PARAMS_AMBIENT 6 11 | #define RENDER_PARAMS_DIFFUSE 9 12 | #define RENDER_PARAMS_LDIR 12 13 | #define RENDER_PARAMS_CONTRAST 14 14 | #define RENDER_PARAMS_GAIN 15 15 | #define RENDER_PARAMS_OFFSET 18 16 | #define RENDER_PARAMS_SPECULAR 21 17 | #define RENDER_PARAMS_SHINENESS 24 18 | 19 | #define RENDER_PARAMS_AMBIENT_DEFAULT 0.5f 20 | #define RENDER_PARAMS_DIFFUSE_DEFAULT 0.5f 21 | #define RENDER_PARAMS_CONTRAST_DEFAULT 1.0f 22 | #define RENDER_PARAMS_GAIN_DEFAULT 1.0f 23 | #define RENDER_PARAMS_OFFSET_DEFAULT 0.0f 24 | #define RENDER_PARAMS_SPECULAR_DEFAULT 80.0f 25 | #define RENDER_PARAMS_SHINENESS_DEFAULT 16.0f 26 | 27 | class RenderServices 28 | { 29 | bool triangleNormalFromVertex(cv::Mat shape, cv::Mat faces, int face_id, int vertex_id, float &nx, float &ny, float &nz); 30 | float triangleNormal(cv::Mat shape, cv::Mat faces, int face_id, float &nx, float &ny, float &nz); 31 | public: 32 | bool estimateFaceNormals(cv::Mat shape, cv::Mat faces, cv::Mat &normals); 33 | bool estimateFaceNormals(cv::Mat shape, cv::Mat faces, cv::Mat &normals, cv::Mat ¢ers, cv::Mat &areas); 34 | bool estimateVertexNormals(cv::Mat shape, cv::Mat faces, cv::Mat &normals); 35 | 36 | bool estimatePointColor(cv::Mat &points, cv::Mat &texs, cv::Mat &normals, std::vector &inds, cv::Mat &visible, cv::Mat &noShadow, float* render_model, cv::Mat &colors); 37 | bool estimatePointColor(cv::Mat &points, cv::Mat &texs, cv::Mat &normals, std::vector &inds, cv::Mat &noShadow, float* render_model, cv::Mat &colors); 38 | bool estimateColor(cv::Mat shape, cv::Mat tex, cv::Mat faces, bool* visible, bool* noShadow, float* render_model, cv::Mat &colors); 39 | bool estimateColor(cv::Mat shape, cv::Mat tex, cv::Mat faces, bool* visible, bool* noShadow, float* render_model, cv::Mat &colors, cv::Mat &normals); 40 | }; 41 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/epnp.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef epnp_h 3 | #define epnp_h 4 | 5 | #include 6 | 7 | class epnp { 8 | public: 9 | epnp(void); 10 | ~epnp(); 11 | 12 | void set_internal_parameters(const double uc, const double vc, 13 | const double fu, const double fv); 14 | 15 | void set_maximum_number_of_correspondences(const int n); 16 | void reset_correspondences(void); 17 | void add_correspondence(const double X, const double Y, const double Z, 18 | const double u, const double v); 19 | 20 | double compute_pose(double R[3][3], double T[3]); 21 | 22 | void relative_error(double & rot_err, double & transl_err, 23 | const double Rtrue[3][3], const double ttrue[3], 24 | const double Rest[3][3], const double test[3]); 25 | 26 | void print_pose(const double R[3][3], const double t[3]); 27 | double reprojection_error(const double R[3][3], const double t[3]); 28 | int qr_solve(CvMat * A, CvMat * b, CvMat * X); 29 | 30 | private: 31 | void choose_control_points(void); 32 | void compute_barycentric_coordinates(void); 33 | void fill_M(CvMat * M, const int row, const double * alphas, const double u, const double v); 34 | void compute_ccs(const double * betas, const double * ut); 35 | void compute_pcs(void); 36 | 37 | void solve_for_sign(void); 38 | 39 | void find_betas_approx_1(const CvMat * L_6x10, const CvMat * Rho, double * betas); 40 | void find_betas_approx_2(const CvMat * L_6x10, const CvMat * Rho, double * betas); 41 | void find_betas_approx_3(const CvMat * L_6x10, const CvMat * Rho, double * betas); 42 | 43 | double dot(const double * v1, const double * v2); 44 | double dist2(const double * p1, const double * p2); 45 | 46 | void compute_rho(double * rho); 47 | void compute_L_6x10(const double * ut, double * l_6x10); 48 | 49 | void gauss_newton(const CvMat * L_6x10, const CvMat * Rho, double current_betas[4]); 50 | void compute_A_and_b_gauss_newton(const double * l_6x10, const double * rho, 51 | double cb[4], CvMat * A, CvMat * b); 52 | 53 | double compute_R_and_t(const double * ut, const double * betas, 54 | double R[3][3], double t[3]); 55 | 56 | void estimate_R_and_t(double R[3][3], double t[3]); 57 | 58 | void copy_R_and_t(const double R_dst[3][3], const double t_dst[3], 59 | double R_src[3][3], double t_src[3]); 60 | 61 | void mat_to_quat(const double R[3][3], double q[4]); 62 | 63 | 64 | double uc, vc, fu, fv; 65 | 66 | double * pws, * us, * alphas, * pcs; 67 | int maximum_number_of_correspondences; 68 | int number_of_correspondences; 69 | 70 | double cws[4][3], ccs[4][3]; 71 | double cws_determinant; 72 | }; 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /face_swap/iris_sfs/utility.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2015 USC, IRIS, Computer vision Lab */ 2 | #pragma once 3 | //#include "cv.h" 4 | #include "highgui.h" 5 | #include 6 | #include 7 | #include 8 | 9 | int splittext(char* str, char** pos); 10 | 11 | Eigen::Matrix3Xd* toMatrix3Xd(cv::Mat mat); 12 | cv::Mat toMat(Eigen::Matrix3Xd emat); 13 | void qr(cv::Mat input, cv::Mat &q, cv::Mat &r); 14 | 15 | cv::Vec3b avSubMatValue8UC3( const CvPoint2D64f* pt, const cv::Mat* mat ); 16 | cv::Vec3d avSubMatValue8UC3_2( const CvPoint2D64f* pt, const cv::Mat* mat ); 17 | 18 | double avSubPixelValue64F( const CvPoint2D64f* pt, const IplImage* img ); 19 | 20 | double avSubPixelValue32F( const CvPoint2D64f* pt, const IplImage* img ); 21 | 22 | double avSubPixelValue8U( const CvPoint2D64f* pt, const IplImage* img ); 23 | 24 | double avSubMatValue64F( const CvPoint2D64f* pt, const cv::Mat* mat ); 25 | 26 | double avSubMatValue32F( const CvPoint2D64f* pt, const cv::Mat* mat ); 27 | 28 | double avSubMatValue8U( const CvPoint2D64f* pt, const cv::Mat* mat ); 29 | 30 | void write_ply4(char* outname, cv::Mat mat_Depth,std::vector colors,std::vector faces); 31 | 32 | void write_ply(char* outname, std::vector points); 33 | void write_ply(char* outname, cv::Mat mat_Depth, std::vector colors); 34 | void write_ply(char* outname, cv::Mat mat_Depth); 35 | void write_ply(char* outname, cv::Mat mat_Depth, cv::Mat mat_Faces); 36 | void write_ply(char* outname, cv::Mat mat_Depth, cv::Mat mat_Color, cv::Mat mat_Faces); 37 | void write_ply(char* outname, cv::Mat mat_Depth,std::vector colors,cv::Mat faces); 38 | 39 | //void write_ply(char* outname, bool* visible, cv::Mat mat_Depth,std::vector colors,cv::Mat faces); 40 | void write_ply(char* outname, int count, bool* visible, float* points_); 41 | void write_ply(char* outname, int count, float* points_); 42 | void write_plyF(char* outname, cv::Mat mat_Depth,std::vector colors,int nFaces, unsigned* faces); 43 | void write_plyFloat(char* outname, cv::Mat mat_Depth); 44 | void write_plyFloat(char* outname, cv::Mat mat_Depth, cv::Mat mat_tex, cv::Mat faces); 45 | 46 | void write_ply(char* outname, Eigen::Matrix3Xd*); 47 | //void write_ply(char* outname, int count, float* points_, float* colors_); 48 | cv::Mat skew(cv::Mat v1); 49 | void groundScale(cv::Mat input, cv::Mat &output, float bgThresh, float gapPc); 50 | cv::Mat findRotation(cv::Mat v1, cv::Mat v2); 51 | -------------------------------------------------------------------------------- /face_swap/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Find all test cpp source files 2 | file(GLOB TEST_CPP_SRC RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" 3 | "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") 4 | 5 | # For each test cpp source file 6 | foreach(TEST_FILE_NAME ${TEST_CPP_SRC}) 7 | message(STATUS "Processing test file: ${TEST_FILE_NAME}") 8 | 9 | # Generate target name 10 | string(REGEX REPLACE ".cpp\$" "" TEST_NAME "${TEST_FILE_NAME}") 11 | message(STATUS "Adding test target: ${TEST_NAME}") 12 | 13 | # Target 14 | add_executable(${TEST_NAME} ${TEST_FILE_NAME}) 15 | target_include_directories(${TEST_NAME} PRIVATE 16 | ${Boost_INCLUDE_DIRS} 17 | ) 18 | target_link_libraries(${TEST_NAME} PRIVATE 19 | face_swap 20 | ${Boost_LIBRARIES} 21 | ) 22 | set_property(TARGET ${TEST_NAME} PROPERTY FOLDER "tests") 23 | 24 | # Installations 25 | install(TARGETS ${TEST_NAME} EXPORT face_swap-targets DESTINATION tests COMPONENT tests) 26 | 27 | endforeach(TEST_FILE_NAME) -------------------------------------------------------------------------------- /face_swap/tests/test_flip.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /face_swap/tests/test_landmarks.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat -------------------------------------------------------------------------------- /face_swap/tests/test_landmarks.cpp: -------------------------------------------------------------------------------- 1 | #include "face_swap/face_detection_landmarks.h" 2 | #include "face_swap/landmarks_utilities.h" 3 | 4 | // std 5 | #include 6 | #include 7 | #include 8 | 9 | // Boost 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | // OpenCV 17 | #include 18 | #include 19 | #include 20 | 21 | using std::cout; 22 | using std::endl; 23 | using std::cerr; 24 | using std::string; 25 | using std::runtime_error; 26 | using namespace boost::program_options; 27 | using namespace boost::filesystem; 28 | 29 | const std::string IMAGE_FILTER = 30 | "(.*\\.(bmp|dib|jpeg|jpg|jpe|jp2|png|pbm|pgm|ppm|sr|ras))"; 31 | 32 | 33 | void getImagesFromDir(const std::string& dir_path, std::vector& img_paths) 34 | { 35 | boost::regex filter(IMAGE_FILTER); 36 | boost::smatch what; 37 | directory_iterator end_itr; // Default ctor yields past-the-end 38 | for (directory_iterator it(dir_path); it != end_itr; ++it) 39 | { 40 | // Skip if not a file 41 | if (!boost::filesystem::is_regular_file(it->status())) continue; 42 | 43 | // Get extension 44 | std::string ext = it->path().extension().string(); 45 | std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); 46 | 47 | // Skip if no match 48 | if (!boost::regex_match(ext, what, filter)) continue; 49 | 50 | img_paths.push_back(it->path().string()); 51 | } 52 | } 53 | 54 | void parseInputPaths(const std::vector& input_paths, std::vector& img_paths) 55 | { 56 | boost::regex filter(IMAGE_FILTER); 57 | boost::smatch what; 58 | 59 | // For each input path 60 | for (size_t i = 0; i < input_paths.size(); ++i) 61 | { 62 | const string& input_path = input_paths[i]; 63 | 64 | // Get extension 65 | std::string ext = path(input_path).extension().string(); 66 | std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); 67 | 68 | if (is_regular_file(input_path) && boost::regex_match(ext, what, filter)) 69 | img_paths.push_back(input_path); 70 | else if (is_directory(input_path)) 71 | getImagesFromDir(input_path, img_paths); 72 | else throw error("input must be contain paths to image files or input directories!"); 73 | } 74 | } 75 | 76 | int main(int argc, char* argv[]) 77 | { 78 | // Parse command line arguments 79 | std::vector input_paths; 80 | string output_path, landmarks_path, cfg_path; 81 | bool preview; 82 | try { 83 | options_description desc("Allowed options"); 84 | desc.add_options() 85 | ("help,h", "display the help message") 86 | ("input,i", value>(&input_paths)->required(), "input paths") 87 | ("output,o", value(&output_path), "output directory") 88 | ("landmarks,l", value(&landmarks_path)->required(), "path to landmarks model") 89 | ("preview,p", value(&preview)->default_value(true), "toggle preview") 90 | ("cfg", value(&cfg_path)->default_value("test_landmarks.cfg"), "configuration file (.cfg)") 91 | ; 92 | variables_map vm; 93 | store(command_line_parser(argc, argv).options(desc). 94 | positional(positional_options_description().add("input", -1)).run(), vm); 95 | 96 | if (vm.count("help")) { 97 | cout << "Usage: test_landmarks [options]" << endl; 98 | cout << desc << endl; 99 | exit(0); 100 | } 101 | 102 | // Read config file 103 | std::ifstream ifs(vm["cfg"].as()); 104 | store(parse_config_file(ifs, desc), vm); 105 | 106 | notify(vm); 107 | 108 | if (!output_path.empty() && !is_directory(output_path)) 109 | throw error("output must be a path to a directory!"); 110 | if (!is_regular_file(landmarks_path)) 111 | throw error("landmarks must be a path to a file!"); 112 | } 113 | catch (const error& e) { 114 | cerr << "Error while parsing command-line arguments: " << e.what() << endl; 115 | cerr << "Use --help to display a list of options." << endl; 116 | exit(1); 117 | } 118 | 119 | try 120 | { 121 | // Parse image paths 122 | std::vector img_paths; 123 | parseInputPaths(input_paths, img_paths); 124 | 125 | // Initialize face detection and landmarks 126 | std::shared_ptr lms = 127 | face_swap::FaceDetectionLandmarks::create(landmarks_path); 128 | 129 | // Initialize timer 130 | boost::timer::cpu_timer timer; 131 | float det_delta_time = 0.0f, lms_delta_time = 0.0f; 132 | 133 | // For each image 134 | for (const string& img_path : img_paths) 135 | { 136 | std::cout << "Processing " << path(img_path).filename() << "..." << std::endl; 137 | 138 | // Read input image 139 | cv::Mat img = cv::imread(img_path); 140 | 141 | // Start measuring time 142 | timer.start(); 143 | 144 | // Detect faces and extract landmarks 145 | std::vector faces; 146 | lms->process(img, faces); 147 | 148 | // Stop measuring time 149 | timer.stop(); 150 | 151 | // Print current timing statistics 152 | lms_delta_time += (timer.elapsed().wall*1.0e-9 - lms_delta_time)*0.1f; 153 | std::cout << "Landmarks timing = " << lms_delta_time << "s (" << 154 | (1.0f / lms_delta_time) << " fps)" << std::endl; 155 | 156 | if(preview) 157 | { 158 | // Render landmarks 159 | cv::Mat out = img.clone(); 160 | face_swap::render(out, faces); 161 | cv::imshow("test_landmarks", out); 162 | cv::waitKey(1); 163 | } 164 | } 165 | } 166 | catch (std::exception& e) 167 | { 168 | cerr << e.what() << endl; 169 | return 1; 170 | } 171 | 172 | return 0; 173 | } 174 | 175 | -------------------------------------------------------------------------------- /face_swap/tests/test_memory.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /face_swap/tests/test_resolution.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /face_swap/tests/test_resolution_batch.cfg: -------------------------------------------------------------------------------- 1 | landmarks = ../data/shape_predictor_68_face_landmarks.dat 2 | model_3dmm_h5 = ../data/BaselFaceModel_mod_wForehead_noEars.h5 3 | model_3dmm_dat = ../data/BaselFace.dat 4 | reg_model = ../data/3dmm_cnn_resnet_101.caffemodel 5 | reg_deploy = ../data/3dmm_cnn_resnet_101_deploy.prototxt 6 | reg_mean = ../data/3dmm_cnn_resnet_101_mean.binaryproto 7 | seg_model = ../data/face_seg_fcn8s.caffemodel 8 | seg_deploy = ../data/face_seg_fcn8s_deploy.prototxt 9 | generic = 0 10 | expressions = 1 11 | gpu = 1 12 | gpu_id = 0 -------------------------------------------------------------------------------- /frontend/faceswap/.buckconfig: -------------------------------------------------------------------------------- 1 | 2 | [android] 3 | target = Google Inc.:Google APIs:23 4 | 5 | [maven_repositories] 6 | central = https://repo1.maven.org/maven2 7 | -------------------------------------------------------------------------------- /frontend/faceswap/.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | ; We fork some components by platform 3 | .*/*[.]android.js 4 | 5 | ; Ignore "BUCK" generated dirs 6 | /\.buckd/ 7 | 8 | ; Ignore unexpected extra "@providesModule" 9 | .*/node_modules/.*/node_modules/fbjs/.* 10 | 11 | ; Ignore duplicate module providers 12 | ; For RN Apps installed via npm, "Libraries" folder is inside 13 | ; "node_modules/react-native" but in the source repo it is in the root 14 | .*/Libraries/react-native/React.js 15 | 16 | ; Ignore polyfills 17 | .*/Libraries/polyfills/.* 18 | 19 | ; Ignore metro 20 | .*/node_modules/metro/.* 21 | 22 | [include] 23 | 24 | [libs] 25 | node_modules/react-native/Libraries/react-native/react-native-interface.js 26 | node_modules/react-native/flow/ 27 | 28 | [options] 29 | emoji=true 30 | 31 | esproposal.optional_chaining=enable 32 | esproposal.nullish_coalescing=enable 33 | 34 | module.system=haste 35 | module.system.haste.use_name_reducers=true 36 | # get basename 37 | module.system.haste.name_reducers='^.*/\([a-zA-Z0-9$_.-]+\.js\(\.flow\)?\)$' -> '\1' 38 | # strip .js or .js.flow suffix 39 | module.system.haste.name_reducers='^\(.*\)\.js\(\.flow\)?$' -> '\1' 40 | # strip .ios suffix 41 | module.system.haste.name_reducers='^\(.*\)\.ios$' -> '\1' 42 | module.system.haste.name_reducers='^\(.*\)\.android$' -> '\1' 43 | module.system.haste.name_reducers='^\(.*\)\.native$' -> '\1' 44 | module.system.haste.paths.blacklist=.*/__tests__/.* 45 | module.system.haste.paths.blacklist=.*/__mocks__/.* 46 | module.system.haste.paths.blacklist=/node_modules/react-native/Libraries/Animated/src/polyfills/.* 47 | module.system.haste.paths.whitelist=/node_modules/react-native/Libraries/.* 48 | 49 | munge_underscores=true 50 | 51 | module.name_mapper='^[./a-zA-Z0-9$_-]+\.\(bmp\|gif\|jpg\|jpeg\|png\|psd\|svg\|webp\|m4v\|mov\|mp4\|mpeg\|mpg\|webm\|aac\|aiff\|caf\|m4a\|mp3\|wav\|html\|pdf\)$' -> 'RelativeImageStub' 52 | 53 | module.file_ext=.js 54 | module.file_ext=.jsx 55 | module.file_ext=.json 56 | module.file_ext=.native.js 57 | 58 | suppress_type=$FlowIssue 59 | suppress_type=$FlowFixMe 60 | suppress_type=$FlowFixMeProps 61 | suppress_type=$FlowFixMeState 62 | 63 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixMe\\($\\|[^(]\\|(\\(\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\) 64 | suppress_comment=\\(.\\|\n\\)*\\$FlowIssue\\((\\(\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\)?:? #[0-9]+ 65 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixedInNextDeploy 66 | suppress_comment=\\(.\\|\n\\)*\\$FlowExpectedError 67 | 68 | [version] 69 | ^0.92.0 70 | -------------------------------------------------------------------------------- /frontend/faceswap/.gitattributes: -------------------------------------------------------------------------------- 1 | *.pbxproj -text 2 | -------------------------------------------------------------------------------- /frontend/faceswap/.gitignore: -------------------------------------------------------------------------------- 1 | # OSX 2 | # 3 | .DS_Store 4 | 5 | # Xcode 6 | # 7 | build/ 8 | *.pbxuser 9 | !default.pbxuser 10 | *.mode1v3 11 | !default.mode1v3 12 | *.mode2v3 13 | !default.mode2v3 14 | *.perspectivev3 15 | !default.perspectivev3 16 | xcuserdata 17 | *.xccheckout 18 | *.moved-aside 19 | DerivedData 20 | *.hmap 21 | *.ipa 22 | *.xcuserstate 23 | project.xcworkspace 24 | 25 | # Android/IntelliJ 26 | # 27 | build/ 28 | .idea 29 | .gradle 30 | local.properties 31 | *.iml 32 | 33 | # node.js 34 | # 35 | node_modules/ 36 | npm-debug.log 37 | yarn-error.log 38 | 39 | # BUCK 40 | buck-out/ 41 | \.buckd/ 42 | *.keystore 43 | 44 | # fastlane 45 | # 46 | # It is recommended to not store the screenshots in the git repo. Instead, use fastlane to re-generate the 47 | # screenshots whenever they are needed. 48 | # For more information about the recommended setup visit: 49 | # https://docs.fastlane.tools/best-practices/source-control/ 50 | 51 | */fastlane/report.xml 52 | */fastlane/Preview.html 53 | */fastlane/screenshots 54 | 55 | # Bundle artifact 56 | *.jsbundle 57 | -------------------------------------------------------------------------------- /frontend/faceswap/.watchmanconfig: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /frontend/faceswap/App.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Sample React Native App 3 | * https://github.com/facebook/react-native 4 | * 5 | * @format 6 | * @flow 7 | */ 8 | 9 | import React, {Component} from 'react'; 10 | 11 | import AppContainer from './src/index' 12 | 13 | 14 | export default class App extends Component { 15 | 16 | render() { 17 | return ( 18 | 19 | ); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /frontend/faceswap/__tests__/App-test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @format 3 | */ 4 | 5 | import 'react-native'; 6 | import React from 'react'; 7 | import App from '../App'; 8 | 9 | // Note: test renderer must be required after react-native. 10 | import renderer from 'react-test-renderer'; 11 | 12 | it('renders correctly', () => { 13 | renderer.create(); 14 | }); 15 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/BUCK: -------------------------------------------------------------------------------- 1 | # To learn about Buck see [Docs](https://buckbuild.com/). 2 | # To run your application with Buck: 3 | # - install Buck 4 | # - `npm start` - to start the packager 5 | # - `cd android` 6 | # - `keytool -genkey -v -keystore keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"` 7 | # - `./gradlew :app:copyDownloadableDepsToLibs` - make all Gradle compile dependencies available to Buck 8 | # - `buck install -r android/app` - compile, install and run application 9 | # 10 | 11 | load(":build_defs.bzl", "create_aar_targets", "create_jar_targets") 12 | 13 | lib_deps = [] 14 | 15 | create_aar_targets(glob(["libs/*.aar"])) 16 | 17 | create_jar_targets(glob(["libs/*.jar"])) 18 | 19 | android_library( 20 | name = "all-libs", 21 | exported_deps = lib_deps, 22 | ) 23 | 24 | android_library( 25 | name = "app-code", 26 | srcs = glob([ 27 | "src/main/java/**/*.java", 28 | ]), 29 | deps = [ 30 | ":all-libs", 31 | ":build_config", 32 | ":res", 33 | ], 34 | ) 35 | 36 | android_build_config( 37 | name = "build_config", 38 | package = "com.faceswap", 39 | ) 40 | 41 | android_resource( 42 | name = "res", 43 | package = "com.faceswap", 44 | res = "src/main/res", 45 | ) 46 | 47 | android_binary( 48 | name = "app", 49 | keystore = "//android/keystores:debug", 50 | manifest = "src/main/AndroidManifest.xml", 51 | package_type = "debug", 52 | deps = [ 53 | ":app-code", 54 | ], 55 | ) 56 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/build_defs.bzl: -------------------------------------------------------------------------------- 1 | """Helper definitions to glob .aar and .jar targets""" 2 | 3 | def create_aar_targets(aarfiles): 4 | for aarfile in aarfiles: 5 | name = "aars__" + aarfile[aarfile.rindex("/") + 1:aarfile.rindex(".aar")] 6 | lib_deps.append(":" + name) 7 | android_prebuilt_aar( 8 | name = name, 9 | aar = aarfile, 10 | ) 11 | 12 | def create_jar_targets(jarfiles): 13 | for jarfile in jarfiles: 14 | name = "jars__" + jarfile[jarfile.rindex("/") + 1:jarfile.rindex(".jar")] 15 | lib_deps.append(":" + name) 16 | prebuilt_jar( 17 | name = name, 18 | binary_jar = jarfile, 19 | ) 20 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # By default, the flags in this file are appended to flags specified 3 | # in /usr/local/Cellar/android-sdk/24.3.3/tools/proguard/proguard-android.txt 4 | # You can edit the include path and order by changing the proguardFiles 5 | # directive in build.gradle. 6 | # 7 | # For more details, see 8 | # http://developer.android.com/guide/developing/tools/proguard.html 9 | 10 | # Add any project specific keep options here: 11 | 12 | # If your project uses WebView with JS, uncomment the following 13 | # and specify the fully qualified class name to the JavaScript interface 14 | # class: 15 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 16 | # public *; 17 | #} 18 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/debug/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 15 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/java/com/faceswap/MainActivity.java: -------------------------------------------------------------------------------- 1 | package com.faceswap; 2 | 3 | import com.facebook.react.ReactActivity; 4 | 5 | public class MainActivity extends ReactActivity { 6 | 7 | /** 8 | * Returns the name of the main component registered from JavaScript. 9 | * This is used to schedule rendering of the component. 10 | */ 11 | @Override 12 | protected String getMainComponentName() { 13 | return "faceswap"; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/java/com/faceswap/MainApplication.java: -------------------------------------------------------------------------------- 1 | package com.faceswap; 2 | 3 | import android.app.Application; 4 | 5 | import com.facebook.react.ReactApplication; 6 | import com.imagepicker.ImagePickerPackage; 7 | import com.swmansion.gesturehandler.react.RNGestureHandlerPackage; 8 | import com.facebook.react.ReactNativeHost; 9 | import com.facebook.react.ReactPackage; 10 | import com.facebook.react.shell.MainReactPackage; 11 | import com.facebook.soloader.SoLoader; 12 | 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | import com.rnfs.RNFSPackage; 17 | 18 | public class MainApplication extends Application implements ReactApplication { 19 | 20 | private final ReactNativeHost mReactNativeHost = new ReactNativeHost(this) { 21 | @Override 22 | public boolean getUseDeveloperSupport() { 23 | return BuildConfig.DEBUG; 24 | } 25 | 26 | @Override 27 | protected List getPackages() { 28 | return Arrays.asList( 29 | new MainReactPackage(), 30 | new ImagePickerPackage(), 31 | new RNGestureHandlerPackage(), 32 | new RNFSPackage() 33 | ); 34 | } 35 | 36 | @Override 37 | protected String getJSMainModuleName() { 38 | return "index"; 39 | } 40 | }; 41 | 42 | @Override 43 | public ReactNativeHost getReactNativeHost() { 44 | return mReactNativeHost; 45 | } 46 | 47 | @Override 48 | public void onCreate() { 49 | super.onCreate(); 50 | SoLoader.init(this, /* native exopackage */ false); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | faceswap 3 | 4 | -------------------------------------------------------------------------------- /frontend/faceswap/android/app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /frontend/faceswap/android/build.gradle: -------------------------------------------------------------------------------- 1 | // Top-level build file where you can add configuration options common to all sub-projects/modules. 2 | 3 | buildscript { 4 | ext { 5 | buildToolsVersion = "28.0.3" 6 | minSdkVersion = 16 7 | compileSdkVersion = 28 8 | targetSdkVersion = 28 9 | supportLibVersion = "28.0.0" 10 | } 11 | repositories { 12 | def REPOSITORY_URL = 'http://maven.aliyun.com/nexus/content/groups/public/' 13 | all { ArtifactRepository repo -> 14 | if(repo instanceof MavenArtifactRepository){ 15 | def url = repo.url.toString() 16 | if (url.startsWith('https://repo1.maven.org/maven2') || url.startsWith('https://jcenter.bintray.com/')) { 17 | remove repo 18 | } 19 | } 20 | } 21 | maven { 22 | url REPOSITORY_URL 23 | } 24 | google() 25 | jcenter() 26 | } 27 | dependencies { 28 | classpath 'com.android.tools.build:gradle:3.3.1' 29 | 30 | // NOTE: Do not place your application dependencies here; they belong 31 | // in the individual module build.gradle files 32 | } 33 | } 34 | 35 | allprojects { 36 | repositories { 37 | mavenLocal() 38 | google() 39 | jcenter() 40 | maven { 41 | // All of React Native (JS, Obj-C sources, Android binaries) is installed from npm 42 | url "$rootDir/../node_modules/react-native/android" 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /frontend/faceswap/android/gradle.properties: -------------------------------------------------------------------------------- 1 | # Project-wide Gradle settings. 2 | 3 | # IDE (e.g. Android Studio) users: 4 | # Gradle settings configured through the IDE *will override* 5 | # any settings specified in this file. 6 | 7 | # For more details on how to configure your build environment visit 8 | # http://www.gradle.org/docs/current/userguide/build_environment.html 9 | 10 | # Specifies the JVM arguments used for the daemon process. 11 | # The setting is particularly useful for tweaking memory settings. 12 | # Default value: -Xmx10248m -XX:MaxPermSize=256m 13 | # org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 14 | 15 | # When configured, Gradle will run in incubating parallel mode. 16 | # This option should only be used with decoupled projects. More details, visit 17 | # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects 18 | # org.gradle.parallel=true 19 | 20 | MYAPP_RELEASE_STORE_FILE=faceswap.keystore 21 | MYAPP_RELEASE_KEY_ALIAS=faceswap 22 | MYAPP_RELEASE_STORE_PASSWORD=123456 23 | MYAPP_RELEASE_KEY_PASSWORD=123456 24 | -------------------------------------------------------------------------------- /frontend/faceswap/android/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fanxiaochen/TransfiguringPortraits/f46f78bffc6c4ace29a191693f445ba52daea152/frontend/faceswap/android/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /frontend/faceswap/android/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | zipStoreBase=GRADLE_USER_HOME 4 | zipStorePath=wrapper/dists 5 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip 6 | -------------------------------------------------------------------------------- /frontend/faceswap/android/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /frontend/faceswap/android/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /frontend/faceswap/android/keystores/BUCK: -------------------------------------------------------------------------------- 1 | keystore( 2 | name = "debug", 3 | properties = "debug.keystore.properties", 4 | store = "debug.keystore", 5 | visibility = [ 6 | "PUBLIC", 7 | ], 8 | ) 9 | -------------------------------------------------------------------------------- /frontend/faceswap/android/keystores/debug.keystore.properties: -------------------------------------------------------------------------------- 1 | key.store=debug.keystore 2 | key.alias=androiddebugkey 3 | key.store.password=android 4 | key.alias.password=android 5 | -------------------------------------------------------------------------------- /frontend/faceswap/android/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'faceswap' 2 | include ':react-native-image-picker' 3 | project(':react-native-image-picker').projectDir = new File(rootProject.projectDir, '../node_modules/react-native-image-picker/android') 4 | include ':react-native-gesture-handler' 5 | project(':react-native-gesture-handler').projectDir = new File(rootProject.projectDir, '../node_modules/react-native-gesture-handler/android') 6 | include ':react-native-fs' 7 | project(':react-native-fs').projectDir = new File(settingsDir, '../node_modules/react-native-fs/android') 8 | 9 | include ':app' 10 | -------------------------------------------------------------------------------- /frontend/faceswap/app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "faceswap", 3 | "displayName": "faceswap" 4 | } -------------------------------------------------------------------------------- /frontend/faceswap/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: ['module:metro-react-native-babel-preset'], 3 | }; 4 | -------------------------------------------------------------------------------- /frontend/faceswap/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo react-native run-android 4 | -------------------------------------------------------------------------------- /frontend/faceswap/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @format 3 | */ 4 | 5 | import {AppRegistry} from 'react-native'; 6 | import App from './App'; 7 | import {name as appName} from './app.json'; 8 | 9 | AppRegistry.registerComponent(appName, () => App); 10 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap-tvOS/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | org.reactjs.native.example.$(PRODUCT_NAME:rfc1034identifier) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | APPL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1 23 | LSRequiresIPhoneOS 24 | 25 | UILaunchStoryboardName 26 | LaunchScreen 27 | UIRequiredDeviceCapabilities 28 | 29 | armv7 30 | 31 | UISupportedInterfaceOrientations 32 | 33 | UIInterfaceOrientationPortrait 34 | UIInterfaceOrientationLandscapeLeft 35 | UIInterfaceOrientationLandscapeRight 36 | 37 | UIViewControllerBasedStatusBarAppearance 38 | 39 | NSLocationWhenInUseUsageDescription 40 | 41 | NSAppTransportSecurity 42 | 43 | 44 | NSExceptionDomains 45 | 46 | localhost 47 | 48 | NSExceptionAllowsInsecureHTTPLoads 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap-tvOSTests/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | org.reactjs.native.example.$(PRODUCT_NAME:rfc1034identifier) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | BNDL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1 23 | 24 | 25 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap.xcodeproj/xcshareddata/xcschemes/faceswap-tvOS.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 29 | 35 | 36 | 37 | 43 | 49 | 50 | 51 | 52 | 53 | 58 | 59 | 61 | 67 | 68 | 69 | 70 | 71 | 77 | 78 | 79 | 80 | 81 | 82 | 92 | 94 | 100 | 101 | 102 | 103 | 104 | 105 | 111 | 113 | 119 | 120 | 121 | 122 | 124 | 125 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap.xcodeproj/xcshareddata/xcschemes/faceswap.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 29 | 35 | 36 | 37 | 43 | 49 | 50 | 51 | 52 | 53 | 58 | 59 | 61 | 67 | 68 | 69 | 70 | 71 | 77 | 78 | 79 | 80 | 81 | 82 | 92 | 94 | 100 | 101 | 102 | 103 | 104 | 105 | 111 | 113 | 119 | 120 | 121 | 122 | 124 | 125 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/AppDelegate.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #import 9 | #import 10 | 11 | @interface AppDelegate : UIResponder 12 | 13 | @property (nonatomic, strong) UIWindow *window; 14 | 15 | @end 16 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/AppDelegate.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #import "AppDelegate.h" 9 | 10 | #import 11 | #import 12 | #import 13 | 14 | @implementation AppDelegate 15 | 16 | - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions 17 | { 18 | RCTBridge *bridge = [[RCTBridge alloc] initWithDelegate:self launchOptions:launchOptions]; 19 | RCTRootView *rootView = [[RCTRootView alloc] initWithBridge:bridge 20 | moduleName:@"faceswap" 21 | initialProperties:nil]; 22 | 23 | rootView.backgroundColor = [[UIColor alloc] initWithRed:1.0f green:1.0f blue:1.0f alpha:1]; 24 | 25 | self.window = [[UIWindow alloc] initWithFrame:[UIScreen mainScreen].bounds]; 26 | UIViewController *rootViewController = [UIViewController new]; 27 | rootViewController.view = rootView; 28 | self.window.rootViewController = rootViewController; 29 | [self.window makeKeyAndVisible]; 30 | return YES; 31 | } 32 | 33 | - (NSURL *)sourceURLForBridge:(RCTBridge *)bridge 34 | { 35 | #if DEBUG 36 | return [[RCTBundleURLProvider sharedSettings] jsBundleURLForBundleRoot:@"index" fallbackResource:nil]; 37 | #else 38 | return [[NSBundle mainBundle] URLForResource:@"main" withExtension:@"jsbundle"]; 39 | #endif 40 | } 41 | 42 | @end 43 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/Base.lproj/LaunchScreen.xib: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 21 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/Images.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "iphone", 5 | "size" : "29x29", 6 | "scale" : "2x" 7 | }, 8 | { 9 | "idiom" : "iphone", 10 | "size" : "29x29", 11 | "scale" : "3x" 12 | }, 13 | { 14 | "idiom" : "iphone", 15 | "size" : "40x40", 16 | "scale" : "2x" 17 | }, 18 | { 19 | "idiom" : "iphone", 20 | "size" : "40x40", 21 | "scale" : "3x" 22 | }, 23 | { 24 | "idiom" : "iphone", 25 | "size" : "60x60", 26 | "scale" : "2x" 27 | }, 28 | { 29 | "idiom" : "iphone", 30 | "size" : "60x60", 31 | "scale" : "3x" 32 | } 33 | ], 34 | "info" : { 35 | "version" : 1, 36 | "author" : "xcode" 37 | } 38 | } -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/Images.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "version" : 1, 4 | "author" : "xcode" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleDisplayName 8 | faceswap 9 | CFBundleExecutable 10 | $(EXECUTABLE_NAME) 11 | CFBundleIdentifier 12 | $(PRODUCT_BUNDLE_IDENTIFIER) 13 | CFBundleInfoDictionaryVersion 14 | 6.0 15 | CFBundleName 16 | $(PRODUCT_NAME) 17 | CFBundlePackageType 18 | APPL 19 | CFBundleShortVersionString 20 | 1.0 21 | CFBundleSignature 22 | ???? 23 | CFBundleVersion 24 | 1 25 | LSRequiresIPhoneOS 26 | 27 | NSLocationWhenInUseUsageDescription 28 | 29 | UILaunchStoryboardName 30 | LaunchScreen 31 | UIRequiredDeviceCapabilities 32 | 33 | armv7 34 | 35 | UISupportedInterfaceOrientations 36 | 37 | UIInterfaceOrientationPortrait 38 | UIInterfaceOrientationLandscapeLeft 39 | UIInterfaceOrientationLandscapeRight 40 | 41 | UIViewControllerBasedStatusBarAppearance 42 | 43 | NSLocationWhenInUseUsageDescription 44 | 45 | NSAppTransportSecurity 46 | 47 | 48 | NSAllowsArbitraryLoads 49 | 50 | NSExceptionDomains 51 | 52 | localhost 53 | 54 | NSExceptionAllowsInsecureHTTPLoads 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswap/main.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #import 9 | 10 | #import "AppDelegate.h" 11 | 12 | int main(int argc, char * argv[]) { 13 | @autoreleasepool { 14 | return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class])); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswapTests/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | BNDL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1 23 | 24 | 25 | -------------------------------------------------------------------------------- /frontend/faceswap/ios/faceswapTests/faceswapTests.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Facebook, Inc. and its affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | #import 9 | #import 10 | 11 | #import 12 | #import 13 | 14 | #define TIMEOUT_SECONDS 600 15 | #define TEXT_TO_LOOK_FOR @"Welcome to React Native!" 16 | 17 | @interface faceswapTests : XCTestCase 18 | 19 | @end 20 | 21 | @implementation faceswapTests 22 | 23 | - (BOOL)findSubviewInView:(UIView *)view matching:(BOOL(^)(UIView *view))test 24 | { 25 | if (test(view)) { 26 | return YES; 27 | } 28 | for (UIView *subview in [view subviews]) { 29 | if ([self findSubviewInView:subview matching:test]) { 30 | return YES; 31 | } 32 | } 33 | return NO; 34 | } 35 | 36 | - (void)testRendersWelcomeScreen 37 | { 38 | UIViewController *vc = [[[RCTSharedApplication() delegate] window] rootViewController]; 39 | NSDate *date = [NSDate dateWithTimeIntervalSinceNow:TIMEOUT_SECONDS]; 40 | BOOL foundElement = NO; 41 | 42 | __block NSString *redboxError = nil; 43 | RCTSetLogFunction(^(RCTLogLevel level, RCTLogSource source, NSString *fileName, NSNumber *lineNumber, NSString *message) { 44 | if (level >= RCTLogLevelError) { 45 | redboxError = message; 46 | } 47 | }); 48 | 49 | while ([date timeIntervalSinceNow] > 0 && !foundElement && !redboxError) { 50 | [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; 51 | [[NSRunLoop mainRunLoop] runMode:NSRunLoopCommonModes beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; 52 | 53 | foundElement = [self findSubviewInView:vc.view matching:^BOOL(UIView *view) { 54 | if ([view.accessibilityLabel isEqualToString:TEXT_TO_LOOK_FOR]) { 55 | return YES; 56 | } 57 | return NO; 58 | }]; 59 | } 60 | 61 | RCTSetLogFunction(RCTDefaultLogFunction); 62 | 63 | XCTAssertNil(redboxError, @"RedBox error: %@", redboxError); 64 | XCTAssertTrue(foundElement, @"Couldn't find element with text '%@' in %d seconds", TEXT_TO_LOOK_FOR, TIMEOUT_SECONDS); 65 | } 66 | 67 | 68 | @end 69 | -------------------------------------------------------------------------------- /frontend/faceswap/metro.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Metro configuration for React Native 3 | * https://github.com/facebook/react-native 4 | * 5 | * @format 6 | */ 7 | 8 | const { resolve } = require('path'); 9 | 10 | module.exports = { 11 | watchFolders: [ 12 | resolve(__dirname, '.'), // your 'native' directory 13 | resolve(__dirname, '../../'), // your '../../' directory 14 | ], 15 | transformer: { 16 | getTransformOptions: async () => ({ 17 | transform: { 18 | experimentalImportSupport: false, 19 | inlineRequires: false, 20 | }, 21 | }), 22 | }, 23 | }; 24 | -------------------------------------------------------------------------------- /frontend/faceswap/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "faceswap", 3 | "version": "0.0.1", 4 | "private": true, 5 | "scripts": { 6 | "start": "node node_modules/react-native/local-cli/cli.js start", 7 | "test": "jest" 8 | }, 9 | "dependencies": { 10 | "axios": "^0.18.0", 11 | "buffer": "^5.2.1", 12 | "image-downloader": "^3.4.2", 13 | "react": "16.8.3", 14 | "react-native": "0.59.3", 15 | "react-native-gesture-handler": "^1.2.0", 16 | "react-native-image-picker": "^0.28.1", 17 | "react-native-uuid": "^1.4.9" 18 | }, 19 | "devDependencies": { 20 | "@babel/core": "7.4.3", 21 | "@babel/runtime": "7.4.3", 22 | "babel-jest": "24.7.1", 23 | "jest": "24.7.1", 24 | "metro-react-native-babel-preset": "0.53.1", 25 | "react-test-renderer": "16.8.3" 26 | }, 27 | "jest": { 28 | "preset": "react-native" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /frontend/faceswap/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd android 4 | sudo ./gradlew clean && sudo ./gradlew assembleRelease 5 | sudo cp app/build/outputs/apk/release/app-release.apk ../faceswap-v0.0.1.apk 6 | cd .. 7 | -------------------------------------------------------------------------------- /frontend/faceswap/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo react-native start --reset-cache 4 | -------------------------------------------------------------------------------- /frontend/faceswap/src/index.js: -------------------------------------------------------------------------------- 1 | 2 | import Waiting from './waiting' 3 | import Swap from './swap' 4 | import Welcome from './welcome' 5 | import Submit from './submit' 6 | 7 | import { 8 | createStackNavigator, 9 | createAppContainer 10 | } from 'react-navigation' 11 | 12 | const AppNavigator = createStackNavigator({ 13 | Home: { 14 | screen: Welcome 15 | }, 16 | Submit: { 17 | screen: Submit, 18 | }, 19 | Wait: { 20 | screen: Waiting, 21 | }, 22 | Swap: { 23 | screen: Swap 24 | } 25 | }, 26 | { 27 | defaultNavigationOptions: { 28 | headerStyle: { 29 | backgroundColor: 'orange' 30 | } 31 | } 32 | }); 33 | 34 | 35 | const AppContainer = createAppContainer(AppNavigator); 36 | 37 | export default AppContainer; 38 | 39 | 40 | -------------------------------------------------------------------------------- /frontend/faceswap/src/submit.js: -------------------------------------------------------------------------------- 1 | 2 | import React, {Component} from 'react' 3 | import { 4 | View, 5 | Button, 6 | TextInput, 7 | Alert, 8 | Dimensions, 9 | StyleSheet 10 | } from 'react-native' 11 | import ImagePicker from 'react-native-image-picker'; 12 | import {submitFormData, uuidGen} from './upload' 13 | 14 | let screenWidth = Dimensions.get('window').width; 15 | let screenHeight = Dimensions.get('window').height; 16 | 17 | const options = { 18 | title: 'Select Avatar', 19 | storageOptions: { 20 | skipBackup: true, 21 | path: 'images', 22 | }, 23 | }; 24 | 25 | export default class Submit extends Component { 26 | constructor(props){ 27 | super(props); 28 | this.state = { 29 | avatarSource: {}, 30 | item: '', 31 | uuid: '' 32 | } 33 | } 34 | 35 | // onTakePhoto() { 36 | // ImagePicker.launchCamera(options, (response) => { 37 | // if (response.didCancel) { 38 | // console.log('User cancelled image picker'); 39 | // } else if (response.error) { 40 | // console.log('ImagePicker Error: ', response.error); 41 | // } else { 42 | // const source = { uri: response.uri }; 43 | // 44 | // // You can also display the image using data: 45 | // // const source = { uri: 'data:image/jpeg;base64,' + response.data }; 46 | // 47 | // this.setState({ 48 | // avatarSource: source, 49 | // }); 50 | // console.log(this.state); 51 | // 52 | // } 53 | // }); 54 | // } 55 | 56 | onTakePhoto() { 57 | ImagePicker.showImagePicker(null, (response) => { 58 | if (response.didCancel) { 59 | console.log('User cancelled image picker'); 60 | } else if (response.error) { 61 | console.log('ImagePicker Error: ', response.error); 62 | } else { 63 | const source = { uri: response.uri }; 64 | 65 | // You can also display the image using data: 66 | // const source = { uri: 'data:image/jpeg;base64,' + response.data }; 67 | 68 | this.setState({ 69 | avatarSource: source, 70 | }); 71 | console.log(this.state); 72 | 73 | } 74 | }); 75 | } 76 | 77 | onSubmit(){ 78 | console.log(this.state) 79 | 80 | if (this.state.item === ''){ 81 | Alert.alert( 82 | 'Error', 83 | 'please type a style', 84 | [ 85 | {text: 'OK', onPress: () => console.log('OK Pressed')}, 86 | ], 87 | {cancelable: false}, 88 | ); 89 | return; 90 | } 91 | 92 | if (Object.entries(this.state.avatarSource).length === 0){ 93 | Alert.alert( 94 | 'Error', 95 | 'please take a selfie', 96 | [ 97 | {text: 'OK', onPress: () => console.log('OK Pressed')}, 98 | ], 99 | {cancelable: false}, 100 | ); 101 | return; 102 | } 103 | 104 | 105 | if (this.state.uuid === ''){ 106 | this.state.uuid = uuidGen.v1(); 107 | } 108 | global.curItem = this.state.item; 109 | global.uuid = this.state.uuid; 110 | 111 | submitFormData(this.state.avatarSource.uri, this.state.item, this.state.uuid); 112 | console.log(global.curItem) 113 | 114 | this.props.navigation.navigate('Wait'); 115 | } 116 | 117 | 118 | render() { 119 | return ( 120 | 121 | this.setState({item})} 126 | /> 127 |