├── .gitignore ├── .vscode └── launch.json ├── README.md ├── adjustGamma.py ├── animegan.py ├── backgrounds ├── beach.jpg ├── office.jpg └── park.jpg ├── bin └── setup_env.sh ├── data ├── husky.jpg ├── husky_AnimeGANv3_PortraitSketch_25.jpg ├── husky_Hayao-60.jpg ├── husky_Hayao_64.jpg ├── husky_Paprika_54.jpg ├── husky_Shinkai_53.jpg ├── porche.jpg └── porche_out.jpg ├── engine.py ├── faceDetection.py ├── faceNet ├── architecture.py ├── convert_to_onnx.py └── faceNet.py ├── faces └── Rokas.png ├── main.py ├── models ├── AnimeGANv3_PortraitSketch_25.onnx ├── Hayao-60.onnx ├── Hayao_64.onnx ├── Paprika_54.onnx ├── Shinkai_53.onnx └── modnet.onnx ├── pencilSketch.py ├── requirements.txt ├── selfieSegmentation.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv 3 | *.mkv 4 | *.mp4 5 | models/face* -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Current File", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "${file}", 12 | "console": "integratedTerminal", 13 | "justMyCode": false 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Background removal with Python 2 | 3 | All this repository is for learning purposes. I cover here how simple is to remove background from selfie view just like Zoom, Google Meets, Skype, and MS Teams. 4 | 5 | ## Installation on Windows: 6 | - Clone this repository. (don't forget to Star it) 7 | - Install virtual environment: ```python -m venv venv``` 8 | - Activate virtual environment: ```venv\Scripts\activate``` 9 | - Install all the requirements: ```pip install -r requirements.txt``` 10 | - (Optional if have Nvidia GPU): install onnxruntime with GPU support: ```pip install onnxruntime-gpu``` 11 | 12 | ## How to run basic background removal: 13 | At this point, when you are looking at this project, I might be already updated this project with more features, but if you want only to run a quick test on your own webcam replace the ```main.py``` code with the following: 14 | ```Python 15 | # main.py 16 | from utils import FPSmetric 17 | from selfieSegmentation import MPSegmentations 18 | from engine import Engine 19 | 20 | if __name__ == '__main__': 21 | fpsMetric = FPSmetric() 22 | segmentationModule = MPSegmentations(threshold=0.3, bg_images_path='', bg_blur_ratio=(45, 45)) 23 | selfieSegmentation = Engine(webcam_id=0, show=True, custom_objects=[segmentationModule, fpsMetric]) 24 | selfieSegmentation.run() 25 | ``` 26 | You can run it by typing ```python main.py``` in a terminal. 27 | 28 | ## Run basic MediaPipe face detection: 29 | ```Python 30 | # main.py 31 | from utils import FPSmetric 32 | from faceDetection import MPFaceDetection 33 | from engine import Engine 34 | 35 | if __name__ == '__main__': 36 | fpsMetric = FPSmetric() 37 | mpFaceDetector = MPFaceDetection() 38 | selfieSegmentation = Engine(webcam_id=0, show=True, custom_objects=[mpFaceDetector, fpsMetric]) 39 | selfieSegmentation.run() 40 | ``` 41 | You can run it by typing ```python main.py``` in a terminal. 42 | 43 | ## Test "Pencil" sketch with Python on saved image: 44 | ```Python 45 | # main.py 46 | from pencilSketch import PencilSketch 47 | from engine import Engine 48 | 49 | if __name__ == '__main__': 50 | pencilSketch = PencilSketch(blur_simga=5) 51 | selfieSegmentation = Engine(image_path='data/porche.jpg', show=True, custom_objects=[pencilSketch]) 52 | selfieSegmentation.run() 53 | ``` 54 | You can run it by typing ```python main.py``` in a terminal. 55 | 56 | ## Test facial recognition example on webcam stream 57 | ```Python 58 | # main.py 59 | from utils import FPSmetric 60 | from engine import Engine 61 | from faceDetection import MPFaceDetection 62 | from faceNet.faceNet import FaceNet 63 | 64 | if __name__ == '__main__': 65 | facenet = FaceNet( 66 | detector = MPFaceDetection(), 67 | onnx_model_path = "models/faceNet.onnx", 68 | anchors = "faces", 69 | force_cpu = True, 70 | ) 71 | engine = Engine(webcam_id=0, show=True, custom_objects=[facenet, FPSmetric()]) 72 | 73 | # save first face crop as anchor, otherwise don't use 74 | while not facenet.detect_save_faces(engine.process_webcam(return_frame=True), output_dir="faces"): 75 | continue 76 | 77 | engine.run() 78 | ``` 79 | You can run it by typing ```python main.py``` in a terminal. 80 | 81 | ## Test AnimeGAN effect with python on webcam stream: 82 | ```Python 83 | from engine import Engine 84 | from animegan import AnimeGAN 85 | 86 | if __name__ == '__main__': 87 | animegan = AnimeGAN("models/Hayao_64.onnx") 88 | engine = Engine(webcam_id=0, show=True, custom_objects=[animegan]) 89 | engine.run() 90 | ``` 91 | You can run it by typing python main.py in a terminal. 92 | 93 | ## Detailed Tutorials: 94 | - [Selfie background remove or blur with Python](https://pylessons.com/remove-background) 95 | - [Real Time CPU face detection tutorial](https://pylessons.com/face-detection) 96 | - [Pencil sketch image with Python](https://pylessons.com/pencil-sketch) 97 | - [Face recognition with Python](https://pylessons.com/face-recognition) 98 | - [AnimeGAN effect with Python](https://pylessons.com/animegan-effect) 99 | -------------------------------------------------------------------------------- /adjustGamma.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | class Adjust_gamma: 5 | def __init__(self, gamma: float = 1.0) -> None: 6 | """build a lookup table mapping the pixel values [0, 255] to 7 | their adjusted gamma values 8 | 9 | Args: 10 | gamma: (float) - value to adjust gamma with 11 | """ 12 | self.invGamma = 1.0 / abs(gamma) 13 | self.table = np.array([((i / 255.0) ** self.invGamma) * 255 for i in np.arange(0, 256)]).astype("uint8") 14 | 15 | def __call__(self, image: np.ndarray): 16 | """apply gamma correction using the lookup table 17 | 18 | Args: 19 | image: (np.ndarray) - image to which to apply the gamma adjust 20 | 21 | Return: 22 | image: (np.ndarray) - image with adjusted gamma 23 | """ 24 | return cv2.LUT(image, self.table) -------------------------------------------------------------------------------- /animegan.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import typing 4 | import numpy as np 5 | import onnxruntime as ort 6 | 7 | class AnimeGAN: 8 | """ Object to image animation using AnimeGAN models 9 | https://github.com/TachibanaYoshino/AnimeGANv2 10 | 11 | onnx models: 12 | 'https://docs.google.com/uc?export=download&id=1VPAPI84qaPUCHKHJLHiMK7BP_JE66xNe' AnimeGAN_Hayao.onnx 13 | 'https://docs.google.com/uc?export=download&id=17XRNQgQoUAnu6SM5VgBuhqSBO4UAVNI1' AnimeGANv2_Hayao.onnx 14 | 'https://docs.google.com/uc?export=download&id=10rQfe4obW0dkNtsQuWg-szC4diBzYFXK' AnimeGANv2_Shinkai.onnx 15 | 'https://docs.google.com/uc?export=download&id=1X3Glf69Ter_n2Tj6p81VpGKx7U4Dq-tI' AnimeGANv2_Paprika.onnx 16 | 17 | """ 18 | def __init__( 19 | self, 20 | model_path: str = '', 21 | downsize_ratio: float = 1.0, 22 | ) -> None: 23 | """ 24 | Args: 25 | model_path: (str) - path to onnx model file 26 | downsize_ratio: (float) - ratio to downsize input frame for faster inference 27 | """ 28 | if not os.path.exists(model_path): 29 | raise Exception(f"Model doesn't exists in {model_path}") 30 | 31 | self.downsize_ratio = downsize_ratio 32 | 33 | providers = ['CUDAExecutionProvider'] if ort.get_device() == "GPU" else ['CPUExecutionProvider'] 34 | 35 | self.ort_sess = ort.InferenceSession(model_path, providers=providers) 36 | 37 | def to_32s(self, x): 38 | return 256 if x < 256 else x - x%32 39 | 40 | def process_frame(self, frame: np.ndarray, x32: bool = True) -> np.ndarray: 41 | """ Function to process frame to fit model input as 32 multiplier and resize to fit model input 42 | 43 | Args: 44 | frame: (np.ndarray) - frame to process 45 | x32: (bool) - if True, resize frame to 32 multiplier 46 | 47 | Returns: 48 | frame: (np.ndarray) - processed frame 49 | """ 50 | h, w = frame.shape[:2] 51 | if x32: # resize image to multiple of 32s 52 | frame = cv2.resize(frame, (self.to_32s(int(w*self.downsize_ratio)), self.to_32s(int(h*self.downsize_ratio)))) 53 | frame = frame.astype(np.float32) / 127.5 - 1.0 54 | return frame 55 | 56 | def post_process(self, frame: np.ndarray, wh: typing.Tuple[int, int]) -> np.ndarray: 57 | """ Convert model float output to uint8 image resized to original frame size 58 | 59 | Args: 60 | frame: (np.ndarray) - AnimeGaAN output frame 61 | wh: (typing.Tuple[int, int]) - original frame size 62 | 63 | Returns: 64 | frame: (np.ndarray) - original size animated image 65 | """ 66 | frame = (frame.squeeze() + 1.) / 2 * 255 67 | frame = frame.astype(np.uint8) 68 | frame = cv2.resize(frame, (wh[0], wh[1])) 69 | return frame 70 | 71 | def __call__(self, frame: np.ndarray) -> np.ndarray: 72 | """Main function to process selfie semgentation on each call 73 | 74 | Args: 75 | frame: (np.ndarray) - frame to excecute face detection on 76 | 77 | Returns: 78 | frame: (np.ndarray) - processed frame with face detection 79 | """ 80 | image = self.process_frame(frame) 81 | outputs = self.ort_sess.run(None, {self.ort_sess._inputs_meta[0].name: np.expand_dims(image, axis=0)}) 82 | frame = self.post_process(outputs[0], frame.shape[:2][::-1]) 83 | 84 | return frame 85 | -------------------------------------------------------------------------------- /backgrounds/beach.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/backgrounds/beach.jpg -------------------------------------------------------------------------------- /backgrounds/office.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/backgrounds/office.jpg -------------------------------------------------------------------------------- /backgrounds/park.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/backgrounds/park.jpg -------------------------------------------------------------------------------- /bin/setup_env.sh: -------------------------------------------------------------------------------- 1 | python -m venv venv 2 | activate() { 3 | . /venv/Scripts/activate 4 | echo "installing requirements to virtual environment" 5 | pip install -r requirements.txt 6 | } 7 | activate -------------------------------------------------------------------------------- /data/husky.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky.jpg -------------------------------------------------------------------------------- /data/husky_AnimeGANv3_PortraitSketch_25.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky_AnimeGANv3_PortraitSketch_25.jpg -------------------------------------------------------------------------------- /data/husky_Hayao-60.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky_Hayao-60.jpg -------------------------------------------------------------------------------- /data/husky_Hayao_64.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky_Hayao_64.jpg -------------------------------------------------------------------------------- /data/husky_Paprika_54.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky_Paprika_54.jpg -------------------------------------------------------------------------------- /data/husky_Shinkai_53.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/husky_Shinkai_53.jpg -------------------------------------------------------------------------------- /data/porche.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/porche.jpg -------------------------------------------------------------------------------- /data/porche_out.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/data/porche_out.jpg -------------------------------------------------------------------------------- /engine.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import stow 3 | import typing 4 | import numpy as np 5 | from tqdm import tqdm 6 | 7 | from selfieSegmentation import MPSegmentation 8 | 9 | class Engine: 10 | """Object to process webcam stream, video source or images 11 | All the processing can be customized and enchanced with custom_objects 12 | """ 13 | def __init__( 14 | self, 15 | image_path: str = "", 16 | video_path: str = "", 17 | webcam_id: int = 0, 18 | show: bool = False, 19 | flip_view: bool = False, 20 | custom_objects: typing.Iterable = [], 21 | output_extension: str = 'out', 22 | start_video_frame: int = 0, 23 | end_video_frame: int = 0, 24 | break_on_end: bool = False, 25 | ) -> None: 26 | """Initialize Engine object for further processing 27 | 28 | Args: 29 | image_path: (str) - path to image to process 30 | video_path: (str) - path to video to process 31 | webcam_id: (int) - ID of the webcam to process 32 | show: (bool) - argument whether to display or not processing 33 | flip_view: (bool) - argument whether to flip view horizontally or not 34 | custom_objects: (typing.Iterable) - custom objects to call every iteration (must have call function) 35 | output_extension: (str) - additional text to add to processed image or video when saving output 36 | start_video_frame: (int) - video frame from which to start applying custom_objects to video 37 | end_video_frame: (int) - last video frame to which apply custom_objects to video 38 | """ 39 | self.video_path = video_path 40 | self.image_path = image_path 41 | self.webcam_id = webcam_id 42 | self.show = show 43 | self.flip_view = flip_view 44 | self.custom_objects = custom_objects 45 | self.output_extension = output_extension 46 | self.start_video_frame = start_video_frame 47 | self.end_video_frame = end_video_frame 48 | self.break_on_end = break_on_end 49 | 50 | def flip(self, frame: np.ndarray) -> np.ndarray: 51 | """Flip given frame horizontally 52 | Args: 53 | frame: (np.ndarray) - frame to be fliped horizontally 54 | 55 | Returns: 56 | frame: (np.ndarray) - fliped frame if self.flip_view = True 57 | """ 58 | if self.flip_view: 59 | return cv2.flip(frame, 1) 60 | 61 | return frame 62 | 63 | def custom_processing(self, frame: np.ndarray) -> np.ndarray: 64 | """Process frame with custom objects (custom object must have call function for each iteration) 65 | Args: 66 | frame: (np.ndarray) - frame to apply custom processing to 67 | 68 | Returns: 69 | frame: (np.ndarray) - custom processed frame 70 | """ 71 | if self.custom_objects: 72 | for custom_object in self.custom_objects: 73 | frame = custom_object(frame) 74 | 75 | return frame 76 | 77 | def display(self, frame: np.ndarray, webcam: bool = False, waitTime: int = 1) -> bool: 78 | """Display current frame if self.show = True 79 | When displaying webcam you can control the background images 80 | 81 | Args: 82 | frame: (np.ndarray) - frame to be displayed 83 | webcam: (bool) - Add aditional function for webcam. Keyboard 'a' for next or 'd' for previous 84 | 85 | Returns: 86 | (bool) - Teturn True if no keyboard "Quit" interruption 87 | """ 88 | if self.show: 89 | cv2.imshow('Remove Background', frame) 90 | k = cv2.waitKey(waitTime) 91 | if k & 0xFF == ord('q'): 92 | cv2.destroyAllWindows() 93 | return False 94 | 95 | if webcam: 96 | if k & 0xFF == ord('a'): 97 | for custom_object in self.custom_objects: 98 | # change background to next with keyboar 'a' button 99 | if isinstance(custom_object, MPSegmentation): 100 | custom_object.change_image(True) 101 | elif k & 0xFF == ord('d'): 102 | for custom_object in self.custom_objects: 103 | # change background to previous with keyboar 'd' button 104 | if isinstance(custom_object, MPSegmentation): 105 | custom_object.change_image(False) 106 | 107 | return True 108 | 109 | def process_image( 110 | self, 111 | image: typing.Union[str, np.ndarray] = None, 112 | output_path: str = None 113 | ) -> np.ndarray: 114 | """The function does to processing with the given image or image path 115 | 116 | Args: 117 | frame: (typing.Union[str, np.ndarray]) - we can pass whether an image path or image numpy buffer 118 | output_path: (str) - we can specify where processed image will be saved 119 | 120 | Returns: 121 | frame: (np.ndarray) - final processed image 122 | """ 123 | if image is not None and isinstance(image, str): 124 | if not stow.exists(image): 125 | raise Exception(f"Given image path doesn't exist {self.image_path}") 126 | else: 127 | extension = stow.extension(image) 128 | if output_path is None: 129 | output_path = image.replace(f".{extension}", f"_{self.output_extension}.{extension}") 130 | image = cv2.imread(image) 131 | 132 | image = self.custom_processing(self.flip(image)) 133 | 134 | cv2.imwrite(output_path, image) 135 | 136 | self.display(image, waitTime=0) 137 | 138 | return image 139 | 140 | def process_webcam(self, return_frame: bool = False) -> typing.Union[None, np.ndarray]: 141 | """Process webcam stream for given webcam_id 142 | """ 143 | # Create a VideoCapture object for given webcam_id 144 | cap = cv2.VideoCapture(self.webcam_id) 145 | while cap.isOpened(): 146 | success, frame = cap.read() 147 | if not success or frame is None: 148 | print("Ignoring empty camera frame.") 149 | continue 150 | 151 | if return_frame: 152 | break 153 | 154 | frame = self.custom_processing(self.flip(frame)) 155 | 156 | if not self.display(frame, webcam=True): 157 | break 158 | 159 | else: 160 | raise Exception(f"Webcam with ID ({self.webcam_id}) can't be opened") 161 | 162 | cap.release() 163 | return frame 164 | 165 | def check_video_frames_range(self, fnum): 166 | """Not to waste resources this function processes only specified range of video frames 167 | 168 | Args: 169 | fnum: (int) - current video frame number 170 | 171 | Returns: 172 | status: (bool) - Return True if skip processing otherwise False 173 | """ 174 | if self.start_video_frame and fnum < self.start_video_frame: 175 | return True 176 | 177 | if self.end_video_frame and fnum > self.end_video_frame: 178 | return True 179 | 180 | return False 181 | 182 | def process_video(self) -> None: 183 | """Process video for given video_path and creates processed video in same path 184 | """ 185 | if not stow.exists(self.video_path): 186 | raise Exception(f"Given video path doesn't exists {self.video_path}") 187 | 188 | # Create a VideoCapture object and read from input file 189 | cap = cv2.VideoCapture(self.video_path) 190 | 191 | # Check if camera opened successfully 192 | if not cap.isOpened(): 193 | raise Exception(f"Error opening video stream or file {self.video_path}") 194 | 195 | # Capture video details 196 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 197 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 198 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 199 | frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 200 | 201 | # Create video writer in the same location as original video 202 | output_path = self.video_path.replace(f".{stow.extension(self.video_path)}", f"_{self.output_extension}.mp4") 203 | out = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*'MP4V'), fps, (width, height)) 204 | 205 | # Read all frames from video 206 | for fnum in tqdm(range(frames)): 207 | # Capture frame-by-frame 208 | success, frame = cap.read() 209 | if not success: 210 | break 211 | 212 | if self.check_video_frames_range(fnum): 213 | out.write(frame) 214 | if self.break_on_end and fnum >= self.end_video_frame: 215 | break 216 | continue 217 | 218 | frame = self.custom_processing(self.flip(frame)) 219 | 220 | out.write(frame) 221 | 222 | if not self.display(frame): 223 | break 224 | 225 | cap.release() 226 | out.release() 227 | 228 | def run(self): 229 | """Main object function to start processing image, video or webcam input 230 | """ 231 | if self.video_path: 232 | self.process_video() 233 | elif self.image_path: 234 | self.process_image(self.image_path) 235 | else: 236 | self.process_webcam() -------------------------------------------------------------------------------- /faceDetection.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import typing 3 | import numpy as np 4 | import mediapipe as mp 5 | 6 | class MPFaceDetection: 7 | """Object to create and do mediapipe face detection, more about it: 8 | https://google.github.io/mediapipe/solutions/face_detection.html 9 | """ 10 | def __init__( 11 | self, 12 | model_selection: bool = 1, 13 | confidence: float = 0.5, 14 | mp_drawing_utils: bool = True, 15 | color: typing.Tuple[int, int, int] = (255, 255, 255), 16 | thickness: int = 2, 17 | ) -> None: 18 | """ 19 | Args: 20 | model_selection: (bool) - 1 - for low distance, 0 - for far distance face detectors. 21 | confidence: (float) - confidence for face detector, when detection are confirmed, range (0.0-1.0). 22 | mp_drawing_utils: (bool) - bool option whether to use mp_drawing utils or or own, Default to True. 23 | color: (typing.Tuple[int, int, int]) - Color for drawing the annotation. Default to the white color. 24 | thickness: (int) - Thickness for drawing the annotation. Default to 2 pixels. 25 | """ 26 | self.mp_drawing_utils = mp_drawing_utils 27 | self.color = color 28 | self.thickness = thickness 29 | self.mp_drawing = mp.solutions.drawing_utils 30 | self.mp_face_detection = mp.solutions.face_detection 31 | self.face_detection = self.mp_face_detection.FaceDetection(model_selection=model_selection, min_detection_confidence=confidence) 32 | 33 | def tlbr(self, frame: np.ndarray, mp_detections: typing.List) -> np.ndarray: 34 | """Return coorinates in typing.Iterable([[Top, Left, Bottom, Right]]) 35 | 36 | Args: 37 | frame: (np.ndarray) - frame on which we want to apply detections 38 | mp_detections: (typing.List) - list of media pipe detections 39 | 40 | Returns: 41 | detections: (np.ndarray) - list of detection in [Top, Left, Bottom, Right] coordinates 42 | """ 43 | detections = [] 44 | frame_height, frame_width, _ = frame.shape 45 | for detection in mp_detections: 46 | height = int(detection.location_data.relative_bounding_box.height * frame_height) 47 | width = int(detection.location_data.relative_bounding_box.width * frame_width) 48 | left = max(0 ,int(detection.location_data.relative_bounding_box.xmin * frame_width)) 49 | top = max(0 ,int(detection.location_data.relative_bounding_box.ymin * frame_height)) 50 | 51 | detections.append([top, left, top + height, left + width]) 52 | 53 | return np.array(detections) 54 | 55 | def __call__(self, frame: np.ndarray, return_tlbr: bool = False) -> np.ndarray: 56 | """Main function to do face detection 57 | 58 | Args: 59 | frame: (np.ndarray) - frame to excecute face detection on 60 | return_tlbr: (bool) - bool option to return coordinates instead of frame with drawn detections 61 | 62 | Returns: 63 | typing.Union[ 64 | frame: (np.ndarray) - processed frame with detected faces, 65 | detections: (typing.List) - detections in [Top, Left, Bottom, Right] 66 | ] 67 | """ 68 | results = self.face_detection.process(frame) 69 | 70 | if return_tlbr: 71 | if results.detections: 72 | return self.tlbr(frame, results.detections) 73 | return [] 74 | 75 | if results.detections: 76 | if self.mp_drawing_utils: 77 | # Draw face detections of each face using media pipe drawing utils. 78 | for detection in results.detections: 79 | self.mp_drawing.draw_detection(frame, detection) 80 | 81 | else: 82 | # Draw face detections of each face using our own tlbr and cv2.rectangle 83 | for tlbr in self.tlbr(frame, results.detections): 84 | cv2.rectangle(frame, tlbr[:2][::-1], tlbr[2:][::-1], self.color, self.thickness) 85 | 86 | return frame -------------------------------------------------------------------------------- /faceNet/architecture.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.layers import Conv2D, Activation, Input, MaxPooling2D, Dense, Dropout, BatchNormalization, Concatenate, Lambda, add, GlobalAveragePooling2D 2 | from tensorflow.keras.models import Model 3 | from tensorflow.keras import backend as K 4 | 5 | def scaling(x, scale): 6 | return x * scale 7 | 8 | def InceptionResNetV2(): 9 | """Architecture from https://github.com/davidsandberg/facenet 10 | """ 11 | inputs = Input(shape=(160, 160, 3)) 12 | x = Conv2D(32, 3, strides=2, padding='valid', use_bias=False, name= 'Conv2d_1a_3x3') (inputs) 13 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_1a_3x3_BatchNorm')(x) 14 | x = Activation('relu', name='Conv2d_1a_3x3_Activation')(x) 15 | x = Conv2D(32, 3, strides=1, padding='valid', use_bias=False, name= 'Conv2d_2a_3x3') (x) 16 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_2a_3x3_BatchNorm')(x) 17 | x = Activation('relu', name='Conv2d_2a_3x3_Activation')(x) 18 | x = Conv2D(64, 3, strides=1, padding='same', use_bias=False, name= 'Conv2d_2b_3x3') (x) 19 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_2b_3x3_BatchNorm')(x) 20 | x = Activation('relu', name='Conv2d_2b_3x3_Activation')(x) 21 | x = MaxPooling2D(3, strides=2, name='MaxPool_3a_3x3')(x) 22 | x = Conv2D(80, 1, strides=1, padding='valid', use_bias=False, name= 'Conv2d_3b_1x1') (x) 23 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_3b_1x1_BatchNorm')(x) 24 | x = Activation('relu', name='Conv2d_3b_1x1_Activation')(x) 25 | x = Conv2D(192, 3, strides=1, padding='valid', use_bias=False, name= 'Conv2d_4a_3x3') (x) 26 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_4a_3x3_BatchNorm')(x) 27 | x = Activation('relu', name='Conv2d_4a_3x3_Activation')(x) 28 | x = Conv2D(256, 3, strides=2, padding='valid', use_bias=False, name= 'Conv2d_4b_3x3') (x) 29 | x = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Conv2d_4b_3x3_BatchNorm')(x) 30 | x = Activation('relu', name='Conv2d_4b_3x3_Activation')(x) 31 | 32 | # 5x Block35 (Inception-ResNet-A block): 33 | branch_0 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_0_Conv2d_1x1') (x) 34 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 35 | branch_0 = Activation('relu', name='Block35_1_Branch_0_Conv2d_1x1_Activation')(branch_0) 36 | branch_1 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_1_Conv2d_0a_1x1') (x) 37 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 38 | branch_1 = Activation('relu', name='Block35_1_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 39 | branch_1 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_1_Conv2d_0b_3x3') (branch_1) 40 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 41 | branch_1 = Activation('relu', name='Block35_1_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 42 | branch_2 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_2_Conv2d_0a_1x1') (x) 43 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 44 | branch_2 = Activation('relu', name='Block35_1_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 45 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_2_Conv2d_0b_3x3') (branch_2) 46 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 47 | branch_2 = Activation('relu', name='Block35_1_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 48 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_1_Branch_2_Conv2d_0c_3x3') (branch_2) 49 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_1_Branch_2_Conv2d_0c_3x3_BatchNorm')(branch_2) 50 | branch_2 = Activation('relu', name='Block35_1_Branch_2_Conv2d_0c_3x3_Activation')(branch_2) 51 | branches = [branch_0, branch_1, branch_2] 52 | mixed = Concatenate(axis=3, name='Block35_1_Concatenate')(branches) 53 | up = Conv2D(256, 1, strides=1, padding='same', use_bias=True, name= 'Block35_1_Conv2d_1x1') (mixed) 54 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.17})(up) 55 | x = add([x, up]) 56 | x = Activation('relu', name='Block35_1_Activation')(x) 57 | 58 | branch_0 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_0_Conv2d_1x1') (x) 59 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 60 | branch_0 = Activation('relu', name='Block35_2_Branch_0_Conv2d_1x1_Activation')(branch_0) 61 | branch_1 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_1_Conv2d_0a_1x1') (x) 62 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 63 | branch_1 = Activation('relu', name='Block35_2_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 64 | branch_1 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_1_Conv2d_0b_3x3') (branch_1) 65 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 66 | branch_1 = Activation('relu', name='Block35_2_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 67 | branch_2 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_2_Conv2d_0a_1x1') (x) 68 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 69 | branch_2 = Activation('relu', name='Block35_2_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 70 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_2_Conv2d_0b_3x3') (branch_2) 71 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 72 | branch_2 = Activation('relu', name='Block35_2_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 73 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_2_Branch_2_Conv2d_0c_3x3') (branch_2) 74 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_2_Branch_2_Conv2d_0c_3x3_BatchNorm')(branch_2) 75 | branch_2 = Activation('relu', name='Block35_2_Branch_2_Conv2d_0c_3x3_Activation')(branch_2) 76 | branches = [branch_0, branch_1, branch_2] 77 | mixed = Concatenate(axis=3, name='Block35_2_Concatenate')(branches) 78 | up = Conv2D(256, 1, strides=1, padding='same', use_bias=True, name= 'Block35_2_Conv2d_1x1') (mixed) 79 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.17})(up) 80 | x = add([x, up]) 81 | x = Activation('relu', name='Block35_2_Activation')(x) 82 | 83 | branch_0 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_0_Conv2d_1x1') (x) 84 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 85 | branch_0 = Activation('relu', name='Block35_3_Branch_0_Conv2d_1x1_Activation')(branch_0) 86 | branch_1 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_1_Conv2d_0a_1x1') (x) 87 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 88 | branch_1 = Activation('relu', name='Block35_3_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 89 | branch_1 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_1_Conv2d_0b_3x3') (branch_1) 90 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 91 | branch_1 = Activation('relu', name='Block35_3_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 92 | branch_2 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_2_Conv2d_0a_1x1') (x) 93 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 94 | branch_2 = Activation('relu', name='Block35_3_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 95 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_2_Conv2d_0b_3x3') (branch_2) 96 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 97 | branch_2 = Activation('relu', name='Block35_3_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 98 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_3_Branch_2_Conv2d_0c_3x3') (branch_2) 99 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_3_Branch_2_Conv2d_0c_3x3_BatchNorm')(branch_2) 100 | branch_2 = Activation('relu', name='Block35_3_Branch_2_Conv2d_0c_3x3_Activation')(branch_2) 101 | branches = [branch_0, branch_1, branch_2] 102 | mixed = Concatenate(axis=3, name='Block35_3_Concatenate')(branches) 103 | up = Conv2D(256, 1, strides=1, padding='same', use_bias=True, name= 'Block35_3_Conv2d_1x1') (mixed) 104 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.17})(up) 105 | x = add([x, up]) 106 | x = Activation('relu', name='Block35_3_Activation')(x) 107 | 108 | branch_0 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_0_Conv2d_1x1') (x) 109 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 110 | branch_0 = Activation('relu', name='Block35_4_Branch_0_Conv2d_1x1_Activation')(branch_0) 111 | branch_1 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_1_Conv2d_0a_1x1') (x) 112 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 113 | branch_1 = Activation('relu', name='Block35_4_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 114 | branch_1 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_1_Conv2d_0b_3x3') (branch_1) 115 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 116 | branch_1 = Activation('relu', name='Block35_4_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 117 | branch_2 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_2_Conv2d_0a_1x1') (x) 118 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 119 | branch_2 = Activation('relu', name='Block35_4_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 120 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_2_Conv2d_0b_3x3') (branch_2) 121 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 122 | branch_2 = Activation('relu', name='Block35_4_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 123 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_4_Branch_2_Conv2d_0c_3x3') (branch_2) 124 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_4_Branch_2_Conv2d_0c_3x3_BatchNorm')(branch_2) 125 | branch_2 = Activation('relu', name='Block35_4_Branch_2_Conv2d_0c_3x3_Activation')(branch_2) 126 | branches = [branch_0, branch_1, branch_2] 127 | mixed = Concatenate(axis=3, name='Block35_4_Concatenate')(branches) 128 | up = Conv2D(256, 1, strides=1, padding='same', use_bias=True, name= 'Block35_4_Conv2d_1x1') (mixed) 129 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.17})(up) 130 | x = add([x, up]) 131 | x = Activation('relu', name='Block35_4_Activation')(x) 132 | 133 | branch_0 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_0_Conv2d_1x1') (x) 134 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 135 | branch_0 = Activation('relu', name='Block35_5_Branch_0_Conv2d_1x1_Activation')(branch_0) 136 | branch_1 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_1_Conv2d_0a_1x1') (x) 137 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 138 | branch_1 = Activation('relu', name='Block35_5_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 139 | branch_1 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_1_Conv2d_0b_3x3') (branch_1) 140 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 141 | branch_1 = Activation('relu', name='Block35_5_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 142 | branch_2 = Conv2D(32, 1, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_2_Conv2d_0a_1x1') (x) 143 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 144 | branch_2 = Activation('relu', name='Block35_5_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 145 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_2_Conv2d_0b_3x3') (branch_2) 146 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 147 | branch_2 = Activation('relu', name='Block35_5_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 148 | branch_2 = Conv2D(32, 3, strides=1, padding='same', use_bias=False, name= 'Block35_5_Branch_2_Conv2d_0c_3x3') (branch_2) 149 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block35_5_Branch_2_Conv2d_0c_3x3_BatchNorm')(branch_2) 150 | branch_2 = Activation('relu', name='Block35_5_Branch_2_Conv2d_0c_3x3_Activation')(branch_2) 151 | branches = [branch_0, branch_1, branch_2] 152 | mixed = Concatenate(axis=3, name='Block35_5_Concatenate')(branches) 153 | up = Conv2D(256, 1, strides=1, padding='same', use_bias=True, name= 'Block35_5_Conv2d_1x1') (mixed) 154 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.17})(up) 155 | x = add([x, up]) 156 | x = Activation('relu', name='Block35_5_Activation')(x) 157 | 158 | # Mixed 6a (Reduction-A block): 159 | branch_0 = Conv2D(384, 3, strides=2, padding='valid', use_bias=False, name= 'Mixed_6a_Branch_0_Conv2d_1a_3x3') (x) 160 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_6a_Branch_0_Conv2d_1a_3x3_BatchNorm')(branch_0) 161 | branch_0 = Activation('relu', name='Mixed_6a_Branch_0_Conv2d_1a_3x3_Activation')(branch_0) 162 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Mixed_6a_Branch_1_Conv2d_0a_1x1') (x) 163 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_6a_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 164 | branch_1 = Activation('relu', name='Mixed_6a_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 165 | branch_1 = Conv2D(192, 3, strides=1, padding='same', use_bias=False, name= 'Mixed_6a_Branch_1_Conv2d_0b_3x3') (branch_1) 166 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_6a_Branch_1_Conv2d_0b_3x3_BatchNorm')(branch_1) 167 | branch_1 = Activation('relu', name='Mixed_6a_Branch_1_Conv2d_0b_3x3_Activation')(branch_1) 168 | branch_1 = Conv2D(256, 3, strides=2, padding='valid', use_bias=False, name= 'Mixed_6a_Branch_1_Conv2d_1a_3x3') (branch_1) 169 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_6a_Branch_1_Conv2d_1a_3x3_BatchNorm')(branch_1) 170 | branch_1 = Activation('relu', name='Mixed_6a_Branch_1_Conv2d_1a_3x3_Activation')(branch_1) 171 | branch_pool = MaxPooling2D(3, strides=2, padding='valid', name='Mixed_6a_Branch_2_MaxPool_1a_3x3')(x) 172 | branches = [branch_0, branch_1, branch_pool] 173 | x = Concatenate(axis=3, name='Mixed_6a')(branches) 174 | 175 | # 10x Block17 (Inception-ResNet-B block): 176 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_1_Branch_0_Conv2d_1x1') (x) 177 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_1_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 178 | branch_0 = Activation('relu', name='Block17_1_Branch_0_Conv2d_1x1_Activation')(branch_0) 179 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_1_Branch_1_Conv2d_0a_1x1') (x) 180 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_1_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 181 | branch_1 = Activation('relu', name='Block17_1_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 182 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_1_Branch_1_Conv2d_0b_1x7') (branch_1) 183 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_1_Branch_1_Conv2d_0b_1x7_BatchNorm')(branch_1) 184 | branch_1 = Activation('relu', name='Block17_1_Branch_1_Conv2d_0b_1x7_Activation')(branch_1) 185 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_1_Branch_1_Conv2d_0c_7x1') (branch_1) 186 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_1_Branch_1_Conv2d_0c_7x1_BatchNorm')(branch_1) 187 | branch_1 = Activation('relu', name='Block17_1_Branch_1_Conv2d_0c_7x1_Activation')(branch_1) 188 | branches = [branch_0, branch_1] 189 | mixed = Concatenate(axis=3, name='Block17_1_Concatenate')(branches) 190 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_1_Conv2d_1x1') (mixed) 191 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 192 | x = add([x, up]) 193 | x = Activation('relu', name='Block17_1_Activation')(x) 194 | 195 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_2_Branch_0_Conv2d_1x1') (x) 196 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_2_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 197 | branch_0 = Activation('relu', name='Block17_2_Branch_0_Conv2d_1x1_Activation')(branch_0) 198 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_2_Branch_2_Conv2d_0a_1x1') (x) 199 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_2_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_1) 200 | branch_1 = Activation('relu', name='Block17_2_Branch_2_Conv2d_0a_1x1_Activation')(branch_1) 201 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_2_Branch_2_Conv2d_0b_1x7') (branch_1) 202 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_2_Branch_2_Conv2d_0b_1x7_BatchNorm')(branch_1) 203 | branch_1 = Activation('relu', name='Block17_2_Branch_2_Conv2d_0b_1x7_Activation')(branch_1) 204 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_2_Branch_2_Conv2d_0c_7x1') (branch_1) 205 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_2_Branch_2_Conv2d_0c_7x1_BatchNorm')(branch_1) 206 | branch_1 = Activation('relu', name='Block17_2_Branch_2_Conv2d_0c_7x1_Activation')(branch_1) 207 | branches = [branch_0, branch_1] 208 | mixed = Concatenate(axis=3, name='Block17_2_Concatenate')(branches) 209 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_2_Conv2d_1x1') (mixed) 210 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 211 | x = add([x, up]) 212 | x = Activation('relu', name='Block17_2_Activation')(x) 213 | 214 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_3_Branch_0_Conv2d_1x1') (x) 215 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_3_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 216 | branch_0 = Activation('relu', name='Block17_3_Branch_0_Conv2d_1x1_Activation')(branch_0) 217 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_3_Branch_3_Conv2d_0a_1x1') (x) 218 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_3_Branch_3_Conv2d_0a_1x1_BatchNorm')(branch_1) 219 | branch_1 = Activation('relu', name='Block17_3_Branch_3_Conv2d_0a_1x1_Activation')(branch_1) 220 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_3_Branch_3_Conv2d_0b_1x7') (branch_1) 221 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_3_Branch_3_Conv2d_0b_1x7_BatchNorm')(branch_1) 222 | branch_1 = Activation('relu', name='Block17_3_Branch_3_Conv2d_0b_1x7_Activation')(branch_1) 223 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_3_Branch_3_Conv2d_0c_7x1') (branch_1) 224 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_3_Branch_3_Conv2d_0c_7x1_BatchNorm')(branch_1) 225 | branch_1 = Activation('relu', name='Block17_3_Branch_3_Conv2d_0c_7x1_Activation')(branch_1) 226 | branches = [branch_0, branch_1] 227 | mixed = Concatenate(axis=3, name='Block17_3_Concatenate')(branches) 228 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_3_Conv2d_1x1') (mixed) 229 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 230 | x = add([x, up]) 231 | x = Activation('relu', name='Block17_3_Activation')(x) 232 | 233 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_4_Branch_0_Conv2d_1x1') (x) 234 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_4_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 235 | branch_0 = Activation('relu', name='Block17_4_Branch_0_Conv2d_1x1_Activation')(branch_0) 236 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_4_Branch_4_Conv2d_0a_1x1') (x) 237 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_4_Branch_4_Conv2d_0a_1x1_BatchNorm')(branch_1) 238 | branch_1 = Activation('relu', name='Block17_4_Branch_4_Conv2d_0a_1x1_Activation')(branch_1) 239 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_4_Branch_4_Conv2d_0b_1x7') (branch_1) 240 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_4_Branch_4_Conv2d_0b_1x7_BatchNorm')(branch_1) 241 | branch_1 = Activation('relu', name='Block17_4_Branch_4_Conv2d_0b_1x7_Activation')(branch_1) 242 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_4_Branch_4_Conv2d_0c_7x1') (branch_1) 243 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_4_Branch_4_Conv2d_0c_7x1_BatchNorm')(branch_1) 244 | branch_1 = Activation('relu', name='Block17_4_Branch_4_Conv2d_0c_7x1_Activation')(branch_1) 245 | branches = [branch_0, branch_1] 246 | mixed = Concatenate(axis=3, name='Block17_4_Concatenate')(branches) 247 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_4_Conv2d_1x1') (mixed) 248 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 249 | x = add([x, up]) 250 | x = Activation('relu', name='Block17_4_Activation')(x) 251 | 252 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_5_Branch_0_Conv2d_1x1') (x) 253 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_5_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 254 | branch_0 = Activation('relu', name='Block17_5_Branch_0_Conv2d_1x1_Activation')(branch_0) 255 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_5_Branch_5_Conv2d_0a_1x1') (x) 256 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_5_Branch_5_Conv2d_0a_1x1_BatchNorm')(branch_1) 257 | branch_1 = Activation('relu', name='Block17_5_Branch_5_Conv2d_0a_1x1_Activation')(branch_1) 258 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_5_Branch_5_Conv2d_0b_1x7') (branch_1) 259 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_5_Branch_5_Conv2d_0b_1x7_BatchNorm')(branch_1) 260 | branch_1 = Activation('relu', name='Block17_5_Branch_5_Conv2d_0b_1x7_Activation')(branch_1) 261 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_5_Branch_5_Conv2d_0c_7x1') (branch_1) 262 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_5_Branch_5_Conv2d_0c_7x1_BatchNorm')(branch_1) 263 | branch_1 = Activation('relu', name='Block17_5_Branch_5_Conv2d_0c_7x1_Activation')(branch_1) 264 | branches = [branch_0, branch_1] 265 | mixed = Concatenate(axis=3, name='Block17_5_Concatenate')(branches) 266 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_5_Conv2d_1x1') (mixed) 267 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 268 | x = add([x, up]) 269 | x = Activation('relu', name='Block17_5_Activation')(x) 270 | 271 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_6_Branch_0_Conv2d_1x1') (x) 272 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_6_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 273 | branch_0 = Activation('relu', name='Block17_6_Branch_0_Conv2d_1x1_Activation')(branch_0) 274 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_6_Branch_6_Conv2d_0a_1x1') (x) 275 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_6_Branch_6_Conv2d_0a_1x1_BatchNorm')(branch_1) 276 | branch_1 = Activation('relu', name='Block17_6_Branch_6_Conv2d_0a_1x1_Activation')(branch_1) 277 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_6_Branch_6_Conv2d_0b_1x7') (branch_1) 278 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_6_Branch_6_Conv2d_0b_1x7_BatchNorm')(branch_1) 279 | branch_1 = Activation('relu', name='Block17_6_Branch_6_Conv2d_0b_1x7_Activation')(branch_1) 280 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_6_Branch_6_Conv2d_0c_7x1') (branch_1) 281 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_6_Branch_6_Conv2d_0c_7x1_BatchNorm')(branch_1) 282 | branch_1 = Activation('relu', name='Block17_6_Branch_6_Conv2d_0c_7x1_Activation')(branch_1) 283 | branches = [branch_0, branch_1] 284 | mixed = Concatenate(axis=3, name='Block17_6_Concatenate')(branches) 285 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_6_Conv2d_1x1') (mixed) 286 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 287 | x = add([x, up]) 288 | x = Activation('relu', name='Block17_6_Activation')(x) 289 | 290 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_7_Branch_0_Conv2d_1x1') (x) 291 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_7_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 292 | branch_0 = Activation('relu', name='Block17_7_Branch_0_Conv2d_1x1_Activation')(branch_0) 293 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_7_Branch_7_Conv2d_0a_1x1') (x) 294 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_7_Branch_7_Conv2d_0a_1x1_BatchNorm')(branch_1) 295 | branch_1 = Activation('relu', name='Block17_7_Branch_7_Conv2d_0a_1x1_Activation')(branch_1) 296 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_7_Branch_7_Conv2d_0b_1x7') (branch_1) 297 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_7_Branch_7_Conv2d_0b_1x7_BatchNorm')(branch_1) 298 | branch_1 = Activation('relu', name='Block17_7_Branch_7_Conv2d_0b_1x7_Activation')(branch_1) 299 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_7_Branch_7_Conv2d_0c_7x1') (branch_1) 300 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_7_Branch_7_Conv2d_0c_7x1_BatchNorm')(branch_1) 301 | branch_1 = Activation('relu', name='Block17_7_Branch_7_Conv2d_0c_7x1_Activation')(branch_1) 302 | branches = [branch_0, branch_1] 303 | mixed = Concatenate(axis=3, name='Block17_7_Concatenate')(branches) 304 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_7_Conv2d_1x1') (mixed) 305 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 306 | x = add([x, up]) 307 | x = Activation('relu', name='Block17_7_Activation')(x) 308 | 309 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_8_Branch_0_Conv2d_1x1') (x) 310 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_8_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 311 | branch_0 = Activation('relu', name='Block17_8_Branch_0_Conv2d_1x1_Activation')(branch_0) 312 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_8_Branch_8_Conv2d_0a_1x1') (x) 313 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_8_Branch_8_Conv2d_0a_1x1_BatchNorm')(branch_1) 314 | branch_1 = Activation('relu', name='Block17_8_Branch_8_Conv2d_0a_1x1_Activation')(branch_1) 315 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_8_Branch_8_Conv2d_0b_1x7') (branch_1) 316 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_8_Branch_8_Conv2d_0b_1x7_BatchNorm')(branch_1) 317 | branch_1 = Activation('relu', name='Block17_8_Branch_8_Conv2d_0b_1x7_Activation')(branch_1) 318 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_8_Branch_8_Conv2d_0c_7x1') (branch_1) 319 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_8_Branch_8_Conv2d_0c_7x1_BatchNorm')(branch_1) 320 | branch_1 = Activation('relu', name='Block17_8_Branch_8_Conv2d_0c_7x1_Activation')(branch_1) 321 | branches = [branch_0, branch_1] 322 | mixed = Concatenate(axis=3, name='Block17_8_Concatenate')(branches) 323 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_8_Conv2d_1x1') (mixed) 324 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 325 | x = add([x, up]) 326 | x = Activation('relu', name='Block17_8_Activation')(x) 327 | 328 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_9_Branch_0_Conv2d_1x1') (x) 329 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_9_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 330 | branch_0 = Activation('relu', name='Block17_9_Branch_0_Conv2d_1x1_Activation')(branch_0) 331 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_9_Branch_9_Conv2d_0a_1x1') (x) 332 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_9_Branch_9_Conv2d_0a_1x1_BatchNorm')(branch_1) 333 | branch_1 = Activation('relu', name='Block17_9_Branch_9_Conv2d_0a_1x1_Activation')(branch_1) 334 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_9_Branch_9_Conv2d_0b_1x7') (branch_1) 335 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_9_Branch_9_Conv2d_0b_1x7_BatchNorm')(branch_1) 336 | branch_1 = Activation('relu', name='Block17_9_Branch_9_Conv2d_0b_1x7_Activation')(branch_1) 337 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_9_Branch_9_Conv2d_0c_7x1') (branch_1) 338 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_9_Branch_9_Conv2d_0c_7x1_BatchNorm')(branch_1) 339 | branch_1 = Activation('relu', name='Block17_9_Branch_9_Conv2d_0c_7x1_Activation')(branch_1) 340 | branches = [branch_0, branch_1] 341 | mixed = Concatenate(axis=3, name='Block17_9_Concatenate')(branches) 342 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_9_Conv2d_1x1') (mixed) 343 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 344 | x = add([x, up]) 345 | x = Activation('relu', name='Block17_9_Activation')(x) 346 | 347 | branch_0 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_10_Branch_0_Conv2d_1x1') (x) 348 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_10_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 349 | branch_0 = Activation('relu', name='Block17_10_Branch_0_Conv2d_1x1_Activation')(branch_0) 350 | branch_1 = Conv2D(128, 1, strides=1, padding='same', use_bias=False, name= 'Block17_10_Branch_10_Conv2d_0a_1x1') (x) 351 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_10_Branch_10_Conv2d_0a_1x1_BatchNorm')(branch_1) 352 | branch_1 = Activation('relu', name='Block17_10_Branch_10_Conv2d_0a_1x1_Activation')(branch_1) 353 | branch_1 = Conv2D(128, [1, 7], strides=1, padding='same', use_bias=False, name= 'Block17_10_Branch_10_Conv2d_0b_1x7') (branch_1) 354 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_10_Branch_10_Conv2d_0b_1x7_BatchNorm')(branch_1) 355 | branch_1 = Activation('relu', name='Block17_10_Branch_10_Conv2d_0b_1x7_Activation')(branch_1) 356 | branch_1 = Conv2D(128, [7, 1], strides=1, padding='same', use_bias=False, name= 'Block17_10_Branch_10_Conv2d_0c_7x1') (branch_1) 357 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block17_10_Branch_10_Conv2d_0c_7x1_BatchNorm')(branch_1) 358 | branch_1 = Activation('relu', name='Block17_10_Branch_10_Conv2d_0c_7x1_Activation')(branch_1) 359 | branches = [branch_0, branch_1] 360 | mixed = Concatenate(axis=3, name='Block17_10_Concatenate')(branches) 361 | up = Conv2D(896, 1, strides=1, padding='same', use_bias=True, name= 'Block17_10_Conv2d_1x1') (mixed) 362 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.1})(up) 363 | x = add([x, up]) 364 | x = Activation('relu', name='Block17_10_Activation')(x) 365 | 366 | # Mixed 7a (Reduction-B block): 8 x 8 x 2080 367 | branch_0 = Conv2D(256, 1, strides=1, padding='same', use_bias=False, name= 'Mixed_7a_Branch_0_Conv2d_0a_1x1') (x) 368 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_0_Conv2d_0a_1x1_BatchNorm')(branch_0) 369 | branch_0 = Activation('relu', name='Mixed_7a_Branch_0_Conv2d_0a_1x1_Activation')(branch_0) 370 | branch_0 = Conv2D(384, 3, strides=2, padding='valid', use_bias=False, name= 'Mixed_7a_Branch_0_Conv2d_1a_3x3') (branch_0) 371 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_0_Conv2d_1a_3x3_BatchNorm')(branch_0) 372 | branch_0 = Activation('relu', name='Mixed_7a_Branch_0_Conv2d_1a_3x3_Activation')(branch_0) 373 | branch_1 = Conv2D(256, 1, strides=1, padding='same', use_bias=False, name= 'Mixed_7a_Branch_1_Conv2d_0a_1x1') (x) 374 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 375 | branch_1 = Activation('relu', name='Mixed_7a_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 376 | branch_1 = Conv2D(256, 3, strides=2, padding='valid', use_bias=False, name= 'Mixed_7a_Branch_1_Conv2d_1a_3x3') (branch_1) 377 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_1_Conv2d_1a_3x3_BatchNorm')(branch_1) 378 | branch_1 = Activation('relu', name='Mixed_7a_Branch_1_Conv2d_1a_3x3_Activation')(branch_1) 379 | branch_2 = Conv2D(256, 1, strides=1, padding='same', use_bias=False, name= 'Mixed_7a_Branch_2_Conv2d_0a_1x1') (x) 380 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_2) 381 | branch_2 = Activation('relu', name='Mixed_7a_Branch_2_Conv2d_0a_1x1_Activation')(branch_2) 382 | branch_2 = Conv2D(256, 3, strides=1, padding='same', use_bias=False, name= 'Mixed_7a_Branch_2_Conv2d_0b_3x3') (branch_2) 383 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_2_Conv2d_0b_3x3_BatchNorm')(branch_2) 384 | branch_2 = Activation('relu', name='Mixed_7a_Branch_2_Conv2d_0b_3x3_Activation')(branch_2) 385 | branch_2 = Conv2D(256, 3, strides=2, padding='valid', use_bias=False, name= 'Mixed_7a_Branch_2_Conv2d_1a_3x3') (branch_2) 386 | branch_2 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Mixed_7a_Branch_2_Conv2d_1a_3x3_BatchNorm')(branch_2) 387 | branch_2 = Activation('relu', name='Mixed_7a_Branch_2_Conv2d_1a_3x3_Activation')(branch_2) 388 | branch_pool = MaxPooling2D(3, strides=2, padding='valid', name='Mixed_7a_Branch_3_MaxPool_1a_3x3')(x) 389 | branches = [branch_0, branch_1, branch_2, branch_pool] 390 | x = Concatenate(axis=3, name='Mixed_7a')(branches) 391 | 392 | # 5x Block8 (Inception-ResNet-C block): 393 | 394 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_1_Branch_0_Conv2d_1x1') (x) 395 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_1_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 396 | branch_0 = Activation('relu', name='Block8_1_Branch_0_Conv2d_1x1_Activation')(branch_0) 397 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_1_Branch_1_Conv2d_0a_1x1') (x) 398 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_1_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 399 | branch_1 = Activation('relu', name='Block8_1_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 400 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_1_Branch_1_Conv2d_0b_1x3') (branch_1) 401 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_1_Branch_1_Conv2d_0b_1x3_BatchNorm')(branch_1) 402 | branch_1 = Activation('relu', name='Block8_1_Branch_1_Conv2d_0b_1x3_Activation')(branch_1) 403 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_1_Branch_1_Conv2d_0c_3x1') (branch_1) 404 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_1_Branch_1_Conv2d_0c_3x1_BatchNorm')(branch_1) 405 | branch_1 = Activation('relu', name='Block8_1_Branch_1_Conv2d_0c_3x1_Activation')(branch_1) 406 | branches = [branch_0, branch_1] 407 | mixed = Concatenate(axis=3, name='Block8_1_Concatenate')(branches) 408 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_1_Conv2d_1x1') (mixed) 409 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.2})(up) 410 | x = add([x, up]) 411 | x = Activation('relu', name='Block8_1_Activation')(x) 412 | 413 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_2_Branch_0_Conv2d_1x1') (x) 414 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_2_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 415 | branch_0 = Activation('relu', name='Block8_2_Branch_0_Conv2d_1x1_Activation')(branch_0) 416 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_2_Branch_2_Conv2d_0a_1x1') (x) 417 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_2_Branch_2_Conv2d_0a_1x1_BatchNorm')(branch_1) 418 | branch_1 = Activation('relu', name='Block8_2_Branch_2_Conv2d_0a_1x1_Activation')(branch_1) 419 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_2_Branch_2_Conv2d_0b_1x3') (branch_1) 420 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_2_Branch_2_Conv2d_0b_1x3_BatchNorm')(branch_1) 421 | branch_1 = Activation('relu', name='Block8_2_Branch_2_Conv2d_0b_1x3_Activation')(branch_1) 422 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_2_Branch_2_Conv2d_0c_3x1') (branch_1) 423 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_2_Branch_2_Conv2d_0c_3x1_BatchNorm')(branch_1) 424 | branch_1 = Activation('relu', name='Block8_2_Branch_2_Conv2d_0c_3x1_Activation')(branch_1) 425 | branches = [branch_0, branch_1] 426 | mixed = Concatenate(axis=3, name='Block8_2_Concatenate')(branches) 427 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_2_Conv2d_1x1') (mixed) 428 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.2})(up) 429 | x = add([x, up]) 430 | x = Activation('relu', name='Block8_2_Activation')(x) 431 | 432 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_3_Branch_0_Conv2d_1x1') (x) 433 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_3_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 434 | branch_0 = Activation('relu', name='Block8_3_Branch_0_Conv2d_1x1_Activation')(branch_0) 435 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_3_Branch_3_Conv2d_0a_1x1') (x) 436 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_3_Branch_3_Conv2d_0a_1x1_BatchNorm')(branch_1) 437 | branch_1 = Activation('relu', name='Block8_3_Branch_3_Conv2d_0a_1x1_Activation')(branch_1) 438 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_3_Branch_3_Conv2d_0b_1x3') (branch_1) 439 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_3_Branch_3_Conv2d_0b_1x3_BatchNorm')(branch_1) 440 | branch_1 = Activation('relu', name='Block8_3_Branch_3_Conv2d_0b_1x3_Activation')(branch_1) 441 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_3_Branch_3_Conv2d_0c_3x1') (branch_1) 442 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_3_Branch_3_Conv2d_0c_3x1_BatchNorm')(branch_1) 443 | branch_1 = Activation('relu', name='Block8_3_Branch_3_Conv2d_0c_3x1_Activation')(branch_1) 444 | branches = [branch_0, branch_1] 445 | mixed = Concatenate(axis=3, name='Block8_3_Concatenate')(branches) 446 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_3_Conv2d_1x1') (mixed) 447 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.2})(up) 448 | x = add([x, up]) 449 | x = Activation('relu', name='Block8_3_Activation')(x) 450 | 451 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_4_Branch_0_Conv2d_1x1') (x) 452 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_4_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 453 | branch_0 = Activation('relu', name='Block8_4_Branch_0_Conv2d_1x1_Activation')(branch_0) 454 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_4_Branch_4_Conv2d_0a_1x1') (x) 455 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_4_Branch_4_Conv2d_0a_1x1_BatchNorm')(branch_1) 456 | branch_1 = Activation('relu', name='Block8_4_Branch_4_Conv2d_0a_1x1_Activation')(branch_1) 457 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_4_Branch_4_Conv2d_0b_1x3') (branch_1) 458 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_4_Branch_4_Conv2d_0b_1x3_BatchNorm')(branch_1) 459 | branch_1 = Activation('relu', name='Block8_4_Branch_4_Conv2d_0b_1x3_Activation')(branch_1) 460 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_4_Branch_4_Conv2d_0c_3x1') (branch_1) 461 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_4_Branch_4_Conv2d_0c_3x1_BatchNorm')(branch_1) 462 | branch_1 = Activation('relu', name='Block8_4_Branch_4_Conv2d_0c_3x1_Activation')(branch_1) 463 | branches = [branch_0, branch_1] 464 | mixed = Concatenate(axis=3, name='Block8_4_Concatenate')(branches) 465 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_4_Conv2d_1x1') (mixed) 466 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.2})(up) 467 | x = add([x, up]) 468 | x = Activation('relu', name='Block8_4_Activation')(x) 469 | 470 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_5_Branch_0_Conv2d_1x1') (x) 471 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_5_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 472 | branch_0 = Activation('relu', name='Block8_5_Branch_0_Conv2d_1x1_Activation')(branch_0) 473 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_5_Branch_5_Conv2d_0a_1x1') (x) 474 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_5_Branch_5_Conv2d_0a_1x1_BatchNorm')(branch_1) 475 | branch_1 = Activation('relu', name='Block8_5_Branch_5_Conv2d_0a_1x1_Activation')(branch_1) 476 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_5_Branch_5_Conv2d_0b_1x3') (branch_1) 477 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_5_Branch_5_Conv2d_0b_1x3_BatchNorm')(branch_1) 478 | branch_1 = Activation('relu', name='Block8_5_Branch_5_Conv2d_0b_1x3_Activation')(branch_1) 479 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_5_Branch_5_Conv2d_0c_3x1') (branch_1) 480 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_5_Branch_5_Conv2d_0c_3x1_BatchNorm')(branch_1) 481 | branch_1 = Activation('relu', name='Block8_5_Branch_5_Conv2d_0c_3x1_Activation')(branch_1) 482 | branches = [branch_0, branch_1] 483 | mixed = Concatenate(axis=3, name='Block8_5_Concatenate')(branches) 484 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_5_Conv2d_1x1') (mixed) 485 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 0.2})(up) 486 | x = add([x, up]) 487 | x = Activation('relu', name='Block8_5_Activation')(x) 488 | 489 | branch_0 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_6_Branch_0_Conv2d_1x1') (x) 490 | branch_0 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_6_Branch_0_Conv2d_1x1_BatchNorm')(branch_0) 491 | branch_0 = Activation('relu', name='Block8_6_Branch_0_Conv2d_1x1_Activation')(branch_0) 492 | branch_1 = Conv2D(192, 1, strides=1, padding='same', use_bias=False, name= 'Block8_6_Branch_1_Conv2d_0a_1x1') (x) 493 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_6_Branch_1_Conv2d_0a_1x1_BatchNorm')(branch_1) 494 | branch_1 = Activation('relu', name='Block8_6_Branch_1_Conv2d_0a_1x1_Activation')(branch_1) 495 | branch_1 = Conv2D(192, [1, 3], strides=1, padding='same', use_bias=False, name= 'Block8_6_Branch_1_Conv2d_0b_1x3') (branch_1) 496 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_6_Branch_1_Conv2d_0b_1x3_BatchNorm')(branch_1) 497 | branch_1 = Activation('relu', name='Block8_6_Branch_1_Conv2d_0b_1x3_Activation')(branch_1) 498 | branch_1 = Conv2D(192, [3, 1], strides=1, padding='same', use_bias=False, name= 'Block8_6_Branch_1_Conv2d_0c_3x1') (branch_1) 499 | branch_1 = BatchNormalization(axis=3, momentum=0.995, epsilon=0.001, scale=False, name='Block8_6_Branch_1_Conv2d_0c_3x1_BatchNorm')(branch_1) 500 | branch_1 = Activation('relu', name='Block8_6_Branch_1_Conv2d_0c_3x1_Activation')(branch_1) 501 | branches = [branch_0, branch_1] 502 | mixed = Concatenate(axis=3, name='Block8_6_Concatenate')(branches) 503 | up = Conv2D(1792, 1, strides=1, padding='same', use_bias=True, name= 'Block8_6_Conv2d_1x1') (mixed) 504 | up = Lambda(scaling, output_shape=K.int_shape(up)[1:], arguments={'scale': 1})(up) 505 | x = add([x, up]) 506 | 507 | # Classification block 508 | x = GlobalAveragePooling2D(name='AvgPool')(x) 509 | x = Dropout(1.0 - 0.8, name='Dropout')(x) 510 | 511 | # Bottleneck 512 | x = Dense(128, use_bias=False, name='Bottleneck')(x) 513 | x = BatchNormalization(momentum=0.995, epsilon=0.001, scale=False, name='Bottleneck_BatchNorm')(x) 514 | 515 | # Create model 516 | model = Model(inputs, x, name='inception_resnet_v1') 517 | 518 | return model -------------------------------------------------------------------------------- /faceNet/convert_to_onnx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | import tf2onnx 4 | from architecture import InceptionResNetV2 5 | 6 | if __name__ == '__main__': 7 | """ weights can be downloaded from https://drive.google.com/drive/folders/1scGoVCQp-cNwKTKOUqevCP1N2LlyXU3l?usp=sharing 8 | Put facenet_keras_weights.h5 file in model folder 9 | """ 10 | facenet_weights_path = "models/facenet_keras_weights.h5" 11 | onnx_model_output_path = "models/faceNet.onnx" 12 | 13 | if not os.path.exists(facenet_weights_path): 14 | raise Exception(f"Model doesn't exists in {facenet_weights_path}, download weights from \ 15 | https://drive.google.com/drive/folders/1scGoVCQp-cNwKTKOUqevCP1N2LlyXU3l?usp=sharing") 16 | 17 | faceNet = InceptionResNetV2() 18 | faceNet.load_weights(facenet_weights_path) 19 | 20 | spec = (tf.TensorSpec(faceNet.inputs[0].shape, tf.float32, name="image_input"),) 21 | tf2onnx.convert.from_keras(faceNet, output_path=onnx_model_output_path, input_signature=spec) -------------------------------------------------------------------------------- /faceNet/faceNet.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import stow 3 | import typing 4 | import numpy as np 5 | import onnxruntime as ort 6 | 7 | class FaceNet: 8 | """FaceNet class object, which can be used for simplified face recognition 9 | """ 10 | def __init__( 11 | self, 12 | detector: object, 13 | onnx_model_path: str = "models/faceNet.onnx", 14 | anchors: typing.Union[str, dict] = 'faces', 15 | force_cpu: bool = False, 16 | threshold: float = 0.5, 17 | color: tuple = (255, 255, 255), 18 | thickness: int = 2, 19 | ) -> None: 20 | """Object for face recognition 21 | Params: 22 | detector: (object) - detector object to detect faces in image 23 | onnx_model_path: (str) - path to onnx model 24 | force_cpu: (bool) - if True, onnx model will be run on CPU 25 | anchors: (str or dict) - path to directory with faces or dictionary with anchor names as keys and anchor encodings as values 26 | threshold: (float) - threshold for face recognition 27 | color: (tuple) - color of bounding box and text 28 | thickness: (int) - thickness of bounding box and text 29 | """ 30 | if not stow.exists(onnx_model_path): 31 | raise Exception(f"Model doesn't exists in {onnx_model_path}") 32 | 33 | self.detector = detector 34 | self.threshold = threshold 35 | self.color = color 36 | self.thickness = thickness 37 | 38 | providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] 39 | 40 | providers = providers if ort.get_device() == "GPU" and not force_cpu else providers[::-1] 41 | 42 | self.ort_sess = ort.InferenceSession(onnx_model_path, providers=providers) 43 | 44 | self.input_shape = self.ort_sess._inputs_meta[0].shape[1:3] 45 | 46 | self.anchors = self.load_anchors(anchors) if isinstance(anchors, str) else anchors 47 | 48 | def normalize(self, img: np.ndarray) -> np.ndarray: 49 | """Normalize image 50 | 51 | Args: 52 | img: (np.ndarray) - image to be normalized 53 | 54 | Returns: 55 | img: (np.ndarray) - normalized image 56 | """ 57 | mean, std = img.mean(), img.std() 58 | return (img - mean) / std 59 | 60 | def l2_normalize(self, x: np.ndarray, axis: int = -1, epsilon: float = 1e-10) -> np.ndarray: 61 | """l2 normalization function 62 | 63 | Args: 64 | x: (np.ndarray) - input array 65 | axis: (int) - axis to normalize 66 | epsilon: (float) - epsilon to avoid division by zero 67 | 68 | Returns: 69 | x: (np.ndarray) - normalized array 70 | """ 71 | output = x / np.sqrt(np.maximum(np.sum(np.square(x), axis=axis, keepdims=True), epsilon)) 72 | return output 73 | 74 | def detect_save_faces(self, image: np.ndarray, output_dir: str = "faces"): 75 | """Detect faces in given image and save them to output_dir 76 | 77 | Args: 78 | image: (np.ndarray) - image to be processed 79 | output_dir: (str) - directory where faces will be saved 80 | 81 | Returns: 82 | bool: (bool) - True if faces were detected and saved 83 | """ 84 | face_crops = [image[t:b, l:r] for t, l, b, r in self.detector(image, return_tlbr=True)] 85 | 86 | if face_crops == []: 87 | return False 88 | 89 | stow.mkdir(output_dir) 90 | 91 | for index, crop in enumerate(face_crops): 92 | output_path = stow.join(output_dir, f"face_{str(index)}.png") 93 | cv2.imwrite(output_path, crop) 94 | print("Crop saved to:", output_path) 95 | 96 | self.anchors = self.load_anchors(output_dir) 97 | 98 | return True 99 | 100 | def load_anchors(self, faces_path: str): 101 | """Generate anchors for given faces path 102 | 103 | Args: 104 | faces_path: (str) - path to directory with faces 105 | 106 | Returns: 107 | anchors: (dict) - dictionary with anchor names as keys and anchor encodings as values 108 | """ 109 | anchors = {} 110 | if not stow.exists(faces_path): 111 | return {} 112 | 113 | for face_path in stow.ls(faces_path): 114 | anchors[stow.basename(face_path)] = self.encode(cv2.imread(face_path.path)) 115 | 116 | return anchors 117 | 118 | def encode(self, face_image: np.ndarray) -> np.ndarray: 119 | """Encode face image with FaceNet model 120 | 121 | Args 122 | face_image: (np.ndarray) - face image to be encoded 123 | 124 | Returns: 125 | face_encoding: (np.ndarray) - face encoding 126 | """ 127 | face = self.normalize(face_image) 128 | face = cv2.resize(face, self.input_shape).astype(np.float32) 129 | 130 | encode = self.ort_sess.run(None, {self.ort_sess._inputs_meta[0].name: np.expand_dims(face, axis=0)})[0][0] 131 | normalized_encode = self.l2_normalize(encode) 132 | 133 | return normalized_encode 134 | 135 | def cosine_distance(self, a: np.ndarray, b: typing.Union[np.ndarray, list]) -> np.ndarray: 136 | """Cosine distance between wectors a and b 137 | 138 | Args: 139 | a: (np.ndarray) - first vector 140 | b: (np.ndarray) - second list of vectors 141 | 142 | Returns: 143 | distance: (float) - cosine distance 144 | """ 145 | if isinstance(a, list): 146 | a = np.array(a) 147 | 148 | if isinstance(b, list): 149 | b = np.array(b) 150 | 151 | return np.dot(a, b.T) / (np.linalg.norm(a) * np.linalg.norm(b)) 152 | 153 | def draw(self, image: np.ndarray, face_crops: dict): 154 | """Draw face crops on image 155 | 156 | Args: 157 | image: (np.ndarray) - image to be drawn on 158 | face_crops: (dict) - dictionary with face crops as values and face names as keys 159 | 160 | Returns: 161 | image: (np.ndarray) - image with drawn face crops 162 | """ 163 | for value in face_crops.values(): 164 | t, l, b, r = value["tlbr"] 165 | cv2.rectangle(image, (l, t), (r, b), self.color, self.thickness) 166 | cv2.putText(image, stow.name(value['name']), (l, t - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, self.color, self.thickness) 167 | 168 | return image 169 | 170 | def __call__(self, frame: np.ndarray) -> np.ndarray: 171 | """Face recognition pipeline 172 | 173 | Args: 174 | frame: (np.ndarray) - image to be processed 175 | 176 | Returns: 177 | frame: (np.ndarray) - image with drawn face recognition results 178 | """ 179 | face_crops = {index: {"name": "Unknown", "tlbr": tlbr} for index, tlbr in enumerate(self.detector(frame, return_tlbr=True))} 180 | for key, value in face_crops.items(): 181 | t, l, b, r = value["tlbr"] 182 | face_encoding = self.encode(frame[t:b, l:r]) 183 | distances = self.cosine_distance(face_encoding, list(self.anchors.values())) 184 | if np.max(distances) > self.threshold: 185 | face_crops[key]["name"] = list(self.anchors.keys())[np.argmax(distances)] 186 | 187 | frame = self.draw(frame, face_crops) 188 | 189 | return frame -------------------------------------------------------------------------------- /faces/Rokas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/faces/Rokas.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from engine import Engine 2 | from animegan import AnimeGAN 3 | 4 | if __name__ == '__main__': 5 | animegan = AnimeGAN("models/Hayao_64.onnx") 6 | engine = Engine(webcam_id=0, show=True, custom_objects=[animegan]) 7 | engine.run() -------------------------------------------------------------------------------- /models/AnimeGANv3_PortraitSketch_25.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/AnimeGANv3_PortraitSketch_25.onnx -------------------------------------------------------------------------------- /models/Hayao-60.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/Hayao-60.onnx -------------------------------------------------------------------------------- /models/Hayao_64.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/Hayao_64.onnx -------------------------------------------------------------------------------- /models/Paprika_54.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/Paprika_54.onnx -------------------------------------------------------------------------------- /models/Shinkai_53.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/Shinkai_53.onnx -------------------------------------------------------------------------------- /models/modnet.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pythonlessons/background_removal/6d8acea0e59f47c1a2e3468a33082686dbef3e0b/models/modnet.onnx -------------------------------------------------------------------------------- /pencilSketch.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import typing 4 | 5 | class PencilSketch: 6 | """Apply pencil sketch effect to an image 7 | """ 8 | def __init__( 9 | self, 10 | blur_simga: int = 5, 11 | ksize: typing.Tuple[int, int] = (0, 0), 12 | sharpen_value: int = None, 13 | kernel: np.ndarray = None, 14 | ) -> None: 15 | """ 16 | Args: 17 | blur_simga: (int) - sigma ratio to apply for cv2.GaussianBlur 18 | ksize: (float) - ratio to apply for cv2.GaussianBlur 19 | sharpen_value: (int) - sharpen value to apply in predefined kernel array 20 | kernel: (np.ndarray) - custom kernel to apply in sharpen function 21 | """ 22 | self.blur_simga = blur_simga 23 | self.ksize = ksize 24 | self.sharpen_value = sharpen_value 25 | self.kernel = np.array([[0, -1, 0], [-1, sharpen_value,-1], [0, -1, 0]]) if kernel == None else kernel 26 | 27 | def dodge(self, front: np.ndarray, back: np.ndarray) -> np.ndarray: 28 | """The formula comes from https://en.wikipedia.org/wiki/Blend_modes 29 | Args: 30 | front: (np.ndarray) - front image to be applied to dodge algorithm 31 | back: (np.ndarray) - back image to be applied to dodge algorithm 32 | 33 | Returns: 34 | image: (np.ndarray) - dodged image 35 | """ 36 | result = back*255.0 / (255.0-front) 37 | result[result>255] = 255 38 | result[back==255] = 255 39 | return result.astype('uint8') 40 | 41 | def sharpen(self, image: np.ndarray) -> np.ndarray: 42 | """Sharpen image by defined kernel size 43 | Args: 44 | image: (np.ndarray) - image to be sharpened 45 | 46 | Returns: 47 | image: (np.ndarray) - sharpened image 48 | """ 49 | if self.sharpen_value is not None and isinstance(self.sharpen_value, int): 50 | inverted = 255 - image 51 | return 255 - cv2.filter2D(src=inverted, ddepth=-1, kernel=self.kernel) 52 | 53 | return image 54 | 55 | def __call__(self, frame: np.ndarray) -> np.ndarray: 56 | """Main function to do pencil sketch 57 | Args: 58 | frame: (np.ndarray) - frame to excecute pencil sketch on 59 | 60 | Returns: 61 | frame: (np.ndarray) - processed frame that is pencil sketch type 62 | """ 63 | grayscale = np.array(np.dot(frame[..., :3], [0.299, 0.587, 0.114]), dtype=np.uint8) 64 | grayscale = np.stack((grayscale,) * 3, axis=-1) # convert 1 channel grayscale image to 3 channels grayscale 65 | 66 | inverted_img = 255 - grayscale 67 | 68 | blur_img = cv2.GaussianBlur(inverted_img, ksize=self.ksize, sigmaX=self.blur_simga) 69 | 70 | final_img = self.dodge(blur_img, grayscale) 71 | 72 | sharpened_image = self.sharpen(final_img) 73 | 74 | return sharpened_image -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | opencv-python 2 | mediapipe 3 | stow 4 | numpy 5 | tqdm 6 | onnxruntime 7 | tf2onnx 8 | tensorflow==2.* -------------------------------------------------------------------------------- /selfieSegmentation.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import stow 3 | import typing 4 | import numpy as np 5 | import mediapipe as mp 6 | 7 | class MPSegmentation: 8 | """Object to create and do mediapipe selfie segmentation, more about it: 9 | https://google.github.io/mediapipe/solutions/selfie_segmentation.html 10 | """ 11 | def __init__( 12 | self, 13 | bg_blur_ratio: typing.Tuple[int, int] = (35, 35), 14 | bg_image: typing.Optional[np.ndarray] = None, 15 | threshold: float = 0.5, 16 | model_selection: bool = 1, 17 | bg_images_path: str = None, 18 | bg_color : typing.Tuple[int, int, int] = None, 19 | ) -> None: 20 | """ 21 | Args: 22 | bg_blur_ratio: (typing.Tuple) = (35, 35) - ratio to apply for cv2.GaussianBlur 23 | bg_image: (typing.Optional) = None - background color to use instead of gray color in background 24 | threshold: (float) = 0.5 - accuracy border threshold separating background and foreground, necessary to play to get the best results 25 | model_selection: (bool) = 1 - general or landscape model selection for segmentations mask 26 | bg_images_path: (str) = None - path to folder for background images 27 | bg_color: (typing.Tuple[int, int, int]) = None - color to replace background with 28 | """ 29 | self.mp_selfie_segmentation = mp.solutions.selfie_segmentation 30 | self.selfie_segmentation = self.mp_selfie_segmentation.SelfieSegmentation(model_selection=model_selection) 31 | 32 | self.bg_blur_ratio = bg_blur_ratio 33 | self.bg_image = bg_image 34 | self.threshold = threshold 35 | self.bg_color = bg_color 36 | 37 | if bg_images_path: 38 | self.bg_images = [cv2.imread(image.path) for image in stow.ls(bg_images_path)] 39 | self.bg_image = self.bg_images[0] 40 | 41 | def change_image(self, prevOrNext: bool = True) -> bool: 42 | """Change image to next or previous ir they are provided 43 | 44 | Args: 45 | prevOrNext: (bool) - argument to change image to next or previous in given list 46 | 47 | Returns: 48 | bool - Return True if successfully changed background image 49 | """ 50 | if not self.bg_images: 51 | return False 52 | 53 | if prevOrNext: 54 | self.bg_images = self.bg_images[1:] + [self.bg_images[0]] 55 | else: 56 | self.bg_images = [self.bg_images[-1]] + self.bg_images[:-1] 57 | self.bg_image = self.bg_images[0] 58 | 59 | return True 60 | 61 | def __call__(self, frame: np.ndarray) -> np.ndarray: 62 | """Main function to process selfie semgentation on each call 63 | 64 | Args: 65 | frame: (np.ndarray) - frame to excecute selfie segmentation on 66 | 67 | Returns: 68 | frame: (np.ndarray) - processed frame with selfie segmentation 69 | """ 70 | results = self.selfie_segmentation.process(frame) 71 | condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > self.threshold 72 | 73 | if self.bg_image is not None: 74 | background = self.bg_image 75 | elif self.bg_color: 76 | background = np.ones(frame.shape, np.uint8)[...,:] * self.bg_color 77 | else: 78 | background = cv2.GaussianBlur(frame, self.bg_blur_ratio, 0) 79 | 80 | frame = np.where(condition, frame, cv2.resize(background, frame.shape[:2][::-1])) 81 | 82 | return frame.astype(np.uint8) -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import typing 3 | import time 4 | import cv2 5 | 6 | class FPSmetric: 7 | """ Measure FPS between calls of this object 8 | """ 9 | def __init__( 10 | self, 11 | range_average: int = 30, 12 | position: typing.Tuple[int, int] = (7, 70), 13 | fontFace: int = cv2.FONT_HERSHEY_SIMPLEX, 14 | fontScale: int = 3, 15 | color: typing.Tuple[int, int, int] = (100, 255, 0), 16 | thickness: int = 3, 17 | lineType: int = cv2.LINE_AA, 18 | ): 19 | """ 20 | Args: 21 | range_average: (int) = 30 - number of how many call should be averaged for a result 22 | position: (typing.Tuple[int, int]) = (7, 70) - position in a frame where to put text 23 | fontFace: (int) = cv2.FONT_HERSHEY_SIMPLEX - cv2 font for text 24 | fontScale: (int) = 3 - size of font 25 | color: (typing.Tuple[int, int, int]) = (100, 255, 0) - RGB color for text 26 | thickness: (int) = 3 - chickness for text 27 | lineType: (int) = cv2.LINE_AA - text line type 28 | """ 29 | self._range_average = range_average 30 | self._frame_time = 0 31 | self._prev_frame_time = 0 32 | self._fps_list = [] 33 | 34 | self.position = position 35 | self.fontFace = fontFace 36 | self.fontScale = fontScale 37 | self.color = color 38 | self.thickness = thickness 39 | self.lineType = lineType 40 | 41 | def __call__(self, frame: np.ndarray = None) -> typing.Union[bool, np.ndarray]: 42 | """Measure duration between each call and return calculated FPS or frame with added FPS on it 43 | 44 | Args: 45 | frame: (np.ndarray) - frame to add FPS text if wanted 46 | 47 | Returns: 48 | fps: (float) - fps number if frame not given otherwise return frame (np.ndarray) 49 | """ 50 | self._prev_frame_time = self._frame_time 51 | self._frame_time = time.time() 52 | if not self._prev_frame_time: 53 | return frame 54 | self._fps_list.append(1/(self._frame_time - self._prev_frame_time)) 55 | self._fps_list = self._fps_list[-self._range_average:] 56 | 57 | fps = float(np.average(self._fps_list)) 58 | 59 | if frame is None: 60 | return fps 61 | 62 | cv2.putText(frame, str(int(fps)), self.position, self.fontFace, self.fontScale, self.color, self.thickness, self.lineType) 63 | return frame --------------------------------------------------------------------------------