├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── data ├── images │ ├── 0--Parade │ │ ├── 0_Parade_Parade_0_829.jpg │ │ └── 0_Parade_Parade_0_901.jpg │ └── 7--Cheering │ │ └── 7_Cheering_Cheering_7_500.jpg ├── local_test_groundtruth.json └── local_test_image_list.txt ├── eval_kit ├── __init__.py ├── client.py └── detector.py ├── local_test.py ├── run_evaluation.py └── sample_detector.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | .idea/ 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This is the sample Dockerfile for the evaluation. 2 | # You can change it to include your own environment for submission. 3 | 4 | FROM mxnet/python:1.4.1_gpu_cu90_py3 5 | 6 | RUN pip3 install easydict boto3 opencv-python pillow 7 | 8 | RUN apt update 9 | RUN apt-get install -y -q libfontconfig1 libxrender1 libglib2.0-0 libsm6 libxext6 ucspi-tcp git 10 | 11 | WORKDIR /app/face_det_eval 12 | RUN git clone https://github.com/yjxiong/mtcnn 13 | 14 | ADD . /app/face_det_eval 15 | 16 | ENV MXNET_CUDNN_AUTOTUNE_DEFAULT=0 17 | 18 | # This command runs the evaluation tool. 19 | # DO NOT MODIFY THE LINE BELOW OTHERWISE THE EVALUATION WILL NOT RUN 20 | CMD python3 run_evaluation.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 yjxiong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WIDER2019FaceDetectionRuntimeContainerExample 2 | This repo provides an example docker container for runtime evaluation for the WIDER 2019 challenge track: face detection accuracy and runtime. 3 | 4 | **Note**: We highly recommend the participants to test their docker images before submission. Please refer to [this section](#Test-the-docker-image-locally) for instructions. 5 | 6 | # BEFORE YOU START: Request Resource Provision 7 | 8 | First create an account on the [challenge website](https://competitions.codalab.org/competitions/22955) as well an [AWS account](https://aws.amazon.com/account/) (in any region except Beijing and Ningxia). Send your AWS account id (12 digits) and an email address to the organizing commitee's email address: `wider-challenge@ie.cuhk.edu.hk`. We will allocate evaluation resources for you. 9 | 10 | # Obtain the the example 11 | 12 | Run the following commands to build the example image 13 | 14 | ```bash 15 | 16 | git clone https://github.com/yjxiong/WIDER2019FaceDetectionRuntimeContainerExample 17 | cd WIDER2019FaceDetectionRuntimeContainerExample 18 | docker build -t wider-challenge- . 19 | ``` 20 | 21 | # Include your own algorithm 22 | 23 | - You algorithm should provide a Python class which implements the interfaces `FaceDetectorBase` in `eval_kit/detection.py`. 24 | - Modify line 27 - 29 in `run_evalution.py` to import your own detector for evaluation. 25 | - Build and submit the image to the ECR repo for your AWS account. 26 | 27 | # Test the docker image locally 28 | 29 | The submission process could take hours. It is slow to debug on the cloud. Here we provide some tools for the participants to locally test the correctness of the algorithms. 30 | 31 | To verify the algorithm can run properly, run the following command 32 | ```bash 33 | nvidia-docker run -it wider-challenge- python3 local_test.py 34 | ``` 35 | It will run the algorithms in the evaluation workflow on some sample images and print out the results. 36 | You can compare your algorithm's output with the groundtruth for the sample images. 37 | 38 | The output will look like 39 | 40 | ``` 41 | ================================================================================ 42 | all image finished, showing verification info below: 43 | ================================================================================ 44 | 45 | INFO:root:Image ID: data/images/0--Parade/0_Parade_Parade_0_901.jpg, Runtime: 0.04310035705566406 46 | INFO:root: gt box: [7, 399, 201, 342] 47 | INFO:root: gt box: [283, 680, 164, 208] 48 | INFO:root: gt box: [712, 351, 292, 374] 49 | INFO:root: prediction box: [290.0172 685.2428 163.99304 210.28986 1. ] 50 | INFO:root: prediction box: [671.7536 314.9785 340.5346 427.6707 0.99999464] 51 | INFO:root: prediction box: [ 16.449345 378.24097 279.26697 400.31653 0.9989442] 52 | INFO:root: 53 | INFO:root:Image ID: data/images/7--Cheering/7_Cheering_Cheering_7_500.jpg, Runtime: 0.04457974433898926 54 | INFO:root: gt box: [442, 140, 312, 456] 55 | INFO:root: prediction box: [420.00894 144.01822 322.09244 405.97415 1. ] 56 | INFO:root: 57 | INFO:root:Image ID: data/images/0--Parade/0_Parade_Parade_0_829.jpg, Runtime: 0.11018848419189453 58 | INFO:root: gt box: [501, 160, 285, 443] 59 | INFO:root: prediction box: [496.86624 167.46695 334.38205 421.00534 1. ] 60 | INFO:root: 61 | INFO:root:Done. Average FPS: 15.162 62 | ``` 63 | 64 | # Submitting the docker image 65 | 66 | Run the following commands (we assume you have set up the [Docker CLI authentication](https://docs.aws.amazon.com/AmazonECR/latest/userguide/Registries.html#registry_auth)) 67 | 68 | Retrieve the login command to use to authenticate your Docker client to your registry. 69 | Use the AWS CLI: 70 | 71 | ```bash 72 | $(aws ecr get-login --no-include-email --region us-west-2 --registry-ids 624814826659) 73 | ``` 74 | 75 | Note: If you receive an "Unknown options: --no-include-email" error when using the AWS CLI, ensure that you have the latest version installed. 76 | 77 | Build your Docker image using the following command. For information on building a Docker file from scratch see the instructions here . You can skip this step if your image is already built: 78 | 79 | ```bash 80 | docker build -t wider-challenge- . 81 | ``` 82 | 83 | After the build completes, tag your image so you can push the image to the repository: 84 | 85 | ```bash 86 | docker tag wider-challenge-:latest 624814826659.dkr.ecr.us-west-2.amazonaws.com/wider-challenge-:latest 87 | ``` 88 | 89 | 90 | Run the following command to push this image to your the AWS ECR repository: 91 | 92 | ```bash 93 | docker push 624814826659.dkr.ecr.us-west-2.amazonaws.com/wider-challenge-:latest 94 | ``` 95 | 96 | After you pushed to the repo, the evaluation will automatically start. In **3 hours** you should receive a email with the evaluation results if the evaluation is successful. 97 | -------------------------------------------------------------------------------- /data/images/0--Parade/0_Parade_Parade_0_829.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjxiong/WIDER2019FaceDetectionRuntimeContainerExample/710076ccc27328701d7170ccb7297bfd1dabf4ca/data/images/0--Parade/0_Parade_Parade_0_829.jpg -------------------------------------------------------------------------------- /data/images/0--Parade/0_Parade_Parade_0_901.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjxiong/WIDER2019FaceDetectionRuntimeContainerExample/710076ccc27328701d7170ccb7297bfd1dabf4ca/data/images/0--Parade/0_Parade_Parade_0_901.jpg -------------------------------------------------------------------------------- /data/images/7--Cheering/7_Cheering_Cheering_7_500.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjxiong/WIDER2019FaceDetectionRuntimeContainerExample/710076ccc27328701d7170ccb7297bfd1dabf4ca/data/images/7--Cheering/7_Cheering_Cheering_7_500.jpg -------------------------------------------------------------------------------- /data/local_test_groundtruth.json: -------------------------------------------------------------------------------- 1 | { 2 | "data/images/0--Parade/0_Parade_Parade_0_901.jpg": [ 3 | [ 4 | 7, 5 | 399, 6 | 201, 7 | 342 8 | ], 9 | [ 10 | 283, 11 | 680, 12 | 164, 13 | 208 14 | ], 15 | [ 16 | 712, 17 | 351, 18 | 292, 19 | 374 20 | ] 21 | ], 22 | "data/images/0--Parade/0_Parade_Parade_0_829.jpg": [ 23 | [ 24 | 501, 25 | 160, 26 | 285, 27 | 443 28 | ] 29 | ], 30 | "data/images/7--Cheering/7_Cheering_Cheering_7_500.jpg": [ 31 | [ 32 | 442, 33 | 140, 34 | 312, 35 | 456 36 | ] 37 | ] 38 | } -------------------------------------------------------------------------------- /data/local_test_image_list.txt: -------------------------------------------------------------------------------- 1 | data/images/0--Parade/0_Parade_Parade_0_829.jpg 2 | data/images/0--Parade/0_Parade_Parade_0_901.jpg 3 | data/images/7--Cheering/7_Cheering_Cheering_7_500.jpg -------------------------------------------------------------------------------- /eval_kit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yjxiong/WIDER2019FaceDetectionRuntimeContainerExample/710076ccc27328701d7170ccb7297bfd1dabf4ca/eval_kit/__init__.py -------------------------------------------------------------------------------- /eval_kit/client.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import cv2 4 | import os 5 | import time 6 | 7 | import logging 8 | import zipfile 9 | try: 10 | import zlib 11 | compression = zipfile.ZIP_DEFLATED 12 | except: 13 | compression = zipfile.ZIP_STORED 14 | 15 | import numpy as np 16 | 17 | from io import BytesIO 18 | 19 | 20 | # EVALUATION SYSTEM SETTINGS 21 | # Do not modify these file otherwise your evaluation will fail. 22 | 23 | WORKSPACE_BUCKET = 'wider2019-eval-workspace' 24 | IMAGE_LIST_PATH = 'test-data/wider2019_face_detection_runtime_eval_image_list.txt' 25 | IMAGE_PREFIX = 'test-images/' 26 | UPLOAD_PREFIX = 'test-upload/' 27 | 28 | 29 | def _get_s3_image_list(s3_bucket, s3_path): 30 | s3_client = boto3.client('s3', 31 | region_name='us-west-2', 32 | endpoint_url='https://s3.us-west-2.amazonaws.com' 33 | ) 34 | 35 | f = BytesIO() 36 | s3_client.download_fileobj(s3_bucket, s3_path, f) 37 | lines = f.getvalue().decode('utf-8').split('\n') 38 | return [x.strip() for x in lines if x != ''] 39 | 40 | 41 | def _download_s3_image(s3_bucket, s3_path): 42 | s3_client = boto3.client('s3', 43 | region_name='us-west-2', 44 | endpoint_url='https://s3.us-west-2.amazonaws.com' 45 | ) 46 | 47 | f = BytesIO() 48 | s3_client.download_fileobj(s3_bucket, s3_path, f) 49 | return cv2.imdecode(np.frombuffer(f.getvalue(), dtype=np.uint8), 50 | cv2.IMREAD_IGNORE_ORIENTATION | cv2.IMREAD_COLOR) 51 | 52 | def _upload_output_to_s3(data, filename, s3_bucket, s3_prefix): 53 | s3_client = boto3.client('s3', 54 | region_name='us-west-2', 55 | endpoint_url='https://s3.us-west-2.amazonaws.com' 56 | ) 57 | local_path = '/tmp/{}'.format(filename) 58 | s3_path = os.path.join(s3_prefix, filename) 59 | json.dump(data, open(local_path, 'w')) 60 | s3_client.upload_file(local_path, s3_bucket, s3_path) 61 | 62 | 63 | def get_job_id(): 64 | return os.environ['WIDER_EVAL_JOB_ID'] 65 | 66 | 67 | def upload_eval_output(output_boxes, output_time, job_id): 68 | """ 69 | This function uploads the testing output to S3 to trigger evaluation. 70 | :param output_boxes: 71 | :param output_time: 72 | :return: 73 | """ 74 | upload_data = { 75 | k: { 76 | "boxes": output_boxes[k].tolist(), 77 | "runtime": output_time[k] 78 | } for k in output_boxes 79 | } 80 | 81 | filename = '{}.json'.format(job_id) 82 | 83 | _upload_output_to_s3(upload_data, filename, WORKSPACE_BUCKET, UPLOAD_PREFIX) 84 | 85 | logging.info("output uploaded to {}".format(filename)) 86 | 87 | def get_image_iter(max_number=None): 88 | """ 89 | This function returns a iterator of input images for the detector 90 | Each iteration provides a tuple of 91 | (image_id: str, image: numpy.ndarray) 92 | the image will be in BGR color format with an array shape of (height, width, 3) 93 | :return: tuple(image_id, image) 94 | """ 95 | 96 | image_list = _get_s3_image_list(WORKSPACE_BUCKET, IMAGE_LIST_PATH) 97 | 98 | if max_number is not None: 99 | image_list = image_list[:max_number] 100 | 101 | logging.info("got image list, {} images".format(len(image_list))) 102 | 103 | for image_id in image_list: 104 | # get image from s3 105 | # decode image and convert to numpy 106 | 107 | st = time.time() 108 | try: 109 | image = _download_s3_image(WORKSPACE_BUCKET, os.path.join(IMAGE_PREFIX, image_id)) 110 | except: 111 | logging.info("Failed to download image: {}".format(os.path.join(IMAGE_PREFIX, image_id))) 112 | raise 113 | elapsed = time.time() - st 114 | logging.info("image down time: {}".format(elapsed)) 115 | yield image_id, image 116 | 117 | 118 | def get_local_image_iter(max_number=None): 119 | """ 120 | This function returns a iterator of input images for the detector. 121 | It is used for local test of participating algorithms 122 | Each iteration provides a tuple of 123 | (image_id: str, image: numpy.ndarray) 124 | the image will be in BGR color format with an array shape of (height, width, 3) 125 | :return: tuple(image_id, image) 126 | """ 127 | image_list = [x.strip() for x in open('data/local_test_image_list.txt')] 128 | 129 | if max_number is not None: 130 | image_list = image_list[:max_number] 131 | 132 | logging.info("got local image list, {} images".format(len(image_list))) 133 | 134 | for image_name in image_list: 135 | # get image from local fs 136 | # decode image and convert to numpy 137 | 138 | st = time.time() 139 | try: 140 | image = cv2.imread(image_name, 141 | cv2.IMREAD_IGNORE_ORIENTATION | cv2.IMREAD_COLOR) 142 | except: 143 | logging.info("Failed to read image: {}".format(image_name)) 144 | raise 145 | elapsed = time.time() - st 146 | logging.info("image down time: {}".format(elapsed)) 147 | yield image_name, image 148 | 149 | 150 | def verify_local_output(output_boxes, output_time): 151 | 152 | gt = json.load(open('data/local_test_groundtruth.json')) 153 | 154 | # print the groundtruth and prediction for the participant to verify 155 | all_time = 0 156 | for k in gt: 157 | 158 | assert k in output_boxes and k in output_time, ValueError("The detector does work on image {}".format(k)) 159 | image_boxes = output_boxes[k] 160 | image_time = output_time[k] 161 | 162 | all_time += image_time 163 | 164 | logging.info("Image ID: {}, Runtime: {}".format(k, image_time)) 165 | for gt_box in gt[k]: 166 | logging.info("\t gt box: {}".format(gt_box)) 167 | for pred_box in image_boxes: 168 | logging.info("\t prediction box: {}".format(pred_box)) 169 | 170 | logging.info(" ") 171 | 172 | average_fps = len(gt) / all_time 173 | logging.info("Done. Average FPS: {:.03f}".format(average_fps)) 174 | 175 | 176 | -------------------------------------------------------------------------------- /eval_kit/detector.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class FaceDetectorBase(ABC): 5 | def __init__(self): 6 | """ 7 | Participants should define their own initialization process. 8 | During this process you can set up your network. The time cost for this step will 9 | not be counted in runtime evaluation 10 | """ 11 | 12 | @abstractmethod 13 | def process_image(self, image): 14 | """ 15 | Process one image, the evaluation toolkit will measure the runtime of every call to this method. 16 | The time cost will include any thing that's between the image input to the final bounding box output. 17 | The image will be given as a numpy array in the shape of (H, W, C) with dtype np.uint8. 18 | The color mode of the image will be **BGR**. 19 | :param image: a numpy array with dtype=np.uint8 representing a image of (H, W, C) shape. 20 | :return: a numpy array of bounding boxes in the following format 21 | [ 22 | [left, top, width, height, confidence], 23 | ... 24 | ], dtype=np.float32 25 | The bounding box locations should be in absolute pixel coordinates. 26 | """ 27 | pass 28 | -------------------------------------------------------------------------------- /local_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script provides a local test routine so you can verify the algorithm works before pushing it to evaluation. 3 | 4 | It runs your face detection algorithm on several local images and verify whether they have obvious issues, e.g: 5 | - Fail to start 6 | - Wrong output format (we expect (left, top, width, height) in pixels) 7 | 8 | It also prints out the runtime for the algorithms for your references. 9 | 10 | 11 | The participants are expected to implement a face detector class. The sample detector illustrates the interface. 12 | Do not modify other part of the evaluation toolkit otherwise the evaluation will fail. 13 | 14 | Author: Yuanjun Xiong 15 | Contact: bitxiong@gmail.com 16 | 17 | WIDER Challenge 2019 18 | """ 19 | 20 | import time 21 | import sys 22 | import logging 23 | 24 | import numpy as np 25 | from eval_kit.client import get_local_image_iter, verify_local_output 26 | 27 | logging.basicConfig(level=logging.INFO) 28 | 29 | ######################################################################################################## 30 | # please change these lines to include your own face detector extending the eval_kit.detector.FaceDetector base class. 31 | sys.path.append("mtcnn") 32 | from sample_detector import SampleMTCNNFaceDetector as WIDERTestFaceDetectorClass 33 | ######################################################################################################## 34 | 35 | 36 | def run_local_test(detector_class, image_iter): 37 | """ 38 | In this function we create the detector instance. And evaluate the wall time for performing face detection. 39 | """ 40 | 41 | # initialize the detector 42 | logging.info("Initializing face detector.") 43 | try: 44 | detector = detector_class() 45 | except: 46 | # send errors to the eval frontend 47 | raise 48 | logging.info("Detector initialized.") 49 | 50 | 51 | # run the images one-by-one and get runtime 52 | overall_time = 0 53 | output_boxes = {} 54 | output_time = {} 55 | eval_cnt = 0 56 | 57 | logging.info("Starting runtime evaluation") 58 | for image_id, image in image_iter: 59 | time_before = time.time() 60 | try: 61 | boxes = detector.process_image(image) 62 | assert isinstance(boxes, np.ndarray) 63 | output_boxes[image_id] = boxes 64 | except: 65 | # send errors to the eval frontend 66 | logging.error("Image id failed: {}".format(image_id)) 67 | raise 68 | elapsed = time.time() - time_before 69 | output_time[image_id] = elapsed 70 | logging.info("image {} run time: {}".format(image_id, elapsed)) 71 | 72 | overall_time += elapsed 73 | eval_cnt += 1 74 | 75 | if eval_cnt % 100 == 0: 76 | logging.info("Finished {} images".format(eval_cnt)) 77 | 78 | logging.info(""" 79 | ================================================================================ 80 | all image finished, showing verification info below: 81 | ================================================================================ 82 | """) 83 | 84 | # verify the algorithm output 85 | verify_local_output(output_boxes, output_time) 86 | 87 | 88 | if __name__ == '__main__': 89 | local_test_image_iter = get_local_image_iter() 90 | run_local_test(WIDERTestFaceDetectorClass, local_test_image_iter) -------------------------------------------------------------------------------- /run_evaluation.py: -------------------------------------------------------------------------------- 1 | """ 2 | The evaluation entry point for WIDER Challenge 2019: Face Detection Accuracy+Runtime Track. 3 | 4 | It will be the entrypoint for the evaluation docker once built. 5 | Basically It downloads a list of images and run the face detector on each image. 6 | Then the runtime and detection output will be reported to the evaluation system. 7 | 8 | The participants are expected to implement a face detector class. The sample detector illustrates the interface. 9 | Do not modify other part of the evaluation toolkit otherwise the evaluation will fail. 10 | 11 | Author: Yuanjun Xiong 12 | Contact: bitxiong@gmail.com 13 | 14 | WIDER Challenge 2019 15 | """ 16 | import time 17 | import sys 18 | import logging 19 | 20 | import numpy as np 21 | from eval_kit.client import upload_eval_output, get_image_iter, get_job_id 22 | 23 | 24 | logging.basicConfig(level=logging.INFO) 25 | 26 | 27 | ######################################################################################################## 28 | # please change these lines to include your own face detector extending the eval_kit.detector.FaceDetector base class. 29 | sys.path.append("mtcnn") 30 | from sample_detector import SampleMTCNNFaceDetector as WIDERTestFaceDetectorClass 31 | ######################################################################################################## 32 | 33 | 34 | def evaluate_runtime(detector_class, image_iter, job_id): 35 | """ 36 | Please DO NOT modify this part of code or the eval_kit 37 | Modification of the evaluation toolkit could result in cancellation of your award. 38 | 39 | In this function we create the detector instance. And evaluate the wall time for performing face detection. 40 | """ 41 | 42 | # initialize the detector 43 | logging.info("Initializing face detector.") 44 | try: 45 | detector = detector_class() 46 | except: 47 | # send errors to the eval frontend 48 | raise 49 | logging.info("Detector initialized.") 50 | 51 | 52 | # run the images one-by-one and get runtime 53 | overall_time = 0 54 | output_boxes = {} 55 | output_time = {} 56 | eval_cnt = 0 57 | 58 | logging.info("Starting runtime evaluation") 59 | for image_id, image in image_iter: 60 | time_before = time.time() 61 | try: 62 | boxes = detector.process_image(image) 63 | assert isinstance(boxes, np.ndarray) 64 | output_boxes[image_id] = boxes 65 | except: 66 | # send errors to the eval frontend 67 | logging.error("Image id failed: {}".format(image_id)) 68 | raise 69 | elapsed = time.time() - time_before 70 | output_time[image_id] = elapsed 71 | logging.info("image {} run time: {}".format(image_id, elapsed)) 72 | 73 | overall_time += elapsed 74 | eval_cnt += 1 75 | 76 | if eval_cnt % 100 == 0: 77 | logging.info("Finished {} images".format(eval_cnt)) 78 | 79 | logging.info("all image finished, uploading evaluation outputs for evaluation.") 80 | # send evaluation output to the server 81 | upload_eval_output(output_boxes, output_time, job_id) 82 | 83 | 84 | if __name__ == '__main__': 85 | job_id = get_job_id() 86 | wider_test_image_iter = get_image_iter() 87 | evaluate_runtime(WIDERTestFaceDetectorClass, wider_test_image_iter, job_id) 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /sample_detector.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is an example of the face detector class which wil be called by the evaluation toolkit 3 | 4 | The implementation is from 5 | https://github.com/Seanlinx/mtcnn/blob/master/demo.py 6 | 7 | """ 8 | 9 | 10 | import numpy as np 11 | import mxnet as mx 12 | import argparse 13 | import cv2 14 | import time 15 | from core.symbol import P_Net, R_Net, O_Net 16 | from core.imdb import IMDB 17 | from config import config 18 | from core.loader import TestLoader 19 | from core.detector import Detector 20 | from core.fcn_detector import FcnDetector 21 | from tools.load_model import load_param 22 | from core.MtcnnDetector import MtcnnDetector 23 | 24 | from eval_kit.detector import FaceDetectorBase 25 | 26 | class SampleMTCNNFaceDetector(FaceDetectorBase): 27 | 28 | def __init__(self): 29 | """ 30 | This is an example face detector used for runtime evaluation in WIDER Challenge 2019. 31 | It takes one BGR image and output a numpy array of face boudning boxes in the format of 32 | [left, top, width, height, confidence] 33 | 34 | Your own face detector should also follow this interface and implement the FaceDetectorBase interface 35 | Please find the detailed requirement of the face detector in FaceDetectorBase 36 | """ 37 | super(SampleMTCNNFaceDetector, self).__init__() 38 | self.detectors = [None, None, None] 39 | 40 | prefix = ['mtcnn/model/pnet', 'mtcnn/model/rnet', 'mtcnn/model/onet'] 41 | epoch = [16, 16, 16] 42 | batch_size = [2048, 256, 16] 43 | slide_window = False 44 | min_face_size = 40 45 | stride = 2 46 | thresh = [0.5, 0.5, 0.7] 47 | 48 | # the evaluation environment provides one NVIDIA P100 GPU 49 | ctx = mx.gpu(0) 50 | 51 | # load pnet model 52 | args, auxs = load_param(prefix[0], epoch[0], convert=True, ctx=ctx) 53 | if slide_window: 54 | PNet = Detector(P_Net("test"), 12, batch_size[0], ctx, args, auxs) 55 | else: 56 | PNet = FcnDetector(P_Net("test"), ctx, args, auxs) 57 | self.detectors[0] = PNet 58 | 59 | # load rnet model 60 | args, auxs = load_param(prefix[1], epoch[0], convert=True, ctx=ctx) 61 | RNet = Detector(R_Net("test"), 24, batch_size[1], ctx, args, auxs) 62 | self.detectors[1] = RNet 63 | 64 | # load onet model 65 | args, auxs = load_param(prefix[2], epoch[2], convert=True, ctx=ctx) 66 | ONet = Detector(O_Net("test"), 48, batch_size[2], ctx, args, auxs) 67 | self.detectors[2] = ONet 68 | 69 | self.mtcnn_detector = MtcnnDetector(detectors=self.detectors, ctx=ctx, min_face_size=min_face_size, 70 | stride=stride, threshold=thresh, slide_window=slide_window) 71 | 72 | def process_image(self, image): 73 | """ 74 | :param image: a numpy.array representing one image with BGR colors 75 | :return: numpy.array of detection results in the format of 76 | [ 77 | [left, top, width, height, confidence], 78 | ... 79 | ], dtype=np.float32 80 | """ 81 | 82 | boxes, boxes_c = self.mtcnn_detector.detect_pnet(image) 83 | if boxes_c is None: 84 | return np.array([], dtype=np.float32) 85 | boxes, boxes_c = self.mtcnn_detector.detect_rnet(image, boxes_c) 86 | if boxes_c is None: 87 | return np.array([], dtype=np.float32) 88 | boxes, boxes_c = self.mtcnn_detector.detect_onet(image, boxes_c) 89 | if boxes_c is None: 90 | return np.array([], dtype=np.float32) 91 | raw_boxes = np.array(boxes_c, dtype=np.float32) 92 | raw_boxes[:, 2] = raw_boxes[:, 2] - raw_boxes[:, 0] 93 | raw_boxes[:, 3] = raw_boxes[:, 3] - raw_boxes[:, 1] 94 | return raw_boxes 95 | 96 | --------------------------------------------------------------------------------