├── HalpeCOCOAPI ├── .gitignore ├── .travis.yml ├── PythonAPI │ ├── MANIFEST.in │ ├── Makefile │ ├── halpecocotools │ │ ├── __init__.py │ │ ├── _mask.pyx │ │ ├── coco.py │ │ ├── cocoeval.py │ │ └── mask.py │ ├── pycocoDemo.ipynb │ ├── pycocoEvalDemo.ipynb │ └── setup.py ├── README.md ├── common │ ├── gason.cpp │ ├── gason.h │ ├── maskApi.c │ └── maskApi.h └── license.txt ├── README.md ├── docs ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg ├── 5.jpg ├── alphapose_136_2.gif ├── face.jpg ├── hand.jpg └── human_model.jpg └── vis.py /HalpeCOCOAPI/.gitignore: -------------------------------------------------------------------------------- 1 | images/ 2 | annotations/ 3 | results/ 4 | external/ 5 | .DS_Store 6 | 7 | MatlabAPI/analyze*/ 8 | MatlabAPI/visualize*/ 9 | MatlabAPI/private/maskApiMex.* 10 | 11 | PythonAPI/pycocotools/__init__.pyc 12 | PythonAPI/pycocotools/_mask.c 13 | PythonAPI/pycocotools/_mask.so 14 | PythonAPI/pycocotools/coco.pyc 15 | PythonAPI/pycocotools/cocoeval.pyc 16 | PythonAPI/pycocotools/mask.pyc 17 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/.travis.yml: -------------------------------------------------------------------------------- 1 | group: travis_latest 2 | language: python 3 | cache: pip 4 | python: 5 | - 2.7 6 | - 3.6 7 | install: 8 | - pip install --upgrade pip 9 | - pip install pycocotools 10 | script: 11 | - true 12 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include common * 2 | include *.pxd 3 | include *.pyx -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | # install cocohalpetools locally 3 | python setup.py build_ext --inplace 4 | rm -rf build 5 | 6 | install: 7 | # install cocohalpetools to the Python site-packages 8 | python setup.py build_ext install 9 | rm -rf build -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/halpecocotools/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/halpecocotools/_mask.pyx: -------------------------------------------------------------------------------- 1 | # distutils: language = c 2 | # distutils: sources = ../common/maskApi.c 3 | 4 | #************************************************************************** 5 | # Microsoft COCO Toolbox. version 2.0 6 | # Data, paper, and tutorials available at: http://mscoco.org/ 7 | # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. 8 | # Licensed under the Simplified BSD License [see coco/license.txt] 9 | #************************************************************************** 10 | 11 | __author__ = 'tsungyi' 12 | 13 | import sys 14 | PYTHON_VERSION = sys.version_info[0] 15 | 16 | # import both Python-level and C-level symbols of Numpy 17 | # the API uses Numpy to interface C and Python 18 | import numpy as np 19 | cimport numpy as np 20 | from libc.stdlib cimport malloc, free 21 | 22 | # intialized Numpy. must do. 23 | np.import_array() 24 | 25 | # import numpy C function 26 | # we use PyArray_ENABLEFLAGS to make Numpy ndarray responsible to memoery management 27 | cdef extern from "numpy/arrayobject.h": 28 | void PyArray_ENABLEFLAGS(np.ndarray arr, int flags) 29 | 30 | # Declare the prototype of the C functions in MaskApi.h 31 | cdef extern from "maskApi.h": 32 | ctypedef unsigned int uint 33 | ctypedef unsigned long siz 34 | ctypedef unsigned char byte 35 | ctypedef double* BB 36 | ctypedef struct RLE: 37 | siz h, 38 | siz w, 39 | siz m, 40 | uint* cnts, 41 | void rlesInit( RLE **R, siz n ) 42 | void rleEncode( RLE *R, const byte *M, siz h, siz w, siz n ) 43 | void rleDecode( const RLE *R, byte *mask, siz n ) 44 | void rleMerge( const RLE *R, RLE *M, siz n, int intersect ) 45 | void rleArea( const RLE *R, siz n, uint *a ) 46 | void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o ) 47 | void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ) 48 | void rleToBbox( const RLE *R, BB bb, siz n ) 49 | void rleFrBbox( RLE *R, const BB bb, siz h, siz w, siz n ) 50 | void rleFrPoly( RLE *R, const double *xy, siz k, siz h, siz w ) 51 | char* rleToString( const RLE *R ) 52 | void rleFrString( RLE *R, char *s, siz h, siz w ) 53 | 54 | # python class to wrap RLE array in C 55 | # the class handles the memory allocation and deallocation 56 | cdef class RLEs: 57 | cdef RLE *_R 58 | cdef siz _n 59 | 60 | def __cinit__(self, siz n =0): 61 | rlesInit(&self._R, n) 62 | self._n = n 63 | 64 | # free the RLE array here 65 | def __dealloc__(self): 66 | if self._R is not NULL: 67 | for i in range(self._n): 68 | free(self._R[i].cnts) 69 | free(self._R) 70 | def __getattr__(self, key): 71 | if key == 'n': 72 | return self._n 73 | raise AttributeError(key) 74 | 75 | # python class to wrap Mask array in C 76 | # the class handles the memory allocation and deallocation 77 | cdef class Masks: 78 | cdef byte *_mask 79 | cdef siz _h 80 | cdef siz _w 81 | cdef siz _n 82 | 83 | def __cinit__(self, h, w, n): 84 | self._mask = malloc(h*w*n* sizeof(byte)) 85 | self._h = h 86 | self._w = w 87 | self._n = n 88 | # def __dealloc__(self): 89 | # the memory management of _mask has been passed to np.ndarray 90 | # it doesn't need to be freed here 91 | 92 | # called when passing into np.array() and return an np.ndarray in column-major order 93 | def __array__(self): 94 | cdef np.npy_intp shape[1] 95 | shape[0] = self._h*self._w*self._n 96 | # Create a 1D array, and reshape it to fortran/Matlab column-major array 97 | ndarray = np.PyArray_SimpleNewFromData(1, shape, np.NPY_UINT8, self._mask).reshape((self._h, self._w, self._n), order='F') 98 | # The _mask allocated by Masks is now handled by ndarray 99 | PyArray_ENABLEFLAGS(ndarray, np.NPY_OWNDATA) 100 | return ndarray 101 | 102 | # internal conversion from Python RLEs object to compressed RLE format 103 | def _toString(RLEs Rs): 104 | cdef siz n = Rs.n 105 | cdef bytes py_string 106 | cdef char* c_string 107 | objs = [] 108 | for i in range(n): 109 | c_string = rleToString( &Rs._R[i] ) 110 | py_string = c_string 111 | objs.append({ 112 | 'size': [Rs._R[i].h, Rs._R[i].w], 113 | 'counts': py_string 114 | }) 115 | free(c_string) 116 | return objs 117 | 118 | # internal conversion from compressed RLE format to Python RLEs object 119 | def _frString(rleObjs): 120 | cdef siz n = len(rleObjs) 121 | Rs = RLEs(n) 122 | cdef bytes py_string 123 | cdef char* c_string 124 | for i, obj in enumerate(rleObjs): 125 | if PYTHON_VERSION == 2: 126 | py_string = str(obj['counts']).encode('utf8') 127 | elif PYTHON_VERSION == 3: 128 | py_string = str.encode(obj['counts']) if type(obj['counts']) == str else obj['counts'] 129 | else: 130 | raise Exception('Python version must be 2 or 3') 131 | c_string = py_string 132 | rleFrString( &Rs._R[i], c_string, obj['size'][0], obj['size'][1] ) 133 | return Rs 134 | 135 | # encode mask to RLEs objects 136 | # list of RLE string can be generated by RLEs member function 137 | def encode(np.ndarray[np.uint8_t, ndim=3, mode='fortran'] mask): 138 | h, w, n = mask.shape[0], mask.shape[1], mask.shape[2] 139 | cdef RLEs Rs = RLEs(n) 140 | rleEncode(Rs._R,mask.data,h,w,n) 141 | objs = _toString(Rs) 142 | return objs 143 | 144 | # decode mask from compressed list of RLE string or RLEs object 145 | def decode(rleObjs): 146 | cdef RLEs Rs = _frString(rleObjs) 147 | h, w, n = Rs._R[0].h, Rs._R[0].w, Rs._n 148 | masks = Masks(h, w, n) 149 | rleDecode(Rs._R, masks._mask, n); 150 | return np.array(masks) 151 | 152 | def merge(rleObjs, intersect=0): 153 | cdef RLEs Rs = _frString(rleObjs) 154 | cdef RLEs R = RLEs(1) 155 | rleMerge(Rs._R, R._R, Rs._n, intersect) 156 | obj = _toString(R)[0] 157 | return obj 158 | 159 | def area(rleObjs): 160 | cdef RLEs Rs = _frString(rleObjs) 161 | cdef uint* _a = malloc(Rs._n* sizeof(uint)) 162 | rleArea(Rs._R, Rs._n, _a) 163 | cdef np.npy_intp shape[1] 164 | shape[0] = Rs._n 165 | a = np.array((Rs._n, ), dtype=np.uint8) 166 | a = np.PyArray_SimpleNewFromData(1, shape, np.NPY_UINT32, _a) 167 | PyArray_ENABLEFLAGS(a, np.NPY_OWNDATA) 168 | return a 169 | 170 | # iou computation. support function overload (RLEs-RLEs and bbox-bbox). 171 | def iou( dt, gt, pyiscrowd ): 172 | def _preproc(objs): 173 | if len(objs) == 0: 174 | return objs 175 | if type(objs) == np.ndarray: 176 | if len(objs.shape) == 1: 177 | objs = objs.reshape((objs[0], 1)) 178 | # check if it's Nx4 bbox 179 | if not len(objs.shape) == 2 or not objs.shape[1] == 4: 180 | raise Exception('numpy ndarray input is only for *bounding boxes* and should have Nx4 dimension') 181 | objs = objs.astype(np.double) 182 | elif type(objs) == list: 183 | # check if list is in box format and convert it to np.ndarray 184 | isbox = np.all(np.array([(len(obj)==4) and ((type(obj)==list) or (type(obj)==np.ndarray)) for obj in objs])) 185 | isrle = np.all(np.array([type(obj) == dict for obj in objs])) 186 | if isbox: 187 | objs = np.array(objs, dtype=np.double) 188 | if len(objs.shape) == 1: 189 | objs = objs.reshape((1,objs.shape[0])) 190 | elif isrle: 191 | objs = _frString(objs) 192 | else: 193 | raise Exception('list input can be bounding box (Nx4) or RLEs ([RLE])') 194 | else: 195 | raise Exception('unrecognized type. The following type: RLEs (rle), np.ndarray (box), and list (box) are supported.') 196 | return objs 197 | def _rleIou(RLEs dt, RLEs gt, np.ndarray[np.uint8_t, ndim=1] iscrowd, siz m, siz n, np.ndarray[np.double_t, ndim=1] _iou): 198 | rleIou( dt._R, gt._R, m, n, iscrowd.data, _iou.data ) 199 | def _bbIou(np.ndarray[np.double_t, ndim=2] dt, np.ndarray[np.double_t, ndim=2] gt, np.ndarray[np.uint8_t, ndim=1] iscrowd, siz m, siz n, np.ndarray[np.double_t, ndim=1] _iou): 200 | bbIou( dt.data, gt.data, m, n, iscrowd.data, _iou.data ) 201 | def _len(obj): 202 | cdef siz N = 0 203 | if type(obj) == RLEs: 204 | N = obj.n 205 | elif len(obj)==0: 206 | pass 207 | elif type(obj) == np.ndarray: 208 | N = obj.shape[0] 209 | return N 210 | # convert iscrowd to numpy array 211 | cdef np.ndarray[np.uint8_t, ndim=1] iscrowd = np.array(pyiscrowd, dtype=np.uint8) 212 | # simple type checking 213 | cdef siz m, n 214 | dt = _preproc(dt) 215 | gt = _preproc(gt) 216 | m = _len(dt) 217 | n = _len(gt) 218 | if m == 0 or n == 0: 219 | return [] 220 | if not type(dt) == type(gt): 221 | raise Exception('The dt and gt should have the same data type, either RLEs, list or np.ndarray') 222 | 223 | # define local variables 224 | cdef double* _iou = 0 225 | cdef np.npy_intp shape[1] 226 | # check type and assign iou function 227 | if type(dt) == RLEs: 228 | _iouFun = _rleIou 229 | elif type(dt) == np.ndarray: 230 | _iouFun = _bbIou 231 | else: 232 | raise Exception('input data type not allowed.') 233 | _iou = malloc(m*n* sizeof(double)) 234 | iou = np.zeros((m*n, ), dtype=np.double) 235 | shape[0] = m*n 236 | iou = np.PyArray_SimpleNewFromData(1, shape, np.NPY_DOUBLE, _iou) 237 | PyArray_ENABLEFLAGS(iou, np.NPY_OWNDATA) 238 | _iouFun(dt, gt, iscrowd, m, n, iou) 239 | return iou.reshape((m,n), order='F') 240 | 241 | def toBbox( rleObjs ): 242 | cdef RLEs Rs = _frString(rleObjs) 243 | cdef siz n = Rs.n 244 | cdef BB _bb = malloc(4*n* sizeof(double)) 245 | rleToBbox( Rs._R, _bb, n ) 246 | cdef np.npy_intp shape[1] 247 | shape[0] = 4*n 248 | bb = np.array((1,4*n), dtype=np.double) 249 | bb = np.PyArray_SimpleNewFromData(1, shape, np.NPY_DOUBLE, _bb).reshape((n, 4)) 250 | PyArray_ENABLEFLAGS(bb, np.NPY_OWNDATA) 251 | return bb 252 | 253 | def frBbox(np.ndarray[np.double_t, ndim=2] bb, siz h, siz w ): 254 | cdef siz n = bb.shape[0] 255 | Rs = RLEs(n) 256 | rleFrBbox( Rs._R, bb.data, h, w, n ) 257 | objs = _toString(Rs) 258 | return objs 259 | 260 | def frPoly( poly, siz h, siz w ): 261 | cdef np.ndarray[np.double_t, ndim=1] np_poly 262 | n = len(poly) 263 | Rs = RLEs(n) 264 | for i, p in enumerate(poly): 265 | np_poly = np.array(p, dtype=np.double, order='F') 266 | rleFrPoly( &Rs._R[i], np_poly.data, int(len(p)/2), h, w ) 267 | objs = _toString(Rs) 268 | return objs 269 | 270 | def frUncompressedRLE(ucRles, siz h, siz w): 271 | cdef np.ndarray[np.uint32_t, ndim=1] cnts 272 | cdef RLE R 273 | cdef uint *data 274 | n = len(ucRles) 275 | objs = [] 276 | for i in range(n): 277 | Rs = RLEs(1) 278 | cnts = np.array(ucRles[i]['counts'], dtype=np.uint32) 279 | # time for malloc can be saved here but it's fine 280 | data = malloc(len(cnts)* sizeof(uint)) 281 | for j in range(len(cnts)): 282 | data[j] = cnts[j] 283 | R = RLE(ucRles[i]['size'][0], ucRles[i]['size'][1], len(cnts), data) 284 | Rs._R[0] = R 285 | objs.append(_toString(Rs)[0]) 286 | return objs 287 | 288 | def frPyObjects(pyobj, h, w): 289 | # encode rle from a list of python objects 290 | if type(pyobj) == np.ndarray: 291 | objs = frBbox(pyobj, h, w) 292 | elif type(pyobj) == list and len(pyobj[0]) == 4: 293 | objs = frBbox(pyobj, h, w) 294 | elif type(pyobj) == list and len(pyobj[0]) > 4: 295 | objs = frPoly(pyobj, h, w) 296 | elif type(pyobj) == list and type(pyobj[0]) == dict \ 297 | and 'counts' in pyobj[0] and 'size' in pyobj[0]: 298 | objs = frUncompressedRLE(pyobj, h, w) 299 | # encode rle from single python object 300 | elif type(pyobj) == list and len(pyobj) == 4: 301 | objs = frBbox([pyobj], h, w)[0] 302 | elif type(pyobj) == list and len(pyobj) > 4: 303 | objs = frPoly([pyobj], h, w)[0] 304 | elif type(pyobj) == dict and 'counts' in pyobj and 'size' in pyobj: 305 | objs = frUncompressedRLE([pyobj], h, w)[0] 306 | else: 307 | raise Exception('input type is not supported.') 308 | return objs 309 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/halpecocotools/coco.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | __version__ = '2.0' 3 | # Interface for accessing the Microsoft COCO dataset. 4 | 5 | # Microsoft COCO is a large image dataset designed for object detection, 6 | # segmentation, and caption generation. pycocotools is a Python API that 7 | # assists in loading, parsing and visualizing the annotations in COCO. 8 | # Please visit http://mscoco.org/ for more information on COCO, including 9 | # for the data, paper, and tutorials. The exact format of the annotations 10 | # is also described on the COCO website. For example usage of the pycocotools 11 | # please see pycocotools_demo.ipynb. In addition to this API, please download both 12 | # the COCO images and annotations in order to run the demo. 13 | 14 | # An alternative to using the API is to load the annotations directly 15 | # into Python dictionary 16 | # Using the API provides additional utility functions. Note that this API 17 | # supports both *instance* and *caption* annotations. In the case of 18 | # captions not all functions are defined (e.g. categories are undefined). 19 | 20 | # The following API functions are defined: 21 | # COCO - COCO api class that loads COCO annotation file and prepare data structures. 22 | # decodeMask - Decode binary mask M encoded via run-length encoding. 23 | # encodeMask - Encode binary mask M using run-length encoding. 24 | # getAnnIds - Get ann ids that satisfy given filter conditions. 25 | # getCatIds - Get cat ids that satisfy given filter conditions. 26 | # getImgIds - Get img ids that satisfy given filter conditions. 27 | # loadAnns - Load anns with the specified ids. 28 | # loadCats - Load cats with the specified ids. 29 | # loadImgs - Load imgs with the specified ids. 30 | # annToMask - Convert segmentation in an annotation to binary mask. 31 | # showAnns - Display the specified annotations. 32 | # loadRes - Load algorithm results and create API for accessing them. 33 | # download - Download COCO images from mscoco.org server. 34 | # Throughout the API "ann"=annotation, "cat"=category, and "img"=image. 35 | # Help on each functions can be accessed by: "help COCO>function". 36 | 37 | # See also COCO>decodeMask, 38 | # COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, 39 | # COCO>getImgIds, COCO>loadAnns, COCO>loadCats, 40 | # COCO>loadImgs, COCO>annToMask, COCO>showAnns 41 | 42 | # The original Microsoft COCO Toolbox is written 43 | # by Piotr Dollar and Tsung-Yi Lin, 2015. 44 | # Licensed under the Simplified BSD License [see coco/license.txt] 45 | # 46 | # Updated and renamed to Halpe COCO Toolbox (halpecocotools) \ 47 | # by Haoyi Zhu in 2021. The Halpe COCO Toolbox is 48 | # developed to support Halpe Full-body dataset. 49 | 50 | import json 51 | import time 52 | import matplotlib.pyplot as plt 53 | from matplotlib.collections import PatchCollection 54 | from matplotlib.patches import Polygon 55 | import numpy as np 56 | import copy 57 | import itertools 58 | from . import mask as maskUtils 59 | import os 60 | from collections import defaultdict 61 | import sys 62 | PYTHON_VERSION = sys.version_info[0] 63 | if PYTHON_VERSION == 2: 64 | from urllib import urlretrieve 65 | elif PYTHON_VERSION == 3: 66 | from urllib.request import urlretrieve 67 | 68 | 69 | def _isArrayLike(obj): 70 | return hasattr(obj, '__iter__') and hasattr(obj, '__len__') 71 | 72 | 73 | class COCO: 74 | def __init__(self, annotation_file=None): 75 | """ 76 | Constructor of Microsoft COCO helper class for reading and visualizing annotations. 77 | :param annotation_file (str): location of annotation file 78 | :param image_folder (str): location to the folder that hosts images. 79 | :return: 80 | """ 81 | # load dataset 82 | self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() 83 | self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) 84 | if not annotation_file == None: 85 | print('loading annotations into memory...') 86 | tic = time.time() 87 | dataset = json.load(open(annotation_file, 'r')) 88 | assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) 89 | print('Done (t={:0.2f}s)'.format(time.time()- tic)) 90 | self.dataset = dataset 91 | self.createIndex() 92 | 93 | def createIndex(self): 94 | # create index 95 | print('creating index...') 96 | anns, cats, imgs = {}, {}, {} 97 | imgToAnns,catToImgs = defaultdict(list),defaultdict(list) 98 | if 'annotations' in self.dataset: 99 | for ann in self.dataset['annotations']: 100 | imgToAnns[ann['image_id']].append(ann) 101 | anns[ann['id']] = ann 102 | 103 | if 'images' in self.dataset: 104 | for img in self.dataset['images']: 105 | imgs[img['id']] = img 106 | 107 | if 'categories' in self.dataset: 108 | for cat in self.dataset['categories']: 109 | cats[cat['id']] = cat 110 | 111 | if 'annotations' in self.dataset and 'categories' in self.dataset: 112 | for ann in self.dataset['annotations']: 113 | catToImgs[ann['category_id']].append(ann['image_id']) 114 | 115 | print('index created!') 116 | 117 | # create class members 118 | self.anns = anns 119 | self.imgToAnns = imgToAnns 120 | self.catToImgs = catToImgs 121 | self.imgs = imgs 122 | self.cats = cats 123 | 124 | def info(self): 125 | """ 126 | Print information about the annotation file. 127 | :return: 128 | """ 129 | for key, value in self.dataset['info'].items(): 130 | print('{}: {}'.format(key, value)) 131 | 132 | def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): 133 | """ 134 | Get ann ids that satisfy given filter conditions. default skips that filter 135 | :param imgIds (int array) : get anns for given imgs 136 | catIds (int array) : get anns for given cats 137 | areaRng (float array) : get anns for given area range (e.g. [0 inf]) 138 | iscrowd (boolean) : get anns for given crowd label (False or True) 139 | :return: ids (int array) : integer array of ann ids 140 | """ 141 | imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] 142 | catIds = catIds if _isArrayLike(catIds) else [catIds] 143 | 144 | if len(imgIds) == len(catIds) == len(areaRng) == 0: 145 | anns = self.dataset['annotations'] 146 | else: 147 | if not len(imgIds) == 0: 148 | lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] 149 | anns = list(itertools.chain.from_iterable(lists)) 150 | else: 151 | anns = self.dataset['annotations'] 152 | anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] 153 | anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] 154 | if not iscrowd == None: 155 | ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] 156 | else: 157 | ids = [ann['id'] for ann in anns] 158 | return ids 159 | 160 | def getCatIds(self, catNms=[], supNms=[], catIds=[]): 161 | """ 162 | filtering parameters. default skips that filter. 163 | :param catNms (str array) : get cats for given cat names 164 | :param supNms (str array) : get cats for given supercategory names 165 | :param catIds (int array) : get cats for given cat ids 166 | :return: ids (int array) : integer array of cat ids 167 | """ 168 | catNms = catNms if _isArrayLike(catNms) else [catNms] 169 | supNms = supNms if _isArrayLike(supNms) else [supNms] 170 | catIds = catIds if _isArrayLike(catIds) else [catIds] 171 | 172 | if len(catNms) == len(supNms) == len(catIds) == 0: 173 | cats = self.dataset['categories'] 174 | else: 175 | cats = self.dataset['categories'] 176 | cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] 177 | cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] 178 | cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] 179 | ids = [cat['id'] for cat in cats] 180 | return ids 181 | 182 | def getImgIds(self, imgIds=[], catIds=[]): 183 | ''' 184 | Get img ids that satisfy given filter conditions. 185 | :param imgIds (int array) : get imgs for given ids 186 | :param catIds (int array) : get imgs with all given cats 187 | :return: ids (int array) : integer array of img ids 188 | ''' 189 | imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] 190 | catIds = catIds if _isArrayLike(catIds) else [catIds] 191 | 192 | if len(imgIds) == len(catIds) == 0: 193 | ids = self.imgs.keys() 194 | else: 195 | ids = set(imgIds) 196 | for i, catId in enumerate(catIds): 197 | if i == 0 and len(ids) == 0: 198 | ids = set(self.catToImgs[catId]) 199 | else: 200 | ids &= set(self.catToImgs[catId]) 201 | return list(ids) 202 | 203 | def loadAnns(self, ids=[]): 204 | """ 205 | Load anns with the specified ids. 206 | :param ids (int array) : integer ids specifying anns 207 | :return: anns (object array) : loaded ann objects 208 | """ 209 | if _isArrayLike(ids): 210 | return [self.anns[id] for id in ids] 211 | elif type(ids) == int: 212 | return [self.anns[ids]] 213 | 214 | def loadCats(self, ids=[]): 215 | """ 216 | Load cats with the specified ids. 217 | :param ids (int array) : integer ids specifying cats 218 | :return: cats (object array) : loaded cat objects 219 | """ 220 | if _isArrayLike(ids): 221 | return [self.cats[id] for id in ids] 222 | elif type(ids) == int: 223 | return [self.cats[ids]] 224 | 225 | def loadImgs(self, ids=[]): 226 | """ 227 | Load anns with the specified ids. 228 | :param ids (int array) : integer ids specifying img 229 | :return: imgs (object array) : loaded img objects 230 | """ 231 | if _isArrayLike(ids): 232 | return [self.imgs[id] for id in ids] 233 | elif type(ids) == int: 234 | return [self.imgs[ids]] 235 | 236 | def showAnns(self, anns): 237 | """ 238 | Display the specified annotations. 239 | :param anns (array of object): annotations to display 240 | :return: None 241 | """ 242 | if len(anns) == 0: 243 | return 0 244 | if 'segmentation' in anns[0] or 'keypoints' in anns[0]: 245 | datasetType = 'instances' 246 | elif 'caption' in anns[0]: 247 | datasetType = 'captions' 248 | else: 249 | raise Exception('datasetType not supported') 250 | if datasetType == 'instances': 251 | ax = plt.gca() 252 | ax.set_autoscale_on(False) 253 | polygons = [] 254 | color = [] 255 | for ann in anns: 256 | c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] 257 | if 'segmentation' in ann: 258 | if type(ann['segmentation']) == list: 259 | # polygon 260 | for seg in ann['segmentation']: 261 | poly = np.array(seg).reshape((int(len(seg)/2), 2)) 262 | polygons.append(Polygon(poly)) 263 | color.append(c) 264 | else: 265 | # mask 266 | t = self.imgs[ann['image_id']] 267 | if type(ann['segmentation']['counts']) == list: 268 | rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) 269 | else: 270 | rle = [ann['segmentation']] 271 | m = maskUtils.decode(rle) 272 | img = np.ones( (m.shape[0], m.shape[1], 3) ) 273 | if ann['iscrowd'] == 1: 274 | color_mask = np.array([2.0,166.0,101.0])/255 275 | if ann['iscrowd'] == 0: 276 | color_mask = np.random.random((1, 3)).tolist()[0] 277 | for i in range(3): 278 | img[:,:,i] = color_mask[i] 279 | ax.imshow(np.dstack( (img, m*0.5) )) 280 | if 'keypoints' in ann and type(ann['keypoints']) == list: 281 | # skeleton 282 | sks = np.array([(0, 1), (0, 2), (1, 3), (2, 4), # Head 283 | (5, 18), (6, 18), (5, 7), (7, 9), (6, 8), (8, 10),# Body 284 | (17, 18), (18, 19), (19, 11), (19, 12), 285 | (11, 13), (12, 14), (13, 15), (14, 16), 286 | (20, 24), (21, 25), (23, 25), (22, 24), (15, 24), (16, 25),# Foot 287 | (26, 27),(27, 28),(28, 29),(29, 30),(30, 31),(31, 32),(32, 33),(33, 34),(34, 35),(35, 36),(36, 37),(37, 38),#Face 288 | (38, 39),(39, 40),(40, 41),(41, 42),(43, 44),(44, 45),(45, 46),(46, 47),(48, 49),(49, 50),(50, 51),(51, 52),#Face 289 | (53, 54),(54, 55),(55, 56),(57, 58),(58, 59),(59, 60),(60, 61),(62, 63),(63, 64),(64, 65),(65, 66),(66, 67),#Face 290 | (68, 69),(69, 70),(70, 71),(71, 72),(72, 73),(74, 75),(75, 76),(76, 77),(77, 78),(78, 79),(79, 80),(80, 81),#Face 291 | (81, 82),(82, 83),(83, 84),(84, 85),(85, 86),(86, 87),(87, 88),(88, 89),(89, 90),(90, 91),(91, 92),(92, 93),#Face 292 | (94,95),(95,96),(96,97),(97,98),(94,99),(99,100),(100,101),(101,102),(94,103),(103,104),(104,105),#LeftHand 293 | (105,106),(94,107),(107,108),(108,109),(109,110),(94,111),(111,112),(112,113),(113,114),#LeftHand 294 | (115,116),(116,117),(117,118),(118,119),(115,120),(120,121),(121,122),(122,123),(115,124),(124,125),#RightHand 295 | (125,126),(126,127),(115,128),(128,129),(129,130),(130,131),(115,132),(132,133),(133,134),(134,135)#RightHand 296 | ]) 297 | kp = np.array(ann['keypoints']) 298 | x = kp[0::3] 299 | y = kp[1::3] 300 | v = kp[2::3] 301 | for i, sk in enumerate(sks): 302 | if np.all(v[sk]>0): 303 | if i < 24: 304 | plt.plot(x[sk],y[sk], linewidth=3, color=c) 305 | else: 306 | plt.plot(x[sk],y[sk], linewidth=1, color='w') 307 | plt.plot(x[0:26][v[0:26]>0], y[0:26][v[0:26]>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) 308 | plt.plot(x[0:26][v[0:26]>1], y[0:26][v[0:26]>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) 309 | 310 | plt.plot(x[26:][v[26:]>0], y[26:][v[26:]>0],'o',markersize=1, markerfacecolor='w', markeredgecolor='w',markeredgewidth=-1) 311 | plt.plot(x[26:][v[26:]>1], y[26:][v[26:]>1],'o',markersize=2, markerfacecolor='w', markeredgecolor='w', markeredgewidth=-1) 312 | p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) 313 | ax.add_collection(p) 314 | p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) 315 | ax.add_collection(p) 316 | elif datasetType == 'captions': 317 | for ann in anns: 318 | print(ann['caption']) 319 | 320 | def loadRes(self, resFile): 321 | """ 322 | Load result file and return a result api object. 323 | :param resFile (str) : file name of result file 324 | :return: res (obj) : result api object 325 | """ 326 | res = COCO() 327 | res.dataset['images'] = [img for img in self.dataset['images']] 328 | 329 | print('Loading and preparing results...') 330 | tic = time.time() 331 | if type(resFile) == str or type(resFile) == unicode: 332 | anns = json.load(open(resFile)) 333 | elif type(resFile) == np.ndarray: 334 | anns = self.loadNumpyAnnotations(resFile) 335 | else: 336 | anns = resFile 337 | assert type(anns) == list, 'results in not an array of objects' 338 | annsImgIds = [ann['image_id'] for ann in anns] 339 | assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ 340 | 'Results do not correspond to current coco set' 341 | if 'caption' in anns[0]: 342 | imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) 343 | res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] 344 | for id, ann in enumerate(anns): 345 | ann['id'] = id+1 346 | elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: 347 | res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) 348 | for id, ann in enumerate(anns): 349 | bb = ann['bbox'] 350 | x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] 351 | if not 'segmentation' in ann: 352 | ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] 353 | ann['area'] = bb[2]*bb[3] 354 | ann['id'] = id+1 355 | ann['iscrowd'] = 0 356 | elif 'segmentation' in anns[0]: 357 | res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) 358 | for id, ann in enumerate(anns): 359 | # now only support compressed RLE format as segmentation results 360 | ann['area'] = maskUtils.area(ann['segmentation']) 361 | if not 'bbox' in ann: 362 | ann['bbox'] = maskUtils.toBbox(ann['segmentation']) 363 | ann['id'] = id+1 364 | ann['iscrowd'] = 0 365 | elif 'keypoints' in anns[0]: 366 | res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) 367 | for id, ann in enumerate(anns): 368 | s = ann['keypoints'] 369 | x = s[0::3] 370 | y = s[1::3] 371 | x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) 372 | ann['area'] = (x1-x0)*(y1-y0) 373 | ann['id'] = id + 1 374 | ann['bbox'] = [x0,y0,x1-x0,y1-y0] 375 | print('DONE (t={:0.2f}s)'.format(time.time()- tic)) 376 | 377 | res.dataset['annotations'] = anns 378 | res.createIndex() 379 | return res 380 | 381 | def download(self, tarDir = None, imgIds = [] ): 382 | ''' 383 | Download COCO images from mscoco.org server. 384 | :param tarDir (str): COCO results directory name 385 | imgIds (list): images to be downloaded 386 | :return: 387 | ''' 388 | if tarDir is None: 389 | print('Please specify target directory') 390 | return -1 391 | if len(imgIds) == 0: 392 | imgs = self.imgs.values() 393 | else: 394 | imgs = self.loadImgs(imgIds) 395 | N = len(imgs) 396 | if not os.path.exists(tarDir): 397 | os.makedirs(tarDir) 398 | for i, img in enumerate(imgs): 399 | tic = time.time() 400 | fname = os.path.join(tarDir, img['file_name']) 401 | if not os.path.exists(fname): 402 | urlretrieve(img['coco_url'], fname) 403 | print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) 404 | 405 | def loadNumpyAnnotations(self, data): 406 | """ 407 | Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} 408 | :param data (numpy.ndarray) 409 | :return: annotations (python nested list) 410 | """ 411 | print('Converting ndarray to lists...') 412 | assert(type(data) == np.ndarray) 413 | print(data.shape) 414 | assert(data.shape[1] == 7) 415 | N = data.shape[0] 416 | ann = [] 417 | for i in range(N): 418 | if i % 1000000 == 0: 419 | print('{}/{}'.format(i,N)) 420 | ann += [{ 421 | 'image_id' : int(data[i, 0]), 422 | 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], 423 | 'score' : data[i, 5], 424 | 'category_id': int(data[i, 6]), 425 | }] 426 | return ann 427 | 428 | def annToRLE(self, ann): 429 | """ 430 | Convert annotation which can be polygons, uncompressed RLE to RLE. 431 | :return: binary mask (numpy 2D array) 432 | """ 433 | t = self.imgs[ann['image_id']] 434 | h, w = t['height'], t['width'] 435 | segm = ann['segmentation'] 436 | if type(segm) == list: 437 | # polygon -- a single object might consist of multiple parts 438 | # we merge all parts into one mask rle code 439 | rles = maskUtils.frPyObjects(segm, h, w) 440 | rle = maskUtils.merge(rles) 441 | elif type(segm['counts']) == list: 442 | # uncompressed RLE 443 | rle = maskUtils.frPyObjects(segm, h, w) 444 | else: 445 | # rle 446 | rle = ann['segmentation'] 447 | return rle 448 | 449 | def annToMask(self, ann): 450 | """ 451 | Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. 452 | :return: binary mask (numpy 2D array) 453 | """ 454 | rle = self.annToRLE(ann) 455 | m = maskUtils.decode(rle) 456 | return m -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/halpecocotools/cocoeval.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tsungyi, Haoyi Zhu' 2 | 3 | import numpy as np 4 | import datetime 5 | import time 6 | from collections import defaultdict 7 | from . import mask as maskUtils 8 | import copy 9 | 10 | class COCOeval: 11 | # Interface for evaluating detection on the Microsoft COCO dataset. 12 | # 13 | # The usage for CocoEval is as follows: 14 | # cocoGt=..., cocoDt=... # load dataset and results 15 | # E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object 16 | # E.params.recThrs = ...; # set parameters as desired 17 | # E.evaluate(); # run per image evaluation 18 | # E.accumulate(); # accumulate per image results 19 | # E.summarize(); # display summary metrics of results 20 | # For example usage see evalDemo.m and http://mscoco.org/. 21 | # 22 | # The evaluation parameters are as follows (defaults in brackets): 23 | # imgIds - [all] N img ids to use for evaluation 24 | # catIds - [all] K cat ids to use for evaluation 25 | # iouThrs - [.5:.05:.95] T=10 IoU thresholds for evaluation 26 | # recThrs - [0:.01:1] R=101 recall thresholds for evaluation 27 | # areaRng - [...] A=4 object area ranges for evaluation 28 | # maxDets - [1 10 100] M=3 thresholds on max detections per image 29 | # iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints' 30 | # iouType replaced the now DEPRECATED useSegm parameter. 31 | # useCats - [1] if true use category labels for evaluation 32 | # Note: if useCats=0 category labels are ignored as in proposal scoring. 33 | # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. 34 | # 35 | # evaluate(): evaluates detections on every image and every category and 36 | # concats the results into the "evalImgs" with fields: 37 | # dtIds - [1xD] id for each of the D detections (dt) 38 | # gtIds - [1xG] id for each of the G ground truths (gt) 39 | # dtMatches - [TxD] matching gt id at each IoU or 0 40 | # gtMatches - [TxG] matching dt id at each IoU or 0 41 | # dtScores - [1xD] confidence of each dt 42 | # gtIgnore - [1xG] ignore flag for each gt 43 | # dtIgnore - [TxD] ignore flag for each dt at each IoU 44 | # 45 | # accumulate(): accumulates the per-image, per-category evaluation 46 | # results in "evalImgs" into the dictionary "eval" with fields: 47 | # params - parameters used for evaluation 48 | # date - date evaluation was performed 49 | # counts - [T,R,K,A,M] parameter dimensions (see above) 50 | # precision - [TxRxKxAxM] precision for every evaluation setting 51 | # recall - [TxKxAxM] max recall for every evaluation setting 52 | # Note: precision and recall==-1 for settings with no gt objects. 53 | # 54 | # See also coco, mask, pycocoDemo, pycocoEvalDemo 55 | # 56 | # The original Microsoft COCO Toolbox is written 57 | # by Piotr Dollar and Tsung-Yi Lin, 2015. 58 | # Licensed under the Simplified BSD License [see coco/license.txt] 59 | # 60 | # Updated and renamed to Halpe COCO Toolbox (cocohalpetools) \ 61 | # by Haoyi Zhu in 2021. The Halpe COCO Toolbox is 62 | # developed to support wholebody human pose datasets, 63 | # including Halpe Full-body and COCO WholeBody. 64 | 65 | def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'): 66 | ''' 67 | Initialize CocoEval using coco APIs for gt and dt 68 | :param cocoGt: coco object with ground truth annotations 69 | :param cocoDt: coco object with detection results 70 | :return: None 71 | ''' 72 | if not iouType: 73 | print('iouType not specified. use default iouType segm') 74 | self.cocoGt = cocoGt # ground truth COCO API 75 | self.cocoDt = cocoDt # detections COCO API 76 | self.params = {} # evaluation parameters 77 | self.evalImgs = defaultdict(list) # per-image per-category evaluation results [KxAxI] elements 78 | self.eval = {} # accumulated evaluation results 79 | self._gts = defaultdict(list) # gt for evaluation 80 | self._dts = defaultdict(list) # dt for evaluation 81 | self.params = Params(iouType=iouType) # parameters 82 | self._paramsEval = {} # parameters for evaluation 83 | self.stats = [] # result summarization 84 | self.ious = {} # ious between all gts and dts 85 | if not cocoGt is None: 86 | self.params.imgIds = sorted(cocoGt.getImgIds()) 87 | self.params.catIds = sorted(cocoGt.getCatIds()) 88 | 89 | 90 | def _prepare(self): 91 | ''' 92 | Prepare ._gts and ._dts for evaluation based on params 93 | :return: None 94 | ''' 95 | def _toMask(anns, coco): 96 | # modify ann['segmentation'] by reference 97 | for ann in anns: 98 | rle = coco.annToRLE(ann) 99 | ann['segmentation'] = rle 100 | p = self.params 101 | if p.useCats: 102 | gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) 103 | dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) 104 | else: 105 | gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) 106 | dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) 107 | 108 | # convert ground truth to mask if iouType == 'segm' 109 | if p.iouType == 'segm': 110 | _toMask(gts, self.cocoGt) 111 | _toMask(dts, self.cocoDt) 112 | # set ignore flag 113 | for gt in gts: 114 | gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0 115 | # gt['ignore'] = 'iscrowd' in gt and gt['iscrowd'] 116 | if p.iouType == 'keypoints': 117 | if len(gt['keypoints']) % 133 == 0 or len(gt['keypoints']) % 136 == 0: 118 | _ignore = gt['ignore'] 119 | gt['ignore'] = dict() 120 | body_kp = np.array(gt['keypoints'][:-116*3]) 121 | foot_kp = np.array(gt['keypoints'][-116*3:-110*3]) 122 | face_kp = np.array(gt['keypoints'][-110*3:-42*3]) 123 | hand_kp = np.array(gt['keypoints'][-42*3:]) 124 | fullbody_kp = np.array(gt['keypoints']) 125 | 126 | vg_body = body_kp[2::3] 127 | vg_foot = foot_kp[2::3] 128 | vg_face = face_kp[2::3] 129 | vg_hand = hand_kp[2::3] 130 | vg_fullbody = fullbody_kp[2::3] 131 | k1_body = np.count_nonzero(vg_body > 0) 132 | k1_foot = np.count_nonzero(vg_foot > 0) 133 | k1_face = np.count_nonzero(vg_face > 0) 134 | k1_hand = np.count_nonzero(vg_hand > 0) 135 | k1_fullbody = np.count_nonzero(vg_fullbody > 0) 136 | 137 | gt['ignore']['body'] = (k1_body == 0) or _ignore 138 | gt['ignore']['foot'] = (k1_foot == 0) or _ignore 139 | gt['ignore']['face'] = (k1_face == 0) or _ignore 140 | gt['ignore']['hand'] = (k1_hand == 0) or _ignore 141 | gt['ignore']['fullbody'] = (k1_fullbody == 0) or _ignore 142 | else: 143 | gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore'] 144 | self._gts = defaultdict(list) # gt for evaluation 145 | self._dts = defaultdict(list) # dt for evaluation 146 | for gt in gts: 147 | self._gts[gt['image_id'], gt['category_id']].append(gt) 148 | for dt in dts: 149 | vd = np.array(dt['keypoints'])[2::3] 150 | if len(vd[vd>0]) == 0: 151 | continue 152 | self._dts[dt['image_id'], dt['category_id']].append(dt) 153 | self.evalImgs = defaultdict(list) # per-image per-category evaluation results 154 | self.eval = {} # accumulated evaluation results 155 | 156 | def evaluate(self): 157 | ''' 158 | Run per image evaluation on given images and store results (a list of dict) in self.evalImgs 159 | :return: None 160 | ''' 161 | tic = time.time() 162 | print('Running per image evaluation...') 163 | p = self.params 164 | # add backward compatibility if useSegm is specified in params 165 | if not p.useSegm is None: 166 | p.iouType = 'segm' if p.useSegm == 1 else 'bbox' 167 | print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) 168 | print('Evaluate annotation type *{}*'.format(p.iouType)) 169 | p.imgIds = list(np.unique(p.imgIds)) 170 | if p.useCats: 171 | p.catIds = list(np.unique(p.catIds)) 172 | p.maxDets = sorted(p.maxDets) 173 | self.params=p 174 | 175 | self._prepare() 176 | # loop through images, area range, max detection number 177 | catIds = p.catIds if p.useCats else [-1] 178 | 179 | if p.iouType == 'segm' or p.iouType == 'bbox': 180 | computeIoU = self.computeIoU 181 | elif p.iouType == 'keypoints': 182 | computeIoU = self.computeOks 183 | self.ious = {(imgId, catId): computeIoU(imgId, catId) \ 184 | for imgId in p.imgIds 185 | for catId in catIds} 186 | 187 | evaluateImg = self.evaluateImg 188 | maxDet = p.maxDets[-1] 189 | self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet) 190 | for catId in catIds 191 | for areaRng in p.areaRng 192 | for imgId in p.imgIds 193 | ] 194 | self._paramsEval = copy.deepcopy(self.params) 195 | toc = time.time() 196 | print('DONE (t={:0.2f}s).'.format(toc-tic)) 197 | 198 | def computeIoU(self, imgId, catId): 199 | p = self.params 200 | if p.useCats: 201 | gt = self._gts[imgId,catId] 202 | dt = self._dts[imgId,catId] 203 | else: 204 | gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] 205 | dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] 206 | if len(gt) == 0 and len(dt) ==0: 207 | return [] 208 | inds = np.argsort([-d['score'] for d in dt], kind='mergesort') 209 | dt = [dt[i] for i in inds] 210 | if len(dt) > p.maxDets[-1]: 211 | dt=dt[0:p.maxDets[-1]] 212 | 213 | if p.iouType == 'segm': 214 | g = [g['segmentation'] for g in gt] 215 | d = [d['segmentation'] for d in dt] 216 | elif p.iouType == 'bbox': 217 | g = [g['bbox'] for g in gt] 218 | d = [d['bbox'] for d in dt] 219 | else: 220 | raise Exception('unknown iouType for iou computation') 221 | 222 | # compute iou between each dt and gt region 223 | iscrowd = [int(o['iscrowd']) for o in gt] 224 | ious = maskUtils.iou(d,g,iscrowd) 225 | return ious 226 | 227 | def computeOks(self, imgId, catId): 228 | parts = ['body', 'foot', 'face', 'hand', 'fullbody'] 229 | _ious = {} 230 | for part in parts: 231 | p = self.params 232 | # dimention here should be Nxm 233 | gts = self._gts[imgId, catId] 234 | dts = self._dts[imgId, catId] 235 | inds = np.argsort([-d['score'] for d in dts], kind='mergesort') 236 | dts = [dts[i] for i in inds] 237 | if len(dts) > p.maxDets[-1]: 238 | dts = dts[0:p.maxDets[-1]] 239 | # if len(gts) == 0 and len(dts) == 0: 240 | if len(gts) == 0 or len(dts) == 0: 241 | _ious[part] = [] 242 | continue 243 | ious = np.zeros((len(dts), len(gts))) 244 | if len(gts[0]['keypoints']) == 136 * 3: 245 | sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89, .8,.8,.8,.89, .89, .89, .89, .89, .89, 246 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 247 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 248 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 249 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 250 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15])/10.0 251 | body_kpnum = 20 252 | foot_kpnum = 6 253 | face_kpnum = 68 254 | hand_kpnum = 21 * 2 255 | 256 | elif len(gts[0]['keypoints']) == 133 * 3: # surport for COCO WholeBody 257 | sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89, .89, .89, .89, .89, .89, .89, 258 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 259 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 260 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 261 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, 262 | .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15, .15])/10.0 263 | body_kpnum = 17 264 | foot_kpnum = 6 265 | face_kpnum = 68 266 | hand_kpnum = 21 * 2 267 | 268 | if part == 'body': 269 | sigmas = sigmas[0 : body_kpnum] 270 | elif part == 'foot': 271 | sigmas = sigmas[body_kpnum : body_kpnum + foot_kpnum] 272 | elif part == 'face': 273 | sigmas = sigmas[body_kpnum + foot_kpnum : body_kpnum + foot_kpnum + face_kpnum] 274 | elif part == 'hand': 275 | sigmas = sigmas[body_kpnum + foot_kpnum + face_kpnum:] 276 | 277 | vars = (sigmas * 2)**2 278 | k = len(sigmas) 279 | from tkinter import _flatten 280 | # compute oks between each detection and ground truth object 281 | for j, gt in enumerate(gts): 282 | # create bounds for ignore regions(double the gt bbox) 283 | if part == 'body': 284 | g = np.array(gt['keypoints'][0 : body_kpnum*3],dtype=np.float32) 285 | elif part == 'foot': 286 | g = np.array(gt['keypoints'][body_kpnum*3 : (body_kpnum + foot_kpnum)*3],dtype=np.float32) 287 | elif part == 'face': 288 | g = np.array(gt['keypoints'][(body_kpnum + foot_kpnum)*3 : (body_kpnum + foot_kpnum + face_kpnum)*3],dtype=np.float32) 289 | elif part == 'hand': 290 | g = np.array(gt['keypoints'][(body_kpnum + foot_kpnum + face_kpnum)*3:],dtype=np.float32) 291 | else: 292 | g = np.array(gt['keypoints'],dtype=np.float32) 293 | 294 | xg = g[0::3]; yg = g[1::3]; vg = g[2::3] 295 | k1 = np.count_nonzero(vg > 0) 296 | bb = gt['bbox'] 297 | x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2 298 | y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2 299 | for i, dt in enumerate(dts): 300 | if part == 'body': 301 | d = np.array(dt['keypoints'][0 : body_kpnum*3],dtype=np.float32) 302 | elif part == 'foot': 303 | d = np.array(dt['keypoints'][body_kpnum*3 : (body_kpnum + foot_kpnum)*3],dtype=np.float32) 304 | elif part == 'face': 305 | d = np.array(dt['keypoints'][(body_kpnum + foot_kpnum)*3 : (body_kpnum + foot_kpnum + face_kpnum)*3],dtype=np.float32) 306 | elif part == 'hand': 307 | d = np.array(dt['keypoints'][(body_kpnum + foot_kpnum + face_kpnum)*3:],dtype=np.float32) 308 | else: 309 | d = np.array(dt['keypoints'], dtype=np.float32) 310 | 311 | xd = d[0::3]; yd = d[1::3] 312 | if k1>0: 313 | # measure the per-keypoint distance if keypoints visible 314 | dx = xd - xg 315 | dy = yd - yg 316 | else: 317 | # measure minimum distance to keypoints in (x0,y0) & (x1,y1) 318 | z = np.zeros((k)) 319 | dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0) 320 | dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0) 321 | e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2 322 | if k1 > 0: 323 | e=e[vg > 0] 324 | ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] 325 | _ious[part] = ious 326 | return _ious 327 | 328 | def evaluateImg(self, imgId, catId, aRng, maxDet): 329 | ''' 330 | perform evaluation for single category and image 331 | :return: dict (single image results) 332 | ''' 333 | if self.params.iouType == 'keypoints': 334 | parts = ['body', 'foot', 'face', 'hand', 'fullbody'] 335 | res = {} 336 | for part in parts: 337 | p = self.params 338 | if p.useCats: 339 | gt = self._gts[imgId,catId] 340 | dt = self._dts[imgId,catId] 341 | else: 342 | gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] 343 | dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] 344 | if len(gt) == 0 and len(dt) ==0: 345 | res[part] = None 346 | continue 347 | 348 | for g in gt: 349 | if g['ignore'][part] or (g['area']aRng[1]): 350 | g['_ignore'] = 1 351 | else: 352 | g['_ignore'] = 0 353 | 354 | # sort dt highest score first, sort gt ignore last 355 | gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort') 356 | gt = [gt[i] for i in gtind] 357 | dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') 358 | dt = [dt[i] for i in dtind[0:maxDet]] 359 | iscrowd = [int(o['iscrowd']) for o in gt] 360 | # load computed ious 361 | ious = self.ious[imgId, catId][part][:, gtind] if len(self.ious[imgId, catId][part]) > 0 else self.ious[imgId, catId][part] 362 | 363 | T = len(p.iouThrs) 364 | G = len(gt) 365 | D = len(dt) 366 | gtm = np.zeros((T,G)) 367 | dtm = np.zeros((T,D)) 368 | gtIg = np.array([g['_ignore'] for g in gt]) 369 | dtIg = np.zeros((T,D)) 370 | if not len(ious)==0: 371 | for tind, t in enumerate(p.iouThrs): 372 | for dind, d in enumerate(dt): 373 | # information about best match so far (m=-1 -> unmatched) 374 | iou = min([t,1-1e-10]) 375 | m = -1 376 | for gind, g in enumerate(gt): 377 | # if this gt already matched, and not a crowd, continue 378 | if gtm[tind,gind]>0 and not iscrowd[gind]: 379 | continue 380 | # if dt matched to reg gt, and on ignore gt, stop 381 | if m>-1 and gtIg[m]==0 and gtIg[gind]==1: 382 | break 383 | # continue to next gt unless better match made 384 | if ious[dind,gind] < iou: 385 | continue 386 | # if match successful and best so far, store appropriately 387 | iou=ious[dind,gind] 388 | m=gind 389 | # if match made store id of match for both dt and gt 390 | if m ==-1: 391 | continue 392 | dtIg[tind,dind] = gtIg[m] 393 | dtm[tind,dind] = gt[m]['id'] 394 | gtm[tind,m] = d['id'] 395 | # set unmatched detections outside of area range to ignore 396 | a = np.array([d['area']aRng[1] for d in dt]).reshape((1, len(dt))) 397 | dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0))) 398 | # store results for given image and category 399 | res[part]={ 400 | 'image_id': imgId, 401 | 'category_id': catId, 402 | 'aRng': aRng, 403 | 'maxDet': maxDet, 404 | 'dtIds': [d['id'] for d in dt], 405 | 'gtIds': [g['id'] for g in gt], 406 | 'dtMatches': dtm, 407 | 'gtMatches': gtm, 408 | 'dtScores': [d['score'] for d in dt], 409 | 'gtIgnore': gtIg, 410 | 'dtIgnore': dtIg, 411 | } 412 | return res 413 | else: 414 | p = self.params 415 | if p.useCats: 416 | gt = self._gts[imgId,catId] 417 | dt = self._dts[imgId,catId] 418 | else: 419 | gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] 420 | dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] 421 | if len(gt) == 0 and len(dt) ==0: 422 | return None 423 | 424 | for g in gt: 425 | if g['ignore'] or (g['area']aRng[1]): 426 | g['_ignore'] = 1 427 | else: 428 | g['_ignore'] = 0 429 | 430 | # sort dt highest score first, sort gt ignore last 431 | gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort') 432 | gt = [gt[i] for i in gtind] 433 | dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') 434 | dt = [dt[i] for i in dtind[0:maxDet]] 435 | iscrowd = [int(o['iscrowd']) for o in gt] 436 | # load computed ious 437 | ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] 438 | 439 | T = len(p.iouThrs) 440 | G = len(gt) 441 | D = len(dt) 442 | gtm = np.zeros((T,G)) 443 | dtm = np.zeros((T,D)) 444 | gtIg = np.array([g['_ignore'] for g in gt]) 445 | dtIg = np.zeros((T,D)) 446 | if not len(ious)==0: 447 | for tind, t in enumerate(p.iouThrs): 448 | for dind, d in enumerate(dt): 449 | # information about best match so far (m=-1 -> unmatched) 450 | iou = min([t,1-1e-10]) 451 | m = -1 452 | for gind, g in enumerate(gt): 453 | # if this gt already matched, and not a crowd, continue 454 | if gtm[tind,gind]>0 and not iscrowd[gind]: 455 | continue 456 | # if dt matched to reg gt, and on ignore gt, stop 457 | if m>-1 and gtIg[m]==0 and gtIg[gind]==1: 458 | break 459 | # continue to next gt unless better match made 460 | if ious[dind,gind] < iou: 461 | continue 462 | # if match successful and best so far, store appropriately 463 | iou=ious[dind,gind] 464 | m=gind 465 | # if match made store id of match for both dt and gt 466 | if m ==-1: 467 | continue 468 | dtIg[tind,dind] = gtIg[m] 469 | dtm[tind,dind] = gt[m]['id'] 470 | gtm[tind,m] = d['id'] 471 | # set unmatched detections outside of area range to ignore 472 | a = np.array([d['area']aRng[1] for d in dt]).reshape((1, len(dt))) 473 | dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0))) 474 | # store results for given image and category 475 | return { 476 | 'image_id': imgId, 477 | 'category_id': catId, 478 | 'aRng': aRng, 479 | 'maxDet': maxDet, 480 | 'dtIds': [d['id'] for d in dt], 481 | 'gtIds': [g['id'] for g in gt], 482 | 'dtMatches': dtm, 483 | 'gtMatches': gtm, 484 | 'dtScores': [d['score'] for d in dt], 485 | 'gtIgnore': gtIg, 486 | 'dtIgnore': dtIg, 487 | } 488 | return res 489 | 490 | def accumulate(self, p = None): 491 | ''' 492 | Accumulate per image evaluation results and store the result in self.eval 493 | :param p: input params for evaluation 494 | :return: None 495 | ''' 496 | print('Accumulating evaluation results...') 497 | tic = time.time() 498 | if not self.evalImgs: 499 | print('Please run evaluate() first') 500 | if self.params.iouType == 'keypoints': 501 | parts = ['body', 'foot', 'face', 'hand', 'fullbody'] 502 | for part in parts: 503 | # allows input customized parameters 504 | if p is None: 505 | p = self.params 506 | p.catIds = p.catIds if p.useCats == 1 else [-1] 507 | T = len(p.iouThrs) 508 | R = len(p.recThrs) 509 | K = len(p.catIds) if p.useCats else 1 510 | A = len(p.areaRng) 511 | M = len(p.maxDets) 512 | precision = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories 513 | recall = -np.ones((T,K,A,M)) 514 | scores = -np.ones((T,R,K,A,M)) 515 | 516 | # create dictionary for future indexing 517 | _pe = self._paramsEval 518 | catIds = _pe.catIds if _pe.useCats else [-1] 519 | setK = set(catIds) 520 | setA = set(map(tuple, _pe.areaRng)) 521 | setM = set(_pe.maxDets) 522 | setI = set(_pe.imgIds) 523 | # get inds to evaluate 524 | k_list = [n for n, k in enumerate(p.catIds) if k in setK] 525 | m_list = [m for n, m in enumerate(p.maxDets) if m in setM] 526 | a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] 527 | i_list = [n for n, i in enumerate(p.imgIds) if i in setI] 528 | I0 = len(_pe.imgIds) 529 | A0 = len(_pe.areaRng) 530 | # retrieve E at each category, area range, and max number of detections 531 | 532 | for k, k0 in enumerate(k_list): 533 | Nk = k0*A0*I0 534 | for a, a0 in enumerate(a_list): 535 | Na = a0*I0 536 | for m, maxDet in enumerate(m_list): 537 | E = [self.evalImgs[Nk + Na + i][part] for i in i_list] 538 | E = [e for e in E if not e is None] 539 | if len(E) == 0: 540 | continue 541 | dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) 542 | 543 | # different sorting method generates slightly different results. 544 | # mergesort is used to be consistent as Matlab implementation. 545 | inds = np.argsort(-dtScores, kind='mergesort') 546 | dtScoresSorted = dtScores[inds] 547 | 548 | dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] 549 | dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] 550 | gtIg = np.concatenate([e['gtIgnore'] for e in E]) 551 | npig = np.count_nonzero(gtIg==0 ) 552 | if npig == 0: 553 | continue 554 | tps = np.logical_and( dtm, np.logical_not(dtIg) ) 555 | fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) ) 556 | 557 | tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) 558 | fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) 559 | for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): 560 | tp = np.array(tp) 561 | fp = np.array(fp) 562 | nd = len(tp) 563 | rc = tp / npig 564 | pr = tp / (fp+tp+np.spacing(1)) 565 | q = np.zeros((R,)) 566 | ss = np.zeros((R,)) 567 | 568 | if nd: 569 | recall[t,k,a,m] = rc[-1] 570 | else: 571 | recall[t,k,a,m] = 0 572 | 573 | # numpy is slow without cython optimization for accessing elements 574 | # use python array gets significant speed improvement 575 | pr = pr.tolist(); q = q.tolist() 576 | 577 | for i in range(nd-1, 0, -1): 578 | if pr[i] > pr[i-1]: 579 | pr[i-1] = pr[i] 580 | 581 | inds = np.searchsorted(rc, p.recThrs, side='left') 582 | try: 583 | for ri, pi in enumerate(inds): 584 | q[ri] = pr[pi] 585 | ss[ri] = dtScoresSorted[pi] 586 | except: 587 | pass 588 | precision[t,:,k,a,m] = np.array(q) 589 | scores[t,:,k,a,m] = np.array(ss) 590 | self.eval[part] = { 591 | 'params': p, 592 | 'counts': [T, R, K, A, M], 593 | 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 594 | 'precision': precision, 595 | 'recall': recall, 596 | 'scores': scores, 597 | } 598 | else: 599 | if p is None: 600 | p = self.params 601 | p.catIds = p.catIds if p.useCats == 1 else [-1] 602 | T = len(p.iouThrs) 603 | R = len(p.recThrs) 604 | K = len(p.catIds) if p.useCats else 1 605 | A = len(p.areaRng) 606 | M = len(p.maxDets) 607 | precision = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories 608 | recall = -np.ones((T,K,A,M)) 609 | scores = -np.ones((T,R,K,A,M)) 610 | 611 | # create dictionary for future indexing 612 | _pe = self._paramsEval 613 | catIds = _pe.catIds if _pe.useCats else [-1] 614 | setK = set(catIds) 615 | setA = set(map(tuple, _pe.areaRng)) 616 | setM = set(_pe.maxDets) 617 | setI = set(_pe.imgIds) 618 | # get inds to evaluate 619 | k_list = [n for n, k in enumerate(p.catIds) if k in setK] 620 | m_list = [m for n, m in enumerate(p.maxDets) if m in setM] 621 | a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] 622 | i_list = [n for n, i in enumerate(p.imgIds) if i in setI] 623 | I0 = len(_pe.imgIds) 624 | A0 = len(_pe.areaRng) 625 | # retrieve E at each category, area range, and max number of detections 626 | 627 | for k, k0 in enumerate(k_list): 628 | Nk = k0*A0*I0 629 | for a, a0 in enumerate(a_list): 630 | Na = a0*I0 631 | for m, maxDet in enumerate(m_list): 632 | E = [self.evalImgs[Nk + Na + i] for i in i_list] 633 | E = [e for e in E if not e is None] 634 | if len(E) == 0: 635 | continue 636 | dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) 637 | 638 | # different sorting method generates slightly different results. 639 | # mergesort is used to be consistent as Matlab implementation. 640 | inds = np.argsort(-dtScores, kind='mergesort') 641 | dtScoresSorted = dtScores[inds] 642 | 643 | dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] 644 | dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] 645 | gtIg = np.concatenate([e['gtIgnore'] for e in E]) 646 | npig = np.count_nonzero(gtIg==0 ) 647 | if npig == 0: 648 | continue 649 | tps = np.logical_and( dtm, np.logical_not(dtIg) ) 650 | fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) ) 651 | 652 | tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) 653 | fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) 654 | for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): 655 | tp = np.array(tp) 656 | fp = np.array(fp) 657 | nd = len(tp) 658 | rc = tp / npig 659 | pr = tp / (fp+tp+np.spacing(1)) 660 | q = np.zeros((R,)) 661 | ss = np.zeros((R,)) 662 | 663 | if nd: 664 | recall[t,k,a,m] = rc[-1] 665 | else: 666 | recall[t,k,a,m] = 0 667 | 668 | # numpy is slow without cython optimization for accessing elements 669 | # use python array gets significant speed improvement 670 | pr = pr.tolist(); q = q.tolist() 671 | 672 | for i in range(nd-1, 0, -1): 673 | if pr[i] > pr[i-1]: 674 | pr[i-1] = pr[i] 675 | 676 | inds = np.searchsorted(rc, p.recThrs, side='left') 677 | try: 678 | for ri, pi in enumerate(inds): 679 | q[ri] = pr[pi] 680 | ss[ri] = dtScoresSorted[pi] 681 | except: 682 | pass 683 | precision[t,:,k,a,m] = np.array(q) 684 | scores[t,:,k,a,m] = np.array(ss) 685 | self.eval = { 686 | 'params': p, 687 | 'counts': [T, R, K, A, M], 688 | 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 689 | 'precision': precision, 690 | 'recall': recall, 691 | 'scores': scores, 692 | } 693 | 694 | toc = time.time() 695 | print('DONE (t={:0.2f}s).'.format( toc-tic)) 696 | 697 | def summarize(self): 698 | ''' 699 | Compute and display summary metrics for evaluation results. 700 | Note this functin can *only* be applied on the default parameter setting 701 | ''' 702 | def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100, part=None): 703 | p = self.params 704 | iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' 705 | titleStr = 'Average Precision' if ap == 1 else 'Average Recall' 706 | typeStr = '(AP)' if ap==1 else '(AR)' 707 | iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ 708 | if iouThr is None else '{:0.2f}'.format(iouThr) 709 | 710 | aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] 711 | mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] 712 | if ap == 1: 713 | # dimension of precision: [TxRxKxAxM] 714 | if part: 715 | s = self.eval[part]['precision'] 716 | else: 717 | s = self.eval['precision'] 718 | # IoU 719 | if iouThr is not None: 720 | t = np.where(iouThr == p.iouThrs)[0] 721 | s = s[t] 722 | s = s[:,:,:,aind,mind] 723 | else: 724 | # dimension of recall: [TxKxAxM] 725 | if part: 726 | s = self.eval[part]['recall'] 727 | else: 728 | s = self.eval['recall'] 729 | if iouThr is not None: 730 | t = np.where(iouThr == p.iouThrs)[0] 731 | s = s[t] 732 | s = s[:,:,aind,mind] 733 | if len(s[s>-1])==0: 734 | mean_s = -1 735 | else: 736 | mean_s = np.mean(s[s>-1]) 737 | print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) 738 | return mean_s 739 | def _summarizeDets(): 740 | stats = np.zeros((12,)) 741 | stats[0] = _summarize(1) 742 | stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) 743 | stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) 744 | stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) 745 | stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) 746 | stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) 747 | stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) 748 | stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) 749 | stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) 750 | stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) 751 | stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) 752 | stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) 753 | return stats 754 | def _summarizeKps(part='fullbody'): 755 | stats = np.zeros((10,)) 756 | stats[0] = _summarize(1, maxDets=20, part=part) 757 | stats[1] = _summarize(1, maxDets=20, iouThr=.5, part=part) 758 | stats[2] = _summarize(1, maxDets=20, iouThr=.75, part=part) 759 | stats[3] = _summarize(1, maxDets=20, areaRng='medium', part=part) 760 | stats[4] = _summarize(1, maxDets=20, areaRng='large', part=part) 761 | stats[5] = _summarize(0, maxDets=20, part=part) 762 | stats[6] = _summarize(0, maxDets=20, iouThr=.5, part=part) 763 | stats[7] = _summarize(0, maxDets=20, iouThr=.75, part=part) 764 | stats[8] = _summarize(0, maxDets=20, areaRng='medium', part=part) 765 | stats[9] = _summarize(0, maxDets=20, areaRng='large', part=part) 766 | return {part: stats} 767 | if not self.eval: 768 | raise Exception('Please run accumulate() first') 769 | iouType = self.params.iouType 770 | if iouType == 'segm' or iouType == 'bbox': 771 | summarize = _summarizeDets 772 | self.stats = summarize() 773 | elif iouType == 'keypoints': 774 | summarize = _summarizeKps 775 | parts = ['body', 'foot', 'face', 'hand', 'fullbody'] 776 | for part in parts: 777 | print(part, ':') 778 | self.stats.append(summarize(part)) 779 | 780 | def __str__(self): 781 | self.summarize() 782 | 783 | class Params: 784 | ''' 785 | Params for coco evaluation api 786 | ''' 787 | def setDetParams(self): 788 | self.imgIds = [] 789 | self.catIds = [] 790 | # np.arange causes trouble. the data point on arange is slightly larger than the true value 791 | self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05) + 1), endpoint=True) 792 | self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01) + 1), endpoint=True) 793 | self.maxDets = [1, 10, 100] 794 | self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] 795 | self.areaRngLbl = ['all', 'small', 'medium', 'large'] 796 | self.useCats = 1 797 | 798 | def setKpParams(self): 799 | self.imgIds = [] 800 | self.catIds = [] 801 | # np.arange causes trouble. the data point on arange is slightly larger than the true value 802 | self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05) + 1), endpoint=True) 803 | self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01) + 1), endpoint=True) 804 | self.maxDets = [20] 805 | self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] 806 | self.areaRngLbl = ['all', 'medium', 'large'] 807 | self.useCats = 1 808 | 809 | def __init__(self, iouType='segm'): 810 | if iouType == 'segm' or iouType == 'bbox': 811 | self.setDetParams() 812 | elif iouType == 'keypoints': 813 | self.setKpParams() 814 | else: 815 | raise Exception('iouType not supported') 816 | self.iouType = iouType 817 | # useSegm is deprecated 818 | self.useSegm = None 819 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/halpecocotools/mask.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tsungyi' 2 | 3 | import pycocotools._mask as _mask 4 | 5 | # Interface for manipulating masks stored in RLE format. 6 | # 7 | # RLE is a simple yet efficient format for storing binary masks. RLE 8 | # first divides a vector (or vectorized image) into a series of piecewise 9 | # constant regions and then for each piece simply stores the length of 10 | # that piece. For example, given M=[0 0 1 1 1 0 1] the RLE counts would 11 | # be [2 3 1 1], or for M=[1 1 1 1 1 1 0] the counts would be [0 6 1] 12 | # (note that the odd counts are always the numbers of zeros). Instead of 13 | # storing the counts directly, additional compression is achieved with a 14 | # variable bitrate representation based on a common scheme called LEB128. 15 | # 16 | # Compression is greatest given large piecewise constant regions. 17 | # Specifically, the size of the RLE is proportional to the number of 18 | # *boundaries* in M (or for an image the number of boundaries in the y 19 | # direction). Assuming fairly simple shapes, the RLE representation is 20 | # O(sqrt(n)) where n is number of pixels in the object. Hence space usage 21 | # is substantially lower, especially for large simple objects (large n). 22 | # 23 | # Many common operations on masks can be computed directly using the RLE 24 | # (without need for decoding). This includes computations such as area, 25 | # union, intersection, etc. All of these operations are linear in the 26 | # size of the RLE, in other words they are O(sqrt(n)) where n is the area 27 | # of the object. Computing these operations on the original mask is O(n). 28 | # Thus, using the RLE can result in substantial computational savings. 29 | # 30 | # The following API functions are defined: 31 | # encode - Encode binary masks using RLE. 32 | # decode - Decode binary masks encoded via RLE. 33 | # merge - Compute union or intersection of encoded masks. 34 | # iou - Compute intersection over union between masks. 35 | # area - Compute area of encoded masks. 36 | # toBbox - Get bounding boxes surrounding encoded masks. 37 | # frPyObjects - Convert polygon, bbox, and uncompressed RLE to encoded RLE mask. 38 | # 39 | # Usage: 40 | # Rs = encode( masks ) 41 | # masks = decode( Rs ) 42 | # R = merge( Rs, intersect=false ) 43 | # o = iou( dt, gt, iscrowd ) 44 | # a = area( Rs ) 45 | # bbs = toBbox( Rs ) 46 | # Rs = frPyObjects( [pyObjects], h, w ) 47 | # 48 | # In the API the following formats are used: 49 | # Rs - [dict] Run-length encoding of binary masks 50 | # R - dict Run-length encoding of binary mask 51 | # masks - [hxwxn] Binary mask(s) (must have type np.ndarray(dtype=uint8) in column-major order) 52 | # iscrowd - [nx1] list of np.ndarray. 1 indicates corresponding gt image has crowd region to ignore 53 | # bbs - [nx4] Bounding box(es) stored as [x y w h] 54 | # poly - Polygon stored as [[x1 y1 x2 y2...],[x1 y1 ...],...] (2D list) 55 | # dt,gt - May be either bounding boxes or encoded masks 56 | # Both poly and bbs are 0-indexed (bbox=[0 0 1 1] encloses first pixel). 57 | # 58 | # Finally, a note about the intersection over union (iou) computation. 59 | # The standard iou of a ground truth (gt) and detected (dt) object is 60 | # iou(gt,dt) = area(intersect(gt,dt)) / area(union(gt,dt)) 61 | # For "crowd" regions, we use a modified criteria. If a gt object is 62 | # marked as "iscrowd", we allow a dt to match any subregion of the gt. 63 | # Choosing gt' in the crowd gt that best matches the dt can be done using 64 | # gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing 65 | # iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) 66 | # For crowd gt regions we use this modified criteria above for the iou. 67 | # 68 | # To compile run "python setup.py build_ext --inplace" 69 | # Please do not contact us for help with compiling. 70 | # 71 | # Microsoft COCO Toolbox. version 2.0 72 | # Data, paper, and tutorials available at: http://mscoco.org/ 73 | # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. 74 | # Licensed under the Simplified BSD License [see coco/license.txt] 75 | 76 | iou = _mask.iou 77 | merge = _mask.merge 78 | frPyObjects = _mask.frPyObjects 79 | 80 | def encode(bimask): 81 | if len(bimask.shape) == 3: 82 | return _mask.encode(bimask) 83 | elif len(bimask.shape) == 2: 84 | h, w = bimask.shape 85 | return _mask.encode(bimask.reshape((h, w, 1), order='F'))[0] 86 | 87 | def decode(rleObjs): 88 | if type(rleObjs) == list: 89 | return _mask.decode(rleObjs) 90 | else: 91 | return _mask.decode([rleObjs])[:,:,0] 92 | 93 | def area(rleObjs): 94 | if type(rleObjs) == list: 95 | return _mask.area(rleObjs) 96 | else: 97 | return _mask.area([rleObjs])[0] 98 | 99 | def toBbox(rleObjs): 100 | if type(rleObjs) == list: 101 | return _mask.toBbox(rleObjs) 102 | else: 103 | return _mask.toBbox([rleObjs])[0] -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/pycocoEvalDemo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "%matplotlib inline\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "from pycocotools.coco import COCO\n", 14 | "from pycocotools.cocoeval import COCOeval\n", 15 | "import numpy as np\n", 16 | "import skimage.io as io\n", 17 | "import pylab\n", 18 | "pylab.rcParams['figure.figsize'] = (10.0, 8.0)" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 2, 24 | "metadata": { 25 | "collapsed": false 26 | }, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "Running demo for *bbox* results.\n" 33 | ] 34 | } 35 | ], 36 | "source": [ 37 | "annType = ['segm','bbox','keypoints']\n", 38 | "annType = annType[1] #specify type here\n", 39 | "prefix = 'person_keypoints' if annType=='keypoints' else 'instances'\n", 40 | "print 'Running demo for *%s* results.'%(annType)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 3, 46 | "metadata": { 47 | "collapsed": false 48 | }, 49 | "outputs": [ 50 | { 51 | "name": "stdout", 52 | "output_type": "stream", 53 | "text": [ 54 | "loading annotations into memory...\n", 55 | "Done (t=8.01s)\n", 56 | "creating index...\n", 57 | "index created!\n" 58 | ] 59 | } 60 | ], 61 | "source": [ 62 | "#initialize COCO ground truth api\n", 63 | "dataDir='../'\n", 64 | "dataType='val2014'\n", 65 | "annFile = '%s/annotations/%s_%s.json'%(dataDir,prefix,dataType)\n", 66 | "cocoGt=COCO(annFile)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 4, 72 | "metadata": { 73 | "collapsed": false 74 | }, 75 | "outputs": [ 76 | { 77 | "name": "stdout", 78 | "output_type": "stream", 79 | "text": [ 80 | "Loading and preparing results... \n", 81 | "DONE (t=0.05s)\n", 82 | "creating index...\n", 83 | "index created!\n" 84 | ] 85 | } 86 | ], 87 | "source": [ 88 | "#initialize COCO detections api\n", 89 | "resFile='%s/results/%s_%s_fake%s100_results.json'\n", 90 | "resFile = resFile%(dataDir, prefix, dataType, annType)\n", 91 | "cocoDt=cocoGt.loadRes(resFile)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": 5, 97 | "metadata": { 98 | "collapsed": false 99 | }, 100 | "outputs": [], 101 | "source": [ 102 | "imgIds=sorted(cocoGt.getImgIds())\n", 103 | "imgIds=imgIds[0:100]\n", 104 | "imgId = imgIds[np.random.randint(100)]" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 6, 110 | "metadata": { 111 | "collapsed": false 112 | }, 113 | "outputs": [ 114 | { 115 | "name": "stdout", 116 | "output_type": "stream", 117 | "text": [ 118 | "Running per image evaluation... \n", 119 | "DONE (t=0.46s).\n", 120 | "Accumulating evaluation results... \n", 121 | "DONE (t=0.38s).\n", 122 | " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.505\n", 123 | " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.697\n", 124 | " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.573\n", 125 | " Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.586\n", 126 | " Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.519\n", 127 | " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.501\n", 128 | " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.387\n", 129 | " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.594\n", 130 | " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.595\n", 131 | " Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.640\n", 132 | " Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.566\n", 133 | " Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.564\n" 134 | ] 135 | } 136 | ], 137 | "source": [ 138 | "# running evaluation\n", 139 | "cocoEval = COCOeval(cocoGt,cocoDt,annType)\n", 140 | "cocoEval.params.imgIds = imgIds\n", 141 | "cocoEval.evaluate()\n", 142 | "cocoEval.accumulate()\n", 143 | "cocoEval.summarize()" 144 | ] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "Python 2", 150 | "language": "python", 151 | "name": "python2" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 2 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython2", 163 | "version": "2.7.10" 164 | } 165 | }, 166 | "nbformat": 4, 167 | "nbformat_minor": 0 168 | } 169 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/PythonAPI/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, Extension 2 | import numpy as np 3 | 4 | # To compile and install locally run "python setup.py build_ext --inplace" 5 | # To install library to Python site-packages run "python setup.py build_ext install" 6 | 7 | ext_modules = [ 8 | Extension( 9 | 'halpecocotools._mask', 10 | sources=['common/maskApi.c', 'halpecocotools/_mask.pyx'], 11 | include_dirs = [np.get_include(), 'common'], 12 | extra_compile_args=[], 13 | ) 14 | ] 15 | 16 | setup( 17 | name='halpecocotools', 18 | packages = ['halpecocotools'], 19 | package_dir = {'halpecocotools': 'halpecocotools'}, 20 | install_requires=[ 21 | 'setuptools>=18.0', 22 | 'cython>=0.27.3', 23 | 'matplotlib>=2.1.0', 24 | ], 25 | version='0.0.0', 26 | description="COCO API for Halpe-Fullbody dataset", 27 | url="https://github.com/HaoyiZhu/HalpeCOCOAPI", 28 | ext_modules= ext_modules 29 | ) 30 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/README.md: -------------------------------------------------------------------------------- 1 | ## Halpe-COCO API: COCO API for [Halpe-FullBody dataset](https://github.com/Fang-Haoshu/Halpe-FullBody) 2 | - We get this API by modifying the [COCO API](https://github.com/cocodataset/cocoapi) to adapt to the Halpe-FullBody dataset. 3 | 4 | ## Details: 5 | - Modify the function showAnns in [PythonAPI/halpecocotools/coco.py](PythonAPI/halpecocotools/coco.py#L236) so that it can show 136 full body keypoints. 6 | - Modify the evaluation code in [PythonAPI/halpecocotools/cocoeval.py](PythonAPI/halpecocotools/cocoeval.py) to adapt to the 136-keypoints case. 7 | 8 | ## To install: 9 | 10 | - Run `pip install halpecocotools`. 11 | 12 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/common/gason.cpp: -------------------------------------------------------------------------------- 1 | // https://github.com/vivkin/gason - pulled January 10, 2016 2 | #include "gason.h" 3 | #include 4 | 5 | #define JSON_ZONE_SIZE 4096 6 | #define JSON_STACK_SIZE 32 7 | 8 | const char *jsonStrError(int err) { 9 | switch (err) { 10 | #define XX(no, str) \ 11 | case JSON_##no: \ 12 | return str; 13 | JSON_ERRNO_MAP(XX) 14 | #undef XX 15 | default: 16 | return "unknown"; 17 | } 18 | } 19 | 20 | void *JsonAllocator::allocate(size_t size) { 21 | size = (size + 7) & ~7; 22 | 23 | if (head && head->used + size <= JSON_ZONE_SIZE) { 24 | char *p = (char *)head + head->used; 25 | head->used += size; 26 | return p; 27 | } 28 | 29 | size_t allocSize = sizeof(Zone) + size; 30 | Zone *zone = (Zone *)malloc(allocSize <= JSON_ZONE_SIZE ? JSON_ZONE_SIZE : allocSize); 31 | if (zone == nullptr) 32 | return nullptr; 33 | zone->used = allocSize; 34 | if (allocSize <= JSON_ZONE_SIZE || head == nullptr) { 35 | zone->next = head; 36 | head = zone; 37 | } else { 38 | zone->next = head->next; 39 | head->next = zone; 40 | } 41 | return (char *)zone + sizeof(Zone); 42 | } 43 | 44 | void JsonAllocator::deallocate() { 45 | while (head) { 46 | Zone *next = head->next; 47 | free(head); 48 | head = next; 49 | } 50 | } 51 | 52 | static inline bool isspace(char c) { 53 | return c == ' ' || (c >= '\t' && c <= '\r'); 54 | } 55 | 56 | static inline bool isdelim(char c) { 57 | return c == ',' || c == ':' || c == ']' || c == '}' || isspace(c) || !c; 58 | } 59 | 60 | static inline bool isdigit(char c) { 61 | return c >= '0' && c <= '9'; 62 | } 63 | 64 | static inline bool isxdigit(char c) { 65 | return (c >= '0' && c <= '9') || ((c & ~' ') >= 'A' && (c & ~' ') <= 'F'); 66 | } 67 | 68 | static inline int char2int(char c) { 69 | if (c <= '9') 70 | return c - '0'; 71 | return (c & ~' ') - 'A' + 10; 72 | } 73 | 74 | static double string2double(char *s, char **endptr) { 75 | char ch = *s; 76 | if (ch == '-') 77 | ++s; 78 | 79 | double result = 0; 80 | while (isdigit(*s)) 81 | result = (result * 10) + (*s++ - '0'); 82 | 83 | if (*s == '.') { 84 | ++s; 85 | 86 | double fraction = 1; 87 | while (isdigit(*s)) { 88 | fraction *= 0.1; 89 | result += (*s++ - '0') * fraction; 90 | } 91 | } 92 | 93 | if (*s == 'e' || *s == 'E') { 94 | ++s; 95 | 96 | double base = 10; 97 | if (*s == '+') 98 | ++s; 99 | else if (*s == '-') { 100 | ++s; 101 | base = 0.1; 102 | } 103 | 104 | unsigned int exponent = 0; 105 | while (isdigit(*s)) 106 | exponent = (exponent * 10) + (*s++ - '0'); 107 | 108 | double power = 1; 109 | for (; exponent; exponent >>= 1, base *= base) 110 | if (exponent & 1) 111 | power *= base; 112 | 113 | result *= power; 114 | } 115 | 116 | *endptr = s; 117 | return ch == '-' ? -result : result; 118 | } 119 | 120 | static inline JsonNode *insertAfter(JsonNode *tail, JsonNode *node) { 121 | if (!tail) 122 | return node->next = node; 123 | node->next = tail->next; 124 | tail->next = node; 125 | return node; 126 | } 127 | 128 | static inline JsonValue listToValue(JsonTag tag, JsonNode *tail) { 129 | if (tail) { 130 | auto head = tail->next; 131 | tail->next = nullptr; 132 | return JsonValue(tag, head); 133 | } 134 | return JsonValue(tag, nullptr); 135 | } 136 | 137 | int jsonParse(char *s, char **endptr, JsonValue *value, JsonAllocator &allocator) { 138 | JsonNode *tails[JSON_STACK_SIZE]; 139 | JsonTag tags[JSON_STACK_SIZE]; 140 | char *keys[JSON_STACK_SIZE]; 141 | JsonValue o; 142 | int pos = -1; 143 | bool separator = true; 144 | JsonNode *node; 145 | *endptr = s; 146 | 147 | while (*s) { 148 | while (isspace(*s)) { 149 | ++s; 150 | if (!*s) break; 151 | } 152 | *endptr = s++; 153 | switch (**endptr) { 154 | case '-': 155 | if (!isdigit(*s) && *s != '.') { 156 | *endptr = s; 157 | return JSON_BAD_NUMBER; 158 | } 159 | case '0': 160 | case '1': 161 | case '2': 162 | case '3': 163 | case '4': 164 | case '5': 165 | case '6': 166 | case '7': 167 | case '8': 168 | case '9': 169 | o = JsonValue(string2double(*endptr, &s)); 170 | if (!isdelim(*s)) { 171 | *endptr = s; 172 | return JSON_BAD_NUMBER; 173 | } 174 | break; 175 | case '"': 176 | o = JsonValue(JSON_STRING, s); 177 | for (char *it = s; *s; ++it, ++s) { 178 | int c = *it = *s; 179 | if (c == '\\') { 180 | c = *++s; 181 | switch (c) { 182 | case '\\': 183 | case '"': 184 | case '/': 185 | *it = c; 186 | break; 187 | case 'b': 188 | *it = '\b'; 189 | break; 190 | case 'f': 191 | *it = '\f'; 192 | break; 193 | case 'n': 194 | *it = '\n'; 195 | break; 196 | case 'r': 197 | *it = '\r'; 198 | break; 199 | case 't': 200 | *it = '\t'; 201 | break; 202 | case 'u': 203 | c = 0; 204 | for (int i = 0; i < 4; ++i) { 205 | if (isxdigit(*++s)) { 206 | c = c * 16 + char2int(*s); 207 | } else { 208 | *endptr = s; 209 | return JSON_BAD_STRING; 210 | } 211 | } 212 | if (c < 0x80) { 213 | *it = c; 214 | } else if (c < 0x800) { 215 | *it++ = 0xC0 | (c >> 6); 216 | *it = 0x80 | (c & 0x3F); 217 | } else { 218 | *it++ = 0xE0 | (c >> 12); 219 | *it++ = 0x80 | ((c >> 6) & 0x3F); 220 | *it = 0x80 | (c & 0x3F); 221 | } 222 | break; 223 | default: 224 | *endptr = s; 225 | return JSON_BAD_STRING; 226 | } 227 | } else if ((unsigned int)c < ' ' || c == '\x7F') { 228 | *endptr = s; 229 | return JSON_BAD_STRING; 230 | } else if (c == '"') { 231 | *it = 0; 232 | ++s; 233 | break; 234 | } 235 | } 236 | if (!isdelim(*s)) { 237 | *endptr = s; 238 | return JSON_BAD_STRING; 239 | } 240 | break; 241 | case 't': 242 | if (!(s[0] == 'r' && s[1] == 'u' && s[2] == 'e' && isdelim(s[3]))) 243 | return JSON_BAD_IDENTIFIER; 244 | o = JsonValue(JSON_TRUE); 245 | s += 3; 246 | break; 247 | case 'f': 248 | if (!(s[0] == 'a' && s[1] == 'l' && s[2] == 's' && s[3] == 'e' && isdelim(s[4]))) 249 | return JSON_BAD_IDENTIFIER; 250 | o = JsonValue(JSON_FALSE); 251 | s += 4; 252 | break; 253 | case 'n': 254 | if (!(s[0] == 'u' && s[1] == 'l' && s[2] == 'l' && isdelim(s[3]))) 255 | return JSON_BAD_IDENTIFIER; 256 | o = JsonValue(JSON_NULL); 257 | s += 3; 258 | break; 259 | case ']': 260 | if (pos == -1) 261 | return JSON_STACK_UNDERFLOW; 262 | if (tags[pos] != JSON_ARRAY) 263 | return JSON_MISMATCH_BRACKET; 264 | o = listToValue(JSON_ARRAY, tails[pos--]); 265 | break; 266 | case '}': 267 | if (pos == -1) 268 | return JSON_STACK_UNDERFLOW; 269 | if (tags[pos] != JSON_OBJECT) 270 | return JSON_MISMATCH_BRACKET; 271 | if (keys[pos] != nullptr) 272 | return JSON_UNEXPECTED_CHARACTER; 273 | o = listToValue(JSON_OBJECT, tails[pos--]); 274 | break; 275 | case '[': 276 | if (++pos == JSON_STACK_SIZE) 277 | return JSON_STACK_OVERFLOW; 278 | tails[pos] = nullptr; 279 | tags[pos] = JSON_ARRAY; 280 | keys[pos] = nullptr; 281 | separator = true; 282 | continue; 283 | case '{': 284 | if (++pos == JSON_STACK_SIZE) 285 | return JSON_STACK_OVERFLOW; 286 | tails[pos] = nullptr; 287 | tags[pos] = JSON_OBJECT; 288 | keys[pos] = nullptr; 289 | separator = true; 290 | continue; 291 | case ':': 292 | if (separator || keys[pos] == nullptr) 293 | return JSON_UNEXPECTED_CHARACTER; 294 | separator = true; 295 | continue; 296 | case ',': 297 | if (separator || keys[pos] != nullptr) 298 | return JSON_UNEXPECTED_CHARACTER; 299 | separator = true; 300 | continue; 301 | case '\0': 302 | continue; 303 | default: 304 | return JSON_UNEXPECTED_CHARACTER; 305 | } 306 | 307 | separator = false; 308 | 309 | if (pos == -1) { 310 | *endptr = s; 311 | *value = o; 312 | return JSON_OK; 313 | } 314 | 315 | if (tags[pos] == JSON_OBJECT) { 316 | if (!keys[pos]) { 317 | if (o.getTag() != JSON_STRING) 318 | return JSON_UNQUOTED_KEY; 319 | keys[pos] = o.toString(); 320 | continue; 321 | } 322 | if ((node = (JsonNode *) allocator.allocate(sizeof(JsonNode))) == nullptr) 323 | return JSON_ALLOCATION_FAILURE; 324 | tails[pos] = insertAfter(tails[pos], node); 325 | tails[pos]->key = keys[pos]; 326 | keys[pos] = nullptr; 327 | } else { 328 | if ((node = (JsonNode *) allocator.allocate(sizeof(JsonNode) - sizeof(char *))) == nullptr) 329 | return JSON_ALLOCATION_FAILURE; 330 | tails[pos] = insertAfter(tails[pos], node); 331 | } 332 | tails[pos]->value = o; 333 | } 334 | return JSON_BREAKING_BAD; 335 | } 336 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/common/gason.h: -------------------------------------------------------------------------------- 1 | // https://github.com/vivkin/gason - pulled January 10, 2016 2 | #pragma once 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | enum JsonTag { 9 | JSON_NUMBER = 0, 10 | JSON_STRING, 11 | JSON_ARRAY, 12 | JSON_OBJECT, 13 | JSON_TRUE, 14 | JSON_FALSE, 15 | JSON_NULL = 0xF 16 | }; 17 | 18 | struct JsonNode; 19 | 20 | #define JSON_VALUE_PAYLOAD_MASK 0x00007FFFFFFFFFFFULL 21 | #define JSON_VALUE_NAN_MASK 0x7FF8000000000000ULL 22 | #define JSON_VALUE_TAG_MASK 0xF 23 | #define JSON_VALUE_TAG_SHIFT 47 24 | 25 | union JsonValue { 26 | uint64_t ival; 27 | double fval; 28 | 29 | JsonValue(double x) 30 | : fval(x) { 31 | } 32 | JsonValue(JsonTag tag = JSON_NULL, void *payload = nullptr) { 33 | assert((uintptr_t)payload <= JSON_VALUE_PAYLOAD_MASK); 34 | ival = JSON_VALUE_NAN_MASK | ((uint64_t)tag << JSON_VALUE_TAG_SHIFT) | (uintptr_t)payload; 35 | } 36 | bool isDouble() const { 37 | return (int64_t)ival <= (int64_t)JSON_VALUE_NAN_MASK; 38 | } 39 | JsonTag getTag() const { 40 | return isDouble() ? JSON_NUMBER : JsonTag((ival >> JSON_VALUE_TAG_SHIFT) & JSON_VALUE_TAG_MASK); 41 | } 42 | uint64_t getPayload() const { 43 | assert(!isDouble()); 44 | return ival & JSON_VALUE_PAYLOAD_MASK; 45 | } 46 | double toNumber() const { 47 | assert(getTag() == JSON_NUMBER); 48 | return fval; 49 | } 50 | char *toString() const { 51 | assert(getTag() == JSON_STRING); 52 | return (char *)getPayload(); 53 | } 54 | JsonNode *toNode() const { 55 | assert(getTag() == JSON_ARRAY || getTag() == JSON_OBJECT); 56 | return (JsonNode *)getPayload(); 57 | } 58 | }; 59 | 60 | struct JsonNode { 61 | JsonValue value; 62 | JsonNode *next; 63 | char *key; 64 | }; 65 | 66 | struct JsonIterator { 67 | JsonNode *p; 68 | 69 | void operator++() { 70 | p = p->next; 71 | } 72 | bool operator!=(const JsonIterator &x) const { 73 | return p != x.p; 74 | } 75 | JsonNode *operator*() const { 76 | return p; 77 | } 78 | JsonNode *operator->() const { 79 | return p; 80 | } 81 | }; 82 | 83 | inline JsonIterator begin(JsonValue o) { 84 | return JsonIterator{o.toNode()}; 85 | } 86 | inline JsonIterator end(JsonValue) { 87 | return JsonIterator{nullptr}; 88 | } 89 | 90 | #define JSON_ERRNO_MAP(XX) \ 91 | XX(OK, "ok") \ 92 | XX(BAD_NUMBER, "bad number") \ 93 | XX(BAD_STRING, "bad string") \ 94 | XX(BAD_IDENTIFIER, "bad identifier") \ 95 | XX(STACK_OVERFLOW, "stack overflow") \ 96 | XX(STACK_UNDERFLOW, "stack underflow") \ 97 | XX(MISMATCH_BRACKET, "mismatch bracket") \ 98 | XX(UNEXPECTED_CHARACTER, "unexpected character") \ 99 | XX(UNQUOTED_KEY, "unquoted key") \ 100 | XX(BREAKING_BAD, "breaking bad") \ 101 | XX(ALLOCATION_FAILURE, "allocation failure") 102 | 103 | enum JsonErrno { 104 | #define XX(no, str) JSON_##no, 105 | JSON_ERRNO_MAP(XX) 106 | #undef XX 107 | }; 108 | 109 | const char *jsonStrError(int err); 110 | 111 | class JsonAllocator { 112 | struct Zone { 113 | Zone *next; 114 | size_t used; 115 | } *head = nullptr; 116 | 117 | public: 118 | JsonAllocator() = default; 119 | JsonAllocator(const JsonAllocator &) = delete; 120 | JsonAllocator &operator=(const JsonAllocator &) = delete; 121 | JsonAllocator(JsonAllocator &&x) : head(x.head) { 122 | x.head = nullptr; 123 | } 124 | JsonAllocator &operator=(JsonAllocator &&x) { 125 | head = x.head; 126 | x.head = nullptr; 127 | return *this; 128 | } 129 | ~JsonAllocator() { 130 | deallocate(); 131 | } 132 | void *allocate(size_t size); 133 | void deallocate(); 134 | }; 135 | 136 | int jsonParse(char *str, char **endptr, JsonValue *value, JsonAllocator &allocator); 137 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/common/maskApi.c: -------------------------------------------------------------------------------- 1 | /************************************************************************** 2 | * Microsoft COCO Toolbox. version 2.0 3 | * Data, paper, and tutorials available at: http://mscoco.org/ 4 | * Code written by Piotr Dollar and Tsung-Yi Lin, 2015. 5 | * Licensed under the Simplified BSD License [see coco/license.txt] 6 | **************************************************************************/ 7 | #include "maskApi.h" 8 | #include 9 | #include 10 | 11 | uint umin( uint a, uint b ) { return (ab) ? a : b; } 13 | 14 | void rleInit( RLE *R, siz h, siz w, siz m, uint *cnts ) { 15 | R->h=h; R->w=w; R->m=m; R->cnts=(m==0)?0:malloc(sizeof(uint)*m); 16 | siz j; if(cnts) for(j=0; jcnts[j]=cnts[j]; 17 | } 18 | 19 | void rleFree( RLE *R ) { 20 | free(R->cnts); R->cnts=0; 21 | } 22 | 23 | void rlesInit( RLE **R, siz n ) { 24 | siz i; *R = (RLE*) malloc(sizeof(RLE)*n); 25 | for(i=0; i0 ) { 61 | c=umin(ca,cb); cc+=c; ct=0; 62 | ca-=c; if(!ca && a0) { 83 | crowd=iscrowd!=NULL && iscrowd[g]; 84 | if(dt[d].h!=gt[g].h || dt[d].w!=gt[g].w) { o[g*m+d]=-1; continue; } 85 | siz ka, kb, a, b; uint c, ca, cb, ct, i, u; int va, vb; 86 | ca=dt[d].cnts[0]; ka=dt[d].m; va=vb=0; 87 | cb=gt[g].cnts[0]; kb=gt[g].m; a=b=1; i=u=0; ct=1; 88 | while( ct>0 ) { 89 | c=umin(ca,cb); if(va||vb) { u+=c; if(va&&vb) i+=c; } ct=0; 90 | ca-=c; if(!ca && athr) keep[j]=0; 105 | } 106 | } 107 | } 108 | 109 | void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ) { 110 | double h, w, i, u, ga, da; siz g, d; int crowd; 111 | for( g=0; gthr) keep[j]=0; 129 | } 130 | } 131 | } 132 | 133 | void rleToBbox( const RLE *R, BB bb, siz n ) { 134 | siz i; for( i=0; id?1:c=dy && xs>xe) || (dxye); 174 | if(flip) { t=xs; xs=xe; xe=t; t=ys; ys=ye; ye=t; } 175 | s = dx>=dy ? (double)(ye-ys)/dx : (double)(xe-xs)/dy; 176 | if(dx>=dy) for( d=0; d<=dx; d++ ) { 177 | t=flip?dx-d:d; u[m]=t+xs; v[m]=(int)(ys+s*t+.5); m++; 178 | } else for( d=0; d<=dy; d++ ) { 179 | t=flip?dy-d:d; v[m]=t+ys; u[m]=(int)(xs+s*t+.5); m++; 180 | } 181 | } 182 | /* get points along y-boundary and downsample */ 183 | free(x); free(y); k=m; m=0; double xd, yd; 184 | x=malloc(sizeof(int)*k); y=malloc(sizeof(int)*k); 185 | for( j=1; jw-1 ) continue; 188 | yd=(double)(v[j]h) yd=h; yd=ceil(yd); 190 | x[m]=(int) xd; y[m]=(int) yd; m++; 191 | } 192 | /* compute rle encoding given y-boundary points */ 193 | k=m; a=malloc(sizeof(uint)*(k+1)); 194 | for( j=0; j0) b[m++]=a[j++]; else { 200 | j++; if(jm, p=0; long x; int more; 207 | char *s=malloc(sizeof(char)*m*6); 208 | for( i=0; icnts[i]; if(i>2) x-=(long) R->cnts[i-2]; more=1; 210 | while( more ) { 211 | char c=x & 0x1f; x >>= 5; more=(c & 0x10) ? x!=-1 : x!=0; 212 | if(more) c |= 0x20; c+=48; s[p++]=c; 213 | } 214 | } 215 | s[p]=0; return s; 216 | } 217 | 218 | void rleFrString( RLE *R, char *s, siz h, siz w ) { 219 | siz m=0, p=0, k; long x; int more; uint *cnts; 220 | while( s[m] ) m++; cnts=malloc(sizeof(uint)*m); m=0; 221 | while( s[p] ) { 222 | x=0; k=0; more=1; 223 | while( more ) { 224 | char c=s[p]-48; x |= (c & 0x1f) << 5*k; 225 | more = c & 0x20; p++; k++; 226 | if(!more && (c & 0x10)) x |= -1 << 5*k; 227 | } 228 | if(m>2) x+=(long) cnts[m-2]; cnts[m++]=(uint) x; 229 | } 230 | rleInit(R,h,w,m,cnts); free(cnts); 231 | } 232 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/common/maskApi.h: -------------------------------------------------------------------------------- 1 | /************************************************************************** 2 | * Microsoft COCO Toolbox. version 2.0 3 | * Data, paper, and tutorials available at: http://mscoco.org/ 4 | * Code written by Piotr Dollar and Tsung-Yi Lin, 2015. 5 | * Licensed under the Simplified BSD License [see coco/license.txt] 6 | **************************************************************************/ 7 | #pragma once 8 | 9 | typedef unsigned int uint; 10 | typedef unsigned long siz; 11 | typedef unsigned char byte; 12 | typedef double* BB; 13 | typedef struct { siz h, w, m; uint *cnts; } RLE; 14 | 15 | /* Initialize/destroy RLE. */ 16 | void rleInit( RLE *R, siz h, siz w, siz m, uint *cnts ); 17 | void rleFree( RLE *R ); 18 | 19 | /* Initialize/destroy RLE array. */ 20 | void rlesInit( RLE **R, siz n ); 21 | void rlesFree( RLE **R, siz n ); 22 | 23 | /* Encode binary masks using RLE. */ 24 | void rleEncode( RLE *R, const byte *mask, siz h, siz w, siz n ); 25 | 26 | /* Decode binary masks encoded via RLE. */ 27 | void rleDecode( const RLE *R, byte *mask, siz n ); 28 | 29 | /* Compute union or intersection of encoded masks. */ 30 | void rleMerge( const RLE *R, RLE *M, siz n, int intersect ); 31 | 32 | /* Compute area of encoded masks. */ 33 | void rleArea( const RLE *R, siz n, uint *a ); 34 | 35 | /* Compute intersection over union between masks. */ 36 | void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o ); 37 | 38 | /* Compute non-maximum suppression between bounding masks */ 39 | void rleNms( RLE *dt, siz n, uint *keep, double thr ); 40 | 41 | /* Compute intersection over union between bounding boxes. */ 42 | void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ); 43 | 44 | /* Compute non-maximum suppression between bounding boxes */ 45 | void bbNms( BB dt, siz n, uint *keep, double thr ); 46 | 47 | /* Get bounding boxes surrounding encoded masks. */ 48 | void rleToBbox( const RLE *R, BB bb, siz n ); 49 | 50 | /* Convert bounding boxes to encoded masks. */ 51 | void rleFrBbox( RLE *R, const BB bb, siz h, siz w, siz n ); 52 | 53 | /* Convert polygon to encoded mask. */ 54 | void rleFrPoly( RLE *R, const double *xy, siz k, siz h, siz w ); 55 | 56 | /* Get compressed string representation of encoded mask. */ 57 | char* rleToString( const RLE *R ); 58 | 59 | /* Convert from compressed string representation of encoded mask. */ 60 | void rleFrString( RLE *R, char *s, siz h, siz w ); 61 | -------------------------------------------------------------------------------- /HalpeCOCOAPI/license.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the FreeBSD Project. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Halpe Full-Body Human Keypoints and HOI-Det dataset 2 | 3 | ## What is Halpe? 4 | **Halpe** is a dataset introduced in [**AlphaPose paper**](http://arxiv.org/abs/2211.03375). It aims at pushing Human Understanding to the extreme. We provide detailed annotation of human keypoints, together with the human-object interaction trplets from HICO-DET. For each person, we annotate 136 keypoints in total, including head,face,body,hand and foot. Below we provide some samples of Halpe dataset. 5 |
6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | ## Download 14 | Train annotations [[Baidu](https://pan.baidu.com/s/1hWX-I-HpXXnLcprFskfriQ) | [Google](https://drive.google.com/file/d/13vj8H0GZ9yugLPhVVWV9fcH-3RyW5Wk5/view?usp=sharing) ] 15 | 16 | Val annotations [[Baidu](https://pan.baidu.com/s/1yDvBkXTSwk20EjiYzimpPw) | [Google](https://drive.google.com/file/d/1FdyCgro2t9_nOhTlMPjEf3c0aLOz9wi6/view?usp=sharing) ] 17 | 18 | Train images from [HICO-DET](https://drive.google.com/open?id=1QZcJmGVlF9f4h-XLWe9Gkmnmj2z1gSnk) 19 | 20 | Val images from [COCO](http://images.cocodataset.org/zips/val2017.zip) 21 | 22 | ## Realtime Demo with tracking 23 | Trained model is available in [AlphaPose](https://github.com/MVIG-SJTU/AlphaPose)! 24 | Check out its [MODEL_ZOO](https://github.com/MVIG-SJTU/AlphaPose/blob/master/docs/MODEL_ZOO.md) 25 |
26 | 27 |
28 | 29 | 30 | ## Keypoints format 31 | We annotate 136 keypoints in total: 32 | ``` 33 | //26 body keypoints 34 | {0, "Nose"}, 35 | {1, "LEye"}, 36 | {2, "REye"}, 37 | {3, "LEar"}, 38 | {4, "REar"}, 39 | {5, "LShoulder"}, 40 | {6, "RShoulder"}, 41 | {7, "LElbow"}, 42 | {8, "RElbow"}, 43 | {9, "LWrist"}, 44 | {10, "RWrist"}, 45 | {11, "LHip"}, 46 | {12, "RHip"}, 47 | {13, "LKnee"}, 48 | {14, "Rknee"}, 49 | {15, "LAnkle"}, 50 | {16, "RAnkle"}, 51 | {17, "Head"}, 52 | {18, "Neck"}, 53 | {19, "Hip"}, 54 | {20, "LBigToe"}, 55 | {21, "RBigToe"}, 56 | {22, "LSmallToe"}, 57 | {23, "RSmallToe"}, 58 | {24, "LHeel"}, 59 | {25, "RHeel"}, 60 | //face 61 | {26-93, 68 Face Keypoints} 62 | //left hand 63 | {94-114, 21 Left Hand Keypoints} 64 | //right hand 65 | {115-135, 21 Right Hand Keypoints} 66 | ``` 67 | Illustration: 68 |
69 |
70 | 26 body keypoints 71 |
72 |
73 |
74 | 68 face keypoints 75 |
76 |
77 |
78 | 21 hand keypoints 79 |
80 | 81 | ## Usage 82 | The annotation is in the same format as COCO dataset. For usage, a good start is to check out the `vis.py`. We also provide related APIs. See [halpecocotools](https://github.com/HaoyiZhu/HalpeCOCOAPI), which can be installed by `pip install halpecocotools`. 83 | 84 | ## Evaluation 85 | We adopt the same evaluation metrics as COCO dataset. 86 | 87 | ## Related resources 88 | A concurrent work [COCO-WholeBody](https://github.com/jin-s13/COCO-WholeBody) also annotate the full body keypoints. And HOI-DET for COCO dataset is also available at [V-COCO](https://github.com/s-gupta/v-coco) 89 | 90 | ## Citation 91 | If the data helps your research, please cite the following paper: 92 | ``` 93 | @article{alphapose, 94 | author = {Fang, Hao-Shu and Li, Jiefeng and Tang, Hongyang and Xu, Chao and Zhu, Haoyi and Xiu, Yuliang and Li, Yong-Lu and Lu, Cewu}, 95 | journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, 96 | title = {AlphaPose: Whole-Body Regional Multi-Person Pose Estimation and Tracking in Real-Time}, 97 | year = {2022} 98 | } 99 | ``` 100 | -------------------------------------------------------------------------------- /docs/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/1.jpg -------------------------------------------------------------------------------- /docs/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/2.jpg -------------------------------------------------------------------------------- /docs/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/3.jpg -------------------------------------------------------------------------------- /docs/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/4.jpg -------------------------------------------------------------------------------- /docs/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/5.jpg -------------------------------------------------------------------------------- /docs/alphapose_136_2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/alphapose_136_2.gif -------------------------------------------------------------------------------- /docs/face.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/face.jpg -------------------------------------------------------------------------------- /docs/hand.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/hand.jpg -------------------------------------------------------------------------------- /docs/human_model.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fang-Haoshu/Halpe-FullBody/70d6014c007c765adc65f5f0145d3f0999ab1135/docs/human_model.jpg -------------------------------------------------------------------------------- /vis.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import cv2 4 | import skimage.io as io 5 | import numpy as np 6 | from tqdm import tqdm 7 | 8 | 9 | l_pair = [ 10 | (0, 1), (0, 2), (1, 3), (2, 4), # Head 11 | (5, 18), (6, 18), (5, 7), (7, 9), (6, 8), (8, 10),# Body 12 | (17, 18), (18, 19), (19, 11), (19, 12), 13 | (11, 13), (12, 14), (13, 15), (14, 16), 14 | (20, 24), (21, 25), (23, 25), (22, 24), (15, 24), (16, 25),# Foot 15 | (26, 27),(27, 28),(28, 29),(29, 30),(30, 31),(31, 32),(32, 33),(33, 34),(34, 35),(35, 36),(36, 37),(37, 38),#Face 16 | (38, 39),(39, 40),(40, 41),(41, 42),(43, 44),(44, 45),(45, 46),(46, 47),(48, 49),(49, 50),(50, 51),(51, 52),#Face 17 | (53, 54),(54, 55),(55, 56),(57, 58),(58, 59),(59, 60),(60, 61),(62, 63),(63, 64),(64, 65),(65, 66),(66, 67),#Face 18 | (68, 69),(69, 70),(70, 71),(71, 72),(72, 73),(74, 75),(75, 76),(76, 77),(77, 78),(78, 79),(79, 80),(80, 81),#Face 19 | (81, 82),(82, 83),(83, 84),(84, 85),(85, 86),(86, 87),(87, 88),(88, 89),(89, 90),(90, 91),(91, 92),(92, 93),#Face 20 | (94,95),(95,96),(96,97),(97,98),(94,99),(99,100),(100,101),(101,102),(94,103),(103,104),(104,105),#LeftHand 21 | (105,106),(94,107),(107,108),(108,109),(109,110),(94,111),(111,112),(112,113),(113,114),#LeftHand 22 | (115,116),(116,117),(117,118),(118,119),(115,120),(120,121),(121,122),(122,123),(115,124),(124,125),#RightHand 23 | (125,126),(126,127),(115,128),(128,129),(129,130),(130,131),(115,132),(132,133),(133,134),(134,135)#RightHand 24 | ] 25 | p_color = [(0, 255, 255), (0, 191, 255), (0, 255, 102), (0, 77, 255), (0, 255, 0), # Nose, LEye, REye, LEar, REar 26 | (77, 255, 255), (77, 255, 204), (77, 204, 255), (191, 255, 77), (77, 191, 255), (191, 255, 77), # LShoulder, RShoulder, LElbow, RElbow, LWrist, RWrist 27 | (204, 77, 255), (77, 255, 204), (191, 77, 255), (77, 255, 191), (127, 77, 255), (77, 255, 127), # LHip, RHip, LKnee, Rknee, LAnkle, RAnkle, Neck 28 | (77, 255, 255), (0, 255, 255), (77, 204, 255), # head, neck, shoulder 29 | (0, 255, 255), (0, 191, 255), (0, 255, 102), (0, 77, 255), (0, 255, 0), (77, 255, 255)] # foot 30 | 31 | line_color = [(0, 215, 255), (0, 255, 204), (0, 134, 255), (0, 255, 50), 32 | (0, 255, 102), (77, 255, 222), (77, 196, 255), (77, 135, 255), (191, 255, 77), (77, 255, 77), 33 | (77, 191, 255), (204, 77, 255), (77, 222, 255), (255, 156, 127), 34 | (0, 127, 255), (255, 127, 77), (0, 77, 255), (255, 77, 36), 35 | (0, 77, 255), (0, 77, 255), (0, 77, 255), (0, 77, 255), (255, 156, 127), (255, 156, 127)] 36 | 37 | 38 | bodyanno = json.load(open('halpe_train_v1.json')) 39 | image_folder = 'path/to/train images' 40 | save_folder = 'save_fig/' 41 | 42 | if not os.path.exists(save_folder): 43 | os.mkdir(save_folder) 44 | 45 | imgs = {} 46 | for img in bodyanno['images']: 47 | imgs[img['id']] = img 48 | 49 | 50 | for hidx, annot in enumerate(tqdm(bodyanno['annotations'])): 51 | if 'keypoints' in annot and type(annot['keypoints']) == list: 52 | imgname = str(imgs[annot['image_id']]['file_name']) 53 | if os.path.exists(os.path.join(save_folder, imgname)): 54 | img = cv2.imread(os.path.join(save_folder, imgname)) 55 | else: 56 | img = cv2.imread(os.path.join(image_folder, imgname)) 57 | part_line = {} 58 | kp = np.array(annot['keypoints']) 59 | kp_x = kp[0::3] 60 | kp_y = kp[1::3] 61 | kp_scores = kp[2::3] 62 | 63 | # Draw keypoints 64 | for n in range(kp_scores.shape[0]): 65 | if kp_scores[n] <= 0.6: 66 | continue 67 | cor_x, cor_y = int(kp_x[n]), int(kp_y[n]) 68 | part_line[n] = (int(cor_x), int(cor_y)) 69 | if n < len(p_color): 70 | cv2.circle(img, (int(cor_x), int(cor_y)), 2, p_color[n], -1) 71 | else: 72 | cv2.circle(img, (int(cor_x), int(cor_y)), 1, (255,255,255), 2) 73 | # Draw limbs 74 | for i, (start_p, end_p) in enumerate(l_pair): 75 | if start_p in part_line and end_p in part_line: 76 | start_xy = part_line[start_p] 77 | end_xy = part_line[end_p] 78 | if i < len(line_color): 79 | cv2.line(img, start_xy, end_xy, line_color[i], 2) 80 | else: 81 | cv2.line(img, start_xy, end_xy, (255,255,255), 1) 82 | 83 | cv2.imwrite(os.path.join(save_folder,imgname),img) 84 | 85 | --------------------------------------------------------------------------------