├── motmetrics ├── apps │ ├── __init__.py │ ├── list_metrics.py │ ├── example.py │ ├── eval_detrac.py │ ├── eval_motchallenge.py │ └── evaluateTracking.py ├── tests │ ├── __init__.py │ ├── test_issue19.py │ ├── test_utils.py │ ├── test_distances.py │ ├── test_io.py │ ├── test_lap.py │ ├── test_mot.py │ └── test_metrics.py ├── etc │ └── mot.png ├── data │ ├── iotest │ │ ├── detrac.mat │ │ ├── motchallenge.txt │ │ ├── vatic.txt │ │ └── detrac.xml │ └── TUD-Campus │ │ ├── test.txt │ │ └── gt.txt ├── math_util.py ├── __init__.py ├── preprocess.py ├── distances.py ├── utils.py ├── lap.py ├── io.py └── mot.py ├── seqmaps └── example.txt ├── imgs ├── 1.png ├── 21.png ├── 22.png ├── 23.png ├── 24.png ├── 25.png └── 3.png ├── requirements_dev.txt ├── test.sh.example ├── requirements.txt ├── MANIFEST.in ├── precommit.sh ├── environment.yml ├── setup.cfg ├── .gitattributes ├── Release.md ├── .gitignore ├── Dockerfile ├── LICENSE ├── setup.py ├── .github └── workflows │ └── python-package.yml ├── Readme.md ├── yourdata └── video2.txt └── pylintrc /motmetrics/apps/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /motmetrics/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /seqmaps/example.txt: -------------------------------------------------------------------------------- 1 | name 2 | MOT16-02 3 | MOT16-04 4 | -------------------------------------------------------------------------------- /imgs/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/1.png -------------------------------------------------------------------------------- /imgs/21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/21.png -------------------------------------------------------------------------------- /imgs/22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/22.png -------------------------------------------------------------------------------- /imgs/23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/23.png -------------------------------------------------------------------------------- /imgs/24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/24.png -------------------------------------------------------------------------------- /imgs/25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/25.png -------------------------------------------------------------------------------- /imgs/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/imgs/3.png -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | flake8-import-order 3 | pytest 4 | pytest-benchmark 5 | -------------------------------------------------------------------------------- /motmetrics/etc/mot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/motmetrics/etc/mot.png -------------------------------------------------------------------------------- /test.sh.example: -------------------------------------------------------------------------------- 1 | python3 -u -m motmetrics.apps.evaluateTracking ./gt_dir/ ./res_dir/ ./seqmaps/example.txt 2 | -------------------------------------------------------------------------------- /motmetrics/data/iotest/detrac.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddz16/py-motmetrics/HEAD/motmetrics/data/iotest/detrac.mat -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.12.1 2 | pandas>=0.23.1 3 | scipy>=0.19.0 4 | xmltodict>=0.12.0 5 | enum34; python_version < '3' 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE requirements.txt 2 | recursive-include motmetrics/data * 3 | recursive-include motmetrics/etc * -------------------------------------------------------------------------------- /precommit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | 5 | pytest --benchmark-disable && \ 6 | flake8 . && \ 7 | pylint motmetrics && \ 8 | pylint --py3k motmetrics 9 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | # conda env create -f environment.yml 2 | name: motmetrics-env 3 | dependencies: 4 | - python=3.6 5 | - numpy 6 | - scipy 7 | - pandas 8 | - pip 9 | - pip: 10 | - pytest -------------------------------------------------------------------------------- /motmetrics/data/iotest/motchallenge.txt: -------------------------------------------------------------------------------- 1 | 1,1,399,182,121,229,1,-1,-1,-1 2 | 1,2,282,201,92,184,1,-1,-1,-1 3 | 2,2,269,202,87,182,1,-1,-1,-1 4 | 2,3,71,151,100,284,1,-1,-1,-1 5 | 2,4,200,206,55,137,1,-1,-1,-1 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | ignore = W503,W504,E501 3 | 4 | [flake8] 5 | ignore = W503,W504,E501 6 | # Options for flake8-import-order: 7 | import-order-style = google 8 | application-import-names = motmetrics 9 | -------------------------------------------------------------------------------- /motmetrics/data/iotest/vatic.txt: -------------------------------------------------------------------------------- 1 | 0 412 0 842 124 0 0 0 0 "worker" 2 | 0 412 10 842 124 1 0 0 1 "pc" "attr1" "attr3" 3 | 1 412 0 842 124 1 0 0 1 "pc" "attr2" 4 | 2 412 0 842 124 2 0 0 1 "worker" "attr4" "attr1" "attr2" 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /Release.md: -------------------------------------------------------------------------------- 1 | 2 | ## Release 3 | 4 | - git flow release start 5 | - version bump motmetrics/__init__.py 6 | - conda env create -f environment.yml 7 | - activate motmetrics-env 8 | - [pip install lapsolver] 9 | - pip install . 10 | - pytest 11 | - deactivate 12 | - conda env remove -n motmetrics-env 13 | - git add, commit 14 | - git flow release finish 15 | - git push 16 | - git push --tags 17 | - git checkout master 18 | - git push 19 | - git checkout develop 20 | - check appveyor, travis and pypi -------------------------------------------------------------------------------- /motmetrics/apps/list_metrics.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """List metrics.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | if __name__ == '__main__': 15 | import motmetrics 16 | 17 | mh = motmetrics.metrics.create() 18 | print(mh.list_metrics_markdown()) 19 | -------------------------------------------------------------------------------- /motmetrics/math_util.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Math utility functions.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import warnings 15 | 16 | import numpy as np 17 | 18 | 19 | def quiet_divide(a, b): 20 | """Quiet divide function that does not warn about (0 / 0).""" 21 | with warnings.catch_warnings(): 22 | warnings.simplefilter('ignore', RuntimeWarning) 23 | return np.true_divide(a, b) 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | *.pyc 49 | .cache/v/cache/lastfailed 50 | *.egg-info/ 51 | build/ 52 | dist/ 53 | .venv/ -------------------------------------------------------------------------------- /motmetrics/__init__.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 9 | 10 | Christoph Heindl, 2017 11 | https://github.com/cheind/py-motmetrics 12 | """ 13 | 14 | from __future__ import absolute_import 15 | from __future__ import division 16 | from __future__ import print_function 17 | 18 | __all__ = [ 19 | "distances", 20 | "io", 21 | "lap", 22 | "metrics", 23 | "utils", 24 | "MOTAccumulator", 25 | ] 26 | 27 | from motmetrics import distances 28 | from motmetrics import io 29 | from motmetrics import lap 30 | from motmetrics import metrics 31 | from motmetrics import utils 32 | from motmetrics.mot import MOTAccumulator 33 | 34 | # Needs to be last line 35 | __version__ = "1.4.0" 36 | -------------------------------------------------------------------------------- /motmetrics/tests/test_issue19.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests issue 19. 9 | 10 | https://github.com/cheind/py-motmetrics/issues/19 11 | """ 12 | 13 | from __future__ import absolute_import 14 | from __future__ import division 15 | from __future__ import print_function 16 | 17 | import numpy as np 18 | 19 | import motmetrics as mm 20 | 21 | 22 | def test_issue19(): 23 | """Tests issue 19.""" 24 | acc = mm.MOTAccumulator() 25 | 26 | g0 = [0, 1] 27 | p0 = [0, 1] 28 | d0 = [[0.2, np.nan], [np.nan, 0.2]] 29 | 30 | g1 = [2, 3] 31 | p1 = [2, 3, 4, 5] 32 | d1 = [[0.28571429, 0.5, 0.0, np.nan], [np.nan, 0.44444444, np.nan, 0.0]] 33 | 34 | acc.update(g0, p0, d0, 0) 35 | acc.update(g1, p1, d1, 1) 36 | 37 | mh = mm.metrics.create() 38 | mh.compute(acc) 39 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | MAINTAINER Avgerinos Christos 4 | 5 | #ARG GT_DIR 6 | #ARG TEST_DIR 7 | 8 | RUN apt-get update \ 9 | && apt-get install -y python3-pip python3-dev vim \ 10 | && cd /usr/local/bin \ 11 | && ln -s /usr/bin/python3 python \ 12 | && pip3 install --upgrade pip 13 | 14 | RUN pip3 install --no-cache-dir numpy scipy 15 | RUN pip install -Iv pandas==0.21.0 16 | RUN mkdir -p /motmetrics/py-motmetrics 17 | RUN mkdir -p /motmetrics/2DMOT2015 18 | 19 | COPY ./py-motmetrics /motmetrics/py-motmetrics 20 | COPY ./data /motmetrics/data 21 | 22 | #RUN pip install motmetrics 23 | RUN pip install -e ./motmetrics/py-motmetrics/ 24 | 25 | #RUN pip install -r motmetrics/py-motmetrics/requirements.txt 26 | 27 | ENV GT_DIR motmetrics/data/train/ 28 | ENV TEST_DIR motmetrics/data/test/ 29 | 30 | #ENTRYPOINT python3 -m motmetrics.apps.eval_motchallenge motmetrics/data/train/ motmetrics/data/test/ && /bin/bash 31 | CMD ["sh", "-c", "python3 -m motmetrics.apps.eval_motchallenge ${GT_DIR} ${TEST_DIR} && /bin/bash"] 32 | 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-2020 Christoph Heindl 4 | Copyright (c) 2018 Toka 5 | Copyright (c) 2019-2020 Jack Valmadre 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | 3 | Christoph Heindl, 2017 4 | https://github.com/cheind/py-motmetrics 5 | """ 6 | import io 7 | import os 8 | 9 | try: 10 | from setuptools import setup 11 | except ImportError: 12 | from distutils.core import setup 13 | 14 | with io.open("requirements.txt") as f: 15 | required = f.read().splitlines() 16 | 17 | with io.open("Readme.md", encoding="utf-8") as f: 18 | long_description = f.read() 19 | 20 | # Handle version number with optional .dev postfix when building a develop branch 21 | # on AppVeyor. 22 | VERSION = io.open("motmetrics/__init__.py").readlines()[-1].split()[-1].strip('"') 23 | BUILD_NUMBER = os.environ.get("APPVEYOR_BUILD_NUMBER", None) 24 | BRANCH_NAME = os.environ.get("APPVEYOR_REPO_BRANCH", "develop") 25 | if BUILD_NUMBER is not None and BRANCH_NAME != "master": 26 | VERSION = "{}.dev{}".format(VERSION, BUILD_NUMBER) 27 | 28 | setup( 29 | name="motmetrics", 30 | version=VERSION, 31 | description="Metrics for multiple object tracker benchmarking.", 32 | author="Christoph Heindl, Jack Valmadre", 33 | url="https://github.com/cheind/py-motmetrics", 34 | license="MIT", 35 | install_requires=required, 36 | packages=["motmetrics", "motmetrics.tests", "motmetrics.apps"], 37 | include_package_data=True, 38 | keywords="tracker MOT evaluation metrics compare", 39 | long_description=long_description, 40 | long_description_content_type="text/markdown", 41 | ) 42 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [develop] 9 | pull_request: 10 | branches: [develop] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | python-version: ["3.8", "3.9", "3.10"] 19 | 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v3 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install flake8 pytest pytest-benchmark 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | pip install lap scipy "ortools<9.4" lapsolver munkres 32 | - name: Lint with flake8 33 | run: | 34 | # stop the build if there are Python syntax errors or undefined names 35 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 36 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 37 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 38 | - name: Test with pytest 39 | run: | 40 | pytest 41 | -------------------------------------------------------------------------------- /motmetrics/data/iotest/detrac.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /motmetrics/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests accumulation of events using utility functions.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import itertools 15 | 16 | import numpy as np 17 | import pandas as pd 18 | 19 | import motmetrics as mm 20 | 21 | 22 | def test_annotations_xor_predictions_present(): 23 | """Tests frames that contain only annotations or predictions.""" 24 | _ = None 25 | anno_tracks = { 26 | 1: [0, 2, 4, 6, _, _, _], 27 | 2: [_, _, 0, 2, 4, _, _], 28 | } 29 | pred_tracks = { 30 | 1: [_, _, 3, 5, 7, 7, 7], 31 | } 32 | anno = _tracks_to_dataframe(anno_tracks) 33 | pred = _tracks_to_dataframe(pred_tracks) 34 | acc = mm.utils.compare_to_groundtruth(anno, pred, 'euc', distfields=['Position'], distth=2) 35 | mh = mm.metrics.create() 36 | metrics = mh.compute(acc, return_dataframe=False, metrics=[ 37 | 'num_objects', 'num_predictions', 'num_unique_objects', 38 | ]) 39 | np.testing.assert_equal(metrics['num_objects'], 7) 40 | np.testing.assert_equal(metrics['num_predictions'], 5) 41 | np.testing.assert_equal(metrics['num_unique_objects'], 2) 42 | 43 | 44 | def _tracks_to_dataframe(tracks): 45 | rows = [] 46 | for track_id, track in tracks.items(): 47 | for frame_id, position in zip(itertools.count(1), track): 48 | if position is None: 49 | continue 50 | rows.append({ 51 | 'FrameId': frame_id, 52 | 'Id': track_id, 53 | 'Position': position, 54 | }) 55 | return pd.DataFrame(rows).set_index(['FrameId', 'Id']) 56 | -------------------------------------------------------------------------------- /motmetrics/tests/test_distances.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests distance computation.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import numpy as np 15 | 16 | import motmetrics as mm 17 | 18 | 19 | def test_norm2squared(): 20 | """Tests norm2squared_matrix.""" 21 | a = np.asfarray([ 22 | [1, 2], 23 | [2, 2], 24 | [3, 2], 25 | ]) 26 | 27 | b = np.asfarray([ 28 | [0, 0], 29 | [1, 1], 30 | ]) 31 | 32 | C = mm.distances.norm2squared_matrix(a, b) 33 | np.testing.assert_allclose( 34 | C, 35 | [ 36 | [5, 1], 37 | [8, 2], 38 | [13, 5] 39 | ] 40 | ) 41 | 42 | C = mm.distances.norm2squared_matrix(a, b, max_d2=5) 43 | np.testing.assert_allclose( 44 | C, 45 | [ 46 | [5, 1], 47 | [np.nan, 2], 48 | [np.nan, 5] 49 | ] 50 | ) 51 | 52 | 53 | def test_norm2squared_empty(): 54 | """Tests norm2squared_matrix with an empty input.""" 55 | a = [] 56 | b = np.asfarray([[0, 0], [1, 1]]) 57 | C = mm.distances.norm2squared_matrix(a, b) 58 | assert C.size == 0 59 | C = mm.distances.norm2squared_matrix(b, a) 60 | assert C.size == 0 61 | 62 | 63 | def test_iou_matrix(): 64 | """Tests iou_matrix.""" 65 | a = np.array([ 66 | [0, 0, 1, 2], 67 | ]) 68 | 69 | b = np.array([ 70 | [0, 0, 1, 2], 71 | [0, 0, 1, 1], 72 | [1, 1, 1, 1], 73 | [0.5, 0, 1, 1], 74 | [0, 1, 1, 1], 75 | ]) 76 | np.testing.assert_allclose( 77 | mm.distances.iou_matrix(a, b), 78 | [[0, 0.5, 1, 0.8, 0.5]], 79 | atol=1e-4 80 | ) 81 | 82 | np.testing.assert_allclose( 83 | mm.distances.iou_matrix(a, b, max_iou=0.5), 84 | [[0, 0.5, np.nan, np.nan, 0.5]], 85 | atol=1e-4 86 | ) 87 | -------------------------------------------------------------------------------- /motmetrics/apps/example.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Example usage.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import numpy as np 15 | 16 | import motmetrics as mm 17 | 18 | if __name__ == '__main__': 19 | 20 | # Create an accumulator that will be updated during each frame 21 | acc = mm.MOTAccumulator(auto_id=True) 22 | 23 | # Each frame a list of ground truth object / hypotheses ids and pairwise distances 24 | # is passed to the accumulator. For now assume that the distance matrix given to us. 25 | 26 | # 2 Matches, 1 False alarm 27 | acc.update( 28 | [1, 2], # Ground truth objects in this frame 29 | [1, 2, 3], # Detector hypotheses in this frame 30 | [[0.1, np.nan, 0.3], # Distances from object 1 to hypotheses 1, 2, 3 31 | [0.5, 0.2, 0.3]] # Distances from object 2 to hypotheses 1, 2, 32 | ) 33 | print(acc.events) 34 | 35 | # 1 Match, 1 Miss 36 | df = acc.update( 37 | [1, 2], 38 | [1], 39 | [[0.2], [0.4]] 40 | ) 41 | print(df) 42 | 43 | # 1 Match, 1 Switch 44 | df = acc.update( 45 | [1, 2], 46 | [1, 3], 47 | [[0.6, 0.2], 48 | [0.1, 0.6]] 49 | ) 50 | print(df) 51 | 52 | # Compute metrics 53 | 54 | mh = mm.metrics.create() 55 | summary = mh.compute(acc, metrics=['num_frames', 'mota', 'motp'], name='acc') 56 | print(summary) 57 | 58 | summary = mh.compute_many( 59 | [acc, acc.events.loc[0:1]], 60 | metrics=['num_frames', 'mota', 'motp'], 61 | names=['full', 'part']) 62 | print(summary) 63 | 64 | strsummary = mm.io.render_summary( 65 | summary, 66 | formatters={'mota': '{:.2%}'.format}, 67 | namemap={'mota': 'MOTA', 'motp': 'MOTP'} 68 | ) 69 | print(strsummary) 70 | 71 | summary = mh.compute_many( 72 | [acc, acc.events.loc[0:1]], 73 | metrics=mm.metrics.motchallenge_metrics, 74 | names=['full', 'part']) 75 | strsummary = mm.io.render_summary( 76 | summary, 77 | formatters=mh.formatters, 78 | namemap=mm.io.motchallenge_metric_names 79 | ) 80 | print(strsummary) 81 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # py-motmetrics modified by ddz 2 | 3 | ## 背景 4 | 5 | 鉴于我在知乎上的文章 6 | [MOT多目标跟踪评价指标及计算代码（持续更新）](https://zhuanlan.zhihu.com/p/405983694) 7 | ，有很多人在评论区指出跑出来的结果有误，大家可以参照我下面的设置和改动来跑你自己的数据。 8 | 9 | ## 数据 10 | 11 | 你可以将你的数据，真实（gt）和测试（test）文件放在`yourdata/`文件夹下，这是一个例子： 12 | 13 | 你现在有两个视频，video1和video2，这两个视频你都进行了多目标跟踪。那么你需要这样组织你的文件夹： 14 | ``` 15 | --video1/gt/gt.txt # video1的gt文件 16 | --video2/gt/gt.txt # video2的gt文件 17 | --video1.txt # video1的test文件 18 | --video2.txt # video2的test文件 19 | ``` 20 | 像这样： 21 | ![img](imgs/1.png) 22 | 23 | 注意，红圈里要一致，绿圈里也要一致。 24 | 25 | 你可以看我这个库里的yourdata文件夹，照葫芦画瓢即可。假如你有很多个视频需要测，同样地组织你的文件夹就行。 26 | 27 | **至于每个文件中的格式，直接参照我的就行，比如：** 28 | ``` 29 | 10,1,660,2116,28,63,-1,-1,-1,-1 30 | 10,2,101,3247,37,61,-1,-1,-1,-1 31 | 10,3,68,517,23,40,-1,-1,-1,-1 32 | ... 33 | ``` 34 | **每行的第一个数是帧号；第二个数是id；后面接着的四个数表示框的位置和大小；然后倒数第四个数是置信度，我这里设置为-1是因为我的跟踪方法根本不输出置信度，如果是你用的跟踪方法输出置信度，则你需要把这个数设为置信度；最后三个数不重要不用管。** 35 | ## 代码改动 36 | 37 | 相比于原始的py-motmetrics库，我只改动了以下几个地方，均是在`motmetrics/apps/eval_motchallenge.py`文件中。 38 | 39 | ### 第一个地方 40 | 我把55、56、97、98行的代码（关于数据集）注释掉了： 41 | 42 | ![img](imgs/21.png) 43 | 44 | 我又在99-102行加了我自己的数据集路径，也就是yourdata下面的gt和test： 45 | 46 | ![img](imgs/22.png) 47 | 48 | 这样做的目的是，直接写死路径，不用给出数据集的路径参数。 49 | 50 | ### 第二个地方 51 | 第109和110行代码中，我添加了min_confidence参数并设置为-1，如下： 52 | 53 | ![img](imgs/23.png) 54 | 55 | 这是什么意思呢？**min_confidence参数的意思是，在进行评估时，只考虑置信度在min_confidence以上的那些跟踪框**。而对于我自己的数据，我的跟踪算法根本不输出置信度，我test文件中的置信度都是-1，所以我把min_confidence参数都设置为了-1。很多人可能是这里出的问题。**你可以根据你自己的跟踪算法究竟考不考虑置信度来设置这个min_confidence参数。** 56 | 57 | ### 第三个地方 58 | 第66-82代码中我改动了compare_dataframes函数，在115-117行中我定义了iou的阈值参数和dist（距离）的阈值参数，并将它们传给compare_dataframes函数，如下： 59 | 60 | ![img](imgs/24.png) 61 | 62 | 这是什么意思呢？**实际上，我们在评估mot指标时，我们会匹配真实框和跟踪框，也就是说，我们会根据iou或者距离来匹配。比如，对于一个真实框和一个跟踪框，它们的iou大于某个阈值，才能匹配成功，算是跟踪成功了，或者我们定义它们的dist（距离）小于某个阈值，才能匹配成功。** 63 | 64 | 由于我的数据比较特殊，如果只看iou的话，很难匹配上。所以我只看框的中心点的距离，但我的数据范围又非常大，所以我将距离阈值设很大（5000）。 65 | 66 | **如果你自己想用，你可以根据你自己数据的特性来更改这两个阈值，以及compare_dataframes函数里的逻辑（红圈中的代码）：** 67 | 68 | ![img](imgs/25.png) 69 | 70 | ## 运行和结果 71 | 72 | 运行： 73 | ``` 74 | python motmetrics/apps/eval_motchallenge.py 75 | ``` 76 | 即可得到： 77 | 78 | ![img](imgs/3.png) 79 | 80 | 各项指标都正常。 81 | 82 | ## motp的问题 83 | 84 | 有很多人问motp为啥那么大，不是0-1之间的数。 85 | 注意，我前面已经说了，我自己的数据比较特殊，如果只看iou的话，两个框很难匹配上。 86 | motp公式如下： 87 | 88 | ![image](https://github.com/ddz16/py-motmetrics/assets/45875562/edbc8af9-f0e6-4ab8-8de5-1d549e19fcf2) 89 | 90 | 其中d表示匹配框之间的距离，一般用**匹配框之间的iou**来计算。因为iou是0-1的，所以motp会小于1。但是，我这里是用**匹配框中心点距离**来计算的d，它的范围并不是0-1，甚至可能很大，所以对应的motp会很大。 91 | 如果你想按照iou来算，得到0-1之间的motp，可以改下面的代码，把这几行去掉就行（注意，这时你得换别的数据，最好是匹配框之间有重叠的数据，别用我给的数据了）： 92 | ![image](https://github.com/ddz16/py-motmetrics/assets/45875562/174dbffa-325f-4b69-9fb1-4d4a19a13bdf) 93 | 这样，你算motp时就纯是用iou来算的了，得到的motp自然也就位于0-1之间了。 94 | -------------------------------------------------------------------------------- /motmetrics/preprocess.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Preprocess data for CLEAR_MOT_M.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | from configparser import ConfigParser 15 | import logging 16 | import time 17 | 18 | import numpy as np 19 | 20 | import motmetrics.distances as mmd 21 | from motmetrics.lap import linear_sum_assignment 22 | 23 | 24 | def preprocessResult(res, gt, inifile): 25 | """Preprocesses data for utils.CLEAR_MOT_M. 26 | 27 | Returns a subset of the predictions. 28 | """ 29 | # pylint: disable=too-many-locals 30 | st = time.time() 31 | labels = [ 32 | 'ped', # 1 33 | 'person_on_vhcl', # 2 34 | 'car', # 3 35 | 'bicycle', # 4 36 | 'mbike', # 5 37 | 'non_mot_vhcl', # 6 38 | 'static_person', # 7 39 | 'distractor', # 8 40 | 'occluder', # 9 41 | 'occluder_on_grnd', # 10 42 | 'occluder_full', # 11 43 | 'reflection', # 12 44 | 'crowd', # 13 45 | ] 46 | distractors = ['person_on_vhcl', 'static_person', 'distractor', 'reflection'] 47 | is_distractor = {i + 1: x in distractors for i, x in enumerate(labels)} 48 | for i in distractors: 49 | is_distractor[i] = 1 50 | seqIni = ConfigParser() 51 | seqIni.read(inifile, encoding='utf8') 52 | F = int(seqIni['Sequence']['seqLength']) 53 | todrop = [] 54 | for t in range(1, F + 1): 55 | if t not in res.index or t not in gt.index: 56 | continue 57 | resInFrame = res.loc[t] 58 | 59 | GTInFrame = gt.loc[t] 60 | A = GTInFrame[['X', 'Y', 'Width', 'Height']].values 61 | B = resInFrame[['X', 'Y', 'Width', 'Height']].values 62 | disM = mmd.iou_matrix(A, B, max_iou=0.5) 63 | le, ri = linear_sum_assignment(disM) 64 | flags = [ 65 | 1 if is_distractor[it['ClassId']] or it['Visibility'] < 0. else 0 66 | for i, (k, it) in enumerate(GTInFrame.iterrows()) 67 | ] 68 | hid = [k for k, it in resInFrame.iterrows()] 69 | for i, j in zip(le, ri): 70 | if not np.isfinite(disM[i, j]): 71 | continue 72 | if flags[i]: 73 | todrop.append((t, hid[j])) 74 | ret = res.drop(labels=todrop) 75 | logging.info('Preprocess take %.3f seconds and remove %d boxes.', 76 | time.time() - st, len(todrop)) 77 | return ret 78 | -------------------------------------------------------------------------------- /motmetrics/tests/test_io.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests IO functions.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import os 15 | 16 | import pandas as pd 17 | 18 | import motmetrics as mm 19 | 20 | DATA_DIR = os.path.join(os.path.dirname(__file__), '../data') 21 | 22 | 23 | def test_load_vatic(): 24 | """Tests VATIC_TXT format.""" 25 | df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/vatic.txt'), fmt=mm.io.Format.VATIC_TXT) 26 | 27 | expected = pd.DataFrame([ 28 | # F,ID,Y,W,H,L,O,G,F,A1,A2,A3,A4 29 | (0, 0, 412, 0, 430, 124, 0, 0, 0, 'worker', 0, 0, 0, 0), 30 | (1, 0, 412, 10, 430, 114, 0, 0, 1, 'pc', 1, 0, 1, 0), 31 | (1, 1, 412, 0, 430, 124, 0, 0, 1, 'pc', 0, 1, 0, 0), 32 | (2, 2, 412, 0, 430, 124, 0, 0, 1, 'worker', 1, 1, 0, 1) 33 | ]) 34 | 35 | assert (df.reset_index().values == expected.values).all() 36 | 37 | 38 | def test_load_motchallenge(): 39 | """Tests MOT15_2D format.""" 40 | df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/motchallenge.txt'), fmt=mm.io.Format.MOT15_2D) 41 | 42 | expected = pd.DataFrame([ 43 | (1, 1, 398, 181, 121, 229, 1, -1, -1), # Note -1 on x and y for correcting matlab 44 | (1, 2, 281, 200, 92, 184, 1, -1, -1), 45 | (2, 2, 268, 201, 87, 182, 1, -1, -1), 46 | (2, 3, 70, 150, 100, 284, 1, -1, -1), 47 | (2, 4, 199, 205, 55, 137, 1, -1, -1), 48 | ]) 49 | 50 | assert (df.reset_index().values == expected.values).all() 51 | 52 | 53 | def test_load_detrac_mat(): 54 | """Tests DETRAC_MAT format.""" 55 | df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/detrac.mat'), fmt=mm.io.Format.DETRAC_MAT) 56 | 57 | expected = pd.DataFrame([ 58 | (1., 1., 745., 356., 148., 115., 1., -1., -1.), 59 | (2., 1., 738., 350., 145., 111., 1., -1., -1.), 60 | (3., 1., 732., 343., 142., 107., 1., -1., -1.), 61 | (4., 1., 725., 336., 139., 104., 1., -1., -1.) 62 | ]) 63 | 64 | assert (df.reset_index().values == expected.values).all() 65 | 66 | 67 | def test_load_detrac_xml(): 68 | """Tests DETRAC_XML format.""" 69 | df = mm.io.loadtxt(os.path.join(DATA_DIR, 'iotest/detrac.xml'), fmt=mm.io.Format.DETRAC_XML) 70 | 71 | expected = pd.DataFrame([ 72 | (1., 1., 744.6, 356.33, 148.2, 115.14, 1., -1., -1.), 73 | (2., 1., 738.2, 349.51, 145.21, 111.29, 1., -1., -1.), 74 | (3., 1., 731.8, 342.68, 142.23, 107.45, 1., -1., -1.), 75 | (4., 1., 725.4, 335.85, 139.24, 103.62, 1., -1., -1.) 76 | ]) 77 | 78 | assert (df.reset_index().values == expected.values).all() 79 | -------------------------------------------------------------------------------- /motmetrics/distances.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Functions for comparing predictions and ground-truth.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import numpy as np 15 | 16 | from motmetrics import math_util 17 | 18 | 19 | def norm2squared_matrix(objs, hyps, max_d2=float('inf')): 20 | """Computes the squared Euclidean distance matrix between object and hypothesis points. 21 | 22 | Params 23 | ------ 24 | objs : NxM array 25 | Object points of dim M in rows 26 | hyps : KxM array 27 | Hypothesis points of dim M in rows 28 | 29 | Kwargs 30 | ------ 31 | max_d2 : float 32 | Maximum tolerable squared Euclidean distance. Object / hypothesis points 33 | with larger distance are set to np.nan signalling do-not-pair. Defaults 34 | to +inf 35 | 36 | Returns 37 | ------- 38 | C : NxK array 39 | Distance matrix containing pairwise distances or np.nan. 40 | """ 41 | 42 | objs = np.atleast_2d(objs).astype(float) 43 | hyps = np.atleast_2d(hyps).astype(float) 44 | 45 | if objs.size == 0 or hyps.size == 0: 46 | return np.empty((0, 0)) 47 | 48 | assert hyps.shape[1] == objs.shape[1], "Dimension mismatch" 49 | 50 | delta = objs[:, np.newaxis] - hyps[np.newaxis, :] 51 | C = np.sum(delta ** 2, axis=-1) 52 | 53 | C[C > max_d2] = np.nan 54 | return C 55 | 56 | 57 | def rect_min_max(r): 58 | min_pt = r[..., :2] 59 | size = r[..., 2:] 60 | max_pt = min_pt + size 61 | return min_pt, max_pt 62 | 63 | 64 | def boxiou(a, b): 65 | """Computes IOU of two rectangles.""" 66 | a_min, a_max = rect_min_max(a) 67 | b_min, b_max = rect_min_max(b) 68 | # Compute intersection. 69 | i_min = np.maximum(a_min, b_min) 70 | i_max = np.minimum(a_max, b_max) 71 | i_size = np.maximum(i_max - i_min, 0) 72 | i_vol = np.prod(i_size, axis=-1) 73 | # Get volume of union. 74 | a_size = np.maximum(a_max - a_min, 0) 75 | b_size = np.maximum(b_max - b_min, 0) 76 | a_vol = np.prod(a_size, axis=-1) 77 | b_vol = np.prod(b_size, axis=-1) 78 | u_vol = a_vol + b_vol - i_vol 79 | return np.where(i_vol == 0, np.zeros_like(i_vol, dtype=np.float64), 80 | math_util.quiet_divide(i_vol, u_vol)) 81 | 82 | 83 | def iou_matrix(objs, hyps, max_iou=1.): 84 | """Computes 'intersection over union (IoU)' distance matrix between object and hypothesis rectangles. 85 | 86 | The IoU is computed as 87 | 88 | IoU(a,b) = 1. - isect(a, b) / union(a, b) 89 | 90 | where isect(a,b) is the area of intersection of two rectangles and union(a, b) the area of union. The 91 | IoU is bounded between zero and one. 0 when the rectangles overlap perfectly and 1 when the overlap is 92 | zero. 93 | 94 | Params 95 | ------ 96 | objs : Nx4 array 97 | Object rectangles (x,y,w,h) in rows 98 | hyps : Kx4 array 99 | Hypothesis rectangles (x,y,w,h) in rows 100 | 101 | Kwargs 102 | ------ 103 | max_iou : float 104 | Maximum tolerable overlap distance. Object / hypothesis points 105 | with larger distance are set to np.nan signalling do-not-pair. Defaults 106 | to 0.5 107 | 108 | Returns 109 | ------- 110 | C : NxK array 111 | Distance matrix containing pairwise distances or np.nan. 112 | """ 113 | 114 | if np.size(objs) == 0 or np.size(hyps) == 0: 115 | return np.empty((0, 0)) 116 | 117 | objs = np.asfarray(objs) 118 | hyps = np.asfarray(hyps) 119 | assert objs.shape[1] == 4 120 | assert hyps.shape[1] == 4 121 | iou = boxiou(objs[:, None], hyps[None, :]) 122 | dist = 1 - iou 123 | return np.where(dist > max_iou, np.nan, dist) 124 | -------------------------------------------------------------------------------- /motmetrics/apps/eval_detrac.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Compute metrics for trackers using DETRAC challenge ground-truth data.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import argparse 15 | from collections import OrderedDict 16 | import glob 17 | import logging 18 | import os 19 | from pathlib import Path 20 | 21 | import motmetrics as mm 22 | 23 | 24 | def parse_args(): 25 | """Defines and parses command-line arguments.""" 26 | parser = argparse.ArgumentParser(description=""" 27 | Compute metrics for trackers using DETRAC challenge ground-truth data. 28 | 29 | Files 30 | ----- 31 | Ground truth files can be in .XML format or .MAT format as provided by http://detrac-db.rit.albany.edu/download 32 | 33 | Test Files for the challenge are reuired to be in MOTchallenge format, they have to comply with the format described in 34 | 35 | Milan, Anton, et al. 36 | "Mot16: A benchmark for multi-object tracking." 37 | arXiv preprint arXiv:1603.00831 (2016). 38 | https://motchallenge.net/ 39 | 40 | Directory Structure 41 | --------- 42 | 43 | Layout for ground truth data 44 | /.txt 45 | /.txt 46 | ... 47 | 48 | OR 49 | /.mat 50 | /.mat 51 | ... 52 | 53 | Layout for test data 54 | /.txt 55 | /.txt 56 | ... 57 | 58 | Sequences of ground truth and test will be matched according to the `` 59 | string.""", formatter_class=argparse.RawTextHelpFormatter) 60 | 61 | parser.add_argument('groundtruths', type=str, help='Directory containing ground truth files.') 62 | parser.add_argument('tests', type=str, help='Directory containing tracker result files') 63 | parser.add_argument('--loglevel', type=str, help='Log level', default='info') 64 | parser.add_argument('--gtfmt', type=str, help='Groundtruth data format', default='detrac-xml') 65 | parser.add_argument('--tsfmt', type=str, help='Test data format', default='mot15-2D') 66 | parser.add_argument('--solver', type=str, help='LAP solver to use') 67 | return parser.parse_args() 68 | 69 | 70 | def compare_dataframes(gts, ts): 71 | """Builds accumulator for each sequence.""" 72 | accs = [] 73 | names = [] 74 | for k, tsacc in ts.items(): 75 | if k in gts: 76 | logging.info('Comparing %s...', k) 77 | accs.append(mm.utils.compare_to_groundtruth(gts[k], tsacc, 'iou', distth=0.5)) 78 | names.append(k) 79 | else: 80 | logging.warning('No ground truth for %s, skipping.', k) 81 | 82 | return accs, names 83 | 84 | 85 | def main(): 86 | # pylint: disable=missing-function-docstring 87 | args = parse_args() 88 | 89 | loglevel = getattr(logging, args.loglevel.upper(), None) 90 | 91 | if loglevel not in ["0","10","20","30","40","50"]: # previous code was always raising error, this code is correct [Ardeshir Shon] 92 | raise ValueError('Invalid log level: {} '.format(args.loglevel)) 93 | loglevel = int(loglevel) # This type casting is needed regarding the logging library documentation 94 | logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s - %(message)s', datefmt='%I:%M:%S') 95 | 96 | if args.solver: 97 | mm.lap.default_solver = args.solver 98 | 99 | gtfiles = glob.glob(os.path.join(args.groundtruths, '*')) 100 | tsfiles = glob.glob(os.path.join(args.tests, '*')) 101 | 102 | logging.info('Found %d groundtruths and %d test files.', len(gtfiles), len(tsfiles)) 103 | logging.info('Available LAP solvers %s', str(mm.lap.available_solvers)) 104 | logging.info('Default LAP solver \'%s\'', mm.lap.default_solver) 105 | logging.info('Loading files.') 106 | 107 | gt = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.gtfmt)) for f in gtfiles]) 108 | ts = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.tsfmt)) for f in tsfiles]) 109 | 110 | mh = mm.metrics.create() 111 | accs, names = compare_dataframes(gt, ts) 112 | 113 | logging.info('Running metrics') 114 | 115 | summary = mh.compute_many(accs, names=names, metrics=mm.metrics.motchallenge_metrics, generate_overall=True) 116 | print(mm.io.render_summary(summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names)) 117 | logging.info('Completed') 118 | 119 | 120 | if __name__ == '__main__': 121 | main() 122 | -------------------------------------------------------------------------------- /motmetrics/apps/eval_motchallenge.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Compute metrics for trackers using MOTChallenge ground-truth data.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import argparse 15 | from collections import OrderedDict 16 | import glob 17 | import logging 18 | import os 19 | from pathlib import Path 20 | 21 | import motmetrics as mm 22 | 23 | 24 | def parse_args(): 25 | """Defines and parses command-line arguments.""" 26 | parser = argparse.ArgumentParser(description=""" 27 | Compute metrics for trackers using MOTChallenge ground-truth data. 28 | 29 | Files 30 | ----- 31 | All file content, ground truth and test files, have to comply with the 32 | format described in 33 | 34 | Milan, Anton, et al. 35 | "Mot16: A benchmark for multi-object tracking." 36 | arXiv preprint arXiv:1603.00831 (2016). 37 | https://motchallenge.net/ 38 | 39 | Structure 40 | --------- 41 | 42 | Layout for ground truth data 43 | //gt/gt.txt 44 | //gt/gt.txt 45 | ... 46 | 47 | Layout for test data 48 | /.txt 49 | /.txt 50 | ... 51 | 52 | Sequences of ground truth and test will be matched according to the `` 53 | string.""", formatter_class=argparse.RawTextHelpFormatter) 54 | 55 | # parser.add_argument('groundtruths', type=str, help='Directory containing ground truth files.') 56 | # parser.add_argument('tests', type=str, help='Directory containing tracker result files') 57 | parser.add_argument('--loglevel', type=str, help='Log level', default='info') 58 | parser.add_argument('--fmt', type=str, help='Data format', default='mot15-2D') 59 | parser.add_argument('--solver', type=str, help='LAP solver to use for matching between frames.') 60 | parser.add_argument('--id_solver', type=str, help='LAP solver to use for ID metrics. Defaults to --solver.') 61 | parser.add_argument('--exclude_id', dest='exclude_id', default=False, action='store_true', 62 | help='Disable ID metrics') 63 | return parser.parse_args() 64 | 65 | 66 | def compare_dataframes(gts, ts, iouth, distth): 67 | """Builds accumulator for each sequence.""" 68 | accs = [] 69 | names = [] 70 | 71 | for k, tsacc in ts.items(): 72 | if k in gts: 73 | if iouth > 0: 74 | logging.info('Comparing IoU distance...') 75 | accs.append(mm.utils.compare_to_groundtruth(gts[k], tsacc, 'iou', distth=1-iouth)) 76 | else: 77 | logging.info('Comparing Euclidean distance...') 78 | accs.append(mm.utils.compare_to_groundtruth(gts[k], tsacc, 'euc', distth=distth)) 79 | names.append(k) 80 | else: 81 | logging.warning('No ground truth for %s, skipping.', k) 82 | return accs, names 83 | 84 | 85 | def main(): 86 | # pylint: disable=missing-function-docstring 87 | args = parse_args() 88 | 89 | loglevel = getattr(logging, args.loglevel.upper(), None) 90 | if not isinstance(loglevel, int): 91 | raise ValueError('Invalid log level: {} '.format(args.loglevel)) 92 | logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s - %(message)s', datefmt='%I:%M:%S') 93 | 94 | if args.solver: 95 | mm.lap.default_solver = args.solver 96 | 97 | # gtfiles = glob.glob(os.path.join(args.groundtruths, '*/gt/gt.txt')) 98 | # tsfiles = [f for f in glob.glob(os.path.join(args.tests, '*.txt')) if not os.path.basename(f).startswith('eval')] 99 | gtfiles = ['yourdata/video1/gt/gt.txt', 100 | 'yourdata/video2/gt/gt.txt'] 101 | tsfiles = ['yourdata/video1.txt', 102 | 'yourdata/video2.txt',] 103 | 104 | logging.info('Found %d groundtruths and %d test files.', len(gtfiles), len(tsfiles)) 105 | logging.info('Available LAP solvers %s', str(mm.lap.available_solvers)) 106 | logging.info('Default LAP solver \'%s\'', mm.lap.default_solver) 107 | logging.info('Loading files.') 108 | 109 | gt = OrderedDict([(Path(f).parts[-3], mm.io.loadtxt(f, fmt=args.fmt, min_confidence=-1)) for f in gtfiles]) 110 | ts = OrderedDict([(os.path.splitext(Path(f).parts[-1])[0], mm.io.loadtxt(f, fmt=args.fmt, min_confidence=-1)) for f in tsfiles]) 111 | 112 | print(gt) 113 | print(ts) 114 | mh = mm.metrics.create() 115 | iou_threshold = 0.0 116 | dist_threshold = 5000 117 | accs, names = compare_dataframes(gt, ts, iou_threshold, dist_threshold) 118 | 119 | metrics = list(mm.metrics.motchallenge_metrics) 120 | if args.exclude_id: 121 | metrics = [x for x in metrics if not x.startswith('id')] 122 | 123 | logging.info('Running metrics') 124 | 125 | if args.id_solver: 126 | mm.lap.default_solver = args.id_solver 127 | summary = mh.compute_many(accs, names=names, metrics=metrics, generate_overall=True) 128 | print(mm.io.render_summary(summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names)) 129 | logging.info('Completed') 130 | 131 | 132 | if __name__ == '__main__': 133 | main() 134 | -------------------------------------------------------------------------------- /motmetrics/utils.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Functions for populating event accumulators.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import numpy as np 15 | 16 | from motmetrics.distances import iou_matrix, norm2squared_matrix 17 | from motmetrics.mot import MOTAccumulator 18 | from motmetrics.preprocess import preprocessResult 19 | 20 | 21 | def compare_to_groundtruth(gt, dt, dist='iou', distfields=None, distth=0.5): 22 | """Compare groundtruth and detector results. 23 | 24 | This method assumes both results are given in terms of DataFrames with at least the following fields 25 | - `FrameId` First level index used for matching ground-truth and test frames. 26 | - `Id` Secondary level index marking available object / hypothesis ids 27 | 28 | Depending on the distance to be used relevant distfields need to be specified. 29 | 30 | Params 31 | ------ 32 | gt : pd.DataFrame 33 | Dataframe for ground-truth 34 | test : pd.DataFrame 35 | Dataframe for detector results 36 | 37 | Kwargs 38 | ------ 39 | dist : str, optional 40 | String identifying distance to be used. Defaults to intersection over union ('iou'). Euclidean 41 | distance ('euc') and squared euclidean distance ('seuc') are also supported. 42 | distfields: array, optional 43 | Fields relevant for extracting distance information. Defaults to ['X', 'Y', 'Width', 'Height'] 44 | distth: float, optional 45 | Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'. 46 | """ 47 | # pylint: disable=too-many-locals 48 | if distfields is None: 49 | distfields = ['X', 'Y', 'Width', 'Height'] 50 | 51 | def compute_iou(a, b): 52 | return iou_matrix(a, b, max_iou=distth) 53 | 54 | def compute_euc(a, b): 55 | return np.sqrt(norm2squared_matrix(a, b, max_d2=distth**2)) 56 | 57 | def compute_seuc(a, b): 58 | return norm2squared_matrix(a, b, max_d2=distth) 59 | 60 | if dist.upper() == 'IOU': 61 | compute_dist = compute_iou 62 | elif dist.upper() == 'EUC': 63 | compute_dist = compute_euc 64 | elif dist.upper() == 'SEUC': 65 | compute_dist = compute_seuc 66 | else: 67 | raise f'Unknown distance metric {dist}. Use "IOU", "EUC" or "SEUC"' 68 | 69 | acc = MOTAccumulator() 70 | 71 | # We need to account for all frames reported either by ground truth or 72 | # detector. In case a frame is missing in GT this will lead to FPs, in 73 | # case a frame is missing in detector results this will lead to FNs. 74 | allframeids = gt.index.union(dt.index).levels[0] 75 | 76 | gt = gt[distfields] 77 | dt = dt[distfields] 78 | fid_to_fgt = dict(iter(gt.groupby('FrameId'))) 79 | fid_to_fdt = dict(iter(dt.groupby('FrameId'))) 80 | 81 | for fid in allframeids: 82 | oids = np.empty(0) 83 | hids = np.empty(0) 84 | dists = np.empty((0, 0)) 85 | if fid in fid_to_fgt: 86 | fgt = fid_to_fgt[fid] 87 | oids = fgt.index.get_level_values('Id') 88 | if fid in fid_to_fdt: 89 | fdt = fid_to_fdt[fid] 90 | hids = fdt.index.get_level_values('Id') 91 | if len(oids) > 0 and len(hids) > 0: 92 | dists = compute_dist(fgt.values, fdt.values) 93 | acc.update(oids, hids, dists, frameid=fid) 94 | 95 | return acc 96 | 97 | 98 | def CLEAR_MOT_M(gt, dt, inifile, dist='iou', distfields=None, distth=0.5, include_all=False, vflag=''): 99 | """Compare groundtruth and detector results. 100 | 101 | This method assumes both results are given in terms of DataFrames with at least the following fields 102 | - `FrameId` First level index used for matching ground-truth and test frames. 103 | - `Id` Secondary level index marking available object / hypothesis ids 104 | 105 | Depending on the distance to be used relevant distfields need to be specified. 106 | 107 | Params 108 | ------ 109 | gt : pd.DataFrame 110 | Dataframe for ground-truth 111 | test : pd.DataFrame 112 | Dataframe for detector results 113 | 114 | Kwargs 115 | ------ 116 | dist : str, optional 117 | String identifying distance to be used. Defaults to intersection over union. 118 | distfields: array, optional 119 | Fields relevant for extracting distance information. Defaults to ['X', 'Y', 'Width', 'Height'] 120 | distth: float, optional 121 | Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'. 122 | """ 123 | # pylint: disable=too-many-locals 124 | if distfields is None: 125 | distfields = ['X', 'Y', 'Width', 'Height'] 126 | 127 | def compute_iou(a, b): 128 | return iou_matrix(a, b, max_iou=distth) 129 | 130 | def compute_euc(a, b): 131 | return norm2squared_matrix(a, b, max_d2=distth) 132 | 133 | compute_dist = compute_iou if dist.upper() == 'IOU' else compute_euc 134 | 135 | acc = MOTAccumulator() 136 | dt = preprocessResult(dt, gt, inifile) 137 | if include_all: 138 | gt = gt[gt['Confidence'] >= 0.99] 139 | else: 140 | gt = gt[(gt['Confidence'] >= 0.99) & (gt['ClassId'] == 1)] 141 | # We need to account for all frames reported either by ground truth or 142 | # detector. In case a frame is missing in GT this will lead to FPs, in 143 | # case a frame is missing in detector results this will lead to FNs. 144 | allframeids = gt.index.union(dt.index).levels[0] 145 | analysis = {'hyp': {}, 'obj': {}} 146 | for fid in allframeids: 147 | oids = np.empty(0) 148 | hids = np.empty(0) 149 | dists = np.empty((0, 0)) 150 | 151 | if fid in gt.index: 152 | fgt = gt.loc[fid] 153 | oids = fgt.index.values 154 | for oid in oids: 155 | oid = int(oid) 156 | if oid not in analysis['obj']: 157 | analysis['obj'][oid] = 0 158 | analysis['obj'][oid] += 1 159 | 160 | if fid in dt.index: 161 | fdt = dt.loc[fid] 162 | hids = fdt.index.values 163 | for hid in hids: 164 | hid = int(hid) 165 | if hid not in analysis['hyp']: 166 | analysis['hyp'][hid] = 0 167 | analysis['hyp'][hid] += 1 168 | 169 | if oids.shape[0] > 0 and hids.shape[0] > 0: 170 | dists = compute_dist(fgt[distfields].values, fdt[distfields].values) 171 | 172 | acc.update(oids, hids, dists, frameid=fid, vf=vflag) 173 | 174 | return acc, analysis 175 | -------------------------------------------------------------------------------- /motmetrics/apps/evaluateTracking.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Compute metrics for trackers using MOTChallenge ground-truth data.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import argparse 15 | from collections import OrderedDict 16 | import io 17 | import logging 18 | import os 19 | import sys 20 | from tempfile import NamedTemporaryFile 21 | import time 22 | 23 | import motmetrics as mm 24 | 25 | 26 | def parse_args(): 27 | """Defines and parses command-line arguments.""" 28 | parser = argparse.ArgumentParser(description=""" 29 | Compute metrics for trackers using MOTChallenge ground-truth data with data preprocess. 30 | 31 | Files 32 | ----- 33 | All file content, ground truth and test files, have to comply with the 34 | format described in 35 | 36 | Milan, Anton, et al. 37 | "Mot16: A benchmark for multi-object tracking." 38 | arXiv preprint arXiv:1603.00831 (2016). 39 | https://motchallenge.net/ 40 | 41 | Structure 42 | --------- 43 | 44 | Layout for ground truth data 45 | //gt/gt.txt 46 | //gt/gt.txt 47 | ... 48 | 49 | Layout for test data 50 | /.txt 51 | /.txt 52 | ... 53 | 54 | Seqmap for test data 55 | [name] 56 | 57 | 58 | ... 59 | 60 | Sequences of ground truth and test will be matched according to the `` 61 | string in the seqmap.""", formatter_class=argparse.RawTextHelpFormatter) 62 | 63 | parser.add_argument('groundtruths', type=str, help='Directory containing ground truth files.') 64 | parser.add_argument('tests', type=str, help='Directory containing tracker result files') 65 | parser.add_argument('seqmap', type=str, help='Text file containing all sequences name') 66 | parser.add_argument('--log', type=str, help='a place to record result and outputfile of mistakes', default='') 67 | parser.add_argument('--loglevel', type=str, help='Log level', default='info') 68 | parser.add_argument('--fmt', type=str, help='Data format', default='mot15-2D') 69 | parser.add_argument('--solver', type=str, help='LAP solver to use') 70 | parser.add_argument('--skip', type=int, default=0, help='skip frames n means choosing one frame for every (n+1) frames') 71 | parser.add_argument('--iou', type=float, default=0.5, help='special IoU threshold requirement for small targets') 72 | return parser.parse_args() 73 | 74 | 75 | def compare_dataframes(gts, ts, vsflag='', iou=0.5): 76 | """Builds accumulator for each sequence.""" 77 | accs = [] 78 | anas = [] 79 | names = [] 80 | for k, tsacc in ts.items(): 81 | if k in gts: 82 | logging.info('Evaluating %s...', k) 83 | if vsflag != '': 84 | fd = io.open(vsflag + '/' + k + '.log', 'w') 85 | else: 86 | fd = '' 87 | acc, ana = mm.utils.CLEAR_MOT_M(gts[k][0], tsacc, gts[k][1], 'iou', distth=iou, vflag=fd) 88 | if fd != '': 89 | fd.close() 90 | accs.append(acc) 91 | anas.append(ana) 92 | names.append(k) 93 | else: 94 | logging.warning('No ground truth for %s, skipping.', k) 95 | 96 | return accs, anas, names 97 | 98 | 99 | def parseSequences(seqmap): 100 | """Loads list of sequences from file.""" 101 | assert os.path.isfile(seqmap), 'Seqmap %s not found.' % seqmap 102 | fd = io.open(seqmap) 103 | res = [] 104 | for row in fd.readlines(): 105 | row = row.strip() 106 | if row == '' or row == 'name' or row[0] == '#': 107 | continue 108 | res.append(row) 109 | fd.close() 110 | return res 111 | 112 | 113 | def generateSkippedGT(gtfile, skip, fmt): 114 | """Generates temporary ground-truth file with some frames skipped.""" 115 | del fmt # unused 116 | tf = NamedTemporaryFile(delete=False, mode='w') 117 | with io.open(gtfile) as fd: 118 | lines = fd.readlines() 119 | for line in lines: 120 | arr = line.strip().split(',') 121 | fr = int(arr[0]) 122 | if fr % (skip + 1) != 1: 123 | continue 124 | pos = line.find(',') 125 | newline = str(fr // (skip + 1) + 1) + line[pos:] 126 | tf.write(newline) 127 | tf.close() 128 | tempfile = tf.name 129 | return tempfile 130 | 131 | 132 | def main(): 133 | # pylint: disable=missing-function-docstring 134 | # pylint: disable=too-many-locals 135 | args = parse_args() 136 | 137 | loglevel = getattr(logging, args.loglevel.upper(), None) 138 | if not isinstance(loglevel, int): 139 | raise ValueError('Invalid log level: {} '.format(args.loglevel)) 140 | logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s - %(message)s', datefmt='%I:%M:%S') 141 | 142 | if args.solver: 143 | mm.lap.default_solver = args.solver 144 | 145 | seqs = parseSequences(args.seqmap) 146 | gtfiles = [os.path.join(args.groundtruths, i, 'gt/gt.txt') for i in seqs] 147 | tsfiles = [os.path.join(args.tests, '%s.txt' % i) for i in seqs] 148 | 149 | for gtfile in gtfiles: 150 | if not os.path.isfile(gtfile): 151 | logging.error('gt File %s not found.', gtfile) 152 | sys.exit(1) 153 | for tsfile in tsfiles: 154 | if not os.path.isfile(tsfile): 155 | logging.error('res File %s not found.', tsfile) 156 | sys.exit(1) 157 | 158 | logging.info('Found %d groundtruths and %d test files.', len(gtfiles), len(tsfiles)) 159 | for seq in seqs: 160 | logging.info('\t%s', seq) 161 | logging.info('Available LAP solvers %s', str(mm.lap.available_solvers)) 162 | logging.info('Default LAP solver \'%s\'', mm.lap.default_solver) 163 | logging.info('Loading files.') 164 | 165 | if args.skip > 0 and 'mot' in args.fmt: 166 | for i, gtfile in enumerate(gtfiles): 167 | gtfiles[i] = generateSkippedGT(gtfile, args.skip, fmt=args.fmt) 168 | 169 | gt = OrderedDict([(seqs[i], (mm.io.loadtxt(f, fmt=args.fmt), os.path.join(args.groundtruths, seqs[i], 'seqinfo.ini'))) for i, f in enumerate(gtfiles)]) 170 | ts = OrderedDict([(seqs[i], mm.io.loadtxt(f, fmt=args.fmt)) for i, f in enumerate(tsfiles)]) 171 | 172 | mh = mm.metrics.create() 173 | st = time.time() 174 | accs, analysis, names = compare_dataframes(gt, ts, args.log, 1. - args.iou) 175 | logging.info('adding frames: %.3f seconds.', time.time() - st) 176 | 177 | logging.info('Running metrics') 178 | 179 | summary = mh.compute_many(accs, anas=analysis, names=names, metrics=mm.metrics.motchallenge_metrics, generate_overall=True) 180 | print(mm.io.render_summary(summary, formatters=mh.formatters, namemap=mm.io.motchallenge_metric_names)) 181 | logging.info('Completed') 182 | 183 | 184 | if __name__ == '__main__': 185 | main() 186 | -------------------------------------------------------------------------------- /motmetrics/tests/test_lap.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests linear assignment problem solvers.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import warnings 15 | 16 | import numpy as np 17 | import pytest 18 | 19 | from motmetrics import lap 20 | 21 | DESIRED_SOLVERS = ['lap', 'lapsolver', 'munkres', 'ortools', 'scipy'] 22 | SOLVERS = lap.available_solvers 23 | 24 | 25 | @pytest.mark.parametrize('solver', DESIRED_SOLVERS) 26 | def test_solver_is_available(solver): 27 | if solver not in lap.available_solvers: 28 | warnings.warn('solver not available: ' + solver) 29 | 30 | 31 | @pytest.mark.parametrize('solver', SOLVERS) 32 | def test_assign_easy(solver): 33 | """Problem that could be solved by a greedy algorithm.""" 34 | costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]]) 35 | costs_copy = costs.copy() 36 | result = lap.linear_sum_assignment(costs, solver=solver) 37 | 38 | expected = np.array([[0, 1, 2], [2, 1, 0]]) 39 | np.testing.assert_equal(result, expected) 40 | np.testing.assert_equal(costs, costs_copy) 41 | 42 | 43 | @pytest.mark.parametrize('solver', SOLVERS) 44 | def test_assign_full(solver): 45 | """Problem that would be incorrect using a greedy algorithm.""" 46 | costs = np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]]) 47 | costs_copy = costs.copy() 48 | result = lap.linear_sum_assignment(costs, solver=solver) 49 | 50 | # Optimal matching is (0, 2), (1, 1), (2, 0) for 6 + 2 + 2. 51 | expected = np.asfarray([[0, 1, 2], [2, 1, 0]]) 52 | np.testing.assert_equal(result, expected) 53 | np.testing.assert_equal(costs, costs_copy) 54 | 55 | 56 | @pytest.mark.parametrize('solver', SOLVERS) 57 | def test_assign_full_negative(solver): 58 | costs = -7 + np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]]) 59 | costs_copy = costs.copy() 60 | result = lap.linear_sum_assignment(costs, solver=solver) 61 | 62 | # Optimal matching is (0, 2), (1, 1), (2, 0) for 5 + 1 + 1. 63 | expected = np.array([[0, 1, 2], [2, 1, 0]]) 64 | np.testing.assert_equal(result, expected) 65 | np.testing.assert_equal(costs, costs_copy) 66 | 67 | 68 | @pytest.mark.parametrize('solver', SOLVERS) 69 | def test_assign_empty(solver): 70 | costs = np.asfarray([[]]) 71 | costs_copy = costs.copy() 72 | result = lap.linear_sum_assignment(costs, solver=solver) 73 | 74 | np.testing.assert_equal(np.size(result), 0) 75 | np.testing.assert_equal(costs, costs_copy) 76 | 77 | 78 | @pytest.mark.parametrize('solver', SOLVERS) 79 | def test_assign_infeasible(solver): 80 | """Tests that minimum-cost solution with most edges is found.""" 81 | costs = np.asfarray([[np.nan, np.nan, 2], 82 | [np.nan, np.nan, 1], 83 | [8, 7, 4]]) 84 | costs_copy = costs.copy() 85 | result = lap.linear_sum_assignment(costs, solver=solver) 86 | 87 | # Optimal matching is (1, 2), (2, 1). 88 | expected = np.array([[1, 2], [2, 1]]) 89 | np.testing.assert_equal(result, expected) 90 | np.testing.assert_equal(costs, costs_copy) 91 | 92 | 93 | @pytest.mark.parametrize('solver', SOLVERS) 94 | def test_assign_disallowed(solver): 95 | costs = np.asfarray([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4]]) 96 | costs_copy = costs.copy() 97 | result = lap.linear_sum_assignment(costs, solver=solver) 98 | 99 | expected = np.array([[0, 1, 2], [0, 2, 1]]) 100 | np.testing.assert_equal(result, expected) 101 | np.testing.assert_equal(costs, costs_copy) 102 | 103 | 104 | @pytest.mark.parametrize('solver', SOLVERS) 105 | def test_assign_non_integer(solver): 106 | costs = (1. / 9) * np.asfarray([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4]]) 107 | costs_copy = costs.copy() 108 | result = lap.linear_sum_assignment(costs, solver=solver) 109 | 110 | expected = np.array([[0, 1, 2], [0, 2, 1]]) 111 | np.testing.assert_equal(result, expected) 112 | np.testing.assert_equal(costs, costs_copy) 113 | 114 | 115 | @pytest.mark.parametrize('solver', SOLVERS) 116 | def test_assign_attractive_disallowed(solver): 117 | """Graph contains an attractive edge that cannot be used.""" 118 | costs = np.asfarray([[-10000, -1], [-1, np.nan]]) 119 | costs_copy = costs.copy() 120 | result = lap.linear_sum_assignment(costs, solver=solver) 121 | 122 | # The optimal solution is (0, 1), (1, 0) for a cost of -2. 123 | # Ensure that the algorithm does not choose the (0, 0) edge. 124 | # This would not be a perfect matching. 125 | expected = np.array([[0, 1], [1, 0]]) 126 | np.testing.assert_equal(result, expected) 127 | np.testing.assert_equal(costs, costs_copy) 128 | 129 | 130 | @pytest.mark.parametrize('solver', SOLVERS) 131 | def test_assign_attractive_broken_ring(solver): 132 | """Graph contains cheap broken ring and expensive unbroken ring.""" 133 | costs = np.asfarray([[np.nan, 1000, np.nan], [np.nan, 1, 1000], [1000, np.nan, 1]]) 134 | costs_copy = costs.copy() 135 | result = lap.linear_sum_assignment(costs, solver=solver) 136 | 137 | # Optimal solution is (0, 1), (1, 2), (2, 0) with cost 1000 + 1000 + 1000. 138 | # Solver might choose (0, 0), (1, 1), (2, 2) with cost inf + 1 + 1. 139 | expected = np.array([[0, 1, 2], [1, 2, 0]]) 140 | np.testing.assert_equal(result, expected) 141 | np.testing.assert_equal(costs, costs_copy) 142 | 143 | 144 | @pytest.mark.parametrize('solver', SOLVERS) 145 | def test_unbalanced_wide(solver): 146 | costs = np.asfarray([[6, 4, 1], [10, 8, 2]]) 147 | costs_copy = costs.copy() 148 | result = lap.linear_sum_assignment(costs, solver=solver) 149 | 150 | expected = np.array([[0, 1], [1, 2]]) 151 | np.testing.assert_equal(result, expected) 152 | np.testing.assert_equal(costs, costs_copy) 153 | 154 | 155 | @pytest.mark.parametrize('solver', SOLVERS) 156 | def test_unbalanced_tall(solver): 157 | costs = np.asfarray([[6, 10], [4, 8], [1, 2]]) 158 | costs_copy = costs.copy() 159 | result = lap.linear_sum_assignment(costs, solver=solver) 160 | 161 | expected = np.array([[1, 2], [0, 1]]) 162 | np.testing.assert_equal(result, expected) 163 | np.testing.assert_equal(costs, costs_copy) 164 | 165 | 166 | @pytest.mark.parametrize('solver', SOLVERS) 167 | def test_unbalanced_disallowed_wide(solver): 168 | costs = np.asfarray([[np.nan, 11, 8], [8, np.nan, 7]]) 169 | costs_copy = costs.copy() 170 | result = lap.linear_sum_assignment(costs, solver=solver) 171 | 172 | expected = np.array([[0, 1], [2, 0]]) 173 | np.testing.assert_equal(result, expected) 174 | np.testing.assert_equal(costs, costs_copy) 175 | 176 | 177 | @pytest.mark.parametrize('solver', SOLVERS) 178 | def test_unbalanced_disallowed_tall(solver): 179 | costs = np.asfarray([[np.nan, 9], [11, np.nan], [8, 7]]) 180 | costs_copy = costs.copy() 181 | result = lap.linear_sum_assignment(costs, solver=solver) 182 | 183 | expected = np.array([[0, 2], [1, 0]]) 184 | np.testing.assert_equal(result, expected) 185 | np.testing.assert_equal(costs, costs_copy) 186 | 187 | 188 | @pytest.mark.parametrize('solver', SOLVERS) 189 | def test_unbalanced_infeasible(solver): 190 | """Tests that minimum-cost solution with most edges is found.""" 191 | costs = np.asfarray([[np.nan, np.nan, 2], 192 | [np.nan, np.nan, 1], 193 | [np.nan, np.nan, 3], 194 | [8, 7, 4]]) 195 | costs_copy = costs.copy() 196 | result = lap.linear_sum_assignment(costs, solver=solver) 197 | 198 | # Optimal matching is (1, 2), (3, 1). 199 | expected = np.array([[1, 3], [2, 1]]) 200 | np.testing.assert_equal(result, expected) 201 | np.testing.assert_equal(costs, costs_copy) 202 | 203 | 204 | def test_change_solver(): 205 | """Tests effect of lap.set_default_solver.""" 206 | 207 | def mysolver(_): 208 | mysolver.called += 1 209 | return np.array([]), np.array([]) 210 | mysolver.called = 0 211 | 212 | costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]]) 213 | 214 | with lap.set_default_solver(mysolver): 215 | lap.linear_sum_assignment(costs) 216 | assert mysolver.called == 1 217 | lap.linear_sum_assignment(costs) 218 | assert mysolver.called == 1 219 | -------------------------------------------------------------------------------- /motmetrics/tests/test_mot.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests behavior of MOTAccumulator.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import numpy as np 15 | import pandas as pd 16 | import pytest 17 | 18 | import motmetrics as mm 19 | 20 | 21 | def test_events(): 22 | """Tests that expected events are created by MOTAccumulator.update().""" 23 | acc = mm.MOTAccumulator() 24 | 25 | # All FP 26 | acc.update([], [1, 2], [], frameid=0) 27 | # All miss 28 | acc.update([1, 2], [], [], frameid=1) 29 | # Match 30 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2) 31 | # Switch 32 | acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3) 33 | # Match. Better new match is available but should prefer history 34 | acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4) 35 | # No data 36 | acc.update([], [], [], frameid=5) 37 | 38 | expect = mm.MOTAccumulator.new_event_dataframe() 39 | expect.loc[(0, 0), :] = ['RAW', np.nan, np.nan, np.nan] 40 | expect.loc[(0, 1), :] = ['RAW', np.nan, 1, np.nan] 41 | expect.loc[(0, 2), :] = ['RAW', np.nan, 2, np.nan] 42 | expect.loc[(0, 3), :] = ['FP', np.nan, 1, np.nan] 43 | expect.loc[(0, 4), :] = ['FP', np.nan, 2, np.nan] 44 | 45 | expect.loc[(1, 0), :] = ['RAW', np.nan, np.nan, np.nan] 46 | expect.loc[(1, 1), :] = ['RAW', 1, np.nan, np.nan] 47 | expect.loc[(1, 2), :] = ['RAW', 2, np.nan, np.nan] 48 | expect.loc[(1, 3), :] = ['MISS', 1, np.nan, np.nan] 49 | expect.loc[(1, 4), :] = ['MISS', 2, np.nan, np.nan] 50 | 51 | expect.loc[(2, 0), :] = ['RAW', np.nan, np.nan, np.nan] 52 | expect.loc[(2, 1), :] = ['RAW', 1, 1, 1.0] 53 | expect.loc[(2, 2), :] = ['RAW', 1, 2, 0.5] 54 | expect.loc[(2, 3), :] = ['RAW', 2, 1, 0.3] 55 | expect.loc[(2, 4), :] = ['RAW', 2, 2, 1.0] 56 | expect.loc[(2, 5), :] = ['MATCH', 1, 2, 0.5] 57 | expect.loc[(2, 6), :] = ['MATCH', 2, 1, 0.3] 58 | 59 | expect.loc[(3, 0), :] = ['RAW', np.nan, np.nan, np.nan] 60 | expect.loc[(3, 1), :] = ['RAW', 1, 1, 0.2] 61 | expect.loc[(3, 2), :] = ['RAW', 2, 2, 0.1] 62 | expect.loc[(3, 3), :] = ['TRANSFER', 1, 1, 0.2] 63 | expect.loc[(3, 4), :] = ['SWITCH', 1, 1, 0.2] 64 | expect.loc[(3, 5), :] = ['TRANSFER', 2, 2, 0.1] 65 | expect.loc[(3, 6), :] = ['SWITCH', 2, 2, 0.1] 66 | 67 | expect.loc[(4, 0), :] = ['RAW', np.nan, np.nan, np.nan] 68 | expect.loc[(4, 1), :] = ['RAW', 1, 1, 5.] 69 | expect.loc[(4, 2), :] = ['RAW', 1, 2, 1.] 70 | expect.loc[(4, 3), :] = ['RAW', 2, 1, 1.] 71 | expect.loc[(4, 4), :] = ['RAW', 2, 2, 5.] 72 | expect.loc[(4, 5), :] = ['MATCH', 1, 1, 5.] 73 | expect.loc[(4, 6), :] = ['MATCH', 2, 2, 5.] 74 | 75 | expect.loc[(5, 0), :] = ['RAW', np.nan, np.nan, np.nan] 76 | 77 | pd.util.testing.assert_frame_equal(acc.events, expect) 78 | 79 | 80 | def test_max_switch_time(): 81 | """Tests max_switch_time option.""" 82 | acc = mm.MOTAccumulator(max_switch_time=1) 83 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b 84 | frameid = acc.update([1, 2], [1, 2], [[0.5, np.nan], [np.nan, 0.5]], frameid=2) # 1->b, 2->a 85 | 86 | df = acc.events.loc[frameid] 87 | assert ((df.Type == 'SWITCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all() 88 | 89 | acc = mm.MOTAccumulator(max_switch_time=1) 90 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b 91 | frameid = acc.update([1, 2], [1, 2], [[0.5, np.nan], [np.nan, 0.5]], frameid=5) # Later frame 1->b, 2->a 92 | 93 | df = acc.events.loc[frameid] 94 | assert ((df.Type == 'MATCH') | (df.Type == 'RAW') | (df.Type == 'TRANSFER')).all() 95 | 96 | 97 | def test_auto_id(): 98 | """Tests auto_id option.""" 99 | acc = mm.MOTAccumulator(auto_id=True) 100 | acc.update([1, 2, 3, 4], [], []) 101 | acc.update([1, 2, 3, 4], [], []) 102 | assert acc.events.index.levels[0][-1] == 1 103 | acc.update([1, 2, 3, 4], [], []) 104 | assert acc.events.index.levels[0][-1] == 2 105 | 106 | with pytest.raises(AssertionError): 107 | acc.update([1, 2, 3, 4], [], [], frameid=5) 108 | 109 | acc = mm.MOTAccumulator(auto_id=False) 110 | with pytest.raises(AssertionError): 111 | acc.update([1, 2, 3, 4], [], []) 112 | 113 | 114 | def test_merge_dataframes(): 115 | """Tests merge_event_dataframes().""" 116 | # pylint: disable=too-many-statements 117 | acc = mm.MOTAccumulator() 118 | 119 | acc.update([], [1, 2], [], frameid=0) 120 | acc.update([1, 2], [], [], frameid=1) 121 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2) 122 | acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3) 123 | 124 | r, mappings = mm.MOTAccumulator.merge_event_dataframes([acc.events, acc.events], return_mappings=True) 125 | 126 | expect = mm.MOTAccumulator.new_event_dataframe() 127 | 128 | expect.loc[(0, 0), :] = ['RAW', np.nan, np.nan, np.nan] 129 | expect.loc[(0, 1), :] = ['RAW', np.nan, mappings[0]['hid_map'][1], np.nan] 130 | expect.loc[(0, 2), :] = ['RAW', np.nan, mappings[0]['hid_map'][2], np.nan] 131 | expect.loc[(0, 3), :] = ['FP', np.nan, mappings[0]['hid_map'][1], np.nan] 132 | expect.loc[(0, 4), :] = ['FP', np.nan, mappings[0]['hid_map'][2], np.nan] 133 | 134 | expect.loc[(1, 0), :] = ['RAW', np.nan, np.nan, np.nan] 135 | expect.loc[(1, 1), :] = ['RAW', mappings[0]['oid_map'][1], np.nan, np.nan] 136 | expect.loc[(1, 2), :] = ['RAW', mappings[0]['oid_map'][2], np.nan, np.nan] 137 | expect.loc[(1, 3), :] = ['MISS', mappings[0]['oid_map'][1], np.nan, np.nan] 138 | expect.loc[(1, 4), :] = ['MISS', mappings[0]['oid_map'][2], np.nan, np.nan] 139 | 140 | expect.loc[(2, 0), :] = ['RAW', np.nan, np.nan, np.nan] 141 | expect.loc[(2, 1), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 1] 142 | expect.loc[(2, 2), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][2], 0.5] 143 | expect.loc[(2, 3), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][1], 0.3] 144 | expect.loc[(2, 4), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 1.0] 145 | expect.loc[(2, 5), :] = ['MATCH', mappings[0]['oid_map'][1], mappings[0]['hid_map'][2], 0.5] 146 | expect.loc[(2, 6), :] = ['MATCH', mappings[0]['oid_map'][2], mappings[0]['hid_map'][1], 0.3] 147 | 148 | expect.loc[(3, 0), :] = ['RAW', np.nan, np.nan, np.nan] 149 | expect.loc[(3, 1), :] = ['RAW', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2] 150 | expect.loc[(3, 2), :] = ['RAW', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1] 151 | expect.loc[(3, 3), :] = ['TRANSFER', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2] 152 | expect.loc[(3, 4), :] = ['SWITCH', mappings[0]['oid_map'][1], mappings[0]['hid_map'][1], 0.2] 153 | expect.loc[(3, 5), :] = ['TRANSFER', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1] 154 | expect.loc[(3, 6), :] = ['SWITCH', mappings[0]['oid_map'][2], mappings[0]['hid_map'][2], 0.1] 155 | 156 | # Merge duplication 157 | expect.loc[(4, 0), :] = ['RAW', np.nan, np.nan, np.nan] 158 | expect.loc[(4, 1), :] = ['RAW', np.nan, mappings[1]['hid_map'][1], np.nan] 159 | expect.loc[(4, 2), :] = ['RAW', np.nan, mappings[1]['hid_map'][2], np.nan] 160 | expect.loc[(4, 3), :] = ['FP', np.nan, mappings[1]['hid_map'][1], np.nan] 161 | expect.loc[(4, 4), :] = ['FP', np.nan, mappings[1]['hid_map'][2], np.nan] 162 | 163 | expect.loc[(5, 0), :] = ['RAW', np.nan, np.nan, np.nan] 164 | expect.loc[(5, 1), :] = ['RAW', mappings[1]['oid_map'][1], np.nan, np.nan] 165 | expect.loc[(5, 2), :] = ['RAW', mappings[1]['oid_map'][2], np.nan, np.nan] 166 | expect.loc[(5, 3), :] = ['MISS', mappings[1]['oid_map'][1], np.nan, np.nan] 167 | expect.loc[(5, 4), :] = ['MISS', mappings[1]['oid_map'][2], np.nan, np.nan] 168 | 169 | expect.loc[(6, 0), :] = ['RAW', np.nan, np.nan, np.nan] 170 | expect.loc[(6, 1), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 1] 171 | expect.loc[(6, 2), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][2], 0.5] 172 | expect.loc[(6, 3), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][1], 0.3] 173 | expect.loc[(6, 4), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 1.0] 174 | expect.loc[(6, 5), :] = ['MATCH', mappings[1]['oid_map'][1], mappings[1]['hid_map'][2], 0.5] 175 | expect.loc[(6, 6), :] = ['MATCH', mappings[1]['oid_map'][2], mappings[1]['hid_map'][1], 0.3] 176 | 177 | expect.loc[(7, 0), :] = ['RAW', np.nan, np.nan, np.nan] 178 | expect.loc[(7, 1), :] = ['RAW', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2] 179 | expect.loc[(7, 2), :] = ['RAW', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1] 180 | expect.loc[(7, 3), :] = ['TRANSFER', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2] 181 | expect.loc[(7, 4), :] = ['SWITCH', mappings[1]['oid_map'][1], mappings[1]['hid_map'][1], 0.2] 182 | expect.loc[(7, 5), :] = ['TRANSFER', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1] 183 | expect.loc[(7, 6), :] = ['SWITCH', mappings[1]['oid_map'][2], mappings[1]['hid_map'][2], 0.1] 184 | 185 | pd.util.testing.assert_frame_equal(r, expect) 186 | -------------------------------------------------------------------------------- /motmetrics/data/TUD-Campus/test.txt: -------------------------------------------------------------------------------- 1 | 1,3,113.84,274.5,57.307,130.05,-1,-1,-1,-1 2 | 1,6,273.05,203.83,77.366,175.56,-1,-1,-1,-1 3 | 1,10,416.68,205.54,91.04,206.59,-1,-1,-1,-1 4 | 1,13,175.02,195.54,60.972,138.36,-1,-1,-1,-1 5 | 2,3,116.37,265.2,62.858,142.64,-1,-1,-1,-1 6 | 2,6,267.86,202.71,77.704,176.33,-1,-1,-1,-1 7 | 2,10,423.95,203.42,91.88,208.5,-1,-1,-1,-1 8 | 2,13,177.14,202.51,58.209,132.09,-1,-1,-1,-1 9 | 3,3,118.93,255.89,68.408,155.24,-1,-1,-1,-1 10 | 3,6,262.73,201.65,78.033,177.08,-1,-1,-1,-1 11 | 3,10,431.14,201.32,92.719,210.4,-1,-1,-1,-1 12 | 3,13,179.21,209.5,55.445,125.82,-1,-1,-1,-1 13 | 4,3,121.53,246.57,73.959,167.83,-1,-1,-1,-1 14 | 4,6,257.67,200.61,78.354,177.81,-1,-1,-1,-1 15 | 4,10,438.16,199.26,93.559,212.31,-1,-1,-1,-1 16 | 4,13,181.25,216.56,52.681,119.55,-1,-1,-1,-1 17 | 5,3,124.22,237.24,79.51,180.43,-1,-1,-1,-1 18 | 5,6,252.68,199.54,78.667,178.52,-1,-1,-1,-1 19 | 5,10,444.94,197.29,94.398,214.21,-1,-1,-1,-1 20 | 5,13,183.24,223.72,49.917,113.27,-1,-1,-1,-1 21 | 6,3,127,227.96,85.061,193.02,-1,-1,-1,-1 22 | 6,6,247.74,198.46,78.972,179.21,-1,-1,-1,-1 23 | 6,10,451.36,195.47,95.238,216.12,-1,-1,-1,-1 24 | 6,13,185.16,230.95,47.154,107,-1,-1,-1,-1 25 | 7,3,129.86,218.75,90.612,205.62,-1,-1,-1,-1 26 | 7,6,242.86,197.44,79.267,179.88,-1,-1,-1,-1 27 | 7,10,457.4,193.8,96.077,218.02,-1,-1,-1,-1 28 | 7,13,187.05,238.21,44.39,100.73,-1,-1,-1,-1 29 | 8,3,132.77,209.65,96.163,218.21,-1,-1,-1,-1 30 | 8,6,237.99,196.55,79.555,180.53,-1,-1,-1,-1 31 | 8,10,463.14,192.21,96.916,219.93,-1,-1,-1,-1 32 | 9,3,135.64,200.65,101.71,230.81,-1,-1,-1,-1 33 | 9,6,233.04,195.92,79.834,181.16,-1,-1,-1,-1 34 | 9,10,468.66,190.65,97.756,221.83,-1,-1,-1,-1 35 | 10,3,138.45,191.79,107.27,243.4,-1,-1,-1,-1 36 | 10,6,227.9,195.56,80.105,181.78,-1,-1,-1,-1 37 | 10,10,474.09,189.09,98.595,223.74,-1,-1,-1,-1 38 | 11,3,141.22,183.1,112.82,256,-1,-1,-1,-1 39 | 11,6,222.43,195.44,80.367,182.37,-1,-1,-1,-1 40 | 11,10,479.5,187.49,99.435,225.64,-1,-1,-1,-1 41 | 12,3,143.96,174.55,118.37,268.6,-1,-1,-1,-1 42 | 12,6,216.54,195.56,80.621,182.95,-1,-1,-1,-1 43 | 12,10,484.99,185.85,100.27,227.55,-1,-1,-1,-1 44 | 13,3,146.68,166.1,123.92,281.19,-1,-1,-1,-1 45 | 13,6,210.1,195.93,80.866,183.51,-1,-1,-1,-1 46 | 13,10,490.67,184.15,101.11,229.45,-1,-1,-1,-1 47 | 14,6,203.1,196.5,81.104,184.04,-1,-1,-1,-1 48 | 14,10,496.59,182.38,101.95,231.36,-1,-1,-1,-1 49 | 15,6,195.62,197.15,81.332,184.56,-1,-1,-1,-1 50 | 15,10,502.81,180.54,102.79,233.26,-1,-1,-1,-1 51 | 16,6,187.79,197.8,81.553,185.06,-1,-1,-1,-1 52 | 16,7,276.17,205.24,60.444,137.16,-1,-1,-1,-1 53 | 16,10,509.37,178.64,103.63,235.16,-1,-1,-1,-1 54 | 17,6,179.76,198.42,81.764,185.54,-1,-1,-1,-1 55 | 17,7,282.02,209.51,58.423,132.58,-1,-1,-1,-1 56 | 17,10,516.23,176.69,104.47,237.07,-1,-1,-1,-1 57 | 18,6,171.65,199.03,81.968,186.01,-1,-1,-1,-1 58 | 18,7,287.93,213.81,56.403,127.99,-1,-1,-1,-1 59 | 18,10,523.34,174.73,105.31,238.97,-1,-1,-1,-1 60 | 19,6,163.55,199.67,82.163,186.45,-1,-1,-1,-1 61 | 19,7,293.99,218.11,54.382,123.4,-1,-1,-1,-1 62 | 19,10,530.57,172.78,106.15,240.88,-1,-1,-1,-1 63 | 20,6,155.46,200.39,82.35,186.87,-1,-1,-1,-1 64 | 20,7,300.27,222.37,52.361,118.82,-1,-1,-1,-1 65 | 21,6,147.3,201.23,82.528,187.28,-1,-1,-1,-1 66 | 21,7,306.86,226.57,50.34,114.23,-1,-1,-1,-1 67 | 22,6,139.04,202.19,82.698,187.66,-1,-1,-1,-1 68 | 22,7,313.74,230.72,48.32,109.65,-1,-1,-1,-1 69 | 23,6,130.69,203.28,82.859,188.03,-1,-1,-1,-1 70 | 23,7,320.84,234.83,46.299,105.06,-1,-1,-1,-1 71 | 24,6,122.3,204.52,83.012,188.38,-1,-1,-1,-1 72 | 24,7,328.08,238.93,44.278,100.48,-1,-1,-1,-1 73 | 24,11,224.23,208.03,71.57,162.41,-1,-1,-1,-1 74 | 25,6,113.93,205.85,83.157,188.71,-1,-1,-1,-1 75 | 25,7,335.33,243.04,42.257,95.892,-1,-1,-1,-1 76 | 25,11,230.36,214.02,68.455,155.34,-1,-1,-1,-1 77 | 26,4,119.05,191.06,80.289,182.2,-1,-1,-1,-1 78 | 26,7,342.57,247.15,40.236,91.306,-1,-1,-1,-1 79 | 26,9,-15.182,261.28,64.106,145.47,-1,-1,-1,-1 80 | 26,11,236.15,218.42,66.058,149.9,-1,-1,-1,-1 81 | 27,4,109.54,188.88,82.744,187.77,-1,-1,-1,-1 82 | 27,7,349.78,251.25,38.216,86.721,-1,-1,-1,-1 83 | 27,9,-16.51,246.39,71.474,162.19,-1,-1,-1,-1 84 | 27,11,241.64,221.42,64.306,145.92,-1,-1,-1,-1 85 | 28,4,100.03,186.72,85.2,193.34,-1,-1,-1,-1 86 | 28,9,-17.899,231.48,78.843,178.91,-1,-1,-1,-1 87 | 28,11,246.79,223.23,63.128,143.25,-1,-1,-1,-1 88 | 29,4,90.482,184.67,87.656,198.91,-1,-1,-1,-1 89 | 29,9,-19.351,216.58,86.212,195.63,-1,-1,-1,-1 90 | 29,11,251.59,224.01,62.458,141.73,-1,-1,-1,-1 91 | 30,4,80.854,182.84,90.111,204.49,-1,-1,-1,-1 92 | 30,9,-20.845,201.69,93.58,212.35,-1,-1,-1,-1 93 | 30,11,255.99,223.9,62.231,141.22,-1,-1,-1,-1 94 | 31,4,71.177,181.25,92.567,210.06,-1,-1,-1,-1 95 | 31,9,-22.364,186.8,100.95,229.07,-1,-1,-1,-1 96 | 31,11,260.09,223.01,62.386,141.57,-1,-1,-1,-1 97 | 32,4,61.563,179.93,95.023,215.63,-1,-1,-1,-1 98 | 32,11,263.94,221.53,62.864,142.65,-1,-1,-1,-1 99 | 33,4,52.124,178.89,97.479,221.2,-1,-1,-1,-1 100 | 33,8,324.5,165.22,108.85,247.01,-1,-1,-1,-1 101 | 33,11,267.58,219.61,63.611,144.35,-1,-1,-1,-1 102 | 34,4,42.892,178.09,99.934,226.78,-1,-1,-1,-1 103 | 34,8,336.28,176.03,105.56,239.53,-1,-1,-1,-1 104 | 34,11,271.09,217.43,64.576,146.54,-1,-1,-1,-1 105 | 35,4,33.841,177.49,102.39,232.35,-1,-1,-1,-1 106 | 35,8,348.07,186.95,102.26,232.06,-1,-1,-1,-1 107 | 35,11,274.48,215.13,65.708,149.11,-1,-1,-1,-1 108 | 36,4,24.941,177.01,104.85,237.92,-1,-1,-1,-1 109 | 36,8,359.8,198.05,98.967,224.58,-1,-1,-1,-1 110 | 36,11,277.84,212.8,66.965,151.96,-1,-1,-1,-1 111 | 37,4,16.117,176.59,107.3,243.49,-1,-1,-1,-1 112 | 37,8,371.47,209.36,95.673,217.1,-1,-1,-1,-1 113 | 37,11,281.24,210.52,68.301,154.99,-1,-1,-1,-1 114 | 38,2,52.423,232.95,80.36,182.36,-1,-1,-1,-1 115 | 38,8,383.09,220.95,92.379,209.63,-1,-1,-1,-1 116 | 38,11,284.69,208.3,69.68,158.12,-1,-1,-1,-1 117 | 39,2,56.427,217.08,87.228,197.94,-1,-1,-1,-1 118 | 39,8,394.66,232.78,89.085,202.15,-1,-1,-1,-1 119 | 39,11,288.21,206.16,71.063,161.26,-1,-1,-1,-1 120 | 40,2,61.088,203.79,92.98,210.99,-1,-1,-1,-1 121 | 40,8,406.23,244.74,85.791,194.68,-1,-1,-1,-1 122 | 40,11,291.81,204.09,72.419,164.34,-1,-1,-1,-1 123 | 41,2,66.422,192.94,97.686,221.67,-1,-1,-1,-1 124 | 41,5,394.42,197.9,97.849,222.04,-1,-1,-1,-1 125 | 41,11,295.46,202.07,73.718,167.28,-1,-1,-1,-1 126 | 42,2,72.446,184.34,101.42,230.14,-1,-1,-1,-1 127 | 42,5,400.86,193.91,102.46,232.51,-1,-1,-1,-1 128 | 42,11,299.14,200,74.933,170.04,-1,-1,-1,-1 129 | 43,2,79.136,177.84,104.25,236.56,-1,-1,-1,-1 130 | 43,5,407.31,190.07,107.08,242.98,-1,-1,-1,-1 131 | 43,11,302.83,197.86,76.041,172.55,-1,-1,-1,-1 132 | 44,2,86.433,173.27,106.24,241.09,-1,-1,-1,-1 133 | 44,5,413.79,186.43,111.69,253.45,-1,-1,-1,-1 134 | 44,11,306.54,195.65,77.02,174.78,-1,-1,-1,-1 135 | 45,2,94.236,170.49,107.48,243.89,-1,-1,-1,-1 136 | 45,5,420.34,182.99,116.3,263.91,-1,-1,-1,-1 137 | 45,11,310.3,193.42,77.854,176.67,-1,-1,-1,-1 138 | 46,2,102.52,169.29,108.02,245.13,-1,-1,-1,-1 139 | 46,5,426.95,179.62,120.92,274.38,-1,-1,-1,-1 140 | 46,11,314.13,191.28,78.529,178.2,-1,-1,-1,-1 141 | 47,2,111.21,169.48,107.95,244.96,-1,-1,-1,-1 142 | 47,5,433.61,176.29,125.53,284.85,-1,-1,-1,-1 143 | 47,11,318.04,189.41,79.034,179.34,-1,-1,-1,-1 144 | 48,2,120.2,170.97,107.33,243.55,-1,-1,-1,-1 145 | 48,5,442.57,183.44,125.53,284.85,-1,-1,-1,-1 146 | 48,11,322.05,187.93,79.36,180.08,-1,-1,-1,-1 147 | 49,1,459.32,237.96,50.475,114.54,-1,-1,-1,-1 148 | 49,2,129.39,173.6,106.23,241.06,-1,-1,-1,-1 149 | 49,11,326.2,186.99,79.503,180.41,-1,-1,-1,-1 150 | 50,1,462.86,233.83,51.899,117.77,-1,-1,-1,-1 151 | 50,2,138.73,177.21,104.73,237.65,-1,-1,-1,-1 152 | 50,11,330.53,186.6,79.461,180.31,-1,-1,-1,-1 153 | 51,1,466.5,230.01,53.199,120.72,-1,-1,-1,-1 154 | 51,2,148.19,181.54,102.89,233.48,-1,-1,-1,-1 155 | 51,11,335.03,186.77,79.236,179.8,-1,-1,-1,-1 156 | 52,1,470.24,226.53,54.374,123.39,-1,-1,-1,-1 157 | 52,2,157.75,186.54,100.79,228.71,-1,-1,-1,-1 158 | 52,11,339.68,187.47,78.833,178.89,-1,-1,-1,-1 159 | 53,1,474.11,223.4,55.425,125.78,-1,-1,-1,-1 160 | 53,2,167.48,192.06,98.496,223.51,-1,-1,-1,-1 161 | 53,11,344.45,188.69,78.259,177.59,-1,-1,-1,-1 162 | 54,1,478.14,220.61,56.352,127.88,-1,-1,-1,-1 163 | 54,2,177.42,197.94,96.081,218.03,-1,-1,-1,-1 164 | 54,11,349.28,190.37,77.525,175.92,-1,-1,-1,-1 165 | 55,1,482.39,218.19,57.154,129.7,-1,-1,-1,-1 166 | 55,2,187.55,204.01,93.617,212.44,-1,-1,-1,-1 167 | 55,11,354.13,192.44,76.646,173.93,-1,-1,-1,-1 168 | 55,12,533.9,309.3,68.825,156.18,-1,-1,-1,-1 169 | 56,1,486.87,216.16,57.832,131.24,-1,-1,-1,-1 170 | 56,2,197.87,210.13,91.174,206.9,-1,-1,-1,-1 171 | 56,11,358.95,194.8,75.638,171.64,-1,-1,-1,-1 172 | 56,12,533.51,279.28,82.049,186.19,-1,-1,-1,-1 173 | 57,1,491.6,214.54,58.386,132.49,-1,-1,-1,-1 174 | 57,2,208.33,216.16,88.824,201.56,-1,-1,-1,-1 175 | 57,11,363.73,197.38,74.522,169.1,-1,-1,-1,-1 176 | 57,12,533.16,249.29,95.273,216.2,-1,-1,-1,-1 177 | 58,1,496.55,213.28,58.815,133.47,-1,-1,-1,-1 178 | 58,2,218.83,221.94,86.637,196.6,-1,-1,-1,-1 179 | 58,11,368.43,200.11,73.32,166.38,-1,-1,-1,-1 180 | 58,12,532.85,219.33,108.5,246.2,-1,-1,-1,-1 181 | 59,1,501.69,212.34,59.119,134.16,-1,-1,-1,-1 182 | 59,2,229.42,227.21,84.684,192.17,-1,-1,-1,-1 183 | 59,11,373.11,202.9,72.061,163.52,-1,-1,-1,-1 184 | 59,12,532.56,189.41,121.72,276.21,-1,-1,-1,-1 185 | 60,1,506.97,211.69,59.3,134.57,-1,-1,-1,-1 186 | 60,2,240.03,231.7,83.037,188.43,-1,-1,-1,-1 187 | 60,11,377.81,205.68,70.773,160.6,-1,-1,-1,-1 188 | 60,12,536.94,180.92,125.53,284.85,-1,-1,-1,-1 189 | 61,1,512.37,211.32,59.355,134.69,-1,-1,-1,-1 190 | 61,2,250.51,235.22,81.767,185.55,-1,-1,-1,-1 191 | 61,11,382.61,208.36,69.489,157.68,-1,-1,-1,-1 192 | 61,12,543.18,181.11,125.53,284.85,-1,-1,-1,-1 193 | 62,1,517.89,211.16,59.287,134.54,-1,-1,-1,-1 194 | 62,2,260.69,237.62,80.944,183.68,-1,-1,-1,-1 195 | 62,11,387.51,210.9,68.246,154.86,-1,-1,-1,-1 196 | 63,1,523.53,211.17,59.094,134.1,-1,-1,-1,-1 197 | 63,2,270.48,238.63,80.641,182.99,-1,-1,-1,-1 198 | 63,11,392.47,213.29,67.081,152.22,-1,-1,-1,-1 199 | 64,1,529.29,211.36,58.776,133.38,-1,-1,-1,-1 200 | 64,2,279.79,238.08,80.928,183.64,-1,-1,-1,-1 201 | 64,11,397.5,215.45,66.039,149.86,-1,-1,-1,-1 202 | 65,1,535.17,211.62,58.334,132.37,-1,-1,-1,-1 203 | 65,2,288.52,235.8,81.876,185.79,-1,-1,-1,-1 204 | 65,11,402.55,217.33,65.163,147.87,-1,-1,-1,-1 205 | 66,1,541.16,211.9,57.768,131.09,-1,-1,-1,-1 206 | 66,2,296.67,231.74,83.556,189.61,-1,-1,-1,-1 207 | 66,11,407.6,218.87,64.503,146.37,-1,-1,-1,-1 208 | 67,1,547.25,212.24,57.077,129.52,-1,-1,-1,-1 209 | 67,2,304.28,225.68,86.04,195.24,-1,-1,-1,-1 210 | 67,11,412.69,220,64.11,145.48,-1,-1,-1,-1 211 | 68,1,553.44,212.71,56.262,127.67,-1,-1,-1,-1 212 | 68,2,311.34,217.45,89.398,202.86,-1,-1,-1,-1 213 | 68,11,417.76,220.6,64.039,145.32,-1,-1,-1,-1 214 | 69,1,559.73,213.37,55.323,125.54,-1,-1,-1,-1 215 | 69,2,317.88,206.93,93.702,212.63,-1,-1,-1,-1 216 | 69,11,422.74,220.5,64.348,146.02,-1,-1,-1,-1 217 | 70,1,566.12,214.27,54.259,123.13,-1,-1,-1,-1 218 | 70,2,323.85,194.07,99.022,224.7,-1,-1,-1,-1 219 | 70,11,427.58,219.49,65.097,147.72,-1,-1,-1,-1 220 | 71,1,572.59,215.45,53.07,120.43,-1,-1,-1,-1 221 | 71,2,329.22,178.75,105.43,239.25,-1,-1,-1,-1 222 | 71,11,432.2,217.39,66.352,150.57,-1,-1,-1,-1 223 | -------------------------------------------------------------------------------- /motmetrics/data/TUD-Campus/gt.txt: -------------------------------------------------------------------------------- 1 | 1,1,399,182,121,229,1,-1,-1,-1 2 | 1,2,282,201,92,184,1,-1,-1,-1 3 | 1,3,63,153,82,288,1,-1,-1,-1 4 | 1,4,192,206,62,137,1,-1,-1,-1 5 | 1,5,125,209,74,157,1,-1,-1,-1 6 | 1,6,162,208,55,145,1,-1,-1,-1 7 | 2,1,399,181,139,235,1,-1,-1,-1 8 | 2,2,269,202,87,182,1,-1,-1,-1 9 | 2,3,71,151,100,284,1,-1,-1,-1 10 | 2,4,200,206,55,137,1,-1,-1,-1 11 | 2,5,127,210,77,157,1,-1,-1,-1 12 | 2,6,157,206,71,143,1,-1,-1,-1 13 | 3,1,419,182,106,227,1,-1,-1,-1 14 | 3,2,271,196,76,190,1,-1,-1,-1 15 | 3,3,70,155,111,286,1,-1,-1,-1 16 | 3,4,209,204,47,139,1,-1,-1,-1 17 | 3,5,136,206,64,160,1,-1,-1,-1 18 | 3,6,162,204,71,142,1,-1,-1,-1 19 | 4,1,428,181,111,237,1,-1,-1,-1 20 | 4,2,262,196,76,185,1,-1,-1,-1 21 | 4,3,80,160,106,280,1,-1,-1,-1 22 | 4,4,218,206,41,138,1,-1,-1,-1 23 | 4,5,131,208,75,154,1,-1,-1,-1 24 | 4,6,164,212,73,131,1,-1,-1,-1 25 | 5,1,439,179,95,238,1,-1,-1,-1 26 | 5,2,264,197,66,187,1,-1,-1,-1 27 | 5,3,84,165,104,267,1,-1,-1,-1 28 | 5,4,227,208,39,139,1,-1,-1,-1 29 | 5,5,136,208,74.364,153.95,1,-1,-1,-1 30 | 5,6,180,211,53,139,1,-1,-1,-1 31 | 6,1,454,179,87,238,1,-1,-1,-1 32 | 6,2,251,194,68,187,1,-1,-1,-1 33 | 6,3,89,164,115,270,1,-1,-1,-1 34 | 6,4,228,208,47,136,1,-1,-1,-1 35 | 6,5,141,209,73.727,153.91,1,-1,-1,-1 36 | 6,6,183,214,53,136,1,-1,-1,-1 37 | 7,1,453,177,81,239,1,-1,-1,-1 38 | 7,2,245,190,69,196,1,-1,-1,-1 39 | 7,3,103,165,110,272,1,-1,-1,-1 40 | 7,4,234,208,48,135.5,1,-1,-1,-1 41 | 7,5,146,209,73.091,153.86,1,-1,-1,-1 42 | 7,6,184,211,56,145,1,-1,-1,-1 43 | 8,1,471,178,76,241,1,-1,-1,-1 44 | 8,2,236,188,70,197,1,-1,-1,-1 45 | 8,3,117,165,101,276,1,-1,-1,-1 46 | 8,4,239,208,49,135,1,-1,-1,-1 47 | 8,5,151,209,72.454,153.82,1,-1,-1,-1 48 | 8,6,190,211,55,144,1,-1,-1,-1 49 | 9,1,464,173,101,244,1,-1,-1,-1 50 | 9,2,232,190,74,195,1,-1,-1,-1 51 | 9,3,125,158,113,283,1,-1,-1,-1 52 | 9,4,245,209,50,134.5,1,-1,-1,-1 53 | 9,5,156,209,71.818,153.77,1,-1,-1,-1 54 | 9,6,193,214,56,138,1,-1,-1,-1 55 | 10,1,479,168,81,251,1,-1,-1,-1 56 | 10,2,224,190,78,194,1,-1,-1,-1 57 | 10,3,134,159,97,283,1,-1,-1,-1 58 | 10,4,251,209,51,134,1,-1,-1,-1 59 | 10,5,161,210,71.182,153.73,1,-1,-1,-1 60 | 11,1,483,169,88,250,1,-1,-1,-1 61 | 11,2,220,194,77,191,1,-1,-1,-1 62 | 11,3,139,154,92,286,1,-1,-1,-1 63 | 11,4,256,209,52,133.5,1,-1,-1,-1 64 | 11,5,166,210,70.546,153.68,1,-1,-1,-1 65 | 12,1,497,167,99,249,1,-1,-1,-1 66 | 12,2,210,195,70,185,1,-1,-1,-1 67 | 12,3,139,155,100,285,1,-1,-1,-1 68 | 12,4,262,209,53,133,1,-1,-1,-1 69 | 12,5,171,210,69.909,153.64,1,-1,-1,-1 70 | 13,1,502,172,100,246,1,-1,-1,-1 71 | 13,2,210,195,73,185,1,-1,-1,-1 72 | 13,3,160,151,90,289,1,-1,-1,-1 73 | 13,4,264,209,55,129,1,-1,-1,-1 74 | 13,5,176,210,69.273,153.59,1,-1,-1,-1 75 | 14,1,499,172,108,241,1,-1,-1,-1 76 | 14,2,203,194,72.2,187.8,1,-1,-1,-1 77 | 14,3,163,149,88,293,1,-1,-1,-1 78 | 14,4,261,208,58,132,1,-1,-1,-1 79 | 14,5,181,211,68.636,153.55,1,-1,-1,-1 80 | 15,1,506,178,109,239,1,-1,-1,-1 81 | 15,2,196,194,71.4,190.6,1,-1,-1,-1 82 | 15,3,179,149,91,291,1,-1,-1,-1 83 | 15,4,268,206,65,149,1,-1,-1,-1 84 | 15,5,186,211,68,153.5,1,-1,-1,-1 85 | 16,1,514,177,117,239,1,-1,-1,-1 86 | 16,2,190,193,70.6,193.4,1,-1,-1,-1 87 | 16,3,182,150,92,292,1,-1,-1,-1 88 | 16,4,279,205,50,139,1,-1,-1,-1 89 | 16,5,191,211,67.364,153.45,1,-1,-1,-1 90 | 17,1,520,176,114,247,1,-1,-1,-1 91 | 17,2,183,193,69.8,196.2,1,-1,-1,-1 92 | 17,3,200,148,93,296,1,-1,-1,-1 93 | 17,4,287,201,44,148,1,-1,-1,-1 94 | 17,5,196,212,66.727,153.41,1,-1,-1,-1 95 | 18,1,522,165,111,263,1,-1,-1,-1 96 | 18,2,176,192,69,199,1,-1,-1,-1 97 | 18,3,196,149,111,299,1,-1,-1,-1 98 | 18,4,293,208,49,139,1,-1,-1,-1 99 | 18,5,201,212,66.091,153.36,1,-1,-1,-1 100 | 19,1,534,168,103,253,1,-1,-1,-1 101 | 19,2,174,185,68,199,1,-1,-1,-1 102 | 19,3,206,157,104,287,1,-1,-1,-1 103 | 19,4,296,213,57,132,1,-1,-1,-1 104 | 19,5,206,212,65.454,153.32,1,-1,-1,-1 105 | 20,1,547,182,88,240,1,-1,-1,-1 106 | 20,2,165,187,62,199,1,-1,-1,-1 107 | 20,3,204,159,118,285,1,-1,-1,-1 108 | 20,4,296,205,60,137,1,-1,-1,-1 109 | 20,5,211,212,64.818,153.27,1,-1,-1,-1 110 | 21,1,565,176,74,247,1,-1,-1,-1 111 | 21,2,159,194,60,191,1,-1,-1,-1 112 | 21,3,215,162,122,282,1,-1,-1,-1 113 | 21,4,301,209,57,135,1,-1,-1,-1 114 | 21,5,216,213,64.182,153.23,1,-1,-1,-1 115 | 22,1,575,170,87,255,1,-1,-1,-1 116 | 22,2,150,188,68,200,1,-1,-1,-1 117 | 22,3,222,163,108,286,1,-1,-1,-1 118 | 22,4,307,208,61,140,1,-1,-1,-1 119 | 22,5,221,213,63.545,153.18,1,-1,-1,-1 120 | 23,1,582,168,81,262,1,-1,-1,-1 121 | 23,2,139,186,69,199,1,-1,-1,-1 122 | 23,3,219,164,119,282,1,-1,-1,-1 123 | 23,4,307,205,68,144,1,-1,-1,-1 124 | 23,5,226,213,62.909,153.14,1,-1,-1,-1 125 | 24,1,585,165,94,269,1,-1,-1,-1 126 | 24,2,131,188,75,200,1,-1,-1,-1 127 | 24,3,243,162,120,289,1,-1,-1,-1 128 | 24,4,310,205,71,142,1,-1,-1,-1 129 | 24,5,231,213,62.273,153.09,1,-1,-1,-1 130 | 24,7,-28,183,76,235,1,-1,-1,-1 131 | 25,2,121,191,87,189,1,-1,-1,-1 132 | 25,3,254,165,97,281,1,-1,-1,-1 133 | 25,4,321,211,55,133,1,-1,-1,-1 134 | 25,5,236,214,61.636,153.05,1,-1,-1,-1 135 | 25,7,-15,179,63,240,1,-1,-1,-1 136 | 26,2,113,190,79,195,1,-1,-1,-1 137 | 26,3,259,155,97,294,1,-1,-1,-1 138 | 26,4,322,208,58,136,1,-1,-1,-1 139 | 26,5,241,214,61,153,1,-1,-1,-1 140 | 26,7,-20,180,83,235,1,-1,-1,-1 141 | 27,2,109,194,88,192,1,-1,-1,-1 142 | 27,3,274,158,88,296,1,-1,-1,-1 143 | 27,4,328,208,57.739,136.26,1,-1,-1,-1 144 | 27,5,242,222,74,150,1,-1,-1,-1 145 | 27,7,-30,182,91,233,1,-1,-1,-1 146 | 28,2,99,196,96,193,1,-1,-1,-1 147 | 28,3,285,153,90,295,1,-1,-1,-1 148 | 28,4,333,208,57.478,136.52,1,-1,-1,-1 149 | 28,5,257,224,55,147,1,-1,-1,-1 150 | 28,7,-21,177,82,236,1,-1,-1,-1 151 | 29,2,88,194,93,188,1,-1,-1,-1 152 | 29,3,287,162,106,283,1,-1,-1,-1 153 | 29,4,339,208,57.217,136.78,1,-1,-1,-1 154 | 29,5,261,218,60,154,1,-1,-1,-1 155 | 29,7,-10,173,74,239.5,1,-1,-1,-1 156 | 30,2,88,188,84,193,1,-1,-1,-1 157 | 30,3,307,157,93,292,1,-1,-1,-1 158 | 30,4,344,209,56.956,137.04,1,-1,-1,-1 159 | 30,5,259,215,65,153,1,-1,-1,-1 160 | 30,7,2,168,66,243,1,-1,-1,-1 161 | 31,2,90,201,84,184,1,-1,-1,-1 162 | 31,3,319,162,95,286,1,-1,-1,-1 163 | 31,4,350,209,56.696,137.3,1,-1,-1,-1 164 | 31,5,264,208,55,162,1,-1,-1,-1 165 | 31,7,3,176,82,235,1,-1,-1,-1 166 | 32,2,84,181,81,205,1,-1,-1,-1 167 | 32,3,319,157,112,287,1,-1,-1,-1 168 | 32,4,355,209,56.435,137.57,1,-1,-1,-1 169 | 32,5,270,215,61,159,1,-1,-1,-1 170 | 32,7,1,174,93,240,1,-1,-1,-1 171 | 33,2,72,188,85,200,1,-1,-1,-1 172 | 33,3,322,162,115,283,1,-1,-1,-1 173 | 33,4,361,209,56.174,137.83,1,-1,-1,-1 174 | 33,5,277,214,52,158,1,-1,-1,-1 175 | 33,7,8,187,100,224,1,-1,-1,-1 176 | 34,2,70,181,74,196,1,-1,-1,-1 177 | 34,3,324,162,122,289,1,-1,-1,-1 178 | 34,4,367,209,55.913,138.09,1,-1,-1,-1 179 | 34,5,278,215,57,155,1,-1,-1,-1 180 | 34,7,22,181,96,227,1,-1,-1,-1 181 | 35,2,62,182,75.2,197,1,-1,-1,-1 182 | 35,3,338,154,121,295,1,-1,-1,-1 183 | 35,4,372,209,55.652,138.35,1,-1,-1,-1 184 | 35,5,288,219,54,151,1,-1,-1,-1 185 | 35,7,33,182,90,235,1,-1,-1,-1 186 | 36,2,54,184,76.4,198,1,-1,-1,-1 187 | 36,3,352,160,111,291,1,-1,-1,-1 188 | 36,4,378,209,55.391,138.61,1,-1,-1,-1 189 | 36,5,299,218,49,153,1,-1,-1,-1 190 | 36,7,31,188,103,221,1,-1,-1,-1 191 | 37,2,47,185,77.6,199,1,-1,-1,-1 192 | 37,3,361,167,106,282,1,-1,-1,-1 193 | 37,4,383,209,55.13,138.87,1,-1,-1,-1 194 | 37,5,301,222,55,153,1,-1,-1,-1 195 | 37,7,43,178,93,238,1,-1,-1,-1 196 | 38,2,39,187,78.8,200,1,-1,-1,-1 197 | 38,3,371,167,107,286,1,-1,-1,-1 198 | 38,4,389,210,54.87,139.13,1,-1,-1,-1 199 | 38,5,303,217,60,157,1,-1,-1,-1 200 | 38,7,49,182,99,230,1,-1,-1,-1 201 | 39,2,31,188,80,201,1,-1,-1,-1 202 | 39,3,385,169,91,281,1,-1,-1,-1 203 | 39,4,394,210,54.609,139.39,1,-1,-1,-1 204 | 39,5,305,213,61,155,1,-1,-1,-1 205 | 39,7,59,178,81,234,1,-1,-1,-1 206 | 40,2,21,179,86,212,1,-1,-1,-1 207 | 40,3,386,155,108,300,1,-1,-1,-1 208 | 40,4,400,210,54.348,139.65,1,-1,-1,-1 209 | 40,5,307,217,64,153,1,-1,-1,-1 210 | 40,7,59,174,92,237,1,-1,-1,-1 211 | 41,2,12,183,83,208,1,-1,-1,-1 212 | 41,3,400,158,96,296,1,-1,-1,-1 213 | 41,4,405,210,54.087,139.91,1,-1,-1,-1 214 | 41,5,314,219,63,155,1,-1,-1,-1 215 | 41,7,71,176,79,235,1,-1,-1,-1 216 | 42,2,10,181,85,207,1,-1,-1,-1 217 | 42,3,404,157,107,300,1,-1,-1,-1 218 | 42,4,411,210,53.826,140.17,1,-1,-1,-1 219 | 42,5,319,220,62,156,1,-1,-1,-1 220 | 42,7,77,177,91,239,1,-1,-1,-1 221 | 43,2,2,186,84,202,1,-1,-1,-1 222 | 43,3,426,159,97,296,1,-1,-1,-1 223 | 43,4,417,210,53.565,140.43,1,-1,-1,-1 224 | 43,5,321,214,65,157,1,-1,-1,-1 225 | 43,7,97,172,80,249,1,-1,-1,-1 226 | 44,2,-3,186,79,209,1,-1,-1,-1 227 | 44,3,430,154,98,303,1,-1,-1,-1 228 | 44,4,422,210,53.304,140.7,1,-1,-1,-1 229 | 44,5,325,214,66,158,1,-1,-1,-1 230 | 44,7,102,171,86,246,1,-1,-1,-1 231 | 45,2,-8,186,74,216,1,-1,-1,-1 232 | 45,3,436,153,110,307,1,-1,-1,-1 233 | 45,4,428,210,53.044,140.96,1,-1,-1,-1 234 | 45,5,329,217,66,153,1,-1,-1,-1 235 | 45,7,97,174,103,240,1,-1,-1,-1 236 | 46,2,-14,186,65,220,1,-1,-1,-1 237 | 46,3,439,158,126,304,1,-1,-1,-1 238 | 46,4,433,211,52.783,141.22,1,-1,-1,-1 239 | 46,5,340,219,54,148,1,-1,-1,-1 240 | 46,7,108,178,114,238,1,-1,-1,-1 241 | 47,2,-24,182,69,221,1,-1,-1,-1 242 | 47,3,449,164,119,294,1,-1,-1,-1 243 | 47,4,439,211,52.522,141.48,1,-1,-1,-1 244 | 47,5,334,211,64,159,1,-1,-1,-1 245 | 47,7,121,177,101,231,1,-1,-1,-1 246 | 47,8,312,204,63,155,1,-1,-1,-1 247 | 48,2,-28,176,76,227,1,-1,-1,-1 248 | 48,3,460,162,125,295,1,-1,-1,-1 249 | 48,4,444,211,52.261,141.74,1,-1,-1,-1 250 | 48,5,342,215,58,150,1,-1,-1,-1 251 | 48,7,127,185,100,231,1,-1,-1,-1 252 | 48,8,318,198,57,161,1,-1,-1,-1 253 | 49,3,478,164,102,291,1,-1,-1,-1 254 | 49,4,450,211,52,142,1,-1,-1,-1 255 | 49,5,345,215,60,157,1,-1,-1,-1 256 | 49,7,132,182,88,236,1,-1,-1,-1 257 | 49,8,312,193,82,171,1,-1,-1,-1 258 | 50,3,481,157,108,300,1,-1,-1,-1 259 | 50,4,450,209,56,142,1,-1,-1,-1 260 | 50,5,356,214,52,154,1,-1,-1,-1 261 | 50,7,140,183,94,235,1,-1,-1,-1 262 | 50,8,328,199,65,162,1,-1,-1,-1 263 | 51,3,494,159,100,303,1,-1,-1,-1 264 | 51,4,455,206,56,143,1,-1,-1,-1 265 | 51,5,352,209,64,165,1,-1,-1,-1 266 | 51,7,154,181,89,238,1,-1,-1,-1 267 | 51,8,328,199,69,165,1,-1,-1,-1 268 | 52,3,497,158,104,301,1,-1,-1,-1 269 | 52,4,460,209,53,139,1,-1,-1,-1 270 | 52,5,358,215,65,150,1,-1,-1,-1 271 | 52,7,165,181,81,238,1,-1,-1,-1 272 | 52,8,330,199,67,166,1,-1,-1,-1 273 | 53,3,505,150,103,307,1,-1,-1,-1 274 | 53,4,467,213,60,140,1,-1,-1,-1 275 | 53,5,365,214,65,158,1,-1,-1,-1 276 | 53,7,173,178,79,241,1,-1,-1,-1 277 | 53,8,333,195,73,169,1,-1,-1,-1 278 | 54,3,508,148,100,310,1,-1,-1,-1 279 | 54,4,473,217,61,131,1,-1,-1,-1 280 | 54,5,365,214,70,158,1,-1,-1,-1 281 | 54,7,185,176,81,249,1,-1,-1,-1 282 | 54,8,342,199,67,166,1,-1,-1,-1 283 | 55,3,514,151,105,306,1,-1,-1,-1 284 | 55,4,482,215,63,130,1,-1,-1,-1 285 | 55,5,367,217,75,151,1,-1,-1,-1 286 | 55,7,196,177,75,242,1,-1,-1,-1 287 | 55,8,349,196,63,162,1,-1,-1,-1 288 | 56,3,529,151,99,308,1,-1,-1,-1 289 | 56,4,482,209,64,144,1,-1,-1,-1 290 | 56,5,379,219,62,151,1,-1,-1,-1 291 | 56,7,201,174,88,240,1,-1,-1,-1 292 | 56,8,352,193,63,173,1,-1,-1,-1 293 | 57,3,538,150,102,310,1,-1,-1,-1 294 | 57,4,486,214,61,131,1,-1,-1,-1 295 | 57,5,382,223,59,151,1,-1,-1,-1 296 | 57,7,208,173,94,249,1,-1,-1,-1 297 | 57,8,366,193,53,172,1,-1,-1,-1 298 | 58,3,539,157,101,305,1,-1,-1,-1 299 | 58,4,491,211,65,140,1,-1,-1,-1 300 | 58,5,386,218,56,152,1,-1,-1,-1 301 | 58,7,213,182,102,239,1,-1,-1,-1 302 | 58,8,366,196,50,175,1,-1,-1,-1 303 | 59,3,553,152,113,304,1,-1,-1,-1 304 | 59,4,499,210,61,142,1,-1,-1,-1 305 | 59,5,393,218,53,152,1,-1,-1,-1 306 | 59,7,223,182,103,237,1,-1,-1,-1 307 | 59,8,371,198,53,166,1,-1,-1,-1 308 | 60,3,553,157,123,300,1,-1,-1,-1 309 | 60,4,506,215,59,134,1,-1,-1,-1 310 | 60,5,397,215,57,152,1,-1,-1,-1 311 | 60,7,231,187,107,236,1,-1,-1,-1 312 | 60,8,372,199,58,163,1,-1,-1,-1 313 | 61,3,565,160,113,308,1,-1,-1,-1 314 | 61,4,517,217,51,131,1,-1,-1,-1 315 | 61,5,402,213,57,154,1,-1,-1,-1 316 | 61,7,236,187,106,235,1,-1,-1,-1 317 | 61,8,372,199,59,169,1,-1,-1,-1 318 | 62,3,572,167,110,295,1,-1,-1,-1 319 | 62,4,514,213,59,138,1,-1,-1,-1 320 | 62,5,411,211,57,160,1,-1,-1,-1 321 | 62,7,251,185,101,242,1,-1,-1,-1 322 | 62,8,375,201,77,161,1,-1,-1,-1 323 | 63,3,575,173,97,290,1,-1,-1,-1 324 | 63,4,518,205,60,144,1,-1,-1,-1 325 | 63,5,409,214,67,160,1,-1,-1,-1 326 | 63,7,260,183,103,244,1,-1,-1,-1 327 | 63,8,377,199,69,169,1,-1,-1,-1 328 | 64,4,528,210,56,142,1,-1,-1,-1 329 | 64,5,418,215,56,152,1,-1,-1,-1 330 | 64,7,273,183,85,240,1,-1,-1,-1 331 | 64,8,378,204,69,165,1,-1,-1,-1 332 | 65,4,534,214,58,139,1,-1,-1,-1 333 | 65,5,423,214,68,158,1,-1,-1,-1 334 | 65,7,280,179,90,242,1,-1,-1,-1 335 | 65,8,383,204,73,161,1,-1,-1,-1 336 | 66,4,537,206,62,143,1,-1,-1,-1 337 | 66,5,421,217,73,154,1,-1,-1,-1 338 | 66,7,297,179,84,246,1,-1,-1,-1 339 | 66,8,383,204,73,162,1,-1,-1,-1 340 | 67,4,542,209,74,147,1,-1,-1,-1 341 | 67,5,434,214,58,158,1,-1,-1,-1 342 | 67,7,306,179,80,239,1,-1,-1,-1 343 | 67,8,391,196,66,175,1,-1,-1,-1 344 | 68,4,547,211,61,140,1,-1,-1,-1 345 | 68,5,434,220,67,151,1,-1,-1,-1 346 | 68,7,315,173,90,253,1,-1,-1,-1 347 | 68,8,403,201,63,167,1,-1,-1,-1 348 | 69,4,557,215,62,138,1,-1,-1,-1 349 | 69,5,441,217,61,155,1,-1,-1,-1 350 | 69,7,325,178,91,250,1,-1,-1,-1 351 | 69,8,399,201,70,177,1,-1,-1,-1 352 | 70,4,561,210,59,142,1,-1,-1,-1 353 | 70,5,446,217,64,155,1,-1,-1,-1 354 | 70,7,326,182,100,235,1,-1,-1,-1 355 | 70,8,403,199,65,169,1,-1,-1,-1 356 | 71,4,561,220,63,133,1,-1,-1,-1 357 | 71,5,449,219,61,153,1,-1,-1,-1 358 | 71,7,335,183,107,243,1,-1,-1,-1 359 | 71,8,416,204,58,164,1,-1,-1,-1 360 | -------------------------------------------------------------------------------- /motmetrics/lap.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tools for solving linear assignment problems.""" 9 | 10 | # pylint: disable=import-outside-toplevel 11 | 12 | from __future__ import absolute_import 13 | from __future__ import division 14 | from __future__ import print_function 15 | 16 | from contextlib import contextmanager 17 | import warnings 18 | 19 | import numpy as np 20 | 21 | 22 | def _module_is_available_py2(name): 23 | try: 24 | imp.find_module(name) 25 | return True 26 | except ImportError: 27 | return False 28 | 29 | 30 | def _module_is_available_py3(name): 31 | return importlib.util.find_spec(name) is not None 32 | 33 | 34 | try: 35 | import importlib.util 36 | except ImportError: 37 | import imp 38 | _module_is_available = _module_is_available_py2 39 | else: 40 | _module_is_available = _module_is_available_py3 41 | 42 | 43 | def linear_sum_assignment(costs, solver=None): 44 | """Solve a linear sum assignment problem (LSA). 45 | 46 | For large datasets solving the minimum cost assignment becomes the dominant runtime part. 47 | We therefore support various solvers out of the box (currently lapsolver, scipy, ortools, munkres) 48 | 49 | Params 50 | ------ 51 | costs : np.array 52 | numpy matrix containing costs. Use NaN/Inf values for unassignable 53 | row/column pairs. 54 | 55 | Kwargs 56 | ------ 57 | solver : callable or str, optional 58 | When str: name of solver to use. 59 | When callable: function to invoke 60 | When None: uses first available solver 61 | """ 62 | costs = np.asarray(costs) 63 | if not costs.size: 64 | return np.array([], dtype=int), np.array([], dtype=int) 65 | 66 | solver = solver or default_solver 67 | 68 | if isinstance(solver, str): 69 | # Try resolve from string 70 | solver = solver_map.get(solver, None) 71 | 72 | assert callable(solver), 'Invalid LAP solver.' 73 | rids, cids = solver(costs) 74 | rids = np.asarray(rids).astype(int) 75 | cids = np.asarray(cids).astype(int) 76 | return rids, cids 77 | 78 | 79 | def add_expensive_edges(costs): 80 | """Replaces non-edge costs (nan, inf) with large number. 81 | 82 | If the optimal solution includes one of these edges, 83 | then the original problem was infeasible. 84 | 85 | Parameters 86 | ---------- 87 | costs : np.ndarray 88 | """ 89 | # The graph is probably already dense if we are doing this. 90 | assert isinstance(costs, np.ndarray) 91 | # The linear_sum_assignment function in scipy does not support missing edges. 92 | # Replace nan with a large constant that ensures it is not chosen. 93 | # If it is chosen, that means the problem was infeasible. 94 | valid = np.isfinite(costs) 95 | if valid.all(): 96 | return costs.copy() 97 | if not valid.any(): 98 | return np.zeros_like(costs) 99 | r = min(costs.shape) 100 | # Assume all edges costs are within [-c, c], c >= 0. 101 | # The cost of an invalid edge must be such that... 102 | # choosing this edge once and the best-possible edge (r - 1) times 103 | # is worse than choosing the worst-possible edge r times. 104 | # l + (r - 1) (-c) > r c 105 | # l > r c + (r - 1) c 106 | # l > (2 r - 1) c 107 | # Choose l = 2 r c + 1 > (2 r - 1) c. 108 | c = np.abs(costs[valid]).max() + 1 # Doesn't hurt to add 1 here. 109 | large_constant = 2 * r * c + 1 110 | return np.where(valid, costs, large_constant) 111 | 112 | 113 | def _exclude_missing_edges(costs, rids, cids): 114 | subset = [ 115 | index for index, (i, j) in enumerate(zip(rids, cids)) 116 | if np.isfinite(costs[i, j]) 117 | ] 118 | return rids[subset], cids[subset] 119 | 120 | 121 | def lsa_solve_scipy(costs): 122 | """Solves the LSA problem using the scipy library.""" 123 | 124 | from scipy.optimize import linear_sum_assignment as scipy_solve 125 | 126 | # scipy (1.3.3) does not support nan or inf values 127 | finite_costs = add_expensive_edges(costs) 128 | rids, cids = scipy_solve(finite_costs) 129 | rids, cids = _exclude_missing_edges(costs, rids, cids) 130 | return rids, cids 131 | 132 | 133 | def lsa_solve_lapsolver(costs): 134 | """Solves the LSA problem using the lapsolver library.""" 135 | from lapsolver import solve_dense 136 | 137 | # Note that lapsolver will add expensive finite edges internally. 138 | # However, older versions did not add a large enough edge. 139 | finite_costs = add_expensive_edges(costs) 140 | rids, cids = solve_dense(finite_costs) 141 | rids, cids = _exclude_missing_edges(costs, rids, cids) 142 | return rids, cids 143 | 144 | 145 | def lsa_solve_munkres(costs): 146 | """Solves the LSA problem using the Munkres library.""" 147 | from munkres import Munkres 148 | 149 | m = Munkres() 150 | # The munkres package may hang if the problem is not feasible. 151 | # Therefore, add expensive edges instead of using munkres.DISALLOWED. 152 | finite_costs = add_expensive_edges(costs) 153 | # Ensure that matrix is square. 154 | finite_costs = _zero_pad_to_square(finite_costs) 155 | indices = np.array(m.compute(finite_costs), dtype=int) 156 | # Exclude extra matches from extension to square matrix. 157 | indices = indices[(indices[:, 0] < costs.shape[0]) 158 | & (indices[:, 1] < costs.shape[1])] 159 | rids, cids = indices[:, 0], indices[:, 1] 160 | rids, cids = _exclude_missing_edges(costs, rids, cids) 161 | return rids, cids 162 | 163 | 164 | def _zero_pad_to_square(costs): 165 | num_rows, num_cols = costs.shape 166 | if num_rows == num_cols: 167 | return costs 168 | n = max(num_rows, num_cols) 169 | padded = np.zeros((n, n), dtype=costs.dtype) 170 | padded[:num_rows, :num_cols] = costs 171 | return padded 172 | 173 | 174 | def lsa_solve_ortools(costs): 175 | """Solves the LSA problem using Google's optimization tools. """ 176 | from ortools.graph import pywrapgraph 177 | 178 | if costs.shape[0] != costs.shape[1]: 179 | # ortools assumes that the problem is square. 180 | # Non-square problem will be infeasible. 181 | # Default to scipy solver rather than add extra zeros. 182 | # (This maintains the same behaviour as previous versions.) 183 | return linear_sum_assignment(costs, solver='scipy') 184 | 185 | rs, cs = np.isfinite(costs).nonzero() # pylint: disable=unbalanced-tuple-unpacking 186 | finite_costs = costs[rs, cs] 187 | scale = find_scale_for_integer_approximation(finite_costs) 188 | if scale != 1: 189 | warnings.warn('costs are not integers; using approximation') 190 | int_costs = np.round(scale * finite_costs).astype(int) 191 | 192 | assignment = pywrapgraph.LinearSumAssignment() 193 | # OR-Tools does not like to receive indices of type np.int64. 194 | rs = rs.tolist() # pylint: disable=no-member 195 | cs = cs.tolist() 196 | int_costs = int_costs.tolist() 197 | for r, c, int_cost in zip(rs, cs, int_costs): 198 | assignment.AddArcWithCost(r, c, int_cost) 199 | 200 | status = assignment.Solve() 201 | try: 202 | _ortools_assert_is_optimal(pywrapgraph, status) 203 | except AssertionError: 204 | # Default to scipy solver rather than add finite edges. 205 | # (This maintains the same behaviour as previous versions.) 206 | return linear_sum_assignment(costs, solver='scipy') 207 | 208 | return _ortools_extract_solution(assignment) 209 | 210 | 211 | def find_scale_for_integer_approximation(costs, base=10, log_max_scale=8, log_safety=2): 212 | """Returns a multiplicative factor to use before rounding to integers. 213 | 214 | Tries to find scale = base ** j (for j integer) such that: 215 | abs(diff(unique(costs))) <= 1 / (scale * safety) 216 | where safety = base ** log_safety. 217 | 218 | Logs a warning if the desired resolution could not be achieved. 219 | """ 220 | costs = np.asarray(costs) 221 | costs = costs[np.isfinite(costs)] # Exclude non-edges (nan, inf) and -inf. 222 | if np.size(costs) == 0: 223 | # No edges with numeric value. Scale does not matter. 224 | return 1 225 | unique = np.unique(costs) 226 | if np.size(unique) == 1: 227 | # All costs have equal values. Scale does not matter. 228 | return 1 229 | try: 230 | _assert_integer(costs) 231 | except AssertionError: 232 | pass 233 | else: 234 | # The costs are already integers. 235 | return 1 236 | 237 | # Find scale = base ** e such that: 238 | # 1 / scale <= tol, or 239 | # e = log(scale) >= -log(tol) 240 | # where tol = min(diff(unique(costs))) 241 | min_diff = np.diff(unique).min() 242 | e = np.ceil(np.log(min_diff) / np.log(base)).astype(int).item() 243 | # Add optional non-negative safety factor to reduce quantization noise. 244 | e += max(log_safety, 0) 245 | # Ensure that we do not reduce the magnitude of the costs. 246 | e = max(e, 0) 247 | # Ensure that the scale is not too large. 248 | if e > log_max_scale: 249 | warnings.warn('could not achieve desired resolution for approximation: ' 250 | 'want exponent %d but max is %d', e, log_max_scale) 251 | e = log_max_scale 252 | scale = base ** e 253 | return scale 254 | 255 | 256 | def _assert_integer(costs): 257 | # Check that costs are not changed by rounding. 258 | # Note: Elements of cost matrix may be nan, inf, -inf. 259 | np.testing.assert_equal(np.round(costs), costs) 260 | 261 | 262 | def _ortools_assert_is_optimal(pywrapgraph, status): 263 | if status == pywrapgraph.LinearSumAssignment.OPTIMAL: 264 | pass 265 | elif status == pywrapgraph.LinearSumAssignment.INFEASIBLE: 266 | raise AssertionError('ortools: infeasible assignment problem') 267 | elif status == pywrapgraph.LinearSumAssignment.POSSIBLE_OVERFLOW: 268 | raise AssertionError('ortools: possible overflow in assignment problem') 269 | else: 270 | raise AssertionError('ortools: unknown status') 271 | 272 | 273 | def _ortools_extract_solution(assignment): 274 | if assignment.NumNodes() == 0: 275 | return np.array([], dtype=int), np.array([], dtype=int) 276 | 277 | pairings = [] 278 | for i in range(assignment.NumNodes()): 279 | pairings.append([i, assignment.RightMate(i)]) 280 | 281 | indices = np.array(pairings, dtype=int) 282 | return indices[:, 0], indices[:, 1] 283 | 284 | 285 | def lsa_solve_lapjv(costs): 286 | """Solves the LSA problem using lap.lapjv().""" 287 | 288 | from lap import lapjv 289 | 290 | # The lap.lapjv function supports +inf edges but there are some issues. 291 | # https://github.com/gatagat/lap/issues/20 292 | # Therefore, replace nans with large finite cost. 293 | finite_costs = add_expensive_edges(costs) 294 | row_to_col, _ = lapjv(finite_costs, return_cost=False, extend_cost=True) 295 | indices = np.array([np.arange(costs.shape[0]), row_to_col], dtype=int).T 296 | # Exclude unmatched rows (in case of unbalanced problem). 297 | indices = indices[indices[:, 1] != -1] # pylint: disable=unsubscriptable-object 298 | rids, cids = indices[:, 0], indices[:, 1] 299 | # Ensure that no missing edges were chosen. 300 | rids, cids = _exclude_missing_edges(costs, rids, cids) 301 | return rids, cids 302 | 303 | 304 | available_solvers = None 305 | default_solver = None 306 | solver_map = None 307 | 308 | 309 | def _init_standard_solvers(): 310 | global available_solvers, default_solver, solver_map # pylint: disable=global-statement 311 | 312 | solvers = [ 313 | ('lapsolver', lsa_solve_lapsolver), 314 | ('lap', lsa_solve_lapjv), 315 | ('scipy', lsa_solve_scipy), 316 | ('munkres', lsa_solve_munkres), 317 | ('ortools', lsa_solve_ortools), 318 | ] 319 | 320 | solver_map = dict(solvers) 321 | 322 | available_solvers = [s[0] for s in solvers if _module_is_available(s[0])] 323 | if len(available_solvers) == 0: 324 | default_solver = None 325 | warnings.warn('No standard LAP solvers found. Consider `pip install lapsolver` or `pip install scipy`', category=RuntimeWarning) 326 | else: 327 | default_solver = available_solvers[0] 328 | 329 | 330 | _init_standard_solvers() 331 | 332 | 333 | @contextmanager 334 | def set_default_solver(newsolver): 335 | """Change the default solver within context. 336 | 337 | Intended usage 338 | 339 | costs = ... 340 | mysolver = lambda x: ... # solver code that returns pairings 341 | 342 | with lap.set_default_solver(mysolver): 343 | rids, cids = lap.linear_sum_assignment(costs) 344 | 345 | Params 346 | ------ 347 | newsolver : callable or str 348 | new solver function 349 | """ 350 | 351 | global default_solver # pylint: disable=global-statement 352 | 353 | oldsolver = default_solver 354 | try: 355 | default_solver = newsolver 356 | yield 357 | finally: 358 | default_solver = oldsolver 359 | -------------------------------------------------------------------------------- /motmetrics/io.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Functions for loading data and writing summaries.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | from enum import Enum 15 | import io 16 | 17 | import numpy as np 18 | import pandas as pd 19 | import scipy.io 20 | import xmltodict 21 | 22 | 23 | class Format(Enum): 24 | """Enumerates supported file formats.""" 25 | 26 | MOT16 = 'mot16' 27 | """Milan, Anton, et al. "Mot16: A benchmark for multi-object tracking." arXiv preprint arXiv:1603.00831 (2016).""" 28 | 29 | MOT15_2D = 'mot15-2D' 30 | """Leal-Taixe, Laura, et al. "MOTChallenge 2015: Towards a benchmark for multi-target tracking." arXiv preprint arXiv:1504.01942 (2015).""" 31 | 32 | VATIC_TXT = 'vatic-txt' 33 | """Vondrick, Carl, Donald Patterson, and Deva Ramanan. "Efficiently scaling up crowdsourced video annotation." International Journal of Computer Vision 101.1 (2013): 184-204. 34 | https://github.com/cvondrick/vatic 35 | """ 36 | 37 | DETRAC_MAT = 'detrac-mat' 38 | """Wen, Longyin et al. "UA-DETRAC: A New Benchmark and Protocol for Multi-Object Detection and Tracking." arXiv preprint arXiv:arXiv:1511.04136 (2016). 39 | http://detrac-db.rit.albany.edu/download 40 | """ 41 | 42 | DETRAC_XML = 'detrac-xml' 43 | """Wen, Longyin et al. "UA-DETRAC: A New Benchmark and Protocol for Multi-Object Detection and Tracking." arXiv preprint arXiv:arXiv:1511.04136 (2016). 44 | http://detrac-db.rit.albany.edu/download 45 | """ 46 | 47 | 48 | def load_motchallenge(fname, **kwargs): 49 | r"""Load MOT challenge data. 50 | 51 | Params 52 | ------ 53 | fname : str 54 | Filename to load data from 55 | 56 | Kwargs 57 | ------ 58 | sep : str 59 | Allowed field separators, defaults to '\s+|\t+|,' 60 | min_confidence : float 61 | Rows with confidence less than this threshold are removed. 62 | Defaults to -1. You should set this to 1 when loading 63 | ground truth MOTChallenge data, so that invalid rectangles in 64 | the ground truth are not considered during matching. 65 | 66 | Returns 67 | ------ 68 | df : pandas.DataFrame 69 | The returned dataframe has the following columns 70 | 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' 71 | The dataframe is indexed by ('FrameId', 'Id') 72 | """ 73 | 74 | sep = kwargs.pop('sep', r'\s+|\t+|,') 75 | min_confidence = kwargs.pop('min_confidence', -1) 76 | df = pd.read_csv( 77 | fname, 78 | sep=sep, 79 | index_col=[0, 1], 80 | skipinitialspace=True, 81 | header=None, 82 | names=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused'], 83 | engine='python' 84 | ) 85 | 86 | # Account for matlab convention. 87 | df[['X', 'Y']] -= (1, 1) 88 | 89 | # Removed trailing column 90 | del df['unused'] 91 | 92 | # Remove all rows without sufficient confidence 93 | return df[df['Confidence'] >= min_confidence] 94 | 95 | 96 | def load_vatictxt(fname, **kwargs): 97 | """Load Vatic text format. 98 | 99 | Loads the vatic CSV text having the following columns per row 100 | 101 | 0 Track ID. All rows with the same ID belong to the same path. 102 | 1 xmin. The top left x-coordinate of the bounding box. 103 | 2 ymin. The top left y-coordinate of the bounding box. 104 | 3 xmax. The bottom right x-coordinate of the bounding box. 105 | 4 ymax. The bottom right y-coordinate of the bounding box. 106 | 5 frame. The frame that this annotation represents. 107 | 6 lost. If 1, the annotation is outside of the view screen. 108 | 7 occluded. If 1, the annotation is occluded. 109 | 8 generated. If 1, the annotation was automatically interpolated. 110 | 9 label. The label for this annotation, enclosed in quotation marks. 111 | 10+ attributes. Each column after this is an attribute set in the current frame 112 | 113 | Params 114 | ------ 115 | fname : str 116 | Filename to load data from 117 | 118 | Returns 119 | ------ 120 | df : pandas.DataFrame 121 | The returned dataframe has the following columns 122 | 'X', 'Y', 'Width', 'Height', 'Lost', 'Occluded', 'Generated', 'ClassId', '', '', ... 123 | where is placeholder for the actual attribute name capitalized (first letter). The order of attribute 124 | columns is sorted in attribute name. The dataframe is indexed by ('FrameId', 'Id') 125 | """ 126 | # pylint: disable=too-many-locals 127 | 128 | sep = kwargs.pop('sep', ' ') 129 | 130 | with io.open(fname) as f: 131 | # First time going over file, we collect the set of all variable activities 132 | activities = set() 133 | for line in f: 134 | for c in line.rstrip().split(sep)[10:]: 135 | activities.add(c) 136 | activitylist = sorted(list(activities)) 137 | 138 | # Second time we construct artificial binary columns for each activity 139 | data = [] 140 | f.seek(0) 141 | for line in f: 142 | fields = line.rstrip().split() 143 | attrs = ['0'] * len(activitylist) 144 | for a in fields[10:]: 145 | attrs[activitylist.index(a)] = '1' 146 | fields = fields[:10] 147 | fields.extend(attrs) 148 | data.append(' '.join(fields)) 149 | 150 | strdata = '\n'.join(data) 151 | 152 | dtype = { 153 | 'Id': np.int64, 154 | 'X': np.float32, 155 | 'Y': np.float32, 156 | 'Width': np.float32, 157 | 'Height': np.float32, 158 | 'FrameId': np.int64, 159 | 'Lost': bool, 160 | 'Occluded': bool, 161 | 'Generated': bool, 162 | 'ClassId': str, 163 | } 164 | 165 | # Remove quotes from activities 166 | activitylist = [a.replace('\"', '').capitalize() for a in activitylist] 167 | 168 | # Add dtypes for activities 169 | for a in activitylist: 170 | dtype[a] = bool 171 | 172 | # Read from CSV 173 | names = ['Id', 'X', 'Y', 'Width', 'Height', 'FrameId', 'Lost', 'Occluded', 'Generated', 'ClassId'] 174 | names.extend(activitylist) 175 | df = pd.read_csv(io.StringIO(strdata), names=names, index_col=['FrameId', 'Id'], header=None, sep=' ') 176 | 177 | # Correct Width and Height which are actually XMax, Ymax in files. 178 | w = df['Width'] - df['X'] 179 | h = df['Height'] - df['Y'] 180 | df['Width'] = w 181 | df['Height'] = h 182 | 183 | return df 184 | 185 | 186 | def load_detrac_mat(fname): 187 | """Loads UA-DETRAC annotations data from mat files 188 | 189 | Competition Site: http://detrac-db.rit.albany.edu/download 190 | 191 | File contains a nested structure of 2d arrays for indexed by frame id 192 | and Object ID. Separate arrays for top, left, width and height are given. 193 | 194 | Params 195 | ------ 196 | fname : str 197 | Filename to load data from 198 | 199 | Kwargs 200 | ------ 201 | Currently none of these arguments used. 202 | 203 | Returns 204 | ------ 205 | df : pandas.DataFrame 206 | The returned dataframe has the following columns 207 | 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' 208 | The dataframe is indexed by ('FrameId', 'Id') 209 | """ 210 | 211 | matData = scipy.io.loadmat(fname) 212 | 213 | frameList = matData['gtInfo'][0][0][4][0] 214 | leftArray = matData['gtInfo'][0][0][0].astype(np.float32) 215 | topArray = matData['gtInfo'][0][0][1].astype(np.float32) 216 | widthArray = matData['gtInfo'][0][0][3].astype(np.float32) 217 | heightArray = matData['gtInfo'][0][0][2].astype(np.float32) 218 | 219 | parsedGT = [] 220 | for f in frameList: 221 | ids = [i + 1 for i, v in enumerate(leftArray[f - 1]) if v > 0] 222 | for i in ids: 223 | row = [] 224 | row.append(f) 225 | row.append(i) 226 | row.append(leftArray[f - 1, i - 1] - widthArray[f - 1, i - 1] / 2) 227 | row.append(topArray[f - 1, i - 1] - heightArray[f - 1, i - 1]) 228 | row.append(widthArray[f - 1, i - 1]) 229 | row.append(heightArray[f - 1, i - 1]) 230 | row.append(1) 231 | row.append(-1) 232 | row.append(-1) 233 | row.append(-1) 234 | parsedGT.append(row) 235 | 236 | df = pd.DataFrame(parsedGT, 237 | columns=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused']) 238 | df.set_index(['FrameId', 'Id'], inplace=True) 239 | 240 | # Account for matlab convention. 241 | df[['X', 'Y']] -= (1, 1) 242 | 243 | # Removed trailing column 244 | del df['unused'] 245 | 246 | return df 247 | 248 | 249 | def load_detrac_xml(fname): 250 | """Loads UA-DETRAC annotations data from xml files 251 | 252 | Competition Site: http://detrac-db.rit.albany.edu/download 253 | 254 | Params 255 | ------ 256 | fname : str 257 | Filename to load data from 258 | 259 | Kwargs 260 | ------ 261 | Currently none of these arguments used. 262 | 263 | Returns 264 | ------ 265 | df : pandas.DataFrame 266 | The returned dataframe has the following columns 267 | 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' 268 | The dataframe is indexed by ('FrameId', 'Id') 269 | """ 270 | 271 | with io.open(fname) as fd: 272 | doc = xmltodict.parse(fd.read()) 273 | frameList = doc['sequence']['frame'] 274 | 275 | parsedGT = [] 276 | for f in frameList: 277 | fid = int(f['@num']) 278 | targetList = f['target_list']['target'] 279 | if not isinstance(targetList, list): 280 | targetList = [targetList] 281 | 282 | for t in targetList: 283 | row = [] 284 | row.append(fid) 285 | row.append(int(t['@id'])) 286 | row.append(float(t['box']['@left'])) 287 | row.append(float(t['box']['@top'])) 288 | row.append(float(t['box']['@width'])) 289 | row.append(float(t['box']['@height'])) 290 | row.append(1) 291 | row.append(-1) 292 | row.append(-1) 293 | row.append(-1) 294 | parsedGT.append(row) 295 | 296 | df = pd.DataFrame(parsedGT, 297 | columns=['FrameId', 'Id', 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility', 'unused']) 298 | df.set_index(['FrameId', 'Id'], inplace=True) 299 | 300 | # Account for matlab convention. 301 | df[['X', 'Y']] -= (1, 1) 302 | 303 | # Removed trailing column 304 | del df['unused'] 305 | 306 | return df 307 | 308 | 309 | def loadtxt(fname, fmt=Format.MOT15_2D, **kwargs): 310 | """Load data from any known format.""" 311 | fmt = Format(fmt) 312 | 313 | switcher = { 314 | Format.MOT16: load_motchallenge, 315 | Format.MOT15_2D: load_motchallenge, 316 | Format.VATIC_TXT: load_vatictxt, 317 | Format.DETRAC_MAT: load_detrac_mat, 318 | Format.DETRAC_XML: load_detrac_xml 319 | } 320 | func = switcher.get(fmt) 321 | return func(fname, **kwargs) 322 | 323 | 324 | def render_summary(summary, formatters=None, namemap=None, buf=None): 325 | """Render metrics summary to console friendly tabular output. 326 | 327 | Params 328 | ------ 329 | summary : pd.DataFrame 330 | Dataframe containing summaries in rows. 331 | 332 | Kwargs 333 | ------ 334 | buf : StringIO-like, optional 335 | Buffer to write to 336 | formatters : dict, optional 337 | Dicionary defining custom formatters for individual metrics. 338 | I.e `{'mota': '{:.2%}'.format}`. You can get preset formatters 339 | from MetricsHost.formatters 340 | namemap : dict, optional 341 | Dictionary defining new metric names for display. I.e 342 | `{'num_false_positives': 'FP'}`. 343 | 344 | Returns 345 | ------- 346 | string 347 | Formatted string 348 | """ 349 | 350 | if namemap is not None: 351 | summary = summary.rename(columns=namemap) 352 | if formatters is not None: 353 | formatters = {namemap.get(c, c): f for c, f in formatters.items()} 354 | 355 | output = summary.to_string( 356 | buf=buf, 357 | formatters=formatters, 358 | ) 359 | 360 | return output 361 | 362 | 363 | motchallenge_metric_names = { 364 | 'idf1': 'IDF1', 365 | 'idp': 'IDP', 366 | 'idr': 'IDR', 367 | 'recall': 'Rcll', 368 | 'precision': 'Prcn', 369 | 'num_unique_objects': 'GT', 370 | 'mostly_tracked': 'MT', 371 | 'partially_tracked': 'PT', 372 | 'mostly_lost': 'ML', 373 | 'num_false_positives': 'FP', 374 | 'num_misses': 'FN', 375 | 'num_switches': 'IDs', 376 | 'num_fragmentations': 'FM', 377 | 'mota': 'MOTA', 378 | 'motp': 'MOTP', 379 | 'num_transfer': 'IDt', 380 | 'num_ascend': 'IDa', 381 | 'num_migrate': 'IDm', 382 | } 383 | """A list mappings for metric names to comply with MOTChallenge.""" 384 | -------------------------------------------------------------------------------- /motmetrics/tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Tests computation of metrics from accumulator.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | import os 15 | 16 | import numpy as np 17 | import pandas as pd 18 | from pytest import approx 19 | 20 | import motmetrics as mm 21 | 22 | DATA_DIR = os.path.join(os.path.dirname(__file__), "../data") 23 | 24 | 25 | def test_metricscontainer_1(): 26 | """Tests registration of events with dependencies.""" 27 | m = mm.metrics.MetricsHost() 28 | m.register(lambda df: 1.0, name="a") 29 | m.register(lambda df: 2.0, name="b") 30 | m.register(lambda df, a, b: a + b, deps=["a", "b"], name="add") 31 | m.register(lambda df, a, b: a - b, deps=["a", "b"], name="sub") 32 | m.register(lambda df, a, b: a * b, deps=["add", "sub"], name="mul") 33 | summary = m.compute( 34 | mm.MOTAccumulator.new_event_dataframe(), metrics=["mul", "add"], name="x" 35 | ) 36 | assert summary.columns.values.tolist() == ["mul", "add"] 37 | assert summary.iloc[0]["mul"] == -3.0 38 | assert summary.iloc[0]["add"] == 3.0 39 | 40 | 41 | def test_metricscontainer_autodep(): 42 | """Tests automatic dependencies from argument names.""" 43 | m = mm.metrics.MetricsHost() 44 | m.register(lambda df: 1.0, name="a") 45 | m.register(lambda df: 2.0, name="b") 46 | m.register(lambda df, a, b: a + b, name="add", deps="auto") 47 | m.register(lambda df, a, b: a - b, name="sub", deps="auto") 48 | m.register(lambda df, add, sub: add * sub, name="mul", deps="auto") 49 | summary = m.compute(mm.MOTAccumulator.new_event_dataframe(), metrics=["mul", "add"]) 50 | assert summary.columns.values.tolist() == ["mul", "add"] 51 | assert summary.iloc[0]["mul"] == -3.0 52 | assert summary.iloc[0]["add"] == 3.0 53 | 54 | 55 | def test_metricscontainer_autoname(): 56 | """Tests automatic names (and dependencies) from inspection.""" 57 | 58 | def constant_a(_): 59 | """Constant a help.""" 60 | return 1.0 61 | 62 | def constant_b(_): 63 | return 2.0 64 | 65 | def add(_, constant_a, constant_b): 66 | return constant_a + constant_b 67 | 68 | def sub(_, constant_a, constant_b): 69 | return constant_a - constant_b 70 | 71 | def mul(_, add, sub): 72 | return add * sub 73 | 74 | m = mm.metrics.MetricsHost() 75 | m.register(constant_a, deps="auto") 76 | m.register(constant_b, deps="auto") 77 | m.register(add, deps="auto") 78 | m.register(sub, deps="auto") 79 | m.register(mul, deps="auto") 80 | 81 | assert m.metrics["constant_a"]["help"] == "Constant a help." 82 | 83 | summary = m.compute(mm.MOTAccumulator.new_event_dataframe(), metrics=["mul", "add"]) 84 | assert summary.columns.values.tolist() == ["mul", "add"] 85 | assert summary.iloc[0]["mul"] == -3.0 86 | assert summary.iloc[0]["add"] == 3.0 87 | 88 | 89 | def test_metrics_with_no_events(): 90 | """Tests metrics when accumulator is empty.""" 91 | acc = mm.MOTAccumulator() 92 | 93 | mh = mm.metrics.create() 94 | metr = mh.compute( 95 | acc, 96 | return_dataframe=False, 97 | return_cached=True, 98 | metrics=[ 99 | "mota", 100 | "motp", 101 | "num_predictions", 102 | "num_objects", 103 | "num_detections", 104 | "num_frames", 105 | ], 106 | ) 107 | assert np.isnan(metr["mota"]) 108 | assert np.isnan(metr["motp"]) 109 | assert metr["num_predictions"] == 0 110 | assert metr["num_objects"] == 0 111 | assert metr["num_detections"] == 0 112 | assert metr["num_frames"] == 0 113 | 114 | 115 | def test_assignment_metrics_with_empty_groundtruth(): 116 | """Tests metrics when there are no ground-truth objects.""" 117 | acc = mm.MOTAccumulator(auto_id=True) 118 | # Empty groundtruth. 119 | acc.update([], [1, 2, 3, 4], []) 120 | acc.update([], [1, 2, 3, 4], []) 121 | acc.update([], [1, 2, 3, 4], []) 122 | acc.update([], [1, 2, 3, 4], []) 123 | 124 | mh = mm.metrics.create() 125 | metr = mh.compute( 126 | acc, 127 | return_dataframe=False, 128 | metrics=[ 129 | "num_matches", 130 | "num_false_positives", 131 | "num_misses", 132 | "idtp", 133 | "idfp", 134 | "idfn", 135 | "num_frames", 136 | ], 137 | ) 138 | assert metr["num_matches"] == 0 139 | assert metr["num_false_positives"] == 16 140 | assert metr["num_misses"] == 0 141 | assert metr["idtp"] == 0 142 | assert metr["idfp"] == 16 143 | assert metr["idfn"] == 0 144 | assert metr["num_frames"] == 4 145 | 146 | 147 | def test_assignment_metrics_with_empty_predictions(): 148 | """Tests metrics when there are no predictions.""" 149 | acc = mm.MOTAccumulator(auto_id=True) 150 | # Empty predictions. 151 | acc.update([1, 2, 3, 4], [], []) 152 | acc.update([1, 2, 3, 4], [], []) 153 | acc.update([1, 2, 3, 4], [], []) 154 | acc.update([1, 2, 3, 4], [], []) 155 | 156 | mh = mm.metrics.create() 157 | metr = mh.compute( 158 | acc, 159 | return_dataframe=False, 160 | metrics=[ 161 | "num_matches", 162 | "num_false_positives", 163 | "num_misses", 164 | "idtp", 165 | "idfp", 166 | "idfn", 167 | "num_frames", 168 | ], 169 | ) 170 | assert metr["num_matches"] == 0 171 | assert metr["num_false_positives"] == 0 172 | assert metr["num_misses"] == 16 173 | assert metr["idtp"] == 0 174 | assert metr["idfp"] == 0 175 | assert metr["idfn"] == 16 176 | assert metr["num_frames"] == 4 177 | 178 | 179 | def test_assignment_metrics_with_both_empty(): 180 | """Tests metrics when there are no ground-truth objects or predictions.""" 181 | acc = mm.MOTAccumulator(auto_id=True) 182 | # Empty groundtruth and empty predictions. 183 | acc.update([], [], []) 184 | acc.update([], [], []) 185 | acc.update([], [], []) 186 | acc.update([], [], []) 187 | 188 | mh = mm.metrics.create() 189 | metr = mh.compute( 190 | acc, 191 | return_dataframe=False, 192 | metrics=[ 193 | "num_matches", 194 | "num_false_positives", 195 | "num_misses", 196 | "idtp", 197 | "idfp", 198 | "idfn", 199 | "num_frames", 200 | ], 201 | ) 202 | assert metr["num_matches"] == 0 203 | assert metr["num_false_positives"] == 0 204 | assert metr["num_misses"] == 0 205 | assert metr["idtp"] == 0 206 | assert metr["idfp"] == 0 207 | assert metr["idfn"] == 0 208 | assert metr["num_frames"] == 4 209 | 210 | 211 | def _extract_counts(acc): 212 | df_map = mm.metrics.events_to_df_map(acc.events) 213 | return mm.metrics.extract_counts_from_df_map(df_map) 214 | 215 | 216 | def test_extract_counts(): 217 | """Tests events_to_df_map() and extract_counts_from_df_map().""" 218 | acc = mm.MOTAccumulator() 219 | # All FP 220 | acc.update([], [1, 2], [], frameid=0) 221 | # All miss 222 | acc.update([1, 2], [], [], frameid=1) 223 | # Match 224 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2) 225 | # Switch 226 | acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3) 227 | # Match. Better new match is available but should prefer history 228 | acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4) 229 | # No data 230 | acc.update([], [], [], frameid=5) 231 | 232 | ocs, hcs, tps = _extract_counts(acc) 233 | 234 | assert ocs == {1: 4, 2: 4} 235 | assert hcs == {1: 4, 2: 4} 236 | expected_tps = { 237 | (1, 1): 3, 238 | (1, 2): 2, 239 | (2, 1): 2, 240 | (2, 2): 3, 241 | } 242 | assert tps == expected_tps 243 | 244 | 245 | def test_extract_pandas_series_issue(): 246 | """Reproduce issue that arises with pd.Series but not pd.DataFrame. 247 | 248 | >>> data = [[0, 1, 0.1], [0, 1, 0.2], [0, 1, 0.3]] 249 | >>> df = pd.DataFrame(data, columns=['x', 'y', 'z']).set_index(['x', 'y']) 250 | >>> df['z'].groupby(['x', 'y']).count() 251 | {(0, 1): 3} 252 | 253 | >>> data = [[0, 1, 0.1], [0, 1, 0.2]] 254 | >>> df = pd.DataFrame(data, columns=['x', 'y', 'z']).set_index(['x', 'y']) 255 | >>> df['z'].groupby(['x', 'y']).count() 256 | {'x': 1, 'y': 1} 257 | 258 | >>> df[['z']].groupby(['x', 'y'])['z'].count().to_dict() 259 | {(0, 1): 2} 260 | """ 261 | acc = mm.MOTAccumulator(auto_id=True) 262 | acc.update([0], [1], [[0.1]]) 263 | acc.update([0], [1], [[0.1]]) 264 | ocs, hcs, tps = _extract_counts(acc) 265 | assert ocs == {0: 2} 266 | assert hcs == {1: 2} 267 | assert tps == {(0, 1): 2} 268 | 269 | 270 | def test_benchmark_extract_counts(benchmark): 271 | """Benchmarks events_to_df_map() and extract_counts_from_df_map().""" 272 | rand = np.random.RandomState(0) 273 | acc = _accum_random_uniform( 274 | rand, 275 | seq_len=100, 276 | num_objs=50, 277 | num_hyps=5000, 278 | objs_per_frame=20, 279 | hyps_per_frame=40, 280 | ) 281 | benchmark(_extract_counts, acc) 282 | 283 | 284 | def _accum_random_uniform( 285 | rand, seq_len, num_objs, num_hyps, objs_per_frame, hyps_per_frame 286 | ): 287 | acc = mm.MOTAccumulator(auto_id=True) 288 | for _ in range(seq_len): 289 | # Choose subset of objects present in this frame. 290 | objs = rand.choice(num_objs, objs_per_frame, replace=False) 291 | # Choose subset of hypotheses present in this frame. 292 | hyps = rand.choice(num_hyps, hyps_per_frame, replace=False) 293 | dist = rand.uniform(size=(objs_per_frame, hyps_per_frame)) 294 | acc.update(objs, hyps, dist) 295 | return acc 296 | 297 | 298 | def test_mota_motp(): 299 | """Tests values of MOTA and MOTP.""" 300 | acc = mm.MOTAccumulator() 301 | 302 | # All FP 303 | acc.update([], [1, 2], [], frameid=0) 304 | # All miss 305 | acc.update([1, 2], [], [], frameid=1) 306 | # Match 307 | acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2) 308 | # Switch 309 | acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3) 310 | # Match. Better new match is available but should prefer history 311 | acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4) 312 | # No data 313 | acc.update([], [], [], frameid=5) 314 | 315 | mh = mm.metrics.create() 316 | metr = mh.compute( 317 | acc, 318 | return_dataframe=False, 319 | return_cached=True, 320 | metrics=[ 321 | "num_matches", 322 | "num_false_positives", 323 | "num_misses", 324 | "num_switches", 325 | "num_detections", 326 | "num_objects", 327 | "num_predictions", 328 | "mota", 329 | "motp", 330 | "num_frames", 331 | ], 332 | ) 333 | 334 | assert metr["num_matches"] == 4 335 | assert metr["num_false_positives"] == 2 336 | assert metr["num_misses"] == 2 337 | assert metr["num_switches"] == 2 338 | assert metr["num_detections"] == 6 339 | assert metr["num_objects"] == 8 340 | assert metr["num_predictions"] == 8 341 | assert metr["mota"] == approx(1.0 - (2 + 2 + 2) / 8) 342 | assert metr["motp"] == approx(11.1 / 6) 343 | assert metr["num_frames"] == 6 344 | 345 | 346 | def test_ids(): 347 | """Test metrics with frame IDs specified manually.""" 348 | acc = mm.MOTAccumulator() 349 | 350 | # No data 351 | acc.update([], [], [], frameid=0) 352 | # Match 353 | acc.update([1, 2], [1, 2], [[1, 0], [0, 1]], frameid=1) 354 | # Switch also Transfer 355 | acc.update([1, 2], [1, 2], [[0.4, np.nan], [np.nan, 0.4]], frameid=2) 356 | # Match 357 | acc.update([1, 2], [1, 2], [[0, 1], [1, 0]], frameid=3) 358 | # Ascend (switch) 359 | acc.update([1, 2], [2, 3], [[1, 0], [0.4, 0.7]], frameid=4) 360 | # Migrate (transfer) 361 | acc.update([1, 3], [2, 3], [[1, 0], [0.4, 0.7]], frameid=5) 362 | # No data 363 | acc.update([], [], [], frameid=6) 364 | 365 | mh = mm.metrics.create() 366 | metr = mh.compute( 367 | acc, 368 | return_dataframe=False, 369 | return_cached=True, 370 | metrics=[ 371 | "num_matches", 372 | "num_false_positives", 373 | "num_misses", 374 | "num_switches", 375 | "num_transfer", 376 | "num_ascend", 377 | "num_migrate", 378 | "num_detections", 379 | "num_objects", 380 | "num_predictions", 381 | "mota", 382 | "motp", 383 | "num_frames", 384 | ], 385 | ) 386 | assert metr["num_matches"] == 7 387 | assert metr["num_false_positives"] == 0 388 | assert metr["num_misses"] == 0 389 | assert metr["num_switches"] == 3 390 | assert metr["num_transfer"] == 3 391 | assert metr["num_ascend"] == 1 392 | assert metr["num_migrate"] == 1 393 | assert metr["num_detections"] == 10 394 | assert metr["num_objects"] == 10 395 | assert metr["num_predictions"] == 10 396 | assert metr["mota"] == approx(1.0 - (0 + 0 + 3) / 10) 397 | assert metr["motp"] == approx(1.6 / 10) 398 | assert metr["num_frames"] == 7 399 | 400 | 401 | def test_correct_average(): 402 | """Tests what is depicted in figure 3 of 'Evaluating MOT Performance'.""" 403 | acc = mm.MOTAccumulator(auto_id=True) 404 | 405 | # No track 406 | acc.update([1, 2, 3, 4], [], []) 407 | acc.update([1, 2, 3, 4], [], []) 408 | acc.update([1, 2, 3, 4], [], []) 409 | acc.update([1, 2, 3, 4], [], []) 410 | 411 | # Track single 412 | acc.update([4], [4], [0]) 413 | acc.update([4], [4], [0]) 414 | acc.update([4], [4], [0]) 415 | acc.update([4], [4], [0]) 416 | 417 | mh = mm.metrics.create() 418 | metr = mh.compute(acc, metrics="mota", return_dataframe=False) 419 | assert metr["mota"] == approx(0.2) 420 | 421 | 422 | def test_motchallenge_files(): 423 | """Tests metrics for sequences TUD-Campus and TUD-Stadtmitte.""" 424 | dnames = [ 425 | "TUD-Campus", 426 | "TUD-Stadtmitte", 427 | ] 428 | 429 | def compute_motchallenge(dname): 430 | df_gt = mm.io.loadtxt(os.path.join(dname, "gt.txt")) 431 | df_test = mm.io.loadtxt(os.path.join(dname, "test.txt")) 432 | return mm.utils.compare_to_groundtruth(df_gt, df_test, "iou", distth=0.5) 433 | 434 | accs = [compute_motchallenge(os.path.join(DATA_DIR, d)) for d in dnames] 435 | 436 | # For testing 437 | # [a.events.to_pickle(n) for (a,n) in zip(accs, dnames)] 438 | 439 | mh = mm.metrics.create() 440 | summary = mh.compute_many( 441 | accs, 442 | metrics=mm.metrics.motchallenge_metrics, 443 | names=dnames, 444 | generate_overall=True, 445 | ) 446 | 447 | print() 448 | print( 449 | mm.io.render_summary( 450 | summary, namemap=mm.io.motchallenge_metric_names, formatters=mh.formatters 451 | ) 452 | ) 453 | # assert ((summary['num_transfer'] - summary['num_migrate']) == (summary['num_switches'] - summary['num_ascend'])).all() # False assertion 454 | summary = summary[mm.metrics.motchallenge_metrics[:15]] 455 | expected = pd.DataFrame( 456 | [ 457 | [ 458 | 0.557659, 459 | 0.729730, 460 | 0.451253, 461 | 0.582173, 462 | 0.941441, 463 | 8.0, 464 | 1, 465 | 6, 466 | 1, 467 | 13, 468 | 150, 469 | 7, 470 | 7, 471 | 0.526462, 472 | 0.277201, 473 | ], 474 | [ 475 | 0.644619, 476 | 0.819760, 477 | 0.531142, 478 | 0.608997, 479 | 0.939920, 480 | 10.0, 481 | 5, 482 | 4, 483 | 1, 484 | 45, 485 | 452, 486 | 7, 487 | 6, 488 | 0.564014, 489 | 0.345904, 490 | ], 491 | [ 492 | 0.624296, 493 | 0.799176, 494 | 0.512211, 495 | 0.602640, 496 | 0.940268, 497 | 18.0, 498 | 6, 499 | 10, 500 | 2, 501 | 58, 502 | 602, 503 | 14, 504 | 13, 505 | 0.555116, 506 | 0.330177, 507 | ], 508 | ] 509 | ) 510 | np.testing.assert_allclose(summary, expected, atol=1e-3) 511 | 512 | 513 | def test_paper_metrics(): 514 | acc = mm.MOTAccumulator(auto_id=True) 515 | acc.update( 516 | [1, 2], # Ground truth objects in this frame 517 | [1, 2, 3], # Detector hypotheses in this frame 518 | [ 519 | [0.1, np.nan, 0.3], # Distances from object 1 to hypotheses 1, 2, 3 520 | [0.5, 0.2, 0.3], # Distances from object 2 to hypotheses 1, 2, 3 521 | ], 522 | ) 523 | acc.update([1, 2], [1], [[0.2], [0.4]]) 524 | acc.update([1, 2], [1, 3], [[0.6, 0.2], [0.1, 0.6]]) 525 | print(acc.events) 526 | mh = mm.metrics.create() 527 | 528 | def my_motp(df: mm.metrics.DataFrameMap): 529 | select = df.full.Type.isin(["MATCH", "SWITCH"]) 530 | subset = df.full[select] 531 | return subset.D.sum() / subset.shape[0] 532 | 533 | mh.register(my_motp, deps="auto") 534 | 535 | summary = mh.compute( 536 | acc, metrics=["num_frames", "precision", "recall"], name="my-acc" 537 | ) 538 | 539 | print(summary) 540 | -------------------------------------------------------------------------------- /yourdata/video2.txt: -------------------------------------------------------------------------------- 1 | 11,0,2490,2483,33,53,-1,-1,-1,-1 2 | 11,1,2972,2784,25,24,-1,-1,-1,-1 3 | 12,0,2490,2487,33,53,-1,-1,-1,-1 4 | 12,1,2972,2784,25,24,-1,-1,-1,-1 5 | 12,2,4032,610,27,45,-1,-1,-1,-1 6 | 12,3,331,2488,37,39,-1,-1,-1,-1 7 | 12,4,2010,151,19,41,-1,-1,-1,-1 8 | 13,0,2491,2492,33,53,-1,-1,-1,-1 9 | 13,1,2970,2783,25,24,-1,-1,-1,-1 10 | 13,2,4033,605,27,45,-1,-1,-1,-1 11 | 13,3,331,2489,37,39,-1,-1,-1,-1 12 | 13,4,2010,153,19,41,-1,-1,-1,-1 13 | 14,0,2492,2497,33,53,-1,-1,-1,-1 14 | 14,1,2971,2782,25,24,-1,-1,-1,-1 15 | 14,2,4032,602,27,45,-1,-1,-1,-1 16 | 14,3,332,2491,37,39,-1,-1,-1,-1 17 | 14,4,2009,154,19,41,-1,-1,-1,-1 18 | 15,0,2493,2501,33,53,-1,-1,-1,-1 19 | 15,1,2970,2782,25,24,-1,-1,-1,-1 20 | 15,2,4033,598,27,45,-1,-1,-1,-1 21 | 15,3,336,2491,37,39,-1,-1,-1,-1 22 | 15,4,2010,151,19,41,-1,-1,-1,-1 23 | 16,0,2494,2504,33,53,-1,-1,-1,-1 24 | 16,1,2970,2781,25,24,-1,-1,-1,-1 25 | 16,2,4033,593,27,45,-1,-1,-1,-1 26 | 16,3,340,2492,37,39,-1,-1,-1,-1 27 | 16,4,2009,151,19,41,-1,-1,-1,-1 28 | 17,0,2495,2508,33,53,-1,-1,-1,-1 29 | 17,1,2969,2781,25,24,-1,-1,-1,-1 30 | 17,2,4033,589,27,45,-1,-1,-1,-1 31 | 17,3,339,2492,37,39,-1,-1,-1,-1 32 | 17,4,2010,150,19,41,-1,-1,-1,-1 33 | 18,0,2496,2513,33,53,-1,-1,-1,-1 34 | 18,1,2968,2780,25,24,-1,-1,-1,-1 35 | 18,2,4035,585,27,45,-1,-1,-1,-1 36 | 18,3,338,2492,37,39,-1,-1,-1,-1 37 | 18,4,2010,150,19,41,-1,-1,-1,-1 38 | 19,0,2495,2519,33,53,-1,-1,-1,-1 39 | 19,1,2968,2780,25,24,-1,-1,-1,-1 40 | 19,2,4035,582,27,45,-1,-1,-1,-1 41 | 19,3,338,2494,37,39,-1,-1,-1,-1 42 | 19,4,2010,147,19,41,-1,-1,-1,-1 43 | 20,0,2495,2522,33,53,-1,-1,-1,-1 44 | 20,1,2967,2779,25,24,-1,-1,-1,-1 45 | 20,2,4035,578,27,45,-1,-1,-1,-1 46 | 20,3,337,2494,37,39,-1,-1,-1,-1 47 | 20,4,2009,145,19,41,-1,-1,-1,-1 48 | 21,0,2494,2527,33,53,-1,-1,-1,-1 49 | 21,1,2966,2779,25,24,-1,-1,-1,-1 50 | 21,2,4035,575,27,45,-1,-1,-1,-1 51 | 21,3,338,2496,37,39,-1,-1,-1,-1 52 | 21,4,2009,144,19,41,-1,-1,-1,-1 53 | 22,0,2495,2531,33,53,-1,-1,-1,-1 54 | 22,1,2967,2778,25,24,-1,-1,-1,-1 55 | 22,2,4036,571,27,45,-1,-1,-1,-1 56 | 22,3,336,2497,37,39,-1,-1,-1,-1 57 | 22,4,2009,145,19,41,-1,-1,-1,-1 58 | 23,0,2495,2537,33,53,-1,-1,-1,-1 59 | 23,1,2967,2778,25,24,-1,-1,-1,-1 60 | 23,2,4036,566,27,45,-1,-1,-1,-1 61 | 23,3,336,2497,37,39,-1,-1,-1,-1 62 | 23,4,2009,144,19,41,-1,-1,-1,-1 63 | 24,0,2494,2540,33,53,-1,-1,-1,-1 64 | 24,1,2965,2777,25,24,-1,-1,-1,-1 65 | 24,2,4036,562,27,45,-1,-1,-1,-1 66 | 24,3,339,2498,37,39,-1,-1,-1,-1 67 | 24,4,2009,143,19,41,-1,-1,-1,-1 68 | 25,0,2495,2543,33,53,-1,-1,-1,-1 69 | 25,1,2965,2777,25,24,-1,-1,-1,-1 70 | 25,2,4036,558,27,45,-1,-1,-1,-1 71 | 25,3,340,2499,37,39,-1,-1,-1,-1 72 | 25,4,2009,140,19,41,-1,-1,-1,-1 73 | 26,0,2496,2547,33,53,-1,-1,-1,-1 74 | 26,1,2964,2777,25,24,-1,-1,-1,-1 75 | 26,2,4036,555,27,45,-1,-1,-1,-1 76 | 26,3,342,2500,37,39,-1,-1,-1,-1 77 | 26,4,2010,139,19,41,-1,-1,-1,-1 78 | 27,0,2498,2551,33,53,-1,-1,-1,-1 79 | 27,1,2965,2777,25,24,-1,-1,-1,-1 80 | 27,2,4036,551,27,45,-1,-1,-1,-1 81 | 27,3,342,2501,37,39,-1,-1,-1,-1 82 | 27,4,2010,138,19,41,-1,-1,-1,-1 83 | 28,0,2498,2556,33,53,-1,-1,-1,-1 84 | 28,1,2964,2776,25,24,-1,-1,-1,-1 85 | 28,2,4037,547,27,45,-1,-1,-1,-1 86 | 28,3,346,2503,37,39,-1,-1,-1,-1 87 | 28,4,2011,137,19,41,-1,-1,-1,-1 88 | 28,8,2068,4038,20,17,-1,-1,-1,-1 89 | 29,0,2499,2560,33,53,-1,-1,-1,-1 90 | 29,1,2964,2776,25,24,-1,-1,-1,-1 91 | 29,2,4038,544,27,45,-1,-1,-1,-1 92 | 29,3,346,2503,37,39,-1,-1,-1,-1 93 | 29,4,2010,137,19,41,-1,-1,-1,-1 94 | 29,8,2072,4040,20,17,-1,-1,-1,-1 95 | 30,0,2500,2565,33,53,-1,-1,-1,-1 96 | 30,1,2963,2775,25,24,-1,-1,-1,-1 97 | 30,2,4038,539,27,45,-1,-1,-1,-1 98 | 30,3,346,2503,37,39,-1,-1,-1,-1 99 | 30,4,2010,137,19,41,-1,-1,-1,-1 100 | 30,8,2077,4041,20,17,-1,-1,-1,-1 101 | 31,0,2501,2569,33,53,-1,-1,-1,-1 102 | 31,1,2962,2775,25,24,-1,-1,-1,-1 103 | 31,2,4039,535,27,45,-1,-1,-1,-1 104 | 31,3,347,2504,37,39,-1,-1,-1,-1 105 | 31,4,2010,136,19,41,-1,-1,-1,-1 106 | 31,8,2077,4042,20,17,-1,-1,-1,-1 107 | 32,0,2502,2574,33,53,-1,-1,-1,-1 108 | 32,1,2964,2774,25,24,-1,-1,-1,-1 109 | 32,2,4040,531,27,45,-1,-1,-1,-1 110 | 32,3,347,2504,37,39,-1,-1,-1,-1 111 | 32,4,2010,135,19,41,-1,-1,-1,-1 112 | 32,8,2082,4044,20,17,-1,-1,-1,-1 113 | 33,0,2504,2577,33,53,-1,-1,-1,-1 114 | 33,1,2964,2774,25,24,-1,-1,-1,-1 115 | 33,2,4041,528,27,45,-1,-1,-1,-1 116 | 33,3,346,2506,37,39,-1,-1,-1,-1 117 | 33,4,2010,133,19,41,-1,-1,-1,-1 118 | 33,8,2079,4044,20,17,-1,-1,-1,-1 119 | 33,10,2400,4054,22,9,-1,-1,-1,-1 120 | 34,0,2506,2582,33,53,-1,-1,-1,-1 121 | 34,1,2962,2774,25,24,-1,-1,-1,-1 122 | 34,2,4041,523,27,45,-1,-1,-1,-1 123 | 34,3,344,2506,37,39,-1,-1,-1,-1 124 | 34,4,2010,134,19,41,-1,-1,-1,-1 125 | 34,8,2083,4046,20,17,-1,-1,-1,-1 126 | 34,10,2445,4054,22,9,-1,-1,-1,-1 127 | 35,0,2508,2586,33,53,-1,-1,-1,-1 128 | 35,1,2961,2773,25,24,-1,-1,-1,-1 129 | 35,2,4041,520,27,45,-1,-1,-1,-1 130 | 35,3,342,2507,37,39,-1,-1,-1,-1 131 | 35,4,2010,132,19,41,-1,-1,-1,-1 132 | 35,8,2172,4041,20,17,-1,-1,-1,-1 133 | 35,10,2481,4053,22,9,-1,-1,-1,-1 134 | 36,0,2506,2590,33,53,-1,-1,-1,-1 135 | 36,1,2959,2773,25,24,-1,-1,-1,-1 136 | 36,2,4040,516,27,45,-1,-1,-1,-1 137 | 36,3,342,2508,37,39,-1,-1,-1,-1 138 | 36,4,2010,131,19,41,-1,-1,-1,-1 139 | 36,8,2273,4036,20,17,-1,-1,-1,-1 140 | 36,10,2526,4053,22,9,-1,-1,-1,-1 141 | 37,0,2505,2595,33,53,-1,-1,-1,-1 142 | 37,1,2959,2772,25,24,-1,-1,-1,-1 143 | 37,2,4041,511,27,45,-1,-1,-1,-1 144 | 37,3,344,2508,37,39,-1,-1,-1,-1 145 | 37,4,2011,130,19,41,-1,-1,-1,-1 146 | 37,8,2313,4035,20,17,-1,-1,-1,-1 147 | 37,10,2569,4052,22,9,-1,-1,-1,-1 148 | 38,0,2504,2599,33,53,-1,-1,-1,-1 149 | 38,1,2958,2772,25,24,-1,-1,-1,-1 150 | 38,2,4043,508,27,45,-1,-1,-1,-1 151 | 38,3,344,2510,37,39,-1,-1,-1,-1 152 | 38,4,2011,129,19,41,-1,-1,-1,-1 153 | 38,8,2345,4033,20,17,-1,-1,-1,-1 154 | 38,10,2601,4051,22,9,-1,-1,-1,-1 155 | 39,0,2504,2603,33,53,-1,-1,-1,-1 156 | 39,1,2957,2771,25,24,-1,-1,-1,-1 157 | 39,2,4042,505,27,45,-1,-1,-1,-1 158 | 39,3,349,2511,37,39,-1,-1,-1,-1 159 | 39,4,2011,127,19,41,-1,-1,-1,-1 160 | 39,8,2375,4031,20,17,-1,-1,-1,-1 161 | 39,10,2632,4050,22,9,-1,-1,-1,-1 162 | 40,0,2505,2608,33,53,-1,-1,-1,-1 163 | 40,1,2957,2771,25,24,-1,-1,-1,-1 164 | 40,2,4042,502,27,45,-1,-1,-1,-1 165 | 40,3,347,2511,37,39,-1,-1,-1,-1 166 | 40,4,2010,127,19,41,-1,-1,-1,-1 167 | 40,8,2403,4028,20,17,-1,-1,-1,-1 168 | 40,10,2661,4049,22,9,-1,-1,-1,-1 169 | 41,0,2507,2612,33,53,-1,-1,-1,-1 170 | 41,1,2956,2770,25,24,-1,-1,-1,-1 171 | 41,2,4042,498,27,45,-1,-1,-1,-1 172 | 41,3,347,2512,37,39,-1,-1,-1,-1 173 | 41,4,2009,128,19,41,-1,-1,-1,-1 174 | 41,8,2435,4026,20,17,-1,-1,-1,-1 175 | 41,10,2698,4048,22,9,-1,-1,-1,-1 176 | 42,0,2509,2616,33,53,-1,-1,-1,-1 177 | 42,1,2956,2770,25,24,-1,-1,-1,-1 178 | 42,2,4043,494,27,45,-1,-1,-1,-1 179 | 42,3,352,2513,37,39,-1,-1,-1,-1 180 | 42,4,2009,128,19,41,-1,-1,-1,-1 181 | 42,8,2459,4024,20,17,-1,-1,-1,-1 182 | 42,10,2724,4046,22,9,-1,-1,-1,-1 183 | 43,0,2510,2620,33,53,-1,-1,-1,-1 184 | 43,1,2957,2770,25,24,-1,-1,-1,-1 185 | 43,2,4044,489,27,45,-1,-1,-1,-1 186 | 43,3,352,2514,37,39,-1,-1,-1,-1 187 | 43,4,2009,128,19,41,-1,-1,-1,-1 188 | 43,8,2487,4022,20,17,-1,-1,-1,-1 189 | 44,0,2509,2625,33,53,-1,-1,-1,-1 190 | 44,1,2956,2769,25,24,-1,-1,-1,-1 191 | 44,2,4045,484,27,45,-1,-1,-1,-1 192 | 44,3,352,2515,37,39,-1,-1,-1,-1 193 | 44,4,2009,126,19,41,-1,-1,-1,-1 194 | 44,8,2513,4019,20,17,-1,-1,-1,-1 195 | 45,0,2509,2629,33,53,-1,-1,-1,-1 196 | 45,1,2955,2769,25,24,-1,-1,-1,-1 197 | 45,2,4045,479,27,45,-1,-1,-1,-1 198 | 45,3,351,2515,37,39,-1,-1,-1,-1 199 | 45,4,2009,125,19,41,-1,-1,-1,-1 200 | 45,8,2533,4016,20,17,-1,-1,-1,-1 201 | 46,0,2510,2633,33,53,-1,-1,-1,-1 202 | 46,1,2954,2768,25,24,-1,-1,-1,-1 203 | 46,2,4044,476,27,45,-1,-1,-1,-1 204 | 46,3,353,2516,37,39,-1,-1,-1,-1 205 | 46,4,2009,123,19,41,-1,-1,-1,-1 206 | 47,0,2512,2637,33,53,-1,-1,-1,-1 207 | 47,1,2953,2767,25,24,-1,-1,-1,-1 208 | 47,2,4045,473,27,45,-1,-1,-1,-1 209 | 47,3,352,2517,37,39,-1,-1,-1,-1 210 | 47,4,2009,123,19,41,-1,-1,-1,-1 211 | 48,0,2514,2642,33,53,-1,-1,-1,-1 212 | 48,1,2954,2767,25,24,-1,-1,-1,-1 213 | 48,2,4045,469,27,45,-1,-1,-1,-1 214 | 48,3,352,2517,37,39,-1,-1,-1,-1 215 | 48,4,2009,122,19,41,-1,-1,-1,-1 216 | 49,0,2514,2647,33,53,-1,-1,-1,-1 217 | 49,1,2955,2767,25,24,-1,-1,-1,-1 218 | 49,2,4047,465,27,45,-1,-1,-1,-1 219 | 49,3,353,2518,37,39,-1,-1,-1,-1 220 | 49,4,2010,122,19,41,-1,-1,-1,-1 221 | 50,0,2514,2651,33,53,-1,-1,-1,-1 222 | 50,1,2953,2766,25,24,-1,-1,-1,-1 223 | 50,2,4048,462,27,45,-1,-1,-1,-1 224 | 50,3,355,2518,37,39,-1,-1,-1,-1 225 | 50,4,2010,120,19,41,-1,-1,-1,-1 226 | 51,0,2514,2656,33,53,-1,-1,-1,-1 227 | 51,1,2953,2766,25,24,-1,-1,-1,-1 228 | 51,2,4048,457,27,45,-1,-1,-1,-1 229 | 51,3,353,2519,37,39,-1,-1,-1,-1 230 | 51,4,2011,121,19,41,-1,-1,-1,-1 231 | 52,0,2513,2660,33,53,-1,-1,-1,-1 232 | 52,1,2952,2765,25,24,-1,-1,-1,-1 233 | 52,2,4048,453,27,45,-1,-1,-1,-1 234 | 52,3,354,2519,37,39,-1,-1,-1,-1 235 | 52,4,2010,122,19,41,-1,-1,-1,-1 236 | 53,0,2512,2664,33,53,-1,-1,-1,-1 237 | 53,1,2951,2765,25,24,-1,-1,-1,-1 238 | 53,2,4048,449,27,45,-1,-1,-1,-1 239 | 53,3,353,2521,37,39,-1,-1,-1,-1 240 | 53,4,2009,121,19,41,-1,-1,-1,-1 241 | 54,0,2513,2668,33,53,-1,-1,-1,-1 242 | 54,1,2952,2765,25,24,-1,-1,-1,-1 243 | 54,2,4048,445,27,45,-1,-1,-1,-1 244 | 54,3,354,2522,37,39,-1,-1,-1,-1 245 | 54,4,2008,119,19,41,-1,-1,-1,-1 246 | 55,0,2514,2672,33,53,-1,-1,-1,-1 247 | 55,1,2952,2764,25,24,-1,-1,-1,-1 248 | 55,2,4049,441,27,45,-1,-1,-1,-1 249 | 55,3,354,2523,37,39,-1,-1,-1,-1 250 | 55,4,2009,116,19,41,-1,-1,-1,-1 251 | 56,0,2515,2677,33,53,-1,-1,-1,-1 252 | 56,1,2951,2764,25,24,-1,-1,-1,-1 253 | 56,2,4049,437,27,45,-1,-1,-1,-1 254 | 56,3,355,2524,37,39,-1,-1,-1,-1 255 | 56,4,2008,116,19,41,-1,-1,-1,-1 256 | 57,0,2516,2680,33,53,-1,-1,-1,-1 257 | 57,1,2951,2763,25,24,-1,-1,-1,-1 258 | 57,2,4050,433,27,45,-1,-1,-1,-1 259 | 57,3,355,2525,37,39,-1,-1,-1,-1 260 | 57,4,2008,116,19,41,-1,-1,-1,-1 261 | 58,0,2516,2686,33,53,-1,-1,-1,-1 262 | 58,1,2951,2763,25,24,-1,-1,-1,-1 263 | 58,2,4051,429,27,45,-1,-1,-1,-1 264 | 58,3,357,2526,37,39,-1,-1,-1,-1 265 | 58,4,2009,116,19,41,-1,-1,-1,-1 266 | 59,0,2517,2690,33,53,-1,-1,-1,-1 267 | 59,1,2950,2763,25,24,-1,-1,-1,-1 268 | 59,2,4051,425,27,45,-1,-1,-1,-1 269 | 59,3,357,2526,37,39,-1,-1,-1,-1 270 | 59,4,2009,113,19,41,-1,-1,-1,-1 271 | 60,0,2517,2694,33,53,-1,-1,-1,-1 272 | 60,1,2948,2762,25,24,-1,-1,-1,-1 273 | 60,2,4051,420,27,45,-1,-1,-1,-1 274 | 60,3,357,2527,37,39,-1,-1,-1,-1 275 | 60,4,2009,115,19,41,-1,-1,-1,-1 276 | 61,0,2518,2698,33,53,-1,-1,-1,-1 277 | 61,1,2947,2762,25,24,-1,-1,-1,-1 278 | 61,2,4052,415,27,45,-1,-1,-1,-1 279 | 61,3,360,2527,37,39,-1,-1,-1,-1 280 | 61,4,2010,116,19,41,-1,-1,-1,-1 281 | 62,0,2519,2702,33,53,-1,-1,-1,-1 282 | 62,1,2945,2761,25,24,-1,-1,-1,-1 283 | 62,2,4052,412,27,45,-1,-1,-1,-1 284 | 62,3,362,2528,37,39,-1,-1,-1,-1 285 | 62,4,2009,113,19,41,-1,-1,-1,-1 286 | 63,0,2521,2707,33,53,-1,-1,-1,-1 287 | 63,1,2946,2761,25,24,-1,-1,-1,-1 288 | 63,2,4052,408,27,45,-1,-1,-1,-1 289 | 63,3,364,2529,37,39,-1,-1,-1,-1 290 | 63,4,2010,112,19,41,-1,-1,-1,-1 291 | 64,0,2522,2711,33,53,-1,-1,-1,-1 292 | 64,1,2945,2760,25,24,-1,-1,-1,-1 293 | 64,2,4053,405,27,45,-1,-1,-1,-1 294 | 64,3,366,2530,37,39,-1,-1,-1,-1 295 | 64,4,2009,112,19,41,-1,-1,-1,-1 296 | 65,0,2524,2717,33,53,-1,-1,-1,-1 297 | 65,1,2943,2760,25,24,-1,-1,-1,-1 298 | 65,2,4054,399,27,45,-1,-1,-1,-1 299 | 65,3,365,2531,37,39,-1,-1,-1,-1 300 | 65,4,2009,111,19,41,-1,-1,-1,-1 301 | 66,0,2523,2722,33,53,-1,-1,-1,-1 302 | 66,1,2944,2760,25,24,-1,-1,-1,-1 303 | 66,2,4054,396,27,45,-1,-1,-1,-1 304 | 66,3,366,2531,37,39,-1,-1,-1,-1 305 | 66,4,2009,109,19,41,-1,-1,-1,-1 306 | 67,0,2523,2726,33,53,-1,-1,-1,-1 307 | 67,1,2945,2759,25,24,-1,-1,-1,-1 308 | 67,2,4055,392,27,45,-1,-1,-1,-1 309 | 67,3,367,2531,37,39,-1,-1,-1,-1 310 | 67,4,2009,106,19,41,-1,-1,-1,-1 311 | 68,0,2523,2729,33,53,-1,-1,-1,-1 312 | 68,1,2944,2758,25,24,-1,-1,-1,-1 313 | 68,2,4056,388,27,45,-1,-1,-1,-1 314 | 68,3,368,2532,37,39,-1,-1,-1,-1 315 | 68,4,2009,105,19,41,-1,-1,-1,-1 316 | 69,0,2524,2733,33,53,-1,-1,-1,-1 317 | 69,1,2944,2758,25,24,-1,-1,-1,-1 318 | 69,2,4055,385,27,45,-1,-1,-1,-1 319 | 69,3,368,2533,37,39,-1,-1,-1,-1 320 | 69,4,2008,105,19,41,-1,-1,-1,-1 321 | 70,0,2525,2739,33,53,-1,-1,-1,-1 322 | 70,1,2944,2757,25,24,-1,-1,-1,-1 323 | 70,2,4055,381,27,45,-1,-1,-1,-1 324 | 70,3,368,2534,37,39,-1,-1,-1,-1 325 | 70,4,2008,103,19,41,-1,-1,-1,-1 326 | 71,0,2526,2743,33,53,-1,-1,-1,-1 327 | 71,1,2944,2757,25,24,-1,-1,-1,-1 328 | 71,2,4056,377,27,45,-1,-1,-1,-1 329 | 71,3,369,2535,37,39,-1,-1,-1,-1 330 | 71,4,2008,102,19,41,-1,-1,-1,-1 331 | 72,0,2529,2748,33,53,-1,-1,-1,-1 332 | 72,1,2943,2756,25,24,-1,-1,-1,-1 333 | 72,2,4056,373,27,45,-1,-1,-1,-1 334 | 72,3,369,2535,37,39,-1,-1,-1,-1 335 | 72,4,2008,103,19,41,-1,-1,-1,-1 336 | 73,0,2528,2753,33,53,-1,-1,-1,-1 337 | 73,1,2942,2756,25,24,-1,-1,-1,-1 338 | 73,2,4056,369,27,45,-1,-1,-1,-1 339 | 73,3,370,2536,37,39,-1,-1,-1,-1 340 | 73,4,2008,102,19,41,-1,-1,-1,-1 341 | 74,0,2528,2756,33,53,-1,-1,-1,-1 342 | 74,1,2942,2755,25,24,-1,-1,-1,-1 343 | 74,2,4056,364,27,45,-1,-1,-1,-1 344 | 74,3,370,2537,37,39,-1,-1,-1,-1 345 | 74,4,2008,103,19,41,-1,-1,-1,-1 346 | 75,0,2529,2760,33,53,-1,-1,-1,-1 347 | 75,1,2941,2755,25,24,-1,-1,-1,-1 348 | 75,2,4057,360,27,45,-1,-1,-1,-1 349 | 75,3,369,2538,37,39,-1,-1,-1,-1 350 | 75,4,2009,102,19,41,-1,-1,-1,-1 351 | 76,0,2529,2764,33,53,-1,-1,-1,-1 352 | 76,1,2940,2754,25,24,-1,-1,-1,-1 353 | 76,2,4058,357,27,45,-1,-1,-1,-1 354 | 76,3,369,2540,37,39,-1,-1,-1,-1 355 | 76,4,2009,100,19,41,-1,-1,-1,-1 356 | 77,0,2530,2768,33,53,-1,-1,-1,-1 357 | 77,1,2942,2754,25,24,-1,-1,-1,-1 358 | 77,2,4058,352,27,45,-1,-1,-1,-1 359 | 77,3,370,2541,37,39,-1,-1,-1,-1 360 | 77,4,2008,99,19,41,-1,-1,-1,-1 361 | 78,0,2531,2772,33,53,-1,-1,-1,-1 362 | 78,1,2941,2753,25,24,-1,-1,-1,-1 363 | 78,2,4059,349,27,45,-1,-1,-1,-1 364 | 78,3,372,2541,37,39,-1,-1,-1,-1 365 | 78,4,2008,98,19,41,-1,-1,-1,-1 366 | 79,0,2532,2777,33,53,-1,-1,-1,-1 367 | 79,1,2940,2753,25,24,-1,-1,-1,-1 368 | 79,2,4059,344,27,45,-1,-1,-1,-1 369 | 79,3,372,2541,37,39,-1,-1,-1,-1 370 | 79,4,2009,99,19,41,-1,-1,-1,-1 371 | 80,0,2533,2782,33,53,-1,-1,-1,-1 372 | 80,1,2940,2753,25,24,-1,-1,-1,-1 373 | 80,2,4059,340,27,45,-1,-1,-1,-1 374 | 80,3,372,2541,37,39,-1,-1,-1,-1 375 | 80,4,2009,97,19,41,-1,-1,-1,-1 376 | 80,46,1474,4049,24,14,-1,-1,-1,-1 377 | 81,0,2533,2786,33,53,-1,-1,-1,-1 378 | 81,1,2940,2752,25,24,-1,-1,-1,-1 379 | 81,2,4059,339,27,45,-1,-1,-1,-1 380 | 81,3,372,2543,37,39,-1,-1,-1,-1 381 | 81,4,2009,96,19,41,-1,-1,-1,-1 382 | 81,46,1460,4045,24,14,-1,-1,-1,-1 383 | 82,0,2536,2791,33,53,-1,-1,-1,-1 384 | 82,1,2940,2752,25,24,-1,-1,-1,-1 385 | 82,2,4059,333,27,45,-1,-1,-1,-1 386 | 82,3,370,2544,37,39,-1,-1,-1,-1 387 | 82,4,2009,95,19,41,-1,-1,-1,-1 388 | 82,46,1449,4042,24,14,-1,-1,-1,-1 389 | 83,0,2535,2795,33,53,-1,-1,-1,-1 390 | 83,1,2938,2752,25,24,-1,-1,-1,-1 391 | 83,2,4060,330,27,45,-1,-1,-1,-1 392 | 83,3,372,2546,37,39,-1,-1,-1,-1 393 | 83,4,2009,93,19,41,-1,-1,-1,-1 394 | 83,46,1447,4037,24,14,-1,-1,-1,-1 395 | 84,0,2538,2799,33,53,-1,-1,-1,-1 396 | 84,1,2938,2751,25,24,-1,-1,-1,-1 397 | 84,2,4060,326,27,45,-1,-1,-1,-1 398 | 84,3,373,2546,37,39,-1,-1,-1,-1 399 | 84,4,2009,92,19,41,-1,-1,-1,-1 400 | 84,46,1441,4033,24,14,-1,-1,-1,-1 401 | 85,0,2539,2803,33,53,-1,-1,-1,-1 402 | 85,1,2938,2750,25,24,-1,-1,-1,-1 403 | 85,2,4062,321,27,45,-1,-1,-1,-1 404 | 85,3,373,2547,37,39,-1,-1,-1,-1 405 | 85,4,2010,92,19,41,-1,-1,-1,-1 406 | 85,46,1433,4029,24,14,-1,-1,-1,-1 407 | 86,0,2540,2808,33,53,-1,-1,-1,-1 408 | 86,1,2937,2750,25,24,-1,-1,-1,-1 409 | 86,2,4062,315,27,45,-1,-1,-1,-1 410 | 86,3,375,2547,37,39,-1,-1,-1,-1 411 | 86,4,2010,91,19,41,-1,-1,-1,-1 412 | 86,46,1426,4025,24,14,-1,-1,-1,-1 413 | 87,0,2539,2812,33,53,-1,-1,-1,-1 414 | 87,1,2936,2750,25,24,-1,-1,-1,-1 415 | 87,2,4062,311,27,45,-1,-1,-1,-1 416 | 87,3,372,2548,37,39,-1,-1,-1,-1 417 | 87,4,2009,90,19,41,-1,-1,-1,-1 418 | 87,46,1422,4021,24,14,-1,-1,-1,-1 419 | 88,0,2539,2817,33,53,-1,-1,-1,-1 420 | 88,1,2936,2749,25,24,-1,-1,-1,-1 421 | 88,2,4062,307,27,45,-1,-1,-1,-1 422 | 88,3,374,2548,37,39,-1,-1,-1,-1 423 | 88,4,2009,89,19,41,-1,-1,-1,-1 424 | 88,46,1416,4017,24,14,-1,-1,-1,-1 425 | 89,0,2540,2820,33,53,-1,-1,-1,-1 426 | 89,1,2936,2749,25,24,-1,-1,-1,-1 427 | 89,2,4063,304,27,45,-1,-1,-1,-1 428 | 89,3,375,2549,37,39,-1,-1,-1,-1 429 | 89,4,2009,92,19,41,-1,-1,-1,-1 430 | 89,46,1410,4014,24,14,-1,-1,-1,-1 431 | 90,0,2540,2823,33,53,-1,-1,-1,-1 432 | 90,1,2935,2748,25,24,-1,-1,-1,-1 433 | 90,2,4064,301,27,45,-1,-1,-1,-1 434 | 90,3,376,2549,37,39,-1,-1,-1,-1 435 | 90,4,2009,89,19,41,-1,-1,-1,-1 436 | 91,0,2540,2828,33,53,-1,-1,-1,-1 437 | 91,1,2934,2748,25,24,-1,-1,-1,-1 438 | 91,2,4065,297,27,45,-1,-1,-1,-1 439 | 91,3,378,2550,37,39,-1,-1,-1,-1 440 | 91,4,2009,88,19,41,-1,-1,-1,-1 441 | 92,0,2543,2832,33,53,-1,-1,-1,-1 442 | 92,1,2934,2747,25,24,-1,-1,-1,-1 443 | 92,2,4065,292,27,45,-1,-1,-1,-1 444 | 92,3,379,2553,37,39,-1,-1,-1,-1 445 | 92,4,2008,87,19,41,-1,-1,-1,-1 446 | 93,0,2545,2837,33,53,-1,-1,-1,-1 447 | 93,1,2934,2746,25,24,-1,-1,-1,-1 448 | 93,2,4066,288,27,45,-1,-1,-1,-1 449 | 93,3,379,2552,37,39,-1,-1,-1,-1 450 | 93,4,2010,86,19,41,-1,-1,-1,-1 451 | 94,0,2545,2841,33,53,-1,-1,-1,-1 452 | 94,1,2933,2746,25,24,-1,-1,-1,-1 453 | 94,2,4066,284,27,45,-1,-1,-1,-1 454 | 94,3,379,2553,37,39,-1,-1,-1,-1 455 | 94,4,2010,86,19,41,-1,-1,-1,-1 456 | 95,0,2546,2845,33,53,-1,-1,-1,-1 457 | 95,1,2933,2746,25,24,-1,-1,-1,-1 458 | 95,2,4067,279,27,45,-1,-1,-1,-1 459 | 95,3,379,2554,37,39,-1,-1,-1,-1 460 | 95,4,2010,84,19,41,-1,-1,-1,-1 461 | 96,0,2546,2850,33,53,-1,-1,-1,-1 462 | 96,1,2934,2745,25,24,-1,-1,-1,-1 463 | 96,2,4066,275,27,45,-1,-1,-1,-1 464 | 96,3,380,2554,37,39,-1,-1,-1,-1 465 | 96,4,2010,85,19,41,-1,-1,-1,-1 466 | 97,0,2546,2854,33,53,-1,-1,-1,-1 467 | 97,1,2934,2745,25,24,-1,-1,-1,-1 468 | 97,2,4067,272,27,45,-1,-1,-1,-1 469 | 97,3,380,2556,37,39,-1,-1,-1,-1 470 | 97,4,2009,83,19,41,-1,-1,-1,-1 471 | 98,0,2547,2858,33,53,-1,-1,-1,-1 472 | 98,1,2933,2745,25,24,-1,-1,-1,-1 473 | 98,2,4067,269,27,45,-1,-1,-1,-1 474 | 98,3,380,2557,37,39,-1,-1,-1,-1 475 | 98,4,2008,83,19,41,-1,-1,-1,-1 476 | 99,0,2547,2863,33,53,-1,-1,-1,-1 477 | 99,1,2931,2744,25,24,-1,-1,-1,-1 478 | 99,2,4067,264,27,45,-1,-1,-1,-1 479 | 99,3,382,2556,37,39,-1,-1,-1,-1 480 | 99,4,2009,81,19,41,-1,-1,-1,-1 481 | 100,0,2549,2867,33,53,-1,-1,-1,-1 482 | 100,1,2930,2744,25,24,-1,-1,-1,-1 483 | 100,2,4066,260,27,45,-1,-1,-1,-1 484 | 100,3,382,2556,37,39,-1,-1,-1,-1 485 | 100,4,2009,81,19,41,-1,-1,-1,-1 486 | 101,0,2550,2871,33,53,-1,-1,-1,-1 487 | 101,1,2930,2743,25,24,-1,-1,-1,-1 488 | 101,2,4066,256,27,45,-1,-1,-1,-1 489 | 101,3,382,2557,37,39,-1,-1,-1,-1 490 | 101,4,2008,81,19,41,-1,-1,-1,-1 491 | 102,0,2551,2875,33,53,-1,-1,-1,-1 492 | 102,1,2931,2743,25,24,-1,-1,-1,-1 493 | 102,2,4067,251,27,45,-1,-1,-1,-1 494 | 102,3,383,2557,37,39,-1,-1,-1,-1 495 | 102,4,2009,81,19,41,-1,-1,-1,-1 496 | 103,0,2552,2879,33,53,-1,-1,-1,-1 497 | 103,1,2929,2742,25,24,-1,-1,-1,-1 498 | 103,2,4068,247,27,45,-1,-1,-1,-1 499 | 103,3,383,2559,37,39,-1,-1,-1,-1 500 | 103,4,2009,80,19,41,-1,-1,-1,-1 501 | 104,0,2553,2883,33,53,-1,-1,-1,-1 502 | 104,1,2930,2742,25,24,-1,-1,-1,-1 503 | 104,2,4068,244,27,45,-1,-1,-1,-1 504 | 104,3,384,2562,37,39,-1,-1,-1,-1 505 | 104,4,2008,78,19,41,-1,-1,-1,-1 506 | 105,0,2554,2888,33,53,-1,-1,-1,-1 507 | 105,1,2929,2741,25,24,-1,-1,-1,-1 508 | 105,2,4068,239,27,45,-1,-1,-1,-1 509 | 105,3,384,2560,37,39,-1,-1,-1,-1 510 | 105,4,2008,77,19,41,-1,-1,-1,-1 511 | 106,0,2556,2892,33,53,-1,-1,-1,-1 512 | 106,1,2930,2741,25,24,-1,-1,-1,-1 513 | 106,2,4069,236,27,45,-1,-1,-1,-1 514 | 106,3,386,2562,37,39,-1,-1,-1,-1 515 | 106,4,2008,75,19,41,-1,-1,-1,-1 516 | 107,0,2556,2896,33,53,-1,-1,-1,-1 517 | 107,1,2929,2741,25,24,-1,-1,-1,-1 518 | 107,2,4069,233,27,45,-1,-1,-1,-1 519 | 107,3,387,2563,37,39,-1,-1,-1,-1 520 | 107,4,2008,75,19,41,-1,-1,-1,-1 521 | 107,61,2948,2746,4,22,-1,-1,-1,-1 522 | 108,0,2556,2900,33,53,-1,-1,-1,-1 523 | 108,1,2927,2740,25,24,-1,-1,-1,-1 524 | 108,2,4069,229,27,45,-1,-1,-1,-1 525 | 108,3,387,2563,37,39,-1,-1,-1,-1 526 | 108,4,2008,75,19,41,-1,-1,-1,-1 527 | 108,61,2948,2746,4,22,-1,-1,-1,-1 528 | 109,0,2558,2904,33,53,-1,-1,-1,-1 529 | 109,1,2928,2739,25,24,-1,-1,-1,-1 530 | 109,2,4070,223,27,45,-1,-1,-1,-1 531 | 109,3,387,2563,37,39,-1,-1,-1,-1 532 | 109,4,2008,74,19,41,-1,-1,-1,-1 533 | 109,61,2949,2746,4,22,-1,-1,-1,-1 534 | 110,0,2560,2908,33,53,-1,-1,-1,-1 535 | 110,1,2927,2739,25,24,-1,-1,-1,-1 536 | 110,2,4071,219,27,45,-1,-1,-1,-1 537 | 110,3,390,2565,37,39,-1,-1,-1,-1 538 | 110,4,2008,73,19,41,-1,-1,-1,-1 539 | 110,61,2948,2748,4,22,-1,-1,-1,-1 540 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 21 | # number of processors available to use. 22 | jobs=1 23 | 24 | # Control the amount of potential inferred values when inferring a single 25 | # object. This can help the performance when dealing with large functions or 26 | # complex, nested conditions. 27 | limit-inference-results=100 28 | 29 | # List of plugins (as comma separated values of python module names) to load, 30 | # usually to register additional checkers. 31 | load-plugins= 32 | 33 | # Pickle collected data for later comparisons. 34 | persistent=yes 35 | 36 | # Specify a configuration file. 37 | #rcfile= 38 | 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit 40 | # user-friendly hints instead of false-positive error messages. 41 | suggestion-mode=yes 42 | 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the 44 | # active Python interpreter and may run arbitrary code. 45 | unsafe-load-any-extension=no 46 | 47 | 48 | [MESSAGES CONTROL] 49 | 50 | # Only show warnings with the listed confidence levels. Leave empty to show 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 52 | confidence= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once). You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use "--disable=all --enable=classes 62 | # --disable=W". 63 | disable=line-too-long, 64 | too-many-arguments, 65 | too-many-branches, 66 | invalid-name, 67 | no-else-return, 68 | useless-object-inheritance, 69 | too-many-instance-attributes, 70 | similarities, 71 | import-error, 72 | 73 | # Enable the message, report, category or checker with the given id(s). You can 74 | # either give multiple identifier separated by comma (,) or put this option 75 | # multiple time (only on the command line, not in the configuration file where 76 | # it should appear only once). See also the "--disable" option for examples. 77 | enable=c-extension-no-member, 78 | # List of options disabled by default: 79 | print-statement, 80 | parameter-unpacking, 81 | unpacking-in-except, 82 | old-raise-syntax, 83 | backtick, 84 | long-suffix, 85 | old-ne-operator, 86 | old-octal-literal, 87 | import-star-module-level, 88 | non-ascii-bytes-literal, 89 | raw-checker-failed, 90 | bad-inline-option, 91 | # locally-disabled, 92 | file-ignored, 93 | # suppressed-message, 94 | useless-suppression, 95 | deprecated-pragma, 96 | use-symbolic-message-instead, 97 | apply-builtin, 98 | basestring-builtin, 99 | buffer-builtin, 100 | cmp-builtin, 101 | coerce-builtin, 102 | execfile-builtin, 103 | file-builtin, 104 | long-builtin, 105 | raw_input-builtin, 106 | reduce-builtin, 107 | standarderror-builtin, 108 | unicode-builtin, 109 | xrange-builtin, 110 | coerce-method, 111 | delslice-method, 112 | getslice-method, 113 | setslice-method, 114 | no-absolute-import, 115 | old-division, 116 | dict-iter-method, 117 | dict-view-method, 118 | next-method-called, 119 | metaclass-assignment, 120 | indexing-exception, 121 | raising-string, 122 | reload-builtin, 123 | oct-method, 124 | hex-method, 125 | nonzero-method, 126 | cmp-method, 127 | input-builtin, 128 | round-builtin, 129 | intern-builtin, 130 | unichr-builtin, 131 | map-builtin-not-iterating, 132 | zip-builtin-not-iterating, 133 | range-builtin-not-iterating, 134 | filter-builtin-not-iterating, 135 | using-cmp-argument, 136 | eq-without-hash, 137 | div-method, 138 | idiv-method, 139 | rdiv-method, 140 | exception-message-attribute, 141 | invalid-str-codec, 142 | sys-max-int, 143 | bad-python3-import, 144 | deprecated-string-function, 145 | deprecated-str-translate-call, 146 | deprecated-itertools-function, 147 | deprecated-types-field, 148 | next-method-defined, 149 | dict-items-not-iterating, 150 | dict-keys-not-iterating, 151 | dict-values-not-iterating, 152 | deprecated-operator-function, 153 | deprecated-urllib-function, 154 | xreadlines-attribute, 155 | deprecated-sys-function, 156 | exception-escape, 157 | comprehension-escape, 158 | 159 | 160 | [REPORTS] 161 | 162 | # Python expression which should return a score less than or equal to 10. You 163 | # have access to the variables 'error', 'warning', 'refactor', and 'convention' 164 | # which contain the number of messages in each category, as well as 'statement' 165 | # which is the total number of statements analyzed. This score is used by the 166 | # global evaluation report (RP0004). 167 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 168 | 169 | # Template used to display messages. This is a python new-style format string 170 | # used to format the message information. See doc for all details. 171 | #msg-template= 172 | 173 | # Set the output format. Available formats are text, parseable, colorized, json 174 | # and msvs (visual studio). You can also give a reporter class, e.g. 175 | # mypackage.mymodule.MyReporterClass. 176 | output-format=text 177 | 178 | # Tells whether to display a full report or only the messages. 179 | reports=no 180 | 181 | # Activate the evaluation score. 182 | score=yes 183 | 184 | 185 | [REFACTORING] 186 | 187 | # Maximum number of nested blocks for function / method body 188 | max-nested-blocks=5 189 | 190 | # Complete name of functions that never returns. When checking for 191 | # inconsistent-return-statements if a never returning function is called then 192 | # it will be considered as an explicit return statement and no message will be 193 | # printed. 194 | never-returning-functions=sys.exit 195 | 196 | 197 | [STRING] 198 | 199 | # This flag controls whether the implicit-str-concat-in-sequence should 200 | # generate a warning on implicit string concatenation in sequences defined over 201 | # several lines. 202 | check-str-concat-over-line-jumps=no 203 | 204 | 205 | [FORMAT] 206 | 207 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 208 | expected-line-ending-format= 209 | 210 | # Regexp for a line that is allowed to be longer than the limit. 211 | ignore-long-lines=^\s*(# )??$ 212 | 213 | # Number of spaces of indent required inside a hanging or continued line. 214 | indent-after-paren=4 215 | 216 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 217 | # tab). 218 | indent-string=' ' 219 | 220 | # Maximum number of characters on a single line. 221 | max-line-length=100 222 | 223 | # Maximum number of lines in a module. 224 | max-module-lines=1000 225 | 226 | # List of optional constructs for which whitespace checking is disabled. `dict- 227 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 228 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 229 | # `empty-line` allows space-only lines. 230 | no-space-check=trailing-comma, 231 | dict-separator 232 | 233 | # Allow the body of a class to be on the same line as the declaration if body 234 | # contains single statement. 235 | single-line-class-stmt=no 236 | 237 | # Allow the body of an if to be on the same line as the test if there is no 238 | # else. 239 | single-line-if-stmt=no 240 | 241 | 242 | [TYPECHECK] 243 | 244 | # List of decorators that produce context managers, such as 245 | # contextlib.contextmanager. Add to this list to register other decorators that 246 | # produce valid context managers. 247 | contextmanager-decorators=contextlib.contextmanager 248 | 249 | # List of members which are set dynamically and missed by pylint inference 250 | # system, and so shouldn't trigger E1101 when accessed. Python regular 251 | # expressions are accepted. 252 | generated-members=(numpy|np)\.random 253 | 254 | # Tells whether missing members accessed in mixin class should be ignored. A 255 | # mixin class is detected if its name ends with "mixin" (case insensitive). 256 | ignore-mixin-members=yes 257 | 258 | # Tells whether to warn about missing members when the owner of the attribute 259 | # is inferred to be None. 260 | ignore-none=yes 261 | 262 | # This flag controls whether pylint should warn about no-member and similar 263 | # checks whenever an opaque object is returned when inferring. The inference 264 | # can return multiple potential results while evaluating a Python object, but 265 | # some branches might not be evaluated, which results in partial inference. In 266 | # that case, it might be useful to still emit no-member and other checks for 267 | # the rest of the inferred objects. 268 | ignore-on-opaque-inference=yes 269 | 270 | # List of class names for which member attributes should not be checked (useful 271 | # for classes with dynamically set attributes). This supports the use of 272 | # qualified names. 273 | ignored-classes=optparse.Values,thread._local,_thread._local 274 | 275 | # List of module names for which member attributes should not be checked 276 | # (useful for modules/projects where namespaces are manipulated during runtime 277 | # and thus existing member attributes cannot be deduced by static analysis). It 278 | # supports qualified module names, as well as Unix pattern matching. 279 | ignored-modules= 280 | 281 | # Show a hint with possible names when a member name was not found. The aspect 282 | # of finding the hint is based on edit distance. 283 | missing-member-hint=yes 284 | 285 | # The minimum edit distance a name should have in order to be considered a 286 | # similar match for a missing member name. 287 | missing-member-hint-distance=1 288 | 289 | # The total number of similar names that should be taken in consideration when 290 | # showing a hint for a missing member. 291 | missing-member-max-choices=1 292 | 293 | # List of decorators that change the signature of a decorated function. 294 | signature-mutators= 295 | 296 | 297 | [MISCELLANEOUS] 298 | 299 | # List of note tags to take in consideration, separated by a comma. 300 | notes=FIXME, 301 | XXX, 302 | TODO 303 | 304 | 305 | [LOGGING] 306 | 307 | # Format style used to check logging format string. `old` means using % 308 | # formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. 309 | logging-format-style=old 310 | 311 | # Logging modules to check that the string format arguments are in logging 312 | # function parameter format. 313 | logging-modules=logging 314 | 315 | 316 | [SPELLING] 317 | 318 | # Limits count of emitted suggestions for spelling mistakes. 319 | max-spelling-suggestions=4 320 | 321 | # Spelling dictionary name. Available dictionaries: none. To make it work, 322 | # install the python-enchant package. 323 | spelling-dict= 324 | 325 | # List of comma separated words that should not be checked. 326 | spelling-ignore-words= 327 | 328 | # A path to a file that contains the private dictionary; one word per line. 329 | spelling-private-dict-file= 330 | 331 | # Tells whether to store unknown words to the private dictionary (see the 332 | # --spelling-private-dict-file option) instead of raising a message. 333 | spelling-store-unknown-words=no 334 | 335 | 336 | [VARIABLES] 337 | 338 | # List of additional names supposed to be defined in builtins. Remember that 339 | # you should avoid defining new builtins when possible. 340 | additional-builtins= 341 | 342 | # Tells whether unused global variables should be treated as a violation. 343 | allow-global-unused-variables=yes 344 | 345 | # List of strings which can identify a callback function by name. A callback 346 | # name must start or end with one of those strings. 347 | callbacks=cb_, 348 | _cb 349 | 350 | # A regular expression matching the name of dummy variables (i.e. expected to 351 | # not be used). 352 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 353 | 354 | # Argument names that match this expression will be ignored. Default to name 355 | # with leading underscore. 356 | ignored-argument-names=_.*|^ignored_|^unused_ 357 | 358 | # Tells whether we should check for unused import in __init__ files. 359 | init-import=no 360 | 361 | # List of qualified module names which can have objects that can redefine 362 | # builtins. 363 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 364 | 365 | 366 | [SIMILARITIES] 367 | 368 | # Ignore comments when computing similarities. 369 | ignore-comments=yes 370 | 371 | # Ignore docstrings when computing similarities. 372 | ignore-docstrings=yes 373 | 374 | # Ignore imports when computing similarities. 375 | ignore-imports=no 376 | 377 | # Minimum lines number of a similarity. 378 | min-similarity-lines=4 379 | 380 | 381 | [BASIC] 382 | 383 | # Naming style matching correct argument names. 384 | argument-naming-style=snake_case 385 | 386 | # Regular expression matching correct argument names. Overrides argument- 387 | # naming-style. 388 | #argument-rgx= 389 | 390 | # Naming style matching correct attribute names. 391 | attr-naming-style=snake_case 392 | 393 | # Regular expression matching correct attribute names. Overrides attr-naming- 394 | # style. 395 | #attr-rgx= 396 | 397 | # Bad variable names which should always be refused, separated by a comma. 398 | bad-names=foo, 399 | bar, 400 | baz, 401 | toto, 402 | tutu, 403 | tata 404 | 405 | # Naming style matching correct class attribute names. 406 | class-attribute-naming-style=any 407 | 408 | # Regular expression matching correct class attribute names. Overrides class- 409 | # attribute-naming-style. 410 | #class-attribute-rgx= 411 | 412 | # Naming style matching correct class names. 413 | class-naming-style=PascalCase 414 | 415 | # Regular expression matching correct class names. Overrides class-naming- 416 | # style. 417 | #class-rgx= 418 | 419 | # Naming style matching correct constant names. 420 | const-naming-style=UPPER_CASE 421 | 422 | # Regular expression matching correct constant names. Overrides const-naming- 423 | # style. 424 | #const-rgx= 425 | 426 | # Minimum line length for functions/classes that require docstrings, shorter 427 | # ones are exempt. 428 | docstring-min-length=10 429 | 430 | # Naming style matching correct function names. 431 | function-naming-style=snake_case 432 | 433 | # Regular expression matching correct function names. Overrides function- 434 | # naming-style. 435 | #function-rgx= 436 | 437 | # Good variable names which should always be accepted, separated by a comma. 438 | good-names=i, 439 | j, 440 | k, 441 | ex, 442 | Run, 443 | _ 444 | 445 | # Include a hint for the correct naming format with invalid-name. 446 | include-naming-hint=no 447 | 448 | # Naming style matching correct inline iteration names. 449 | inlinevar-naming-style=any 450 | 451 | # Regular expression matching correct inline iteration names. Overrides 452 | # inlinevar-naming-style. 453 | #inlinevar-rgx= 454 | 455 | # Naming style matching correct method names. 456 | method-naming-style=snake_case 457 | 458 | # Regular expression matching correct method names. Overrides method-naming- 459 | # style. 460 | #method-rgx= 461 | 462 | # Naming style matching correct module names. 463 | module-naming-style=snake_case 464 | 465 | # Regular expression matching correct module names. Overrides module-naming- 466 | # style. 467 | #module-rgx= 468 | 469 | # Colon-delimited sets of names that determine each other's naming style when 470 | # the name regexes allow several styles. 471 | name-group= 472 | 473 | # Regular expression which should only match function or class names that do 474 | # not require a docstring. 475 | no-docstring-rgx=^_ 476 | 477 | # List of decorators that produce properties, such as abc.abstractproperty. Add 478 | # to this list to register other decorators that produce valid properties. 479 | # These decorators are taken in consideration only for invalid-name. 480 | property-classes=abc.abstractproperty 481 | 482 | # Naming style matching correct variable names. 483 | variable-naming-style=snake_case 484 | 485 | # Regular expression matching correct variable names. Overrides variable- 486 | # naming-style. 487 | #variable-rgx= 488 | 489 | 490 | [CLASSES] 491 | 492 | # List of method names used to declare (i.e. assign) instance attributes. 493 | defining-attr-methods=__init__, 494 | __new__, 495 | setUp, 496 | __post_init__ 497 | 498 | # List of member names, which should be excluded from the protected access 499 | # warning. 500 | exclude-protected=_asdict, 501 | _fields, 502 | _replace, 503 | _source, 504 | _make 505 | 506 | # List of valid names for the first argument in a class method. 507 | valid-classmethod-first-arg=cls 508 | 509 | # List of valid names for the first argument in a metaclass class method. 510 | valid-metaclass-classmethod-first-arg=cls 511 | 512 | 513 | [DESIGN] 514 | 515 | # Maximum number of arguments for function / method. 516 | max-args=5 517 | 518 | # Maximum number of attributes for a class (see R0902). 519 | max-attributes=7 520 | 521 | # Maximum number of boolean expressions in an if statement (see R0916). 522 | max-bool-expr=5 523 | 524 | # Maximum number of branch for function / method body. 525 | max-branches=12 526 | 527 | # Maximum number of locals for function / method body. 528 | max-locals=15 529 | 530 | # Maximum number of parents for a class (see R0901). 531 | max-parents=7 532 | 533 | # Maximum number of public methods for a class (see R0904). 534 | max-public-methods=20 535 | 536 | # Maximum number of return / yield for function / method body. 537 | max-returns=6 538 | 539 | # Maximum number of statements in function / method body. 540 | max-statements=50 541 | 542 | # Minimum number of public methods for a class (see R0903). 543 | min-public-methods=2 544 | 545 | 546 | [IMPORTS] 547 | 548 | # List of modules that can be imported at any level, not just the top level 549 | # one. 550 | allow-any-import-level= 551 | 552 | # Allow wildcard imports from modules that define __all__. 553 | allow-wildcard-with-all=no 554 | 555 | # Analyse import fallback blocks. This can be used to support both Python 2 and 556 | # 3 compatible code, which means that the block might have code that exists 557 | # only in one or another interpreter, leading to false positives when analysed. 558 | analyse-fallback-blocks=no 559 | 560 | # Deprecated modules which should not be used, separated by a comma. 561 | deprecated-modules=optparse,tkinter.tix 562 | 563 | # Create a graph of external dependencies in the given file (report RP0402 must 564 | # not be disabled). 565 | ext-import-graph= 566 | 567 | # Create a graph of every (i.e. internal and external) dependencies in the 568 | # given file (report RP0402 must not be disabled). 569 | import-graph= 570 | 571 | # Create a graph of internal dependencies in the given file (report RP0402 must 572 | # not be disabled). 573 | int-import-graph= 574 | 575 | # Force import order to recognize a module as part of the standard 576 | # compatibility libraries. 577 | known-standard-library= 578 | 579 | # Force import order to recognize a module as part of a third party library. 580 | known-third-party=enchant 581 | 582 | # Couples of modules and preferred modules, separated by a comma. 583 | preferred-modules= 584 | 585 | 586 | [EXCEPTIONS] 587 | 588 | # Exceptions that will emit a warning when being caught. Defaults to 589 | # "BaseException, Exception". 590 | overgeneral-exceptions=BaseException, 591 | Exception 592 | -------------------------------------------------------------------------------- /motmetrics/mot.py: -------------------------------------------------------------------------------- 1 | # py-motmetrics - Metrics for multiple object tracker (MOT) benchmarking. 2 | # https://github.com/cheind/py-motmetrics/ 3 | # 4 | # MIT License 5 | # Copyright (c) 2017-2020 Christoph Heindl, Jack Valmadre and others. 6 | # See LICENSE file for terms. 7 | 8 | """Accumulate tracking events frame by frame.""" 9 | 10 | from __future__ import absolute_import 11 | from __future__ import division 12 | from __future__ import print_function 13 | 14 | from collections import OrderedDict 15 | import itertools 16 | 17 | import numpy as np 18 | import pandas as pd 19 | 20 | from motmetrics.lap import linear_sum_assignment 21 | 22 | _INDEX_FIELDS = ['FrameId', 'Event'] 23 | _EVENT_FIELDS = ['Type', 'OId', 'HId', 'D'] 24 | 25 | 26 | class MOTAccumulator(object): 27 | """Manage tracking events. 28 | 29 | This class computes per-frame tracking events from a given set of object / hypothesis 30 | ids and pairwise distances. Indended usage 31 | 32 | import motmetrics as mm 33 | acc = mm.MOTAccumulator() 34 | acc.update(['a', 'b'], [0, 1, 2], dists, frameid=0) 35 | ... 36 | acc.update(['d'], [6,10], other_dists, frameid=76) 37 | summary = mm.metrics.summarize(acc) 38 | print(mm.io.render_summary(summary)) 39 | 40 | Update is called once per frame and takes objects / hypothesis ids and a pairwise distance 41 | matrix between those (see distances module for support). Per frame max(len(objects), len(hypothesis)) 42 | events are generated. Each event type is one of the following 43 | - `'MATCH'` a match between a object and hypothesis was found 44 | - `'SWITCH'` a match between a object and hypothesis was found but differs from previous assignment (hypothesisid != previous) 45 | - `'MISS'` no match for an object was found 46 | - `'FP'` no match for an hypothesis was found (spurious detections) 47 | - `'RAW'` events corresponding to raw input 48 | - `'TRANSFER'` a match between a object and hypothesis was found but differs from previous assignment (objectid != previous) 49 | - `'ASCEND'` a match between a object and hypothesis was found but differs from previous assignment (hypothesisid is new) 50 | - `'MIGRATE'` a match between a object and hypothesis was found but differs from previous assignment (objectid is new) 51 | 52 | Events are tracked in a pandas Dataframe. The dataframe is hierarchically indexed by (`FrameId`, `EventId`), 53 | where `FrameId` is either provided during the call to `update` or auto-incremented when `auto_id` is set 54 | true during construction of MOTAccumulator. `EventId` is auto-incremented. The dataframe has the following 55 | columns 56 | - `Type` one of `('MATCH', 'SWITCH', 'MISS', 'FP', 'RAW')` 57 | - `OId` object id or np.nan when `'FP'` or `'RAW'` and object is not present 58 | - `HId` hypothesis id or np.nan when `'MISS'` or `'RAW'` and hypothesis is not present 59 | - `D` distance or np.nan when `'FP'` or `'MISS'` or `'RAW'` and either object/hypothesis is absent 60 | 61 | From the events and associated fields the entire tracking history can be recovered. Once the accumulator 62 | has been populated with per-frame data use `metrics.summarize` to compute statistics. See `metrics.compute_metrics` 63 | for a list of metrics computed. 64 | 65 | References 66 | ---------- 67 | 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." 68 | EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. 69 | 2. Milan, Anton, et al. "Mot16: A benchmark for multi-object tracking." arXiv preprint arXiv:1603.00831 (2016). 70 | 3. Li, Yuan, Chang Huang, and Ram Nevatia. "Learning to associate: Hybridboosted multi-target tracker for crowded scene." 71 | Computer Vision and Pattern Recognition, 2009. CVPR 2009. IEEE Conference on. IEEE, 2009. 72 | """ 73 | 74 | def __init__(self, auto_id=False, max_switch_time=float('inf')): 75 | """Create a MOTAccumulator. 76 | 77 | Params 78 | ------ 79 | auto_id : bool, optional 80 | Whether or not frame indices are auto-incremented or provided upon 81 | updating. Defaults to false. Not specifying a frame-id when this value 82 | is true results in an error. Specifying a frame-id when this value is 83 | false also results in an error. 84 | 85 | max_switch_time : scalar, optional 86 | Allows specifying an upper bound on the timespan an unobserved but 87 | tracked object is allowed to generate track switch events. Useful if groundtruth 88 | objects leaving the field of view keep their ID when they reappear, 89 | but your tracker is not capable of recognizing this (resulting in 90 | track switch events). The default is that there is no upper bound 91 | on the timespan. In units of frame timestamps. When using auto_id 92 | in units of count. 93 | """ 94 | 95 | # Parameters of the accumulator. 96 | self.auto_id = auto_id 97 | self.max_switch_time = max_switch_time 98 | 99 | # Accumulator state. 100 | self._events = None 101 | self._indices = None 102 | self.m = None 103 | self.res_m = None 104 | self.last_occurrence = None 105 | self.last_match = None 106 | self.hypHistory = None 107 | self.dirty_events = None 108 | self.cached_events_df = None 109 | 110 | self.reset() 111 | 112 | def reset(self): 113 | """Reset the accumulator to empty state.""" 114 | 115 | self._events = {field: [] for field in _EVENT_FIELDS} 116 | self._indices = {field: [] for field in _INDEX_FIELDS} 117 | self.m = {} # Pairings up to current timestamp 118 | self.res_m = {} # Result pairings up to now 119 | self.last_occurrence = {} # Tracks most recent occurance of object 120 | self.last_match = {} # Tracks most recent match of object 121 | self.hypHistory = {} 122 | self.dirty_events = True 123 | self.cached_events_df = None 124 | 125 | def _append_to_indices(self, frameid, eid): 126 | self._indices['FrameId'].append(frameid) 127 | self._indices['Event'].append(eid) 128 | 129 | def _append_to_events(self, typestr, oid, hid, distance): 130 | self._events['Type'].append(typestr) 131 | self._events['OId'].append(oid) 132 | self._events['HId'].append(hid) 133 | self._events['D'].append(distance) 134 | 135 | def update(self, oids, hids, dists, frameid=None, vf=''): 136 | """Updates the accumulator with frame specific objects/detections. 137 | 138 | This method generates events based on the following algorithm [1]: 139 | 1. Try to carry forward already established tracks. If any paired object / hypothesis 140 | from previous timestamps are still visible in the current frame, create a 'MATCH' 141 | event between them. 142 | 2. For the remaining constellations minimize the total object / hypothesis distance 143 | error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous 144 | match create a 'SWITCH' else a 'MATCH' event. 145 | 3. Create 'MISS' events for all remaining unassigned objects. 146 | 4. Create 'FP' events for all remaining unassigned hypotheses. 147 | 148 | Params 149 | ------ 150 | oids : N array 151 | Array of object ids. 152 | hids : M array 153 | Array of hypothesis ids. 154 | dists: NxM array 155 | Distance matrix. np.nan values to signal do-not-pair constellations. 156 | See `distances` module for support methods. 157 | 158 | Kwargs 159 | ------ 160 | frameId : id 161 | Unique frame id. Optional when MOTAccumulator.auto_id is specified during 162 | construction. 163 | vf: file to log details 164 | Returns 165 | ------- 166 | frame_events : pd.DataFrame 167 | Dataframe containing generated events 168 | 169 | References 170 | ---------- 171 | 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." 172 | EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. 173 | """ 174 | # pylint: disable=too-many-locals, too-many-statements 175 | 176 | self.dirty_events = True 177 | oids = np.asarray(oids) 178 | oids_masked = np.zeros_like(oids, dtype=np.bool_) 179 | hids = np.asarray(hids) 180 | hids_masked = np.zeros_like(hids, dtype=np.bool_) 181 | dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy() 182 | 183 | if frameid is None: 184 | assert self.auto_id, 'auto-id is not enabled' 185 | if len(self._indices['FrameId']) > 0: 186 | frameid = self._indices['FrameId'][-1] + 1 187 | else: 188 | frameid = 0 189 | else: 190 | assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' 191 | 192 | eid = itertools.count() 193 | 194 | # 0. Record raw events 195 | 196 | no = len(oids) 197 | nh = len(hids) 198 | 199 | # Add a RAW event simply to ensure the frame is counted. 200 | self._append_to_indices(frameid, next(eid)) 201 | self._append_to_events('RAW', np.nan, np.nan, np.nan) 202 | 203 | # There must be at least one RAW event per object and hypothesis. 204 | # Record all finite distances as RAW events. 205 | valid_i, valid_j = np.where(np.isfinite(dists)) 206 | valid_dists = dists[valid_i, valid_j] 207 | for i, j, dist_ij in zip(valid_i, valid_j, valid_dists): 208 | self._append_to_indices(frameid, next(eid)) 209 | self._append_to_events('RAW', oids[i], hids[j], dist_ij) 210 | # Add a RAW event for objects and hypotheses that were present but did 211 | # not overlap with anything. 212 | used_i = np.unique(valid_i) 213 | used_j = np.unique(valid_j) 214 | unused_i = np.setdiff1d(np.arange(no), used_i) 215 | unused_j = np.setdiff1d(np.arange(nh), used_j) 216 | for oid in oids[unused_i]: 217 | self._append_to_indices(frameid, next(eid)) 218 | self._append_to_events('RAW', oid, np.nan, np.nan) 219 | for hid in hids[unused_j]: 220 | self._append_to_indices(frameid, next(eid)) 221 | self._append_to_events('RAW', np.nan, hid, np.nan) 222 | 223 | if oids.size * hids.size > 0: 224 | # 1. Try to re-establish tracks from previous correspondences 225 | for i in range(oids.shape[0]): 226 | # No need to check oids_masked[i] here. 227 | if oids[i] not in self.m: 228 | continue 229 | 230 | hprev = self.m[oids[i]] 231 | j, = np.where(~hids_masked & (hids == hprev)) 232 | if j.shape[0] == 0: 233 | continue 234 | j = j[0] 235 | 236 | if np.isfinite(dists[i, j]): 237 | o = oids[i] 238 | h = hids[j] 239 | oids_masked[i] = True 240 | hids_masked[j] = True 241 | self.m[oids[i]] = hids[j] 242 | 243 | self._append_to_indices(frameid, next(eid)) 244 | self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) 245 | self.last_match[o] = frameid 246 | self.hypHistory[h] = frameid 247 | 248 | # 2. Try to remaining objects/hypotheses 249 | dists[oids_masked, :] = np.nan 250 | dists[:, hids_masked] = np.nan 251 | 252 | rids, cids = linear_sum_assignment(dists) 253 | 254 | for i, j in zip(rids, cids): 255 | if not np.isfinite(dists[i, j]): 256 | continue 257 | 258 | o = oids[i] 259 | h = hids[j] 260 | is_switch = (o in self.m and 261 | self.m[o] != h and 262 | abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) 263 | cat1 = 'SWITCH' if is_switch else 'MATCH' 264 | if cat1 == 'SWITCH': 265 | if h not in self.hypHistory: 266 | subcat = 'ASCEND' 267 | self._append_to_indices(frameid, next(eid)) 268 | self._append_to_events(subcat, oids[i], hids[j], dists[i, j]) 269 | # ignore the last condition temporarily 270 | is_transfer = (h in self.res_m and 271 | self.res_m[h] != o) 272 | # is_transfer = (h in self.res_m and 273 | # self.res_m[h] != o and 274 | # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) 275 | cat2 = 'TRANSFER' if is_transfer else 'MATCH' 276 | if cat2 == 'TRANSFER': 277 | if o not in self.last_match: 278 | subcat = 'MIGRATE' 279 | self._append_to_indices(frameid, next(eid)) 280 | self._append_to_events(subcat, oids[i], hids[j], dists[i, j]) 281 | self._append_to_indices(frameid, next(eid)) 282 | self._append_to_events(cat2, oids[i], hids[j], dists[i, j]) 283 | if vf != '' and (cat1 != 'MATCH' or cat2 != 'MATCH'): 284 | if cat1 == 'SWITCH': 285 | vf.write('%s %d %d %d %d %d\n' % (subcat[:2], o, self.last_match[o], self.m[o], frameid, h)) 286 | if cat2 == 'TRANSFER': 287 | vf.write('%s %d %d %d %d %d\n' % (subcat[:2], h, self.hypHistory[h], self.res_m[h], frameid, o)) 288 | self.hypHistory[h] = frameid 289 | self.last_match[o] = frameid 290 | self._append_to_indices(frameid, next(eid)) 291 | self._append_to_events(cat1, oids[i], hids[j], dists[i, j]) 292 | oids_masked[i] = True 293 | hids_masked[j] = True 294 | self.m[o] = h 295 | self.res_m[h] = o 296 | 297 | # 3. All remaining objects are missed 298 | for o in oids[~oids_masked]: 299 | self._append_to_indices(frameid, next(eid)) 300 | self._append_to_events('MISS', o, np.nan, np.nan) 301 | if vf != '': 302 | vf.write('FN %d %d\n' % (frameid, o)) 303 | 304 | # 4. All remaining hypotheses are false alarms 305 | for h in hids[~hids_masked]: 306 | self._append_to_indices(frameid, next(eid)) 307 | self._append_to_events('FP', np.nan, h, np.nan) 308 | if vf != '': 309 | vf.write('FP %d %d\n' % (frameid, h)) 310 | 311 | # 5. Update occurance state 312 | for o in oids: 313 | self.last_occurrence[o] = frameid 314 | 315 | return frameid 316 | 317 | @property 318 | def events(self): 319 | if self.dirty_events: 320 | self.cached_events_df = MOTAccumulator.new_event_dataframe_with_data(self._indices, self._events) 321 | self.dirty_events = False 322 | return self.cached_events_df 323 | 324 | @property 325 | def mot_events(self): 326 | df = self.events 327 | return df[df.Type != 'RAW'] 328 | 329 | @staticmethod 330 | def new_event_dataframe(): 331 | """Create a new DataFrame for event tracking.""" 332 | idx = pd.MultiIndex(levels=[[], []], codes=[[], []], names=['FrameId', 'Event']) 333 | cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE']) 334 | df = pd.DataFrame( 335 | OrderedDict([ 336 | ('Type', pd.Series(cats)), # Type of event. One of FP (false positive), MISS, SWITCH, MATCH 337 | ('OId', pd.Series(dtype=float)), # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways. 338 | ('HId', pd.Series(dtype=float)), # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways. 339 | ('D', pd.Series(dtype=float)), # Distance or NaN when FP or MISS 340 | ]), 341 | index=idx 342 | ) 343 | return df 344 | 345 | @staticmethod 346 | def new_event_dataframe_with_data(indices, events): 347 | """Create a new DataFrame filled with data. 348 | 349 | Params 350 | ------ 351 | indices: dict 352 | dict of lists with fields 'FrameId' and 'Event' 353 | events: dict 354 | dict of lists with fields 'Type', 'OId', 'HId', 'D' 355 | """ 356 | 357 | if len(events) == 0: 358 | return MOTAccumulator.new_event_dataframe() 359 | 360 | raw_type = pd.Categorical( 361 | events['Type'], 362 | categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'], 363 | ordered=False) 364 | series = [ 365 | pd.Series(raw_type, name='Type'), 366 | pd.Series(events['OId'], dtype=float, name='OId'), 367 | pd.Series(events['HId'], dtype=float, name='HId'), 368 | pd.Series(events['D'], dtype=float, name='D') 369 | ] 370 | 371 | idx = pd.MultiIndex.from_arrays( 372 | [indices[field] for field in _INDEX_FIELDS], 373 | names=_INDEX_FIELDS) 374 | df = pd.concat(series, axis=1) 375 | df.index = idx 376 | return df 377 | 378 | @staticmethod 379 | def merge_analysis(anas, infomap): 380 | # pylint: disable=missing-function-docstring 381 | res = {'hyp': {}, 'obj': {}} 382 | mapp = {'hyp': 'hid_map', 'obj': 'oid_map'} 383 | for ana, infom in zip(anas, infomap): 384 | if ana is None: 385 | return None 386 | for t in ana.keys(): 387 | which = mapp[t] 388 | if np.nan in infom[which]: 389 | res[t][int(infom[which][np.nan])] = 0 390 | if 'nan' in infom[which]: 391 | res[t][int(infom[which]['nan'])] = 0 392 | for _id, cnt in ana[t].items(): 393 | if _id not in infom[which]: 394 | _id = str(_id) 395 | res[t][int(infom[which][_id])] = cnt 396 | return res 397 | 398 | @staticmethod 399 | def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, return_mappings=False): 400 | """Merge dataframes. 401 | 402 | Params 403 | ------ 404 | dfs : list of pandas.DataFrame or MotAccumulator 405 | A list of event containers to merge 406 | 407 | Kwargs 408 | ------ 409 | update_frame_indices : boolean, optional 410 | Ensure that frame indices are unique in the merged container 411 | update_oids : boolean, unique 412 | Ensure that object ids are unique in the merged container 413 | update_hids : boolean, unique 414 | Ensure that hypothesis ids are unique in the merged container 415 | return_mappings : boolean, unique 416 | Whether or not to return mapping information 417 | 418 | Returns 419 | ------- 420 | df : pandas.DataFrame 421 | Merged event data frame 422 | """ 423 | 424 | mapping_infos = [] 425 | new_oid = itertools.count() 426 | new_hid = itertools.count() 427 | 428 | r = MOTAccumulator.new_event_dataframe() 429 | for df in dfs: 430 | 431 | if isinstance(df, MOTAccumulator): 432 | df = df.events 433 | 434 | copy = df.copy() 435 | infos = {} 436 | 437 | # Update index 438 | if update_frame_indices: 439 | # pylint: disable=cell-var-from-loop 440 | next_frame_id = max(r.index.get_level_values(0).max() + 1, r.index.get_level_values(0).unique().shape[0]) 441 | if np.isnan(next_frame_id): 442 | next_frame_id = 0 443 | copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1])) 444 | infos['frame_offset'] = next_frame_id 445 | 446 | # Update object / hypothesis ids 447 | if update_oids: 448 | # pylint: disable=cell-var-from-loop 449 | oid_map = dict([oid, str(next(new_oid))] for oid in copy['OId'].dropna().unique()) 450 | copy['OId'] = copy['OId'].map(lambda x: oid_map[x], na_action='ignore') 451 | infos['oid_map'] = oid_map 452 | 453 | if update_hids: 454 | # pylint: disable=cell-var-from-loop 455 | hid_map = dict([hid, str(next(new_hid))] for hid in copy['HId'].dropna().unique()) 456 | copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore') 457 | infos['hid_map'] = hid_map 458 | 459 | r = r.append(copy) 460 | mapping_infos.append(infos) 461 | 462 | if return_mappings: 463 | return r, mapping_infos 464 | else: 465 | return r 466 | --------------------------------------------------------------------------------