├── .binder └── requirements.txt ├── .clabot ├── .gitattributes ├── .github ├── stale.yml └── workflows │ ├── deploy-guide-dev.yml │ ├── deploy-guide.yml │ └── update-stable.yml ├── .gitignore ├── .pylintrc ├── .travis.yml ├── CLAC.txt ├── CLAI.txt ├── CONTRIBUTING.md ├── Check_targets_script.ipynb ├── EVAL_pose.ipynb ├── HISTORY.md ├── LICENSE ├── LICENSE.AGPL ├── MANIFEST.in ├── README.md ├── ROC_ball.ipynb ├── ball_metrics.py ├── cpp └── examples │ └── cifcafdecoder │ ├── CMakeLists.txt │ ├── aiapp.hpp │ ├── image_based.hpp │ ├── main.cpp │ ├── math_helpers.cpp │ ├── math_helpers.hpp │ ├── object_detection.hpp │ ├── openpifpaf_postprocessor.cpp │ ├── openpifpaf_postprocessor.hpp │ ├── sample_data_0_8.cpp │ └── sample_data_0_8.hpp ├── docs ├── coco │ ├── 000000081988.jpg │ ├── 000000081988.jpg.decoder_flame.svg │ ├── 000000081988.jpg.predictions.json │ └── 000000081988.jpg.predictions.png ├── datasets.md ├── eval_logs.md ├── predict.ipynb ├── scale_distribution.ipynb ├── skeleton_coco.png ├── skeleton_dense.png ├── skeleton_kinematic_tree.png └── wave3.gif ├── guide ├── _config.yml ├── _config_dev.yml ├── _toc.yml ├── artwork.py ├── cli_help.ipynb ├── coco │ ├── 000000081988.jpg │ ├── 000000114907.jpg │ ├── 000000147740.jpg │ ├── 000000188465.jpg │ └── 000000541055.jpg ├── datasets.ipynb ├── dev.ipynb ├── examples.ipynb ├── faq.ipynb ├── intro.md ├── moduledocs.ipynb ├── predict_api.ipynb ├── predict_cli.ipynb ├── profile.ipynb ├── references.bib └── train.ipynb ├── oks.py ├── oks.sh ├── openpifpaf ├── __init__.py ├── annotation.py ├── benchmark.py ├── count_ops.py ├── datasets │ ├── __init__.py │ ├── build_deepsport_ball_dataset.py │ ├── coco.py │ ├── coco_mrcnn.ipynb │ ├── collate.py │ ├── constants.py │ ├── deepsport.py │ ├── factory.py │ ├── headmeta.py │ ├── image_list.py │ ├── keemotion.py │ └── multidataset.py ├── decoder │ ├── __init__.py │ ├── caf_scored.py │ ├── caf_seeds.py │ ├── cif_hr.py │ ├── cif_seeds.py │ ├── factory.py │ ├── field_config.py │ ├── generator │ │ ├── __init__.py │ │ ├── cifcaf.py │ │ ├── cifcent.py │ │ ├── cifdet.py │ │ ├── cifpan.py │ │ ├── cifpanball.py │ │ ├── cifseg.py │ │ └── generator.py │ ├── instance_scorer.py │ ├── nms.py │ ├── occupancy.py │ ├── profiler.py │ ├── profiler_autograd.py │ └── utils.py ├── encoder │ ├── __init__.py │ ├── annrescaler.py │ ├── annrescaler_ball.py │ ├── caf.py │ ├── cif.py │ ├── cif_ball.py │ ├── cifdet.py │ ├── factory.py │ ├── pan.py │ └── seg.py ├── eval_coco.py ├── eval_coco_abolfazl.py ├── eval_oks_pq.py ├── export_onnx.py ├── functional.c ├── functional.pyx ├── logs.py ├── migrate.py ├── network │ ├── __init__.py │ ├── aspp.py │ ├── basenetworks.py │ ├── factory.py │ ├── heads.py │ ├── losses.py │ ├── nets.py │ ├── panoptic_deeplab.py │ ├── panoptic_losses.py │ └── trainer.py ├── oks_abolfazl.py ├── optimize.py ├── panoptic_deeplab │ └── decoder │ │ ├── __init__.py │ │ ├── aspp.py │ │ ├── conv_module.py │ │ ├── deeplabv3.py │ │ ├── deeplabv3plus.py │ │ └── panoptic_deeplab.py ├── predict.py ├── predict_pan.py ├── show │ ├── __init__.py │ ├── animation_frame.py │ ├── canvas.py │ ├── cli.py │ ├── fields.py │ └── painters.py ├── train.py ├── train_instance_scorer.py ├── transforms │ ├── __init__.py │ ├── annotations.py │ ├── compose.py │ ├── crop.py │ ├── crop_keemotion.py │ ├── hflip.py │ ├── image.py │ ├── minsize.py │ ├── multi_scale.py │ ├── pad.py │ ├── preprocess.py │ ├── random.py │ ├── rotate.py │ ├── scale.py │ ├── unclipped.py │ ├── utils.py │ └── zoomscale.py ├── utils.py ├── video.py └── visualizer │ ├── __init__.py │ ├── base.py │ ├── caf.py │ ├── cif.py │ ├── cifdet.py │ ├── cifhr.py │ ├── cli.py │ ├── occupancy.py │ ├── seeds.py │ └── seg.py ├── requirements.txt ├── run ├── setup ├── setup.cfg ├── setup.py ├── tests ├── coco │ ├── images │ │ └── puppy_dog.jpg │ └── train1.json ├── test_clis.py ├── test_forward.py ├── test_help.py ├── test_image_scale.py ├── test_input_processing.py ├── test_localization.py ├── test_multiprocessing.py ├── test_network.py ├── test_onnx_export.py ├── test_scale_loss.py ├── test_train.py └── test_transforms.py ├── versioneer.py └── visdrone_debug /.binder/requirements.txt: -------------------------------------------------------------------------------- 1 | .[dev,test] 2 | -------------------------------------------------------------------------------- /.clabot: -------------------------------------------------------------------------------- 1 | { 2 | "contributors": ["svenkreiss", "bertoni9", "george-adaimi", "junedgar", "mdenna-nviso", "thancaocuong", "ganler"], 3 | "message": "Thanks for your Pull Request. We require contributors to sign our Contributor License Agreement (CLA), and we do not have {{usersWithoutCLA}} on file. You retain the copyright to your contribution and this will allow us to use it, too. In order for us to review and merge your code, please follow the short steps in CONTRIBUTING.md to get yourself added as a CLA'ed contributor." 4 | } 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb filter=nbstripout 2 | *.ipynb diff=jupyternotebook 3 | *.ipynb merge=jupyternotebook 4 | docs/** filter= diff= 5 | openpifpaf/_version.py export-subst 6 | cpp/examples/cifcafdecoder/sample_data* linguist-vendored 7 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 90 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 14 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | - good-first-issue 10 | # Label to use when marking an issue as stale 11 | staleLabel: stale 12 | # Comment to post when marking an issue as stale. Set to `false` to disable 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. Set to `false` to disable 18 | closeComment: false 19 | -------------------------------------------------------------------------------- /.github/workflows/deploy-guide-dev.yml: -------------------------------------------------------------------------------- 1 | name: deploy-guide-dev 2 | 3 | # Only run this when the master branch changes 4 | on: 5 | push: 6 | branches: 7 | - dev 8 | 9 | # This job installs dependencies, build the guide, and pushes it to `gh-pages` 10 | jobs: 11 | deploy-guide-dev: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | # Install dependencies 17 | - name: Set up Python 3.7 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: 3.7 21 | 22 | - name: Install dependencies 23 | run: | 24 | pip install .[dev,test] 25 | sudo apt install jq 26 | 27 | # make sure KernelSpec in all notebooks is python3 28 | - name: Normalize KernelSpec 29 | run: | 30 | for fn in guide/*.ipynb; do 31 | jq '(.metadata.kernelspec.name,.metadata.kernelspec.display_name)="python3"' ${fn} > ${fn}_ 32 | mv ${fn}_ ${fn} 33 | done 34 | 35 | # Build the guide 36 | - name: Build the artwork for guide 37 | run: cd guide && python artwork.py 38 | 39 | - name: Build the guide 40 | run: cd guide && cp _config_dev.yml _config.yml && jupyter-book build . 41 | 42 | # Push the book's HTML to gh-pages 43 | - name: GitHub Pages action 44 | uses: peaceiris/actions-gh-pages@v3.7.0-8 45 | with: 46 | github_token: ${{ secrets.GITHUB_TOKEN }} 47 | publish_dir: ./guide/_build/html 48 | destination_dir: dev 49 | -------------------------------------------------------------------------------- /.github/workflows/deploy-guide.yml: -------------------------------------------------------------------------------- 1 | name: deploy-guide 2 | 3 | # Only run this when the master branch changes 4 | on: 5 | # push: 6 | # branches: 7 | # - master 8 | # - guide 9 | release: 10 | types: 11 | - released 12 | 13 | # This job installs dependencies, build the guide, and pushes it to `gh-pages` 14 | jobs: 15 | deploy-guide: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | # Install dependencies 21 | - name: Set up Python 3.7 22 | uses: actions/setup-python@v1 23 | with: 24 | python-version: 3.7 25 | 26 | - name: Install dependencies 27 | run: | 28 | pip install .[dev,test] 29 | sudo apt install jq 30 | 31 | # make sure KernelSpec in all notebooks is python3 32 | - name: Normalize KernelSpec 33 | run: | 34 | for fn in guide/*.ipynb; do 35 | jq '(.metadata.kernelspec.name,.metadata.kernelspec.display_name)="python3"' ${fn} > ${fn}_ 36 | mv ${fn}_ ${fn} 37 | done 38 | 39 | # Build the guide 40 | - name: Build the artwork for guide 41 | run: cd guide && python artwork.py 42 | 43 | - name: Build the guide 44 | run: cd guide && jupyter-book build . 45 | 46 | # Push the book's HTML to gh-pages 47 | - name: GitHub Pages action 48 | uses: peaceiris/actions-gh-pages@v3.5.9 49 | with: 50 | github_token: ${{ secrets.GITHUB_TOKEN }} 51 | publish_dir: ./guide/_build/html 52 | -------------------------------------------------------------------------------- /.github/workflows/update-stable.yml: -------------------------------------------------------------------------------- 1 | name: update-stable 2 | 3 | on: 4 | release: 5 | types: 6 | - released 7 | 8 | jobs: 9 | update-stable: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | ref: main 15 | fetch-depth: 0 16 | - name: Push to stable branch 17 | run: | 18 | git pull 19 | git checkout stable 20 | git merge main -m "merge main" 21 | git push 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /env 2 | slurm*.out 3 | 4 | # system 5 | .DS_Store 6 | .vscode 7 | *.so 8 | 9 | # Python 10 | build/ 11 | venv*/ 12 | *.egg-info 13 | .eggs/ 14 | __pycache__ 15 | dist/ 16 | .pytest_cache/ 17 | .ipynb_checkpoints/ 18 | openpifpaf/functional.cpython-* 19 | openpifpaf/functional.html 20 | 21 | # guide 22 | guide/_build/ 23 | guide/coco/*.predictions.png 24 | guide/coco/*.predictions.json 25 | guide/logo.png 26 | guide/favicon.png 27 | guide/profile_decoder.1.json 28 | guide/profile_decoder.prof 29 | guide/profile_decoder_flame.svg 30 | 31 | # data 32 | data-*/ 33 | data_*/ 34 | docs-private/ 35 | data-mscoco 36 | *debug.png 37 | *.pl 38 | *.pl-* 39 | *.pl.* 40 | *.pkl 41 | *.pkl-* 42 | *.pkl.* 43 | *.log 44 | *.zip 45 | outputs/ 46 | shared/ 47 | decoder.prof 48 | decoder_flame.svg 49 | *.onnx 50 | debug.*.png 51 | test-clis-*/ 52 | data 53 | pretrained 54 | coco_debug 55 | outputs 56 | visualrelation_debug 57 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [BASIC] 2 | 3 | variable-rgx=[a-z0-9_]{1,30}$ 4 | good-names=ap,ar,ax,b,d,f,g,gt,h,i,im,lr,n,p,r,s,t,t1,t2,th,v,vs,w,wh,x,x1,x2,xs,y,ys,xy 5 | 6 | 7 | [IMPORTS] 8 | 9 | allow-any-import-level=pycocotools,pycocotools.coco 10 | 11 | 12 | [SIMILARITIES] 13 | 14 | # Minimum lines number of a similarity. 15 | min-similarity-lines=15 16 | 17 | # Ignore comments when computing similarities. 18 | ignore-comments=yes 19 | 20 | # Ignore docstrings when computing similarities. 21 | ignore-docstrings=yes 22 | 23 | # Ignore imports when computing similarities. 24 | ignore-imports=yes 25 | 26 | 27 | [TYPECHECK] 28 | 29 | # List of members which are set dynamically and missed by pylint inference 30 | # system, and so shouldn't trigger E1101 when accessed. Python regular 31 | # expressions are accepted. 32 | generated-members=numpy.*,torch.*,cv2.*,openpifpaf.functional.* 33 | 34 | ignored-modules=openpifpaf.functional,pycocotools 35 | 36 | 37 | # for pytorch: not-callable 38 | # for pytorch 1.6.0: abstract-method 39 | disable=missing-docstring,too-many-arguments,too-many-instance-attributes,too-many-locals,too-few-public-methods,not-callable,abstract-method 40 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: python 3 | cache: pip 4 | matrix: 5 | fast_finish: true 6 | include: 7 | - name: "Python 3.7 on Linux (PyTorch 1.4)" 8 | python: 3.7 9 | before_install: 10 | - pip3 install --upgrade pip setuptools 11 | - pip3 install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 12 | - name: "Python 3.7 on Linux with training" 13 | if: branch = master 14 | python: 3.7 15 | env: 16 | - PIFPAFTRAINING=1 17 | before_install: 18 | - pip3 install --upgrade pip setuptools 19 | - pip3 install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 20 | - pip3 install Cython numpy 21 | - "pip3 install --editable .[train]" 22 | - name: "Python 3.8 on Linux with training" 23 | python: 3.8 24 | env: 25 | - PIFPAFTRAINING=1 26 | before_install: 27 | - pip3 install --upgrade pip setuptools 28 | - pip3 install torch==1.5.1+cpu torchvision==0.6.1+cpu -f https://download.pytorch.org/whl/torch_stable.html 29 | - pip3 install Cython numpy 30 | - "pip3 install --editable .[train] --use-feature=2020-resolver" 31 | - name: "Python 3.6 on Linux (PyTorch 1.4)" 32 | if: branch = master 33 | python: 3.6 34 | before_install: 35 | - pip3 install --upgrade pip setuptools 36 | - pip3 install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 37 | - name: "Python 3.7 on Linux (PyTorch 1.3)" 38 | if: branch = master 39 | python: 3.7 40 | before_install: 41 | - pip3 install --upgrade pip setuptools 42 | - pip3 install torch==1.3.1+cpu torchvision==0.4.2+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html 43 | - name: "Python 3.7 on Linux (PyTorch 1.5.1)" 44 | if: branch = master 45 | python: 3.7 46 | before_install: 47 | - pip3 install --upgrade pip setuptools 48 | - pip3 install torch==1.5.1+cpu torchvision==0.6.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html 49 | - name: "Python 3.7 on Linux (PyTorch 1.6)" 50 | if: branch = master 51 | python: 3.7 52 | before_install: 53 | - pip3 install --upgrade pip setuptools 54 | - pip3 install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html 55 | - name: "Python 3.8 on Linux (PyTorch 1.5.1)" 56 | if: branch = master 57 | python: 3.8 58 | before_install: 59 | - pip3 install --upgrade pip setuptools 60 | - pip3 install torch==1.5.1+cpu torchvision==0.6.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html 61 | - name: "Python 3.8-dev on Linux (PyTorch 1.5.1)" 62 | if: branch = master 63 | python: 3.8-dev 64 | before_install: 65 | - pip3 install --upgrade pip setuptools 66 | - pip3 install torch==1.5.1+cpu torchvision==0.6.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html 67 | - name: "macOS (PyTorch 1.4)" 68 | if: branch = master 69 | os: osx 70 | osx_image: xcode11 71 | language: shell # 'language: python' is an error on Travis CI macOS 72 | before_install: 73 | - pip3 install --upgrade pip setuptools 74 | - pip3 install torch==1.4.0 torchvision==0.5.0 -f https://download.pytorch.org/whl/torch_stable.html 75 | - name: "Windows (PyTorch 1.4)" 76 | if: branch = master 77 | os: windows 78 | language: shell # 'language: python' is an error on Travis CI Windows 79 | before_install: 80 | - choco install python --version 3.7.4 81 | - python --version 82 | - python -m pip install --upgrade pip 83 | - pip3 install --upgrade pip setuptools 84 | - pip3 install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 85 | env: PATH=/c/Python37:/c/Python37/Scripts:$PATH 86 | install: 87 | - "pip3 install --editable .[test]" 88 | script: 89 | - pylint openpifpaf --disable=fixme 90 | - pylint tests/*.py --disable=fixme 91 | - pytest -vv 92 | - | 93 | if [ $PIFPAFTRAINING == "1" ]; then 94 | pytest --nbval-lax --current-env docs/predict.ipynb; 95 | cd guide && pytest --nbval-lax --current-env *.ipynb; 96 | fi 97 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Create a pull request with your suggested changes. Make sure your code passes 2 | `pytest` and `pylint` checks. 3 | 4 | E-mail a signed copy of the 5 | [CLAI](https://github.com/vita-epfl/openpifpaf/blob/main/CLAI.txt) 6 | (and if applicable the 7 | [CLAC](https://github.com/vita-epfl/openpifpaf/blob/main/CLAC.txt)) 8 | as PDF file to research@svenkreiss.com. 9 | 10 | Thanks. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019-2020 by Sven Kreiss and contributors. All rights reserved. 2 | 3 | This project and all its files are licensed under 4 | GNU AGPLv3 or later version. 5 | 6 | If this license is not suitable for your business or project 7 | please contact EPFL-TTO (https://tto.epfl.ch/) and UCLouvain-TTO (https://uclouvain.be/en/research/ltto) for a full commercial license. 8 | 9 | This software may not be used to harm any person deliberately. 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include openpifpaf/functional.pyx 2 | include openpifpaf/functional.c 3 | include versioneer.py 4 | include openpifpaf/_version.py 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeepSportLab 2 | 3 | 4 | > DeepSportLab: a Unified Framework for BallDetection, Player Instance Segmentationand Pose Estimation in Team Sports Scenes 5 | > 6 | > This paper presents a unified framework to(i)locate the ball,(ii)predict the pose, and(iii)segment the instance mask of players in team sports scenes. Those problems are ofhigh interest in automated sports analytics, production, and broadcast. A common prac-tice is to individually solve each problem by exploiting universal state-of-the-art models,e.g., Panoptic-DeepLab for player segmentation. In addition to the increased complexityresulting from the multiplication of single-task models, the use of the off-the-shelf mod-els also impedes the performance due to the complexity and specificity of the team sportsscenes, such as strong occlusion and motion blur. To circumvent those limitations, ourpaper proposes to train a single model that simultaneously predicts the ball and the playermask and pose by combining the part intensity fields and the spatial embeddings princi-ples. Part intensity fields provide the ball and player location, as well as player joints lo-cation. Spatial embeddings are then exploited to associate player instance pixels to theirrespective player center, but also to group player joints into skeletons. We demonstratethe effectiveness of the proposed model on the DeepSport basketball dataset, achievingcomparable performance to the SoA models addressing each individual task separately. 7 | 8 | # Commercial License 9 | 10 | Part of this software is available for licensing via the EPFL Technology Transfer 11 | Office (https://tto.epfl.ch/, info.tto@epfl.ch). 12 | 13 | The rest is available for licensing via the UCLouvain Technology Transfer 14 | Office (https://uclouvain.be/en/research/ltto, LTTO@uclouvain.be). 15 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_STANDARD 11) 2 | 3 | set(BIN "decoder-example") 4 | file(GLOB SRC *.cpp) 5 | 6 | add_executable(${BIN} ${SRC}) 7 | 8 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/aiapp.hpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Ai-app base interface and types 3 | /// 4 | /// \copyright 2018 NVISO SA. All rights reserved. 5 | /// \license This project is released under the XXXXXX License. 6 | /// 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | namespace lpdnn { 16 | namespace ai_app { 17 | 18 | /// Aiapp Blob 19 | /// This could be improved to allow referring to existing data 20 | /// thus avoding unneeded data-copy, for example by using shared_ptr. 21 | struct Blob { 22 | /// Data dimensions. Mandatory if the blob represents a tensor. 23 | std::vector dim; 24 | 25 | /// Data. Mandatory if the blob represents a tensor. 26 | std::vector data; 27 | 28 | /// Optional raw representation. 29 | std::vector raw; 30 | 31 | /// Optional CBOR representation when data is structured. 32 | std::vector cbor; 33 | 34 | /// Optional additional information 35 | /// (eg, description of internal representation: "NCHW,8bits,dp3"). 36 | std::string info; 37 | }; 38 | 39 | /// AI-App interface 40 | class Aiapp { 41 | public: 42 | virtual ~Aiapp() {} 43 | 44 | /// @return the ai-class id for this aiapp 45 | virtual const char* class_id() const = 0; 46 | 47 | /// @return the implementation id for this aiapp 48 | virtual const char* impl_id() const = 0; 49 | 50 | /// Initialization options 51 | /// \param cfg: configuration string, typically in JSON format. 52 | /// \return: true if success 53 | virtual bool init(const std::string& cfg) = 0; 54 | 55 | /// Set runtime options for the specified component 56 | /// \param opt: runtime options, typically in JSON format. 57 | /// \param name: subcomponent name 58 | /// \return: true if success 59 | virtual bool set_options(const std::string& opt, 60 | const std::string& name = "") = 0; 61 | 62 | /// Introspection methods 63 | /// \{ 64 | 65 | /// \return: names of all direct subcomponents of the specified component 66 | virtual std::vector components( 67 | const std::string& name = "") const = 0; 68 | 69 | /// \return output(s) of the specified component 70 | virtual std::vector output(const std::string& name = "") const = 0; 71 | 72 | /// \return metrics of the specified component and all its subcomponents 73 | virtual std::string metrics(const std::string& name = "") const = 0; 74 | 75 | /// set end-of-execution at the end of the specified component 76 | /// if name is empty any exit-point previously set is removed 77 | virtual bool set_exit_after(const std::string& name = "") = 0; 78 | 79 | /// \} 80 | }; 81 | 82 | /// AiApp standard processing components 83 | /// Each ai-app can contain other sub-components. 84 | /// Each subcomponent can be identified by a pathname, for example: 85 | /// "preprocessing.normalize" 86 | /// "inference.net1.conv23" 87 | struct Component { 88 | /// Standard component names. Their use is not mandatory but 89 | /// allows an ai-app to be supported by existing tools. 90 | static constexpr char const* preprocessing = "preprocessing"; 91 | static constexpr char const* inference = "inference"; 92 | static constexpr char const* postprocessing = "postprocessing"; 93 | 94 | /// Ai-app interface parameters 95 | static constexpr char const* interface = "interface"; 96 | 97 | /// Name separator in a component pathname string. 98 | /// Component names can't contain the separator except possibly for the leafs 99 | static constexpr char separator = '.'; 100 | 101 | /// Concatenate component names in a component pathname 102 | static std::string join(const std::string& path, const std::string& comp) { 103 | return path + separator + comp; 104 | } 105 | }; 106 | 107 | /// AiApp Metrics 108 | struct Metrics { 109 | /// Standard metrics. All timings are in microseconds. 110 | static constexpr char const* init_time = "init_time"; 111 | static constexpr char const* inference_time = "inference_time"; 112 | static constexpr char const* inference_cpu_time = "inference_cpu_time"; 113 | }; 114 | 115 | } // namespace ai_app 116 | } // namespace lpdnn 117 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/image_based.hpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Ai-app interface and types for image-based ai-apps 3 | /// 4 | /// \copyright 2018 NVISO SA. All rights reserved. 5 | /// \license This project is released under the XXXXXX License. 6 | /// 7 | 8 | #pragma once 9 | 10 | #include "aiapp.hpp" 11 | 12 | namespace lpdnn { 13 | namespace ai_app { 14 | 15 | /// 2-dimensional size 16 | struct Dim2d { 17 | int x; 18 | int y; 19 | }; 20 | 21 | /// Rectangle 22 | struct Rect { 23 | Dim2d origin; 24 | Dim2d size; 25 | 26 | bool empty() const { return size.x <= 0 || size.y <= 0; } 27 | }; 28 | 29 | /// Landmarks 30 | struct Landmark { 31 | Dim2d position; 32 | float confidence; /// Negative value if N/A 33 | }; 34 | 35 | struct Landmarks { 36 | /// Landmark specification identifier 37 | std::string type; 38 | /// Landmark points 39 | std::vector points; 40 | }; 41 | 42 | /// Image representation. 43 | /// The data of a RAW image consists of *y scanlines of *x pixels, 44 | /// with each pixel consisting of N interleaved 8-bit components; the first 45 | /// pixel pointed to is top-left-most in the image. There is no padding between 46 | /// image scanlines or between pixels, regardless of format. The number of 47 | /// components N is 3 for RGB images, 4 for RGBA, 1 for grayscale. 48 | /// Support for 8bits RGB format is MANDATORY for all image-processing AiApps. 49 | /// An image can be constructed from a std::vector, or a std::string 50 | /// or raw data pointer and size. When passing rvalues vector or strings, the 51 | /// image will take ownership of the data, otherwise will just keep reference. 52 | class Image { 53 | protected: 54 | /// Contains image data if we have ownership of it 55 | std::vector _image_content; 56 | 57 | public: 58 | /// Image format 59 | enum class Format { 60 | raw_grayscale = 1, /// 8bits grayscale 61 | raw_rgb8 = 3, /// 8bits RGB *MANDATORY* 62 | raw_rgba8 = 4, /// 8bits RGBA 63 | 64 | encoded = 256, /// Standard JPEG/BMP/PNG/TIFF format 65 | 66 | custom = 512 /// Custom format. Use attributes field for more details. 67 | }; 68 | 69 | /// Don't take data ownership. 70 | /// img_dim parameter can be omitted in case of encoded images since 71 | /// this information will be extracted from the image content itself. 72 | Image(Format img_format, const std::vector& data, Dim2d img_dim = {}) 73 | : Image(img_format, data.data(), data.size(), img_dim) {} 74 | 75 | /// Take data ownership 76 | Image(Format img_format, std::vector&& data, Dim2d img_dim = {}) 77 | : _image_content(std::move(data)), 78 | format{img_format}, 79 | dim(img_dim), 80 | data{_image_content.data()}, 81 | data_size{_image_content.size()} {} 82 | 83 | /// Don't take data ownership. 84 | Image(Format img_format, const std::string& data, Dim2d img_dim = {}) 85 | : Image(img_format, (uint8_t*)data.c_str(), data.size(), img_dim) {} 86 | 87 | /// Take data ownership 88 | Image(Format img_format, std::string&& data, Dim2d img_dim = {}) 89 | : Image(img_format, 90 | std::vector((uint8_t*)data.c_str(), 91 | (uint8_t*)data.c_str() + data.size()), 92 | img_dim) { 93 | data.clear(); 94 | } 95 | 96 | /// Don't take data ownership 97 | /// img_data_size is mandatory in case of encoded images. 98 | Image(Format img_format, const uint8_t* img_data, size_t img_data_size, 99 | Dim2d img_dim = {}) 100 | : format{img_format}, 101 | dim(img_dim), 102 | data{img_data}, 103 | data_size{img_data_size} {} 104 | 105 | /// Utility factory methods 106 | static Image encoded(const std::vector& data) { 107 | return Image(Format::encoded, data); 108 | } 109 | 110 | /// Image format 111 | Format format; 112 | 113 | /// Image dimensions (for raw images) 114 | Dim2d dim; 115 | 116 | /// Region of interest inside the image (all if empty) 117 | Rect roi{}; 118 | 119 | /// Custom attributes. 120 | /// This is ai-app specific and allows to specify custom data formats. 121 | std::string attributes; 122 | 123 | /// Pointer to image data (no ownership of the data). 124 | const uint8_t* data; 125 | 126 | /// Size of image data. Mandatory for encoded images. 127 | size_t data_size; 128 | 129 | /// Additional optional information about the image. 130 | /// May be required by some aiapps. 131 | Landmarks landmarks; 132 | }; 133 | 134 | /// Abstract image-based AiApp 135 | class Image_based : virtual public Aiapp { 136 | public: 137 | /// @return supported image formats (ordered by preference) 138 | virtual std::vector image_formats() const = 0; 139 | }; 140 | 141 | } // namespace ai_app 142 | } // namespace lpdnn 143 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/main.cpp: -------------------------------------------------------------------------------- 1 | #include "openpifpaf_postprocessor.hpp" 2 | #include "sample_data_0_8.hpp" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using namespace std; 10 | using namespace lpdnn; 11 | 12 | static void app_display_objects(const ai_app::Object_detection::Result& result, float threshold) { 13 | 14 | // Check if nothing found 15 | int numBoxes = result.items.size(); 16 | if (numBoxes == 1 && result.items[0].confidence < 0) numBoxes = 0; 17 | 18 | cout << "Object detection: " << numBoxes << " objects found." << endl; 19 | 20 | // Add bounding boxes and labels on image 21 | for (size_t i = 0; i < numBoxes; i++) { 22 | const auto& item = result.items[i]; 23 | // Skip detection if confidence is too low 24 | if (item.confidence < threshold) continue; 25 | 26 | const auto& bb = item.bounding_box; 27 | auto ix = item.class_index; 28 | string label_text; 29 | stringstream stream; 30 | stream << fixed << setprecision(2) << item.confidence; 31 | string conf_text = stream.str(); 32 | cout << "r: x=" << bb.origin.x << ",y=" << bb.origin.y << " dx=" << bb.size.x << ",dy=" << bb.size.y << " (" 33 | << conf_text << ")"; 34 | if (!item.landmarks.points.empty()) { 35 | cout << " landmarks "; 36 | for (const auto lm : item.landmarks.points) { 37 | cout << " [" << lm.position.x << "," << lm.position.y; 38 | if (lm.confidence >= 0) 39 | cout << " (" << fixed << setprecision(2) << lm.confidence << ")"; 40 | cout << "]"; 41 | } 42 | } 43 | cout << endl; 44 | 45 | } 46 | } 47 | 48 | int main(int argc, char** argv) { 49 | lpdnn::aiapp_impl::OpenPifPafPostprocessor pp; 50 | const ai_app::Rect tileCoordinates { {0, -27}, {640, 481} }; 51 | ai_app::Object_detection::Result res = pp.postprocess_0_8( 52 | inputWidth, inputHeight, tensorWidth, tensorHeight, tileCoordinates, 53 | pif_c, pif_r, pif_b, pif_s, paf_c, paf_r1, paf_r2, paf_b1, paf_b2 54 | ); 55 | 56 | cout << "Expected result: " << 57 | "2 objects found.\n" 58 | "r: x=296,y=146 dx=195,dy=223 (0.48) landmarks [0,-27 (0.00)] [0,-27 (0.00)] [0,-27 (0.00)] [388,150 (0.52)] [405,146 (0.55)] [372,178 (0.82)] [423,178 (0.85)] [338,187 (0.72)] [461,196 (0.90)] [296,191 (0.72)] [491,181 (0.86)] [387,256 (0.74)] [421,257 (0.85)] [383,324 (0.65)] [411,315 (0.70)] [386,369 (0.59)] [403,364 (0.58)]\n" 59 | "r: x=63,y=303 dx=78,dy=64 (0.27) landmarks [87,315 (0.42)] [92,309 (0.43)] [83,311 (0.33)] [102,303 (0.53)] [0,-27 (0.00)] [124,315 (0.76)] [73,320 (0.61)] [141,353 (0.68)] [63,352 (0.48)] [117,366 (0.49)] [75,364 (0.37)] [131,363 (0.67)] [94,366 (0.58)] [130,365 (0.31)] [71,367 (0.26)] [0,-27 (0.00)] [0,-27 (0.00)]\n" << endl; 60 | 61 | app_display_objects(res, 0.1); 62 | 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/math_helpers.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "math_helpers.hpp" 3 | #include 4 | 5 | #ifdef __APPLE__ 6 | #define MATH_HELPERS_ACCELERATE 1 7 | #else 8 | #define MATH_HELPERS_ACCELERATE 0 9 | #endif 10 | 11 | #if MATH_HELPERS_ACCELERATE 12 | #include 13 | #else 14 | #include 15 | #endif 16 | 17 | void vfill(float*x, unsigned long n, float v) { 18 | #if MATH_HELPERS_ACCELERATE 19 | vDSP_vfill(&v, x, 1, n); 20 | #else 21 | // Slow version 22 | for (unsigned long i = 0; i < n; ++i) { 23 | x[i] = v; 24 | } 25 | #endif 26 | } 27 | 28 | void vadd(const float *a, const float *b, float *c, unsigned long n) { 29 | #if MATH_HELPERS_ACCELERATE 30 | vDSP_vadd(a, 1, b, 1, c, 1, n); 31 | #else 32 | // Slow version 33 | for (unsigned long i = 0; i < n; ++i) { 34 | c[i] = a[i] + b[i]; 35 | } 36 | #endif 37 | } 38 | 39 | void vexp(float *x, unsigned long n) { 40 | #if MATH_HELPERS_ACCELERATE 41 | int n_ = (int)n; 42 | vvexpf(x, x, &n_); 43 | #else 44 | // Slow version 45 | for (unsigned long i = 0; i < n; ++i) { 46 | x[i] = std::exp(x[i]); 47 | } 48 | #endif 49 | } 50 | 51 | void vmul(const float *a, const float *b, float *c, unsigned long n) { 52 | #if MATH_HELPERS_ACCELERATE 53 | vDSP_vmul(a, 1, b, 1, c, 1, n); 54 | #else 55 | // Slow version 56 | for (unsigned long i = 0; i < n; ++i) { 57 | c[i] = a[i] * b[i]; 58 | } 59 | #endif 60 | } 61 | 62 | void vsmul(const float *a, float b, float *c, unsigned long n) { 63 | #if MATH_HELPERS_ACCELERATE 64 | vDSP_vsmul(a, 1, &b, c, 1, n); 65 | #else 66 | // Slow version 67 | for (unsigned long i = 0; i < n; ++i) { 68 | c[i] = a[i] * b; 69 | } 70 | #endif 71 | } 72 | 73 | float vargmax(const float *x, unsigned long n, int* i) { 74 | assert(n > 0); 75 | #if MATH_HELPERS_ACCELERATE 76 | float maxValue = 0.0f; 77 | vDSP_Length maxIndex = 0; 78 | vDSP_maxvi(x, 1, &maxValue, &maxIndex, n); 79 | *i = (int)maxIndex; 80 | return maxValue; 81 | #else 82 | // Slow version 83 | float maxValue = x[0]; 84 | unsigned long maxIndex = 0; 85 | for (unsigned long i = 1; i < n; ++i) { 86 | if (x[i] > maxValue) { 87 | maxValue = x[i]; 88 | maxIndex = i; 89 | } 90 | } 91 | *i = (int)maxIndex; 92 | return maxValue; 93 | #endif 94 | } 95 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/math_helpers.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // x[i] = v 4 | void vfill(float*x, unsigned long n, float v); 5 | 6 | // c[i] = a[i] + b[i] 7 | void vadd(const float *a, const float *b, float *c, unsigned long n); 8 | 9 | // x[i] = exp(x[i]) 10 | void vexp(float *x, unsigned long n); 11 | 12 | // c[i] = a[i] * b[i] 13 | void vmul(const float *a, const float *b, float *c, unsigned long n); 14 | 15 | // c[i] = a[i] * b 16 | void vsmul(const float *a, float b, float *c, unsigned long n); 17 | 18 | // out = max(x) 19 | // i = argmax(x) 20 | float vargmax(const float *x, unsigned long n, int* i); 21 | 22 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/object_detection.hpp: -------------------------------------------------------------------------------- 1 | /// 2 | /// Ai-app interface for object detection 3 | /// 4 | /// \copyright 2018 NVISO SA. All rights reserved. 5 | /// \license This project is released under the XXXXXX License. 6 | /// 7 | 8 | #pragma once 9 | 10 | #include "image_based.hpp" 11 | 12 | namespace lpdnn { 13 | namespace ai_app { 14 | 15 | /// Object detection AiApp 16 | class Object_detection : virtual public Image_based { 17 | public: 18 | struct Result { 19 | struct Item { 20 | float confidence; 21 | int class_index; 22 | Rect bounding_box; 23 | Landmarks landmarks; 24 | }; 25 | 26 | bool success{}; 27 | std::vector items; 28 | }; 29 | 30 | /// Set minimum detectable object size 31 | /// @return true if success 32 | virtual bool set_min_size(Dim2d minSize) = 0; 33 | 34 | /// Set maximum detectable object size 35 | /// @return true if success 36 | virtual bool set_max_size(Dim2d maxSize) = 0; 37 | 38 | /// Perform inference. 39 | virtual Result execute(const Image& input) = 0; 40 | 41 | /// @return Names of classes 42 | virtual std::vector classes() = 0; 43 | 44 | /// @return our aiapp class id 45 | const char* class_id() const override { return ai_class_id; } 46 | static constexpr char const* ai_class_id = "com_bonseyes::object_detection"; 47 | }; 48 | 49 | } // namespace ai_app 50 | } // namespace lpdnn 51 | -------------------------------------------------------------------------------- /cpp/examples/cifcafdecoder/sample_data_0_8.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SAMPLE_DATA_HPP 2 | #define SAMPLE_DATA_HPP 3 | 4 | constexpr int inputWidth = 129; 5 | constexpr int inputHeight = 97; 6 | constexpr int tensorWidth = 17; 7 | constexpr int tensorHeight = 13; 8 | constexpr int tensorSize = tensorWidth * tensorHeight; 9 | extern const float pif_c[17 * tensorSize]; 10 | extern const float pif_r[17 * 2 * tensorSize]; 11 | extern const float pif_b[17 * tensorSize]; 12 | extern const float pif_s[17 * tensorSize]; 13 | extern const float paf_c[19 * tensorSize]; 14 | extern const float paf_r1[19 * 2 * tensorSize]; 15 | extern const float paf_r2[19 * 2 * tensorSize]; 16 | extern const float paf_b1[19 * tensorSize]; 17 | extern const float paf_b2[19 * tensorSize]; 18 | 19 | #endif // SAMPLE_DATA_HPP 20 | -------------------------------------------------------------------------------- /docs/coco/000000081988.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/coco/000000081988.jpg -------------------------------------------------------------------------------- /docs/coco/000000081988.jpg.predictions.json: -------------------------------------------------------------------------------- 1 | [{"keypoints": [0.0, -3.0, 0.0, 0.0, -3.0, 0.0, 0.0, -3.0, 0.0, 387.92, 149.78, 0.93, 412.08, 150.24, 0.93, 376.8, 176.01, 0.93, 426.97, 178.2, 0.98, 337.21, 191.21, 0.87, 463.56, 192.44, 0.94, 300.38, 193.76, 0.85, 495.94, 184.82, 0.94, 386.74, 254.44, 0.92, 415.89, 253.74, 0.91, 384.01, 315.45, 0.85, 409.0, 316.42, 0.91, 386.61, 367.7, 0.43, 405.5, 366.93, 0.76], "bbox": [288.19, 142.82, 219.72, 240.64], "score": 0.776, "category_id": 1}, {"keypoints": [81.58, 317.15, 0.73, 85.77, 312.54, 0.78, 80.37, 312.16, 0.56, 100.37, 308.25, 0.93, 0.0, -3.0, 0.0, 123.9, 319.06, 0.86, 78.38, 322.67, 0.86, 147.34, 348.24, 0.77, 58.47, 350.95, 0.89, 125.12, 355.55, 0.57, 52.27, 380.7, 0.88, 122.76, 361.79, 0.77, 94.69, 362.32, 0.83, 151.85, 360.71, 0.74, 75.55, 368.06, 0.71, 106.22, 379.11, 0.64, 100.08, 379.35, 0.41], "bbox": [46.68, 305.1, 112.81, 82.21], "score": 0.753, "category_id": 1}, {"keypoints": [359.09, 299.52, 0.83, 363.81, 294.76, 0.77, 354.5, 294.49, 0.81, 370.47, 296.34, 0.63, 347.7, 297.22, 0.87, 381.71, 318.43, 0.68, 339.75, 323.07, 0.91, 387.56, 350.85, 0.46, 335.39, 354.76, 0.85, 375.73, 362.42, 0.28, 335.38, 365.91, 0.83, 375.9, 367.7, 0.71, 349.92, 368.55, 0.83, 392.69, 362.36, 0.36, 331.14, 374.4, 0.44, 331.18, 378.04, 0.13, 331.31, 385.84, 0.29], "bbox": [322.79, 292.3, 77.57, 101.51], "score": 0.694, "category_id": 1}, {"keypoints": [236.44, 323.1, 0.33, 237.9, 318.75, 0.21, 233.31, 320.55, 0.47, 0.0, -3.0, 0.0, 220.71, 316.44, 0.51, 241.45, 318.09, 0.76, 202.48, 318.92, 0.69, 242.91, 350.49, 0.83, 196.83, 349.11, 0.66, 239.85, 381.06, 0.63, 193.86, 378.72, 0.37, 221.92, 337.97, 0.57, 199.3, 338.08, 0.5, 226.29, 366.83, 0.3, 194.64, 374.38, 0.21, 0.0, -3.0, 0.0, 0.0, -3.0, 0.0], "bbox": [188.89, 312.82, 58.95, 72.39], "score": 0.505, "category_id": 1}, {"keypoints": [491.15, 348.81, 0.23, 493.69, 344.94, 0.37, 488.61, 345.01, 0.14, 503.85, 334.63, 0.41, 0.0, -3.0, 0.0, 522.43, 335.09, 0.4, 489.47, 335.32, 0.53, 553.67, 355.01, 0.26, 489.03, 354.4, 0.79, 562.96, 379.67, 0.12, 487.72, 378.05, 0.96, 540.78, 344.05, 0.45, 522.94, 345.88, 0.42, 521.85, 380.68, 0.5, 498.82, 377.06, 0.43, 566.02, 385.18, 0.41, 531.69, 375.85, 0.19], "bbox": [483.6, 329.7, 89.18, 62.24], "score": 0.486, "category_id": 1}] -------------------------------------------------------------------------------- /docs/coco/000000081988.jpg.predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/coco/000000081988.jpg.predictions.png -------------------------------------------------------------------------------- /docs/datasets.md: -------------------------------------------------------------------------------- 1 | # Datasets 2 | 3 | Download MSCOCO data: 4 | 5 | ```sh 6 | mkdir data-mscoco 7 | cd data-mscoco 8 | gsutil ls gs://images.cocodataset.org # to list available directories 9 | 10 | mkdir -p images/val2017 11 | gsutil -m rsync gs://images.cocodataset.org/val2017 images/val2017 12 | 13 | mkdir -p images/train2017 14 | gsutil -m rsync gs://images.cocodataset.org/train2017 images/train2017 15 | 16 | gsutil cp gs://images.cocodataset.org/annotations/annotations_trainval2017.zip . 17 | # or 18 | wget http://images.cocodataset.org/annotations/annotations_trainval2017.zip 19 | unzip annotations_trainval2017.zip 20 | wget http://images.cocodataset.org/annotations/image_info_test2017.zip 21 | unzip image_info_test2017.zip 22 | 23 | # test images: run inside of images directory 24 | wget http://images.cocodataset.org/zips/test2017.zip 25 | unzip test2017.zip 26 | ``` 27 | 28 | Download MPII data: 29 | 30 | ```sh 31 | mkdir data-mpii 32 | cd data-mpii 33 | wget https://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/mpii_human_pose_v1.tar.gz 34 | wget https://datasets.d2.mpi-inf.mpg.de/andriluka14cvpr/mpii_human_pose_v1_u12_2.zip 35 | ``` 36 | 37 | 38 | Download NightOwls: 39 | 40 | ```sh 41 | mkdir data-nightowls 42 | cd data-nightowls 43 | wget http://www.robots.ox.ac.uk/\~vgg/data/nightowls/python/nightowls_validation.json 44 | wget http://www.robots.ox.ac.uk/\~vgg/data/nightowls/python/nightowls_validation.zip 45 | unzip nightowls_validation.zip 46 | ``` 47 | -------------------------------------------------------------------------------- /docs/skeleton_coco.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/skeleton_coco.png -------------------------------------------------------------------------------- /docs/skeleton_dense.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/skeleton_dense.png -------------------------------------------------------------------------------- /docs/skeleton_kinematic_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/skeleton_kinematic_tree.png -------------------------------------------------------------------------------- /docs/wave3.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/docs/wave3.gif -------------------------------------------------------------------------------- /guide/_config.yml: -------------------------------------------------------------------------------- 1 | # Book settings 2 | title: OpenPifPaf Guide 3 | author: Sven Kreiss 4 | email: research@svenkreiss.com 5 | logo: logo.png 6 | discription: Guide for OpenPifPaf. 7 | 8 | # latex: 9 | # latex_documents: 10 | # targetname: book.tex 11 | 12 | html: 13 | favicon: favicon.png 14 | 15 | # Information about where the book exists on the web 16 | repository: 17 | url : https://github.com/vita-epfl/openpifpaf # Online location of your book 18 | path_to_book : guide # Optional path to your book, relative to the repository root 19 | branch : stable # Which branch of the repository should be used when creating links (optional) 20 | 21 | launch_buttons: 22 | binderhub_url : "https://mybinder.org" # The URL for your BinderHub (e.g., https://mybinder.org) 23 | -------------------------------------------------------------------------------- /guide/_config_dev.yml: -------------------------------------------------------------------------------- 1 | # Book settings 2 | title: OpenPifPaf Guide DEV 3 | author: Sven Kreiss 4 | email: research@svenkreiss.com 5 | logo: logo.png 6 | discription: Guide for OpenPifPaf (dev version). 7 | 8 | # latex: 9 | # latex_documents: 10 | # targetname: book.tex 11 | 12 | html: 13 | favicon: favicon.png 14 | extra_navbar: This is the dev version of the Guide. Here is the stable version. 15 | 16 | # Information about where the book exists on the web 17 | repository: 18 | url : https://github.com/vita-epfl/openpifpaf # Online location of your book 19 | path_to_book : guide # Optional path to your book, relative to the repository root 20 | branch : dev # Which branch of the repository should be used when creating links (optional) 21 | 22 | launch_buttons: 23 | binderhub_url : "https://mybinder.org" # The URL for your BinderHub (e.g., https://mybinder.org) 24 | -------------------------------------------------------------------------------- /guide/_toc.yml: -------------------------------------------------------------------------------- 1 | - file: intro 2 | 3 | - part: Getting Started 4 | chapters: 5 | - file: datasets 6 | - file: train 7 | - file: predict_cli 8 | - file: cli_help 9 | - file: examples 10 | - file: faq 11 | 12 | - part: API 13 | chapters: 14 | - file: predict_api 15 | - file: moduledocs 16 | 17 | - part: Development 18 | chapters: 19 | - file: dev 20 | - file: profile 21 | - title: History 22 | url: https://github.com/vita-epfl/openpifpaf/blob/main/HISTORY.md 23 | - title: GitHub 24 | url: https://github.com/vita-epfl/openpifpaf 25 | -------------------------------------------------------------------------------- /guide/artwork.py: -------------------------------------------------------------------------------- 1 | import openpifpaf 2 | from openpifpaf.datasets.constants import ( 3 | COCO_KEYPOINTS, 4 | COCO_PERSON_SKELETON, 5 | COCO_UPRIGHT_POSE, 6 | ) 7 | 8 | 9 | def main(): 10 | ann = openpifpaf.Annotation(keypoints=COCO_KEYPOINTS, skeleton=COCO_PERSON_SKELETON) 11 | ann.set(COCO_UPRIGHT_POSE) 12 | 13 | # favicon 14 | keypoint_painter = openpifpaf.show.KeypointPainter( 15 | color_connections=True, linewidth=48, markersize=0) 16 | openpifpaf.datasets.constants.draw_ann( 17 | ann, 18 | keypoint_painter=keypoint_painter, 19 | aspect='equal', 20 | margin=0.8, 21 | frameon=False, 22 | filename='favicon.png', 23 | ) 24 | 25 | # logo 26 | keypoint_painter = openpifpaf.show.KeypointPainter( 27 | color_connections=True, linewidth=12) 28 | openpifpaf.datasets.constants.draw_ann( 29 | ann, 30 | keypoint_painter=keypoint_painter, 31 | frameon=False, 32 | filename='logo.png', 33 | ) 34 | 35 | 36 | if __name__ == '__main__': 37 | main() 38 | -------------------------------------------------------------------------------- /guide/cli_help.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[Sven Kreiss](https://www.svenkreiss.com/), 2020\n", 8 | "\n", 9 | "# CLI `--help`\n", 10 | "\n", 11 | "This is a reference for all the `--help` message from all of OpenPifPaf's command line interfaces (CLIs)." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "(cli-help-predict)=\n", 19 | "## predict" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "!python3 -m openpifpaf.predict --help" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "(cli-help-video)=\n", 36 | "## video" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "!python3 -m openpifpaf.video --help" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "## train" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "!python3 -m openpifpaf.train --help" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## eval_coco" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "!python3 -m openpifpaf.eval_coco --help" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "## export_onnx" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "!python3 -m openpifpaf.export_onnx --help" 94 | ] 95 | }, 96 | { 97 | "cell_type": "markdown", 98 | "metadata": {}, 99 | "source": [ 100 | "## benchmark" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "!python3 -m openpifpaf.benchmark --help" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "## logs" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "!python3 -m openpifpaf.logs --help" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": null, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [] 134 | } 135 | ], 136 | "metadata": { 137 | "kernelspec": { 138 | "display_name": "python3", 139 | "name": "python3" 140 | }, 141 | "language_info": { 142 | "codemirror_mode": { 143 | "name": "ipython", 144 | "version": 3 145 | }, 146 | "file_extension": ".py", 147 | "mimetype": "text/x-python", 148 | "name": "python", 149 | "nbconvert_exporter": "python", 150 | "pygments_lexer": "ipython3", 151 | "version": "3.7.4-final" 152 | } 153 | }, 154 | "nbformat": 4, 155 | "nbformat_minor": 2 156 | } 157 | -------------------------------------------------------------------------------- /guide/coco/000000081988.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/guide/coco/000000081988.jpg -------------------------------------------------------------------------------- /guide/coco/000000114907.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/guide/coco/000000114907.jpg -------------------------------------------------------------------------------- /guide/coco/000000147740.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/guide/coco/000000147740.jpg -------------------------------------------------------------------------------- /guide/coco/000000188465.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/guide/coco/000000188465.jpg -------------------------------------------------------------------------------- /guide/coco/000000541055.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/guide/coco/000000541055.jpg -------------------------------------------------------------------------------- /guide/dev.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[Sven Kreiss](https://www.svenkreiss.com/), 2020\n", 8 | "\n", 9 | "# Contribute\n", 10 | "\n", 11 | "Before we can accept contributions, you need to become a CLAed contributor.\n", 12 | "E-mail a signed copy of the\n", 13 | "[CLAI](https://github.com/vita-epfl/openpifpaf/blob/main/CLAI.txt)\n", 14 | "(and if applicable the\n", 15 | "[CLAC](https://github.com/vita-epfl/openpifpaf/blob/main/CLAC.txt))\n", 16 | "as PDF file to research@svenkreiss.com." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": {}, 22 | "source": [ 23 | "(modify-code)=\n", 24 | "## Modify Code\n", 25 | "\n", 26 | "For development of the openpifpaf source code itself, you need to clone this repository and then:\n", 27 | "\n", 28 | "```sh\n", 29 | "pip3 install numpy cython\n", 30 | "pip3 install --editable '.[dev,train,test]'\n", 31 | "```\n", 32 | "\n", 33 | "The last command installs the Python package in the current directory\n", 34 | "(signified by the dot) with the optional dependencies needed for training and\n", 35 | "testing. If you modify `functional.pyx`, run this last command again which\n", 36 | "recompiles the static code.\n", 37 | "\n", 38 | "Develop your features in separate feature branches. \n", 39 | "Create a pull request with your suggested changes. Make sure your code passes\n", 40 | "`pytest` and `pylint` checks." 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "## Things to Contribute\n", 48 | "\n", 49 | "This is a research project and changing fast. Contributions can be in many areas:\n", 50 | "* Add a new dataset?\n", 51 | "* Add a new base network? \n", 52 | "* Try a different loss? \n", 53 | "* Try a better multi-task strategy?\n", 54 | "* Try a different head architecture? \n", 55 | "* Add a new task? \n", 56 | "* Run on new hardware (mobile phones, embedded devices, ...)? \n", 57 | "* Improve training schedule/procedure?\n", 58 | "* Use it to build an app?\n", 59 | "* Improve documentation (!!)\n", 60 | "* ..." 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "metadata": {}, 66 | "source": [ 67 | "## Build Guide\n", 68 | "\n", 69 | "```sh\n", 70 | "cd guide\n", 71 | "jb build .\n", 72 | "```\n", 73 | "\n", 74 | "If you encounter issues with the kernel spec in the notebooks, this is a piece of code that is used to normalize the kernel spec names in the guide geployment script:\n", 75 | "\n", 76 | "```sh\n", 77 | "for fn in *.ipynb; do\n", 78 | " jq '(.metadata.kernelspec.name,.metadata.kernelspec.display_name)=\"python3\"' ${fn} > ${fn}_\n", 79 | " mv ${fn}_ ${fn}\n", 80 | "done\n", 81 | "```" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "## Build Environment" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "!pip freeze" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "## Scratch" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "# Uncomment when using on Google Colab:\n", 114 | "# !pip install --upgrade openpifpaf==0.11.1" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "!python -m openpifpaf.predict --version" 124 | ] 125 | } 126 | ], 127 | "metadata": { 128 | "kernelspec": { 129 | "display_name": "python3", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.7.4-final" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 2 147 | } 148 | -------------------------------------------------------------------------------- /guide/examples.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Examples\n", 8 | "\n", 9 | "Example predictions from the COCO val set.\n", 10 | "These are the first images with more than five person annotations that were shared with [CC-BY-2.0].\n", 11 | "These examples are not cherry-picked.\n", 12 | "\n", 13 | "All examples are predicted with the fast model `shufflenetv2k16w`.
\n", 14 | "For more accurate results, try `--checkpoint=shufflenetv2k30w`.\n", 15 | "\n", 16 | "The examples below are generated with the shown command which is documented in {doc}`predict_cli`.\n", 17 | "\n", 18 | "[CC-BY-2.0]: https://creativecommons.org/licenses/by/2.0/" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "!python -m openpifpaf.predict coco/000000188465.jpg --image-output --long-edge=1025" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "![predicted](coco/000000188465.jpg.predictions.png)
\n", 35 | "Image credit: [Photo](https://www.flickr.com/photos/jelonekphotography/4593328722/) by Brian Jelonek shared under [CC-BY-2.0]." 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "!python -m openpifpaf.predict coco/000000114907.jpg --image-output" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "![predicted](coco/000000114907.jpg.predictions.png)
\n", 52 | "Image credit: [Photo](https://www.flickr.com/photos/fortrucker/7825237468/) by Fort Rucker shared under [CC-BY-2.0]." 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "!python -m openpifpaf.predict coco/000000147740.jpg --image-output" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "![predicted](coco/000000147740.jpg.predictions.png)
\n", 69 | "Image credit: [Photo](https://www.flickr.com/photos/cumidanciki/8026451116/) by CCFoodTravel.com shared under [CC-BY-2.0]." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "!python -m openpifpaf.predict coco/000000541055.jpg --image-output" 79 | ] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "![predicted](coco/000000541055.jpg.predictions.png)
\n", 86 | "Image credit: [Photo](https://www.flickr.com/photos/12407269@N07/3465325303/) by Jarrod Doll shared under [CC-BY-2.0]." 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [] 95 | } 96 | ], 97 | "metadata": { 98 | "kernelspec": { 99 | "display_name": "python3", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.7.4-final" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 2 117 | } 118 | -------------------------------------------------------------------------------- /guide/faq.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# FAQ\n", 8 | "\n", 9 | "Frequently Asked Questions (FAQ)." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Why evaluate with 641px instead of 640px?\n", 17 | "\n", 18 | "OpenPifPaf uses the standard convention of PyTorch models to pad convolutions. \n", 19 | "Let's start with an example: a single layer of a 3x3 conv with stride 2 that is padded with 1. For an output feature map of size 2x2, the input must be of size 3x3. This generalization holds: the input must be of size `(nx * stride + 1, ny * stride + 1)`.\n", 20 | "The models that OpenPifPaf uses have an intermediate layer with stride 16. Therefore, good input image sizes are multiples of 16 plus 1. \n", 21 | "\n", 22 | "It is usually not a problem if the input size is not perfect. There will just be a small margin on the right side and bottom of the image that is not \"covered\" by a feature map cell." 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "## Predict is slow\n", 30 | "\n", 31 | "Check whether your installation of PyTorch can access CUDA for GPU processing.\n", 32 | "If the output of the command below is False, then PyTorch cannot make use of your GPU and OpenPifPaf falls back to CPU processing." 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "!python -c \"import torch; print(torch.cuda.is_available())\"" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "You can also run `predict` with the `--debug` option. Compare your output with the output in {doc}`predict_cli` to understand which part of the process is slow for you. For a fair comparison, also use `--disable-cuda` because the reference in this documentation is created without CUDA." 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "## Python.h is missing\n", 56 | "\n", 57 | "Installation fails with an exception complaining about a missing `Python.h` file. This means that you need the development files for Python itself. On Ubuntu systems, you can get this with `sudo apt-get install python3-dev`. For more operating systems, there is a good [StackOverflow post](https://stackoverflow.com/questions/21530577/fatal-error-python-h-no-such-file-or-directory) on the topic." 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": {}, 64 | "outputs": [], 65 | "source": [] 66 | } 67 | ], 68 | "metadata": { 69 | "kernelspec": { 70 | "display_name": "python3", 71 | "name": "python3" 72 | }, 73 | "language_info": { 74 | "codemirror_mode": { 75 | "name": "ipython", 76 | "version": 3 77 | }, 78 | "file_extension": ".py", 79 | "mimetype": "text/x-python", 80 | "name": "python", 81 | "nbconvert_exporter": "python", 82 | "pygments_lexer": "ipython3", 83 | "version": "3.7.4-final" 84 | } 85 | }, 86 | "nbformat": 4, 87 | "nbformat_minor": 2 88 | } 89 | -------------------------------------------------------------------------------- /guide/moduledocs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[Sven Kreiss](https://www.svenkreiss.com/), 2020" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import openpifpaf\n", 17 | "import openpifpaf.encoder" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "print(openpifpaf.__doc__)" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "# Modules" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "## openpifpaf.datasets" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "print(openpifpaf.datasets.__doc__)" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "metadata": {}, 55 | "source": [ 56 | "## openpifpaf.decoder" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "print(openpifpaf.decoder.__doc__)" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "## openpifpaf.encoder" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "print(openpifpaf.encoder.__doc__)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "metadata": {}, 87 | "source": [ 88 | "## openpifpaf.network" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [ 97 | "print(openpifpaf.network.__doc__)" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "## openpifpaf.show" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "print(openpifpaf.show.__doc__)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": {}, 119 | "source": [ 120 | "## openpifpaf.transforms" 121 | ] 122 | }, 123 | { 124 | "cell_type": "code", 125 | "execution_count": null, 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "print(openpifpaf.transforms.__doc__)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "metadata": {}, 135 | "source": [ 136 | "## openpifpaf.visualizer" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "print(openpifpaf.visualizer.__doc__)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [] 154 | } 155 | ], 156 | "metadata": { 157 | "kernelspec": { 158 | "display_name": "python3", 159 | "name": "python3" 160 | }, 161 | "language_info": { 162 | "codemirror_mode": { 163 | "name": "ipython", 164 | "version": 3 165 | }, 166 | "file_extension": ".py", 167 | "mimetype": "text/x-python", 168 | "name": "python", 169 | "nbconvert_exporter": "python", 170 | "pygments_lexer": "ipython3", 171 | "version": "3.7.4-final" 172 | } 173 | }, 174 | "nbformat": 4, 175 | "nbformat_minor": 2 176 | } 177 | -------------------------------------------------------------------------------- /guide/predict_cli.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[Sven Kreiss](https://www.svenkreiss.com/), 2020\n", 8 | "\n", 9 | "# Prediction\n", 10 | "\n", 11 | "Use OpenPifPaf from the command line to run multi-person pose estimation on images.\n", 12 | "For programmatic predictions, please refer to {doc}`predict_api`.\n", 13 | "Below is a short intro to running prediction on videos.\n", 14 | "\n", 15 | "Run `openpifpaf.predict` on an image:" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": null, 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "!python -m openpifpaf.predict coco/000000081988.jpg --image-output --json-output --debug" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "This command produced two outputs: an image and a json file.\n", 32 | "You can provide file or folder arguments to the `--image-output` and `--json-output` flags.\n", 33 | "Here, we used the default which created these two files:\n", 34 | "\n", 35 | "```sh\n", 36 | "coco/000000081988.jpg.predictions.png\n", 37 | "coco/000000081988.jpg.predictions.json\n", 38 | "```\n", 39 | "\n", 40 | "Here is the image:\n", 41 | "\n", 42 | "![predictions](coco/000000081988.jpg.predictions.png)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "And below is the json output. The json data is a list where each entry in the list corresponds to one pose annotation. In this case, there are five entries corresponding to the five people in the image. Each annotation contains information on `\"keypoints\"`, `\"bbox\"`, `\"score\"` and `\"category_id\"`.\n", 50 | "\n", 51 | "All coordinates are in pixel coordinates. The `\"keypoints\"` entry is in COCO format with triples of `(x, y, c)` (`c` for confidence) for every joint as listed under {ref}`coco-person-keypoints`. The pixel coordinates have sub-pixel accuracy, i.e. 10.5 means the joint is between pixel 10 and 11.\n", 52 | "In rare cases, joints can be localized outside the field of view and then the pixel coordinates can be negative. When `c` is zero, the joint was not detected.\n", 53 | "\n", 54 | "The `\"bbox\"` (bounding box) format is `(x, y, w, h)`: the $(x, y)$ coordinate of the top-left corner followed by width and height.\n", 55 | "\n", 56 | "The `\"score\"` is a number between zero and one." 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "!python -m json.tool coco/000000081988.jpg.predictions.json" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "metadata": {}, 71 | "source": [ 72 | "## Optional Arguments\n", 73 | "\n", 74 | "* `--show`: show interactive matplotlib output\n", 75 | "* `--debug-images`: enable debug messages and debug plots\n", 76 | "\n", 77 | "Full list of arguments is available with `--help`: {ref}`CLI help for predict `." 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": {}, 83 | "source": [ 84 | "## Video\n", 85 | "\n", 86 | "```sh\n", 87 | "python3 -m openpifpaf.video --source myvideotoprocess.mp4 --video-output --json-output\n", 88 | "```\n", 89 | "\n", 90 | "Requires OpenCV. The `--video-output` option also requires matplotlib.\n", 91 | "Replace `myvideotoprocess.mp4` with `0` for webcam0 or other OpenCV compatible sources.\n", 92 | "The full list of arguments is available with `--help`: {ref}`CLI help for video `." 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "Image credit: \"[Learning to surf](https://www.flickr.com/photos/fotologic/6038911779/in/photostream/)\" by fotologic which is licensed under [CC-BY-2.0].\n", 100 | "\n", 101 | "[CC-BY-2.0]: https://creativecommons.org/licenses/by/2.0/" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": null, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [] 110 | } 111 | ], 112 | "metadata": { 113 | "kernelspec": { 114 | "display_name": "python3", 115 | "name": "python3" 116 | }, 117 | "language_info": { 118 | "codemirror_mode": { 119 | "name": "ipython", 120 | "version": 3 121 | }, 122 | "file_extension": ".py", 123 | "mimetype": "text/x-python", 124 | "name": "python", 125 | "nbconvert_exporter": "python", 126 | "pygments_lexer": "ipython3", 127 | "version": "3.7.4-final" 128 | } 129 | }, 130 | "nbformat": 4, 131 | "nbformat_minor": 2 132 | } 133 | -------------------------------------------------------------------------------- /guide/profile.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "[Sven Kreiss](https://www.svenkreiss.com/), 2020\n", 8 | "\n", 9 | "# Profile" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "## Profile Decoder" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": { 23 | "tags": [] 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "!python -m openpifpaf.predict coco/000000081988.jpg --no-download-progress --debug --profile-decoder" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "!flameprof profile_decoder.prof > profile_decoder_flame.svg" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "tags": [] 43 | }, 44 | "source": [ 45 | "![decoder flame graph](profile_decoder_flame.svg)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "There is a second output that is generated from the Autograd Profiler. This can only be viewed in the Chrome browser: \n", 53 | "* open `chrome://tracing`\n", 54 | "* click \"Load\" in the top left corner\n", 55 | "* select `decoder_profile.1.json`" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [] 64 | } 65 | ], 66 | "metadata": { 67 | "kernelspec": { 68 | "display_name": "python3", 69 | "name": "python3" 70 | }, 71 | "language_info": { 72 | "codemirror_mode": { 73 | "name": "ipython", 74 | "version": 3 75 | }, 76 | "file_extension": ".py", 77 | "mimetype": "text/x-python", 78 | "name": "python", 79 | "nbconvert_exporter": "python", 80 | "pygments_lexer": "ipython3", 81 | "version": 3 82 | } 83 | }, 84 | "nbformat": 4, 85 | "nbformat_minor": 2 86 | } 87 | -------------------------------------------------------------------------------- /guide/references.bib: -------------------------------------------------------------------------------- 1 | @InProceedings{kreiss2019pifpaf, 2 | author = {Kreiss, Sven and Bertoni, Lorenzo and Alahi, Alexandre}, 3 | title = {{PifPaf: Composite Fields for Human Pose Estimation}}, 4 | booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, 5 | month = {June}, 6 | year = {2019} 7 | } 8 | -------------------------------------------------------------------------------- /oks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #SBATCH --output="OKS-%j.log" 4 | #SBATCH --ntasks=1 5 | #SBATCH --cpus-per-task=2 6 | #SBATCH -w belldevcv01 7 | # auie #SBATCH --gres="gpu:1" 8 | 9 | python oks.py $@ 10 | -------------------------------------------------------------------------------- /openpifpaf/__init__.py: -------------------------------------------------------------------------------- 1 | """An open implementation of PifPaf.""" 2 | 3 | __version__ = '0.11.6' 4 | 5 | from .annotation import Annotation, AnnotationDet 6 | from . import datasets 7 | from . import decoder 8 | from . import encoder 9 | from . import network 10 | from . import optimize 11 | from . import transforms 12 | # from .dataset_utilities import dataset_utilities 13 | -------------------------------------------------------------------------------- /openpifpaf/count_ops.py: -------------------------------------------------------------------------------- 1 | """Export a checkpoint as an ONNX model. 2 | 3 | Applies onnx utilities to improve the exported model and 4 | also tries to simplify the model with onnx-simplifier. 5 | 6 | https://github.com/onnx/onnx/blob/master/docs/PythonAPIOverview.md 7 | https://github.com/daquexian/onnx-simplifier 8 | """ 9 | 10 | import argparse 11 | 12 | import torch 13 | 14 | import openpifpaf 15 | 16 | try: 17 | import thop 18 | except ImportError: 19 | raise Exception('need to install thop (pip install thop) for this script') 20 | 21 | 22 | def count(checkpoint): 23 | dummy_input = torch.randn(1, 3, 641, 641) 24 | model, _ = openpifpaf.network.factory(checkpoint=checkpoint) 25 | return thop.profile(model, inputs=(dummy_input, )) 26 | 27 | 28 | class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, 29 | argparse.RawDescriptionHelpFormatter): 30 | pass 31 | 32 | 33 | def main(): 34 | parser = argparse.ArgumentParser( 35 | prog='python3 -m openpifpaf.count_ops', 36 | description=__doc__, 37 | formatter_class=CustomFormatter, 38 | ) 39 | parser.add_argument('--version', action='version', 40 | version='OpenPifPaf {version}'.format(version=openpifpaf.__version__)) 41 | 42 | parser.add_argument('--checkpoint') 43 | args = parser.parse_args() 44 | 45 | gmacs, params = count(args.checkpoint) 46 | print('GMACs = {0:.2f}, million params = {1:.2f}'.format(gmacs / 1e9, params / 1e6)) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /openpifpaf/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | """Datasets and tools to load data in batches.""" 2 | 3 | from .coco import Coco 4 | from .keemotion import Keemotion 5 | from .collate import collate_images_anns_meta, collate_images_targets_meta, collate_images_targets_inst_meta, collate_images_targets_inst_meta_eval 6 | from .factory import train_cli, train_configure, train_factory 7 | from . import headmeta 8 | from .image_list import ImageList, PilImageList 9 | from .multidataset import MultiDataset 10 | 11 | -------------------------------------------------------------------------------- /openpifpaf/datasets/build_deepsport_ball_dataset.py: -------------------------------------------------------------------------------- 1 | from tqdm.auto import tqdm 2 | 3 | from dataset_utilities.providers import AWSSession 4 | from dataset_utilities.ds.instants_dataset import InstantsDataset, ViewsDataset, BuildCameraViews, AddBallAnnotation, DownloadFlags 5 | from mlworkflow import FilteredDataset, TransformedDataset, PickledDataset 6 | 7 | session = AWSSession(profile_name="abolfazl@ucl").as_role("basketball-instants-dataset-ConsumerRole") 8 | 9 | # # predicate = lambda instant_key, instant: len([a for a in instant.annotations if a.type == "player"]) > 0 10 | # predicate = lambda instant_key, instant: instant.annotated_human_masks 11 | 12 | # instants_dataset = InstantsDataset( 13 | # sport="basketball", 14 | # local_storage="/scratch/mistasse/abolfazl/keemotion", 15 | # session=session, 16 | # predicate=predicate, 17 | # progress_wrapper=tqdm, 18 | # download_flags=DownloadFlags.WITH_CALIB_FILE|DownloadFlags.WITH_HUMAN_SEGMENTATION_MASKS|DownloadFlags.WITH_IMAGES 19 | # ) 20 | 21 | # # instants_dataset.download() 22 | # views_dataset = ViewsDataset(instants_dataset, view_builder=BuildCameraViews()) 23 | # dataset = TransformedDataset(views_dataset, [AddBallAnnotation()]) 24 | # # dataset = FilteredDataset(dataset, predicate=lambda k,v: k.camera == v.ball.camera) # keep only cameras views in which there is a ball annotated. The rest is skipped. 25 | 26 | # # dataset = FilteredDataset(dataset, predicate=lambda k,v: k.camera == v.ball.camera and v.human_masks is not None) # keep only cameras views for which there's a human masks and the ball 27 | # # dataset = FilteredDataset(dataset, predicate=lambda k,v: v.human_masks is not None) # keep only cameras views for which there's a human masks 28 | 29 | 30 | # # with open("/data/mistasse/abolfazl/keemotion/pickled/camera_views_with_human_masks_large.pickle", "wb") as f: 31 | # with open("/scratch/mistasse/abolfazl/keemotion/pickled/camera_views_with_human_masks_ball_mask.pickle", "wb") as f: 32 | # PickledDataset.create(dataset, f, yield_keys_wrapper=tqdm) 33 | # # python3 -m openpifpaf.train --lr=0.0001 --momentum=0.98 --epochs=150 --lr-decay 130 140 --lr-decay-epochs=10 --batch-size=32 --square-edge=385 --weight-decay=1e-5 --update-batchnorm-runningstatistics --basenet=shufflenetv2k16w --headnets cifball --dataset deepsport --deepsport-pickled-dataset /scratch/gva/views_camera_with_ball2.pickle 34 | 35 | 36 | 37 | # predicate = lambda instant_key, instant: instant.annotated_human_masks 38 | # df = DownloadFlags.WITH_HUMAN_SEGMENTATION_MASKS | DownloadFlags.WITH_IMAGES | DownloadFlags.WITH_CALIB_FILE 39 | # instants_dataset = InstantsDataset(sport="basketball", session=session, 40 | # local_storage="/scratch/mistasse/abolfazl/keemotion", progress_wrapper=tqdm, 41 | # predicate=predicate, download_flags = df) 42 | # # build a dataset of balls centered in the image with a margin of 50cm around the ball 43 | # views_dataset = ViewsDataset(instants_dataset, view_builder=BuildCameraViews()) 44 | # ds = TransformedDataset(views_dataset, [AddBallAnnotation()]) 45 | # PickledDataset.create(ds, "/scratch/mistasse/abolfazl/keemotion/pickled/camera_views_with_human_masks_ball_mask.pickle") 46 | 47 | 48 | 49 | 50 | # predicate = lambda instant_key, instant: instant.annotated_human_masks 51 | df = DownloadFlags.WITH_HUMAN_SEGMENTATION_MASKS | DownloadFlags.WITH_IMAGES | DownloadFlags.WITH_CALIB_FILE 52 | instants_dataset = InstantsDataset(sport="basketball", session=session, 53 | local_storage="/scratch/mistasse/abolfazl/keemotion", progress_wrapper=tqdm, 54 | download_flags = df, 55 | predicate=lambda k,v: v.annotated_human_masks) 56 | # build a dataset of balls centered in the image with a margin of 50cm around the ball 57 | views_dataset = ViewsDataset(instants_dataset, view_builder=BuildCameraViews()) 58 | ds = TransformedDataset(views_dataset, [AddBallAnnotation()]) 59 | # ds = FilteredDataset(ds, predicate=lambda k,v: k.camera == v.ball.camera) 60 | PickledDataset.create(ds, "/scratch/mistasse/abolfazl/keemotion/pickled/camera_views_with_human_masks_ball_mask_v2.pickle") 61 | -------------------------------------------------------------------------------- /openpifpaf/datasets/coco_mrcnn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [] 23 | } 24 | ], 25 | "metadata": { 26 | "kernelspec": { 27 | "display_name": "Python 3", 28 | "language": "python", 29 | "name": "python3" 30 | }, 31 | "language_info": { 32 | "codemirror_mode": { 33 | "name": "ipython", 34 | "version": 3 35 | }, 36 | "file_extension": ".py", 37 | "mimetype": "text/x-python", 38 | "name": "python", 39 | "nbconvert_exporter": "python", 40 | "pygments_lexer": "ipython3", 41 | "version": "3.7.3" 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 4 46 | } 47 | -------------------------------------------------------------------------------- /openpifpaf/datasets/collate.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def collate_images_anns_meta(batch): 5 | images = torch.utils.data.dataloader.default_collate([b[0] for b in batch]) 6 | anns = [b[1] for b in batch] 7 | metas = [b[2] for b in batch] 8 | return images, anns, metas 9 | 10 | 11 | def collate_images_targets_meta(batch): 12 | images = torch.utils.data.dataloader.default_collate([b[0] for b in batch]) 13 | targets = torch.utils.data.dataloader.default_collate([b[1] for b in batch]) 14 | metas = [b[2] for b in batch] 15 | return images, targets, metas 16 | 17 | ### AMA 18 | def collate_images_targets_inst_meta(batch): 19 | images = torch.utils.data.dataloader.default_collate([b[0] for b in batch]) 20 | targets = torch.utils.data.dataloader.default_collate([b[1] for b in batch]) 21 | # masks = [b[2] for b in batch] 22 | metas = [b[2] for b in batch] 23 | return images,targets, metas 24 | 25 | def collate_images_targets_inst_meta_eval(batch): 26 | images = torch.utils.data.dataloader.default_collate([b[0] for b in batch]) 27 | # targets = torch.utils.data.dataloader.default_collate([b[1] for b in batch]) 28 | # masks = [b[2] for b in batch] 29 | anns = [b[1] for b in batch] 30 | metas = [b[2] for b in batch] 31 | targets = torch.utils.data.dataloader.default_collate([b[3] for b in batch]) 32 | # targets = [b[3] for b in batch] 33 | return images,anns, metas, targets 34 | 35 | def collate_images_targets_inst_meta_views(batch): 36 | images = torch.utils.data.dataloader.default_collate([b[0] for b in batch]) 37 | targets = torch.utils.data.dataloader.default_collate([b[1] for b in batch]) 38 | # masks = [b[2] for b in batch] 39 | metas = [b[2] for b in batch] 40 | views = [b[3] for b in batch] 41 | keys = [b[4] for b in batch] 42 | return images,targets, metas, views, keys -------------------------------------------------------------------------------- /openpifpaf/datasets/image_list.py: -------------------------------------------------------------------------------- 1 | import PIL 2 | import torch 3 | 4 | 5 | from .. import transforms 6 | 7 | 8 | class ImageList(torch.utils.data.Dataset): 9 | def __init__(self, image_paths, preprocess=None): 10 | self.image_paths = image_paths 11 | self.preprocess = preprocess or transforms.EVAL_TRANSFORM 12 | 13 | def __getitem__(self, index): 14 | image_path = self.image_paths[index] 15 | with open(image_path, 'rb') as f: 16 | image = PIL.Image.open(f).convert('RGB') 17 | 18 | anns = [] 19 | mask = [] 20 | 21 | image, anns, meta = self.preprocess(image, anns, None) 22 | meta.update({ 23 | 'dataset_index': index, 24 | 'file_name': image_path, 25 | }) 26 | 27 | return image, anns, meta 28 | 29 | def __len__(self): 30 | return len(self.image_paths) 31 | 32 | 33 | class PilImageList(torch.utils.data.Dataset): 34 | def __init__(self, images, preprocess=None): 35 | self.images = images 36 | self.preprocess = preprocess or transforms.EVAL_TRANSFORM 37 | 38 | def __getitem__(self, index): 39 | image = self.images[index].copy().convert('RGB') 40 | 41 | anns = [] 42 | image, anns, meta = self.preprocess(image, anns, None) 43 | meta.update({ 44 | 'dataset_index': index, 45 | 'file_name': 'pilimage{}'.format(index), 46 | }) 47 | 48 | return image, anns, meta 49 | 50 | def __len__(self): 51 | return len(self.images) 52 | -------------------------------------------------------------------------------- /openpifpaf/datasets/multidataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class MultiDataset(object): 4 | def __init__(self, heads, dataloaders): # datloaders: list of tuples of train dl and val dl 5 | 6 | self.heads = heads 7 | 8 | self.train_dl = dataloaders 9 | cifs = ['cif', 'cifcent', 'cifcentball'] 10 | 11 | 12 | def __iter__(self): 13 | self.train_iters = [iter(tr_dl) for tr_dl in self.train_dl] 14 | return self 15 | 16 | def __next__(self): 17 | try: 18 | list_scenes = [next(tr_iter) for tr_iter in self.train_iters] 19 | except StopIteration: 20 | raise StopIteration 21 | 22 | 23 | 24 | data = torch.cat([img for img,_,_ in list_scenes], dim=0) 25 | meta = [] 26 | for _,_,met in list_scenes: 27 | meta += met 28 | 29 | targets = [] 30 | 31 | for h_ix, head in enumerate(self.heads): 32 | if head in ['cif', 'cifcent', 'cifcentball']: 33 | 34 | cif_conf = torch.cat([trg[h_ix][0] for _,trg,_ in list_scenes], dim=0) 35 | cif_vec = torch.cat([trg[h_ix][1] for _,trg,_ in list_scenes], dim=0) 36 | cif_scale = torch.cat([trg[h_ix][2] for _,trg,_ in list_scenes], dim=0) 37 | targets.append((cif_conf, cif_vec, cif_scale)) 38 | elif head == 'pan': 39 | # print('data pan') 40 | target_pan = dict() 41 | target_pan['semantic'] = torch.cat([trg[h_ix]['semantic'] for _,trg,_ in list_scenes], dim=0) 42 | target_pan['offset'] = torch.cat([trg[h_ix]['offset'] for _,trg,_ in list_scenes], dim=0) 43 | target_pan['semantic_weights'] = torch.cat([trg[h_ix]['semantic_weights'] for _,trg,_ in list_scenes], dim=0) 44 | target_pan['offset_weights'] = torch.cat([trg[h_ix]['offset_weights'] for _,trg,_ in list_scenes], dim=0) 45 | targets.append(target_pan) 46 | elif head == 'cent': 47 | 48 | cif_conf = torch.cat([trg[h_ix][0] for _,trg,_ in list_scenes], dim=0) 49 | cif_vec = torch.cat([trg[h_ix][1] for _,trg,_ in list_scenes], dim=0) 50 | cif_scale = torch.cat([trg[h_ix][2] for _,trg,_ in list_scenes], dim=0) 51 | targets.append((cif_conf, cif_vec, cif_scale)) 52 | elif head == 'ball': 53 | 54 | cif_conf = torch.cat([trg[h_ix][0] for _,trg,_ in list_scenes], dim=0) 55 | cif_vec = torch.cat([trg[h_ix][1] for _,trg,_ in list_scenes], dim=0) 56 | cif_scale = torch.cat([trg[h_ix][2] for _,trg,_ in list_scenes], dim=0) 57 | targets.append((cif_conf, cif_vec, cif_scale)) 58 | 59 | 60 | return data, targets, meta 61 | 62 | 63 | def __len__(self): 64 | return min([len(d) for d in self.train_dl]) -------------------------------------------------------------------------------- /openpifpaf/decoder/__init__.py: -------------------------------------------------------------------------------- 1 | """Collections of decoders: fields to annotations.""" 2 | 3 | from .caf_scored import CafScored 4 | from .cif_hr import CifHr, CifDetHr 5 | from .cif_seeds import CifSeeds 6 | from .factory import cli, configure, factory_decode, factory_from_args 7 | from .field_config import FieldConfig 8 | from .generator.cifcaf import CifCaf 9 | from .generator.cifcent import CifCent 10 | from .generator.cifdet import CifDet 11 | from .generator.generator import Generator 12 | from . import nms 13 | from .occupancy import Occupancy 14 | from .profiler import Profiler 15 | from .profiler_autograd import ProfilerAutograd 16 | -------------------------------------------------------------------------------- /openpifpaf/decoder/caf_scored.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import numpy as np 5 | 6 | # pylint: disable=import-error 7 | from ..functional import scalar_values 8 | from .field_config import FieldConfig 9 | 10 | LOG = logging.getLogger(__name__) 11 | 12 | 13 | class CafScored: 14 | default_score_th = 0.1 15 | 16 | def __init__(self, cifhr, config: FieldConfig, skeleton, *, score_th=None, cif_floor=0.1): 17 | self.cifhr = cifhr 18 | self.config = config 19 | self.skeleton = skeleton 20 | self.score_th = score_th or self.default_score_th 21 | self.cif_floor = cif_floor 22 | 23 | self.forward = None 24 | self.backward = None 25 | 26 | def directed(self, caf_i, forward): 27 | if forward: 28 | return self.forward[caf_i], self.backward[caf_i] 29 | 30 | return self.backward[caf_i], self.forward[caf_i] 31 | 32 | def fill_caf(self, caf, stride, min_distance=0.0, max_distance=None): 33 | start = time.perf_counter() 34 | 35 | if self.forward is None: 36 | self.forward = [np.empty((9, 0), dtype=caf.dtype) for _ in caf] 37 | self.backward = [np.empty((9, 0), dtype=caf.dtype) for _ in caf] 38 | 39 | for caf_i, nine in enumerate(caf): 40 | assert nine.shape[0] == 9 41 | 42 | mask = nine[0] > self.score_th 43 | if not np.any(mask): 44 | continue 45 | nine = nine[:, mask] 46 | 47 | if min_distance: 48 | dist = np.linalg.norm(nine[1:3] - nine[5:7], axis=0) 49 | mask_dist = dist > min_distance / stride 50 | nine = nine[:, mask_dist] 51 | 52 | if max_distance: 53 | dist = np.linalg.norm(nine[1:3] - nine[5:7], axis=0) 54 | mask_dist = dist < max_distance / stride 55 | nine = nine[:, mask_dist] 56 | 57 | nine = np.copy(nine) 58 | nine[(1, 2, 3, 4, 5, 6, 7, 8), :] *= stride 59 | scores = nine[0] 60 | 61 | j1i = self.skeleton[caf_i][0] - 1 62 | if self.cif_floor < 1.0 and j1i < len(self.cifhr): 63 | cifhr_b = scalar_values(self.cifhr[j1i], nine[1], nine[2], default=0.0) 64 | scores_b = scores * (self.cif_floor + (1.0 - self.cif_floor) * cifhr_b) 65 | else: 66 | scores_b = scores 67 | mask_b = scores_b > self.score_th 68 | d9_b = np.copy(nine[:, mask_b][(0, 5, 6, 7, 8, 1, 2, 3, 4), :]) 69 | d9_b[0] = scores_b[mask_b] 70 | self.backward[caf_i] = np.concatenate((self.backward[caf_i], d9_b), axis=1) 71 | 72 | j2i = self.skeleton[caf_i][1] - 1 73 | if self.cif_floor < 1.0 and j2i < len(self.cifhr): 74 | cifhr_f = scalar_values(self.cifhr[j2i], nine[5], nine[6], default=0.0) 75 | scores_f = scores * (self.cif_floor + (1.0 - self.cif_floor) * cifhr_f) 76 | else: 77 | scores_f = scores 78 | mask_f = scores_f > self.score_th 79 | d9_f = np.copy(nine[:, mask_f]) 80 | d9_f[0] = scores_f[mask_f] 81 | self.forward[caf_i] = np.concatenate((self.forward[caf_i], d9_f), axis=1) 82 | 83 | LOG.debug('scored caf (%d, %d) in %.3fs', 84 | sum(f.shape[1] for f in self.forward), 85 | sum(b.shape[1] for b in self.backward), 86 | time.perf_counter() - start) 87 | return self 88 | 89 | def fill(self, fields): 90 | for caf_i, stride, min_distance, max_distance in zip( 91 | self.config.caf_indices, 92 | self.config.caf_strides, 93 | self.config.caf_min_distances, 94 | self.config.caf_max_distances): 95 | self.fill_caf(fields[caf_i], stride, 96 | min_distance=min_distance, max_distance=max_distance) 97 | 98 | return self 99 | -------------------------------------------------------------------------------- /openpifpaf/decoder/caf_seeds.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import numpy as np 5 | 6 | LOG = logging.getLogger(__name__) 7 | 8 | 9 | class CafSeeds: 10 | def __init__(self, seed_threshold, *, keypoints, skeleton, 11 | score_scale=1.0, 12 | debug_visualizer=None): 13 | self.seed_threshold = seed_threshold 14 | self.score_scale = score_scale 15 | self.keypoints = keypoints 16 | self.skeleton_m1 = np.array(skeleton) - 1 17 | self.debug_visualizer = debug_visualizer 18 | 19 | LOG.debug('seed threshold = %f', self.seed_threshold) 20 | 21 | self.seeds = [] 22 | self.seed_values = [] 23 | 24 | def fill(self, caf, stride=1.0): 25 | start = time.perf_counter() 26 | 27 | for field_i, p in enumerate(caf): 28 | p = p[:, :, p[0][0] > self.seed_threshold] 29 | (v1, x1, y1, _, s1), (__, x2, y2, ___, s2) = p 30 | 31 | j1i, j2i = self.skeleton_m1[field_i] 32 | new_seeds = np.zeros((len(v1), len(self.keypoints), 4), dtype=np.float32) 33 | for new_seed, vv, xx1, yy1, ss1, xx2, yy2, ss2 in zip( 34 | new_seeds, v1, x1, y1, s1, x2, y2, s2): 35 | new_seed[j1i] = xx1, yy1, ss1, vv 36 | new_seed[j2i] = xx2, yy2, ss2, vv 37 | self.seed_values.append(vv) 38 | 39 | new_seeds[:, :, 0:3] *= stride 40 | self.seeds.append(new_seeds) 41 | 42 | LOG.debug('seeds %d, %.3fs', sum(len(s) for s in self.seeds), time.perf_counter() - start) 43 | return self 44 | 45 | def get(self): 46 | if self.debug_visualizer: 47 | self.debug_visualizer.seeds(self.seeds) 48 | 49 | order = np.argsort(self.seed_values)[::-1] 50 | return np.concatenate(self.seeds, axis=0)[order] 51 | 52 | def fill_sequence(self, cafs, strides=None): 53 | if strides is None: 54 | strides = [1.0 for _ in cafs] 55 | for caf, stride in zip(cafs, strides): 56 | self.fill(caf, stride) 57 | 58 | return self 59 | -------------------------------------------------------------------------------- /openpifpaf/decoder/cif_hr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import numpy as np 5 | import imageio 6 | 7 | # pylint: disable=import-error 8 | from ..functional import scalar_square_add_gauss_with_max 9 | from .field_config import FieldConfig 10 | from .. import visualizer 11 | 12 | LOG = logging.getLogger(__name__) 13 | 14 | 15 | class CifHr: 16 | neighbors = 16 17 | v_threshold = 0.1 18 | debug_visualizer = visualizer.CifHr() 19 | 20 | def __init__(self, config: FieldConfig): 21 | self.config = config 22 | self.accumulated = None 23 | 24 | def fill_cif(self, cif, stride, min_scale=0.0): 25 | return self.fill_multiple([cif], stride, min_scale) 26 | 27 | def accumulate(self, len_cifs, t, p, stride, min_scale): 28 | p = p[:, p[0] > self.v_threshold] 29 | if min_scale: 30 | p = p[:, p[4] > min_scale / stride] 31 | 32 | v, x, y, _, scale = p 33 | x = x * stride 34 | y = y * stride 35 | sigma = np.maximum(1.0, 0.5 * scale * stride) 36 | 37 | # Occupancy covers 2sigma. 38 | # Restrict this accumulation to 1sigma so that seeds for the same joint 39 | # are properly suppressed. 40 | scalar_square_add_gauss_with_max( 41 | t, x, y, sigma, v / self.neighbors / len_cifs, truncate=1.0) 42 | 43 | def fill_multiple(self, cifs, stride, min_scale=0.0): 44 | start = time.perf_counter() 45 | 46 | if self.accumulated is None: 47 | shape = ( 48 | cifs[0].shape[0], 49 | int((cifs[0].shape[2] - 1) * stride + 1), 50 | int((cifs[0].shape[3] - 1) * stride + 1), 51 | ) 52 | ta = np.zeros(shape, dtype=np.float32) 53 | else: 54 | ta = np.zeros(self.accumulated.shape, dtype=np.float32) 55 | 56 | for cif in cifs: 57 | for t, p in zip(ta, cif): 58 | self.accumulate(len(cifs), t, p, stride, min_scale) 59 | 60 | if self.accumulated is None: 61 | self.accumulated = ta 62 | else: 63 | self.accumulated = np.maximum(ta, self.accumulated) 64 | 65 | kp_id = 0 # ball is sole keypoint 66 | pad_left, pad_top, pad_right, pad_bottom = 0, 4, 1, 5 # padding copied from logs 67 | _, height, width = self.accumulated.shape 68 | v_start = pad_top 69 | v_stop = height-pad_bottom 70 | h_start = pad_left 71 | h_stop = width-pad_right 72 | LOG.debug("accumulated hr heatmap can be created by uncommenting the following line") 73 | 74 | 75 | LOG.debug('target_intensities %.3fs', time.perf_counter() - start) 76 | return self 77 | 78 | def fill(self, fields): 79 | if len(self.config.cif_indices) == 10: 80 | for cif_i1, cif_i2, stride, min_scale in zip(self.config.cif_indices[:5], 81 | self.config.cif_indices[5:], 82 | self.config.cif_strides[:5], 83 | self.config.cif_min_scales[:5]): 84 | self.fill_multiple([fields[cif_i1], fields[cif_i2]], stride, min_scale=min_scale) 85 | else: 86 | for cif_i, stride, min_scale in zip(self.config.cif_indices, 87 | self.config.cif_strides, 88 | self.config.cif_min_scales): 89 | self.fill_cif(fields[cif_i], stride, min_scale=min_scale) 90 | 91 | self.debug_visualizer.predicted(self.accumulated) 92 | return self 93 | 94 | 95 | class CifDetHr(CifHr): 96 | def accumulate(self, len_cifs, t, p, stride, min_scale): 97 | p = p[:, p[0] > self.v_threshold] 98 | if min_scale: 99 | p = p[:, p[4] > min_scale / stride] 100 | p = p[:, p[5] > min_scale / stride] 101 | 102 | v, x, y, _, w, h, _ = p 103 | x = x * stride 104 | y = y * stride 105 | sigma = np.maximum(1.0, 0.1 * np.minimum(w, h) * stride) 106 | 107 | # Occupancy covers 2sigma. 108 | # Restrict this accumulation to 1sigma so that seeds for the same joint 109 | # are properly suppressed. 110 | scalar_square_add_gauss_with_max( 111 | t, x, y, sigma, v / self.neighbors / len_cifs, truncate=1.0) 112 | -------------------------------------------------------------------------------- /openpifpaf/decoder/cif_seeds.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | # pylint: disable=import-error 5 | from ..functional import scalar_values 6 | from .field_config import FieldConfig 7 | from .cif_hr import CifHr 8 | from .. import visualizer 9 | 10 | LOG = logging.getLogger(__name__) 11 | 12 | 13 | class CifSeeds: 14 | threshold = None 15 | score_scale = 1.0 16 | debug_visualizer = visualizer.Seeds() 17 | 18 | def __init__(self, cifhr: CifHr, config: FieldConfig): 19 | self.cifhr = cifhr 20 | self.config = config 21 | self.seeds = [] 22 | 23 | def fill_cif(self, cif, stride, *, min_scale=0.0, seed_mask=None): 24 | start = time.perf_counter() 25 | 26 | sv = 0.0 27 | 28 | for field_i, p in enumerate(cif): 29 | if seed_mask is not None and not seed_mask[field_i]: 30 | continue 31 | p = p[:, p[0] > self.threshold] 32 | if min_scale: 33 | p = p[:, p[4] > min_scale / stride] 34 | c, x, y, _, s = p 35 | 36 | start_sv = time.perf_counter() 37 | v = scalar_values(self.cifhr[field_i], x * stride, y * stride, default=0.0) 38 | v = 0.9 * v + 0.1 * c 39 | sv += time.perf_counter() - start_sv 40 | 41 | if self.score_scale != 1.0: 42 | v = v * self.score_scale 43 | m = v > self.threshold 44 | x, y, v, s = x[m] * stride, y[m] * stride, v[m], s[m] * stride 45 | 46 | for vv, xx, yy, ss in zip(v, x, y, s): 47 | self.seeds.append((vv, field_i, xx, yy, ss)) 48 | 49 | LOG.debug('seeds %d, %.3fs (C++ %.3fs)', len(self.seeds), time.perf_counter() - start, sv) 50 | return self 51 | 52 | def get(self): 53 | self.debug_visualizer.predicted(self.seeds) 54 | return sorted(self.seeds, reverse=True) 55 | 56 | def fill(self, fields): 57 | for cif_i, stride, min_scale in zip(self.config.cif_indices, 58 | self.config.cif_strides, 59 | self.config.cif_min_scales): 60 | self.fill_cif(fields[cif_i], stride, 61 | min_scale=min_scale, 62 | seed_mask=self.config.seed_mask) 63 | 64 | return self 65 | 66 | 67 | class CifDetSeeds(CifSeeds): 68 | def fill_cif(self, cif, stride, *, min_scale=0.0, seed_mask=None): 69 | start = time.perf_counter() 70 | 71 | for field_i, p in enumerate(cif): 72 | if seed_mask is not None and not seed_mask[field_i]: 73 | continue 74 | p = p[:, p[0] > self.threshold] 75 | if min_scale: 76 | p = p[:, p[4] > min_scale / stride] 77 | p = p[:, p[5] > min_scale / stride] 78 | c, x, y, _, w, h, _ = p 79 | v = scalar_values(self.cifhr[field_i], x * stride, y * stride, default=0.0) 80 | v = 0.9 * v + 0.1 * c 81 | if self.score_scale != 1.0: 82 | v = v * self.score_scale 83 | m = v > self.threshold 84 | x, y, v, w, h = x[m] * stride, y[m] * stride, v[m], w[m] * stride, h[m] * stride 85 | 86 | for vv, xx, yy, ww, hh in zip(v, x, y, w, h): 87 | self.seeds.append((vv, field_i, xx, yy, ww, hh)) 88 | 89 | LOG.debug('seeds %d, %.3fs', len(self.seeds), time.perf_counter() - start) 90 | return self 91 | -------------------------------------------------------------------------------- /openpifpaf/decoder/field_config.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import List 3 | 4 | 5 | @dataclasses.dataclass 6 | class FieldConfig: 7 | cif_indices: List[int] = dataclasses.field(default_factory=lambda: [0]) 8 | caf_indices: List[int] = dataclasses.field(default_factory=lambda: [1]) 9 | cif_strides: List[int] = dataclasses.field(default_factory=lambda: [8]) 10 | caf_strides: List[int] = dataclasses.field(default_factory=lambda: [8]) 11 | cif_min_scales: List[float] = dataclasses.field(default_factory=lambda: [0.0]) 12 | caf_min_distances: List[float] = dataclasses.field(default_factory=lambda: [0.0]) 13 | caf_max_distances: List[float] = dataclasses.field(default_factory=lambda: [None]) 14 | seed_mask: List[int] = None 15 | confidence_scales: List[float] = None 16 | cif_visualizers: list = None 17 | caf_visualizers: list = None 18 | 19 | def verify(self): 20 | assert len(self.cif_strides) == len(self.cif_indices) 21 | assert len(self.cif_strides) == len(self.cif_min_scales) 22 | 23 | assert len(self.caf_strides) == len(self.caf_indices) 24 | assert len(self.caf_strides) == len(self.caf_min_distances) 25 | assert len(self.caf_strides) == len(self.caf_max_distances) 26 | -------------------------------------------------------------------------------- /openpifpaf/decoder/generator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/openpifpaf/decoder/generator/__init__.py -------------------------------------------------------------------------------- /openpifpaf/decoder/generator/cifcent.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import logging 3 | import time 4 | 5 | from .generator import Generator 6 | from ...annotation import AnnotationCent 7 | from ..field_config import FieldConfig 8 | from ..cif_hr import CifHr 9 | from ..cif_seeds import CifSeeds 10 | from .. import nms 11 | from ..occupancy import Occupancy 12 | from ... import visualizer 13 | import numpy as np 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class CifCent(Generator): 18 | occupancy_visualizer = visualizer.Occupancy() 19 | 20 | def __init__(self, field_config: FieldConfig, keypoints, *, worker_pool=None): 21 | super().__init__(worker_pool) 22 | self.field_config = field_config 23 | self.keypoints = keypoints 24 | 25 | self.timers = defaultdict(float) 26 | 27 | def __call__(self, fields): 28 | start = time.perf_counter() 29 | 30 | if self.field_config.cif_visualizers: 31 | for vis, cif_i in zip(self.field_config.cif_visualizers, self.field_config.cif_indices): 32 | vis.predicted(fields[cif_i]) 33 | 34 | cifhr = CifHr(self.field_config).fill(fields) 35 | ball_from_mask = [np.unravel_index(np.argmax(cifhr.accumulated[0]), cifhr.accumulated[0].shape)] 36 | ball_fyxv = [ 37 | (0, y, x, cifhr.accumulated[0][y,x]) 38 | for y, x in ball_from_mask 39 | ] 40 | 41 | 42 | # seeds = CifSeeds(cifhr.accumulated, self.field_config).fill(fields) 43 | # occupied = Occupancy(cifhr.accumulated.shape, 2, min_scale=2.0) 44 | 45 | annotations = [] 46 | # def mark_occupied(ann): 47 | # for joint_i, xyv in enumerate(ann.data): 48 | # if xyv[2] == 0.0: 49 | # continue 50 | 51 | # width = ann.joint_scales[joint_i] 52 | # occupied.set(joint_i, xyv[0], xyv[1], width) # width = 2 * sigma 53 | 54 | 55 | # for v, f, x, y, s in seeds.get(): 56 | # if occupied.get(f, x, y): 57 | # continue 58 | 59 | # ann = AnnotationCent(self.keypoints).add(f, (x, y, v)) 60 | # ann.joint_scales[f] = s 61 | # # self._grow(ann, caf_scored) 62 | # annotations.append(ann) 63 | # mark_occupied(ann) 64 | 65 | # # for v, f, x, y, w, h in seeds.get(): 66 | # # if occupied.get(f, x, y): 67 | # # continue 68 | # # ann = AnnotationDet(self.categories).set(f, v, (x - w/2.0, y - h/2.0, w, h)) 69 | # # annotations.append(ann) 70 | # # occupied.set(f, x, y, 0.1 * min(w, h)) 71 | 72 | # self.occupancy_visualizer.predicted(occupied) 73 | 74 | # annotations = nms.Detection().annotations(annotations) 75 | # annotations = sorted(annotations, key=lambda a: -a.score) 76 | 77 | LOG.info('annotations %d, decoder = %.3fs', len(annotations), time.perf_counter() - start) 78 | return annotations 79 | -------------------------------------------------------------------------------- /openpifpaf/decoder/generator/cifdet.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import logging 3 | import time 4 | 5 | from .generator import Generator 6 | from ...annotation import AnnotationDet 7 | from ..field_config import FieldConfig 8 | from ..cif_hr import CifDetHr 9 | from ..cif_seeds import CifDetSeeds 10 | from .. import nms 11 | from ..occupancy import Occupancy 12 | from ... import visualizer 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class CifDet(Generator): 18 | occupancy_visualizer = visualizer.Occupancy() 19 | 20 | def __init__(self, field_config: FieldConfig, categories, *, worker_pool=None): 21 | super().__init__(worker_pool) 22 | self.field_config = field_config 23 | self.categories = categories 24 | 25 | self.timers = defaultdict(float) 26 | 27 | def __call__(self, fields): 28 | start = time.perf_counter() 29 | 30 | if self.field_config.cif_visualizers: 31 | for vis, cif_i in zip(self.field_config.cif_visualizers, self.field_config.cif_indices): 32 | vis.predicted(fields[cif_i]) 33 | 34 | cifhr = CifDetHr(self.field_config).fill(fields) 35 | seeds = CifDetSeeds(cifhr.accumulated, self.field_config).fill(fields) 36 | occupied = Occupancy(cifhr.accumulated.shape, 2, min_scale=2.0) 37 | 38 | annotations = [] 39 | for v, f, x, y, w, h in seeds.get(): 40 | if occupied.get(f, x, y): 41 | continue 42 | ann = AnnotationDet(self.categories).set(f, v, (x - w/2.0, y - h/2.0, w, h)) 43 | annotations.append(ann) 44 | occupied.set(f, x, y, 0.1 * min(w, h)) 45 | 46 | self.occupancy_visualizer.predicted(occupied) 47 | 48 | annotations = nms.Detection().annotations(annotations) 49 | # annotations = sorted(annotations, key=lambda a: -a.score) 50 | 51 | LOG.info('annotations %d, decoder = %.3fs', len(annotations), time.perf_counter() - start) 52 | return annotations 53 | -------------------------------------------------------------------------------- /openpifpaf/decoder/instance_scorer.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import json 3 | 4 | import numpy as np 5 | import torch 6 | 7 | 8 | class InstanceScorer(torch.nn.Module): 9 | def __init__(self, in_features=35): 10 | super(InstanceScorer, self).__init__() 11 | self.compute_layers = torch.nn.Sequential( 12 | torch.nn.Linear(in_features, 64), 13 | torch.nn.Tanh(), 14 | torch.nn.Linear(64, 64), 15 | torch.nn.Tanh(), 16 | torch.nn.Linear(64, 64), 17 | torch.nn.Tanh(), 18 | torch.nn.Linear(64, 1), 19 | torch.nn.Sigmoid(), 20 | ) 21 | 22 | def forward(self, x): # pylint: disable=arguments-differ 23 | return self.compute_layers(x - 0.5) 24 | 25 | def from_annotation(self, ann): 26 | v = torch.tensor([ann.scale()] + 27 | ann.data[:, 2].tolist() + 28 | ann.joint_scales.tolist()).float() 29 | with torch.no_grad(): 30 | return float(self.forward(v).item()) 31 | 32 | 33 | class InstanceScoreRecorder(object): 34 | def __init__(self): 35 | """Drop in replacement for InstanceScorer that records the 36 | ground truth dataset instead.""" 37 | self.data = [] 38 | self.next_gt = None 39 | 40 | def set_gt(self, gt): 41 | assert self.next_gt is None 42 | self.next_gt = gt 43 | 44 | def from_annotation(self, annotations): 45 | gt = copy.deepcopy(self.next_gt) 46 | for ann in annotations: 47 | kps = ann.data 48 | 49 | matched = None 50 | for ann_gt in gt: 51 | kps_gt = ann_gt['keypoints'] 52 | mask = kps_gt[:, 2] > 0 53 | if not np.any(mask): 54 | continue 55 | 56 | diff = kps[mask, :2] - kps_gt[mask, :2] 57 | dist = np.mean(np.abs(diff)) 58 | if dist > 10.0: 59 | continue 60 | 61 | matched = ann_gt 62 | break 63 | 64 | if matched is None: 65 | self.data.append((ann, 0)) 66 | continue 67 | 68 | # found a match 69 | self.data.append((ann, 1)) 70 | gt.remove(matched) 71 | 72 | self.next_gt = None 73 | 74 | def write_data(self, filename): 75 | with open(filename, 'w') as f: 76 | for ann, y in self.data: 77 | f.write(json.dumps({ 78 | 'keypoints': ann.data.tolist(), 79 | 'joint_scales': ann.joint_scales.tolist(), 80 | 'score': ann.score(), 81 | 'scale': ann.scale(), 82 | 'target': y, 83 | })) 84 | f.write('\n') 85 | -------------------------------------------------------------------------------- /openpifpaf/decoder/nms.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import numpy as np 5 | 6 | from .occupancy import Occupancy 7 | 8 | LOG = logging.getLogger(__name__) 9 | 10 | 11 | class Keypoints: 12 | suppression = 0.0 13 | instance_threshold = 0.0 14 | keypoint_threshold = 0.0 15 | occupancy_visualizer = None 16 | 17 | def annotations(self, anns): 18 | start = time.perf_counter() 19 | 20 | for ann in anns: 21 | ann.data[ann.data[:, 2] < self.keypoint_threshold] = 0.0 22 | anns = [ann for ann in anns if ann.score() >= self.instance_threshold] 23 | 24 | if not anns: 25 | return anns 26 | 27 | occupied = Occupancy(( 28 | len(anns[0].data), 29 | int(max(np.max(ann.data[:, 1]) for ann in anns) + 1), 30 | int(max(np.max(ann.data[:, 0]) for ann in anns) + 1), 31 | ), 2, min_scale=4) 32 | 33 | anns = sorted(anns, key=lambda a: -a.score()) 34 | for ann in anns: 35 | assert ann.joint_scales is not None 36 | assert len(occupied) == len(ann.data) 37 | for f, (xyv, joint_s) in enumerate(zip(ann.data, ann.joint_scales)): 38 | v = xyv[2] 39 | if v == 0.0: 40 | continue 41 | 42 | if occupied.get(f, xyv[0], xyv[1]): 43 | xyv[2] *= self.suppression 44 | else: 45 | occupied.set(f, xyv[0], xyv[1], joint_s) # joint_s = 2 * sigma 46 | 47 | if self.occupancy_visualizer is not None: 48 | LOG.debug('Occupied fields after NMS') 49 | self.occupancy_visualizer.predicted(occupied) 50 | 51 | for ann in anns: 52 | ann.data[ann.data[:, 2] < self.keypoint_threshold] = 0.0 53 | anns = [ann for ann in anns if ann.score() >= self.instance_threshold] 54 | anns = sorted(anns, key=lambda a: -a.score()) 55 | 56 | LOG.debug('nms = %.3fs', time.perf_counter() - start) 57 | return anns 58 | 59 | 60 | class Detection: 61 | suppression = 0.1 62 | suppression_soft = 0.3 63 | instance_threshold = 0.1 64 | iou_threshold = 0.7 65 | iou_threshold_soft = 0.5 66 | 67 | @staticmethod 68 | def bbox_iou(box, other_boxes): 69 | box = np.expand_dims(box, 0) 70 | x1 = np.maximum(box[:, 0], other_boxes[:, 0]) 71 | y1 = np.maximum(box[:, 1], other_boxes[:, 1]) 72 | x2 = np.minimum(box[:, 0] + box[:, 2], other_boxes[:, 0] + other_boxes[:, 2]) 73 | y2 = np.minimum(box[:, 1] + box[:, 3], other_boxes[:, 1] + other_boxes[:, 3]) 74 | inter_area = np.maximum(0.0, x2 - x1) * np.maximum(0.0, y2 - y1) 75 | box_area = box[:, 2] * box[:, 3] 76 | other_areas = other_boxes[:, 2] * other_boxes[:, 3] 77 | return inter_area / (box_area + other_areas - inter_area + 1e-5) 78 | 79 | def annotations(self, anns): 80 | start = time.perf_counter() 81 | 82 | anns = [ann for ann in anns if ann.score >= self.instance_threshold] 83 | if not anns: 84 | return anns 85 | anns = sorted(anns, key=lambda a: -a.score) 86 | 87 | all_boxes = np.stack([ann.bbox for ann in anns]) 88 | for ann_i, ann in enumerate(anns[1:], start=1): 89 | mask = [ann.score >= self.instance_threshold for ann in anns[:ann_i]] 90 | ious = self.bbox_iou(ann.bbox, all_boxes[:ann_i][mask]) 91 | max_iou = np.max(ious) 92 | 93 | if max_iou > self.iou_threshold: 94 | ann.score *= self.suppression 95 | elif max_iou > self.iou_threshold_soft: 96 | ann.score *= self.suppression_soft 97 | 98 | anns = [ann for ann in anns if ann.score >= self.instance_threshold] 99 | anns = sorted(anns, key=lambda a: -a.score) 100 | 101 | LOG.debug('nms = %.3fs', time.perf_counter() - start) 102 | return anns 103 | -------------------------------------------------------------------------------- /openpifpaf/decoder/occupancy.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import numpy as np 3 | 4 | from .utils import scalar_square_add_single 5 | from ..functional import scalar_nonzero_clipped_with_reduction 6 | 7 | LOG = logging.getLogger(__name__) 8 | 9 | 10 | class Occupancy(): 11 | def __init__(self, shape, reduction, *, min_scale=None): 12 | assert len(shape) == 3 13 | if min_scale is None: 14 | min_scale = reduction 15 | assert min_scale >= reduction 16 | 17 | self.reduction = reduction 18 | self.min_scale = min_scale 19 | self.min_scale_reduced = min_scale / reduction 20 | 21 | self.occupancy = np.zeros(( 22 | shape[0], 23 | int(shape[1] / reduction), 24 | int(shape[2] / reduction), 25 | ), dtype=np.uint8) 26 | LOG.debug('shape = %s, min_scale = %d', self.occupancy.shape, self.min_scale_reduced) 27 | 28 | def __len__(self): 29 | return len(self.occupancy) 30 | 31 | def set(self, f, x, y, sigma): 32 | """Setting needs to be centered at the rounded (x, y).""" 33 | if f >= len(self.occupancy): 34 | return 35 | 36 | xi = round(x / self.reduction) 37 | yi = round(y / self.reduction) 38 | si = round(max(self.min_scale_reduced, sigma / self.reduction)) 39 | scalar_square_add_single(self.occupancy[f], xi, yi, si, 1) 40 | 41 | def get(self, f, x, y): 42 | """Getting needs to be done at the floor of (x, y).""" 43 | if f >= len(self.occupancy): 44 | return 1.0 45 | 46 | # floor is done in scalar_nonzero_clipped below 47 | return scalar_nonzero_clipped_with_reduction(self.occupancy[f], x, y, self.reduction) 48 | -------------------------------------------------------------------------------- /openpifpaf/decoder/profiler.py: -------------------------------------------------------------------------------- 1 | import cProfile 2 | import io 3 | import logging 4 | import pstats 5 | 6 | LOG = logging.getLogger(__name__) 7 | 8 | 9 | class Profiler: 10 | def __init__(self, function_to_profile, *, profile=None, out_name=None): 11 | if profile is None: 12 | profile = cProfile.Profile() 13 | 14 | self.function_to_profile = function_to_profile 15 | self.profile = profile 16 | self.out_name = out_name 17 | 18 | def __call__(self, *args, **kwargs): 19 | self.profile.enable() 20 | 21 | result = self.function_to_profile(*args, **kwargs) 22 | 23 | self.profile.disable() 24 | iostream = io.StringIO() 25 | ps = pstats.Stats(self.profile, stream=iostream) 26 | ps = ps.sort_stats('tottime') 27 | ps.print_stats() 28 | if self.out_name: 29 | LOG.info('writing profile file %s', self.out_name) 30 | ps.dump_stats(self.out_name) 31 | print(iostream.getvalue()) 32 | 33 | return result 34 | -------------------------------------------------------------------------------- /openpifpaf/decoder/profiler_autograd.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import torch 4 | 5 | LOG = logging.getLogger(__name__) 6 | 7 | 8 | class ProfilerAutograd: 9 | trace_counter = 0 10 | 11 | def __init__(self, function_to_profile, *, device, out_name=None): 12 | if not out_name: 13 | out_name = 'pytorch_chrome_trace.json' 14 | 15 | self.function_to_profile = function_to_profile 16 | self.device = device 17 | 18 | self.out_name = out_name 19 | 20 | def __call__(self, *args, **kwargs): 21 | with torch.autograd.profiler.profile(use_cuda=str(self.device) == 'cuda') as prof: 22 | result = self.function_to_profile(*args, **kwargs) 23 | print(prof.key_averages()) 24 | 25 | self.__class__.trace_counter += 1 26 | tracefilename = '{}.{}.json'.format( 27 | self.out_name.replace('.json', '').replace('.prof', ''), 28 | self.trace_counter, 29 | ) 30 | LOG.info('writing trace file %s', tracefilename) 31 | prof.export_chrome_trace(tracefilename) 32 | 33 | return result 34 | -------------------------------------------------------------------------------- /openpifpaf/decoder/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for decoders.""" 2 | 3 | import functools 4 | import numpy as np 5 | 6 | 7 | @functools.lru_cache(maxsize=16) 8 | def index_field(shape): 9 | yx = np.indices(shape, dtype=np.float32) 10 | xy = np.flip(yx, axis=0) 11 | return xy 12 | 13 | 14 | def sparse_bilinear_kernel(coord, value): 15 | l = coord.astype(int) 16 | g = np.meshgrid(*((ll, ll + 1) for ll in l)) 17 | g = list(zip(*(gg.reshape(-1) for gg in g))) 18 | 19 | v = [np.prod(1.0 - np.abs(coord-corner)) * value for corner in g] 20 | return g, v 21 | 22 | 23 | class Sparse2DGaussianField(object): 24 | def __init__(self, data=None, nearest_neighbors=25): 25 | if data is None: 26 | data = np.zeros((0, 3)) 27 | 28 | self.nearest_neighbors = nearest_neighbors 29 | self.data = data 30 | 31 | def value(self, xy, sigma): 32 | mask = np.logical_and( 33 | np.logical_and(self.data[0] > xy[0] - 2*sigma, 34 | self.data[0] < xy[0] + 2*sigma), 35 | np.logical_and(self.data[1] > xy[1] - 2*sigma, 36 | self.data[1] < xy[1] + 2*sigma), 37 | ) 38 | diff = np.expand_dims(xy, -1) - self.data[:2, mask] 39 | if diff.shape[1] == 0: 40 | return 0.0 41 | 42 | gauss_1d = np.exp(-0.5 * diff**2 / sigma**2) 43 | gauss = np.prod(gauss_1d, axis=0) 44 | 45 | v = np.sum(gauss * self.data[2, mask]) 46 | return np.tanh(v * 3.0 / self.nearest_neighbors) 47 | 48 | def values(self, xys, sigmas): 49 | assert xys.shape[-1] == 2 50 | if xys.shape[0] == 0: 51 | return np.zeros((0,)) 52 | 53 | if isinstance(sigmas, float): 54 | sigmas = np.full((xys.shape[0],), sigmas) 55 | if hasattr(sigmas, 'shape') and sigmas.shape[0] == 1 and xys.shape[0] > 1: 56 | sigmas = np.full((xys.shape[0],), sigmas[0]) 57 | 58 | return np.stack([self.value(xy, sigma) for xy, sigma in zip(xys, sigmas)]) 59 | 60 | 61 | def scalar_square_add_single(field, x, y, sigma, value): 62 | minx = max(0, int(x - sigma)) 63 | miny = max(0, int(y - sigma)) 64 | maxx = max(minx + 1, min(field.shape[1], int(x + sigma) + 1)) 65 | maxy = max(miny + 1, min(field.shape[0], int(y + sigma) + 1)) 66 | field[miny:maxy, minx:maxx] += value 67 | -------------------------------------------------------------------------------- /openpifpaf/encoder/__init__.py: -------------------------------------------------------------------------------- 1 | """Convert a set of keypoint coordinates into target fields. 2 | 3 | Takes an annotation from a dataset and turns it into the 4 | ground truth for a field. 5 | """ 6 | 7 | from .annrescaler import AnnRescaler 8 | from .annrescaler_ball import AnnRescalerBall 9 | from .annrescaler_cent import AnnRescalerCent 10 | from .factory import cli, configure, factory, factory_head 11 | from .caf import Caf 12 | from .cif import Cif 13 | from .pan import PanopticTargetGenerator 14 | -------------------------------------------------------------------------------- /openpifpaf/migrate.py: -------------------------------------------------------------------------------- 1 | """Migrate a model.""" 2 | 3 | import argparse 4 | import logging 5 | 6 | import torch 7 | 8 | from . import network, __version__ 9 | 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser( 13 | prog='python3 -m openpifpaf.migrate', 14 | description=__doc__, 15 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 16 | ) 17 | parser.add_argument('--version', action='version', 18 | version='OpenPifPaf {version}'.format(version=__version__)) 19 | 20 | parser.add_argument('--debug', default=False, action='store_true') 21 | parser.add_argument('--output') 22 | network.cli(parser) 23 | args = parser.parse_args() 24 | 25 | network.configure(args) 26 | 27 | if args.checkpoint is None: 28 | raise Exception('checkpoint must be provided for old model to migrate from') 29 | 30 | if args.output is None: 31 | args.output = args.checkpoint + '.out.pkl' 32 | 33 | logging.basicConfig(level=logging.INFO if not args.debug else logging.DEBUG) 34 | 35 | # load old model 36 | checkpoint = torch.load(args.checkpoint) 37 | 38 | # create a new model 39 | args.checkpoint = None 40 | new_model, _ = network.factory_from_args(args) 41 | 42 | # transfer state from old to new model 43 | new_model.load_state_dict(checkpoint['model'].state_dict()) 44 | checkpoint['model'] = new_model 45 | torch.save(checkpoint, args.output) 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /openpifpaf/network/__init__.py: -------------------------------------------------------------------------------- 1 | """Backbone networks, head networks and tools for training.""" 2 | 3 | from .factory import cli, configure, factory, factory_from_args, local_checkpoint_path 4 | from .trainer import Trainer 5 | from . import losses 6 | 7 | from .panoptic_losses import RegularCE, OhemCE, DeepLabCE 8 | from torch import nn 9 | from .heads import index_field_torch 10 | 11 | -------------------------------------------------------------------------------- /openpifpaf/network/aspp.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Reference: https://github.com/pytorch/vision/blob/master/torchvision/models/segmentation/deeplabv3.py 3 | # Modified by Bowen Cheng (bcheng9@illinois.edu) 4 | # ------------------------------------------------------------------------------ 5 | 6 | import torch 7 | from torch import nn 8 | from torch.nn import functional as F 9 | 10 | __all__ = ["ASPP"] 11 | 12 | 13 | class ASPPConv(nn.Sequential): 14 | def __init__(self, in_channels, out_channels, dilation): 15 | modules = [ 16 | nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False), 17 | nn.BatchNorm2d(out_channels), 18 | nn.ReLU() 19 | ] 20 | super(ASPPConv, self).__init__(*modules) 21 | 22 | 23 | class ASPPPooling(nn.Module): 24 | def __init__(self, in_channels, out_channels): 25 | super(ASPPPooling, self).__init__() 26 | self.aspp_pooling = nn.Sequential( 27 | nn.AdaptiveAvgPool2d(1), 28 | nn.Conv2d(in_channels, out_channels, 1, bias=False), 29 | nn.ReLU() 30 | ) 31 | 32 | def set_image_pooling(self, pool_size=None): 33 | if pool_size is None: 34 | self.aspp_pooling[0] = nn.AdaptiveAvgPool2d(1) 35 | else: 36 | self.aspp_pooling[0] = nn.AvgPool2d(kernel_size=pool_size, stride=1) 37 | 38 | def forward(self, x): 39 | size = x.shape[-2:] 40 | x = self.aspp_pooling(x) 41 | return F.interpolate(x, size=size, mode='bilinear', align_corners=True) 42 | 43 | 44 | class ASPP(nn.Module): 45 | def __init__(self, in_channels, out_channels, atrous_rates): 46 | super(ASPP, self).__init__() 47 | # out_channels = 256 48 | modules = [] 49 | modules.append(nn.Sequential( 50 | nn.Conv2d(in_channels, out_channels, 1, bias=False), 51 | nn.BatchNorm2d(out_channels), 52 | nn.ReLU())) 53 | 54 | rate1, rate2, rate3 = tuple(atrous_rates) 55 | modules.append(ASPPConv(in_channels, out_channels, rate1)) 56 | modules.append(ASPPConv(in_channels, out_channels, rate2)) 57 | modules.append(ASPPConv(in_channels, out_channels, rate3)) 58 | modules.append(ASPPPooling(in_channels, out_channels)) 59 | 60 | self.convs = nn.ModuleList(modules) 61 | 62 | self.project = nn.Sequential( 63 | nn.Conv2d(5 * out_channels, out_channels, 1, bias=False), 64 | nn.BatchNorm2d(out_channels), 65 | nn.ReLU(), 66 | nn.Dropout(0.5)) 67 | 68 | def set_image_pooling(self, pool_size): 69 | self.convs[-1].set_image_pooling(pool_size) 70 | 71 | def forward(self, x): 72 | res = [] 73 | for conv in self.convs: 74 | res.append(conv(x)) 75 | res = torch.cat(res, dim=1) 76 | return self.project(res) 77 | -------------------------------------------------------------------------------- /openpifpaf/panoptic_deeplab/decoder/__init__.py: -------------------------------------------------------------------------------- 1 | from .aspp import ASPP 2 | from .deeplabv3 import DeepLabV3Decoder 3 | from .deeplabv3plus import DeepLabV3PlusDecoder 4 | from .panoptic_deeplab import PanopticDeepLabDecoder 5 | -------------------------------------------------------------------------------- /openpifpaf/panoptic_deeplab/decoder/aspp.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Reference: https://github.com/pytorch/vision/blob/master/torchvision/models/segmentation/deeplabv3.py 3 | # Modified by Bowen Cheng (bcheng9@illinois.edu) 4 | # ------------------------------------------------------------------------------ 5 | 6 | import torch 7 | from torch import nn 8 | from torch.nn import functional as F 9 | 10 | __all__ = ["ASPP"] 11 | 12 | 13 | class ASPPConv(nn.Sequential): 14 | def __init__(self, in_channels, out_channels, dilation): 15 | modules = [ 16 | nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False), 17 | nn.BatchNorm2d(out_channels), 18 | nn.ReLU() 19 | ] 20 | super(ASPPConv, self).__init__(*modules) 21 | 22 | 23 | class ASPPPooling(nn.Module): 24 | def __init__(self, in_channels, out_channels): 25 | super(ASPPPooling, self).__init__() 26 | self.aspp_pooling = nn.Sequential( 27 | nn.AdaptiveAvgPool2d(1), 28 | nn.Conv2d(in_channels, out_channels, 1, bias=False), 29 | nn.ReLU() 30 | ) 31 | 32 | def set_image_pooling(self, pool_size=None): 33 | if pool_size is None: 34 | self.aspp_pooling[0] = nn.AdaptiveAvgPool2d(1) 35 | else: 36 | self.aspp_pooling[0] = nn.AvgPool2d(kernel_size=pool_size, stride=1) 37 | 38 | def forward(self, x): 39 | size = x.shape[-2:] 40 | x = self.aspp_pooling(x) 41 | return F.interpolate(x, size=size, mode='bilinear', align_corners=True) 42 | 43 | 44 | class ASPP(nn.Module): 45 | def __init__(self, in_channels, out_channels, atrous_rates): 46 | super(ASPP, self).__init__() 47 | # out_channels = 256 48 | modules = [] 49 | modules.append(nn.Sequential( 50 | nn.Conv2d(in_channels, out_channels, 1, bias=False), 51 | nn.BatchNorm2d(out_channels), 52 | nn.ReLU())) 53 | 54 | rate1, rate2, rate3 = tuple(atrous_rates) 55 | modules.append(ASPPConv(in_channels, out_channels, rate1)) 56 | modules.append(ASPPConv(in_channels, out_channels, rate2)) 57 | modules.append(ASPPConv(in_channels, out_channels, rate3)) 58 | modules.append(ASPPPooling(in_channels, out_channels)) 59 | 60 | self.convs = nn.ModuleList(modules) 61 | 62 | self.project = nn.Sequential( 63 | nn.Conv2d(5 * out_channels, out_channels, 1, bias=False), 64 | nn.BatchNorm2d(out_channels), 65 | nn.ReLU(), 66 | nn.Dropout(0.5)) 67 | 68 | def set_image_pooling(self, pool_size): 69 | self.convs[-1].set_image_pooling(pool_size) 70 | 71 | def forward(self, x): 72 | res = [] 73 | for conv in self.convs: 74 | res.append(conv(x)) 75 | res = torch.cat(res, dim=1) 76 | return self.project(res) 77 | -------------------------------------------------------------------------------- /openpifpaf/panoptic_deeplab/decoder/conv_module.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # Common modules. 3 | # Written by Bowen Cheng (bcheng9@illinois.edu) 4 | # ------------------------------------------------------------------------------ 5 | 6 | from functools import partial 7 | 8 | import torch 9 | from torch import nn 10 | from torch.nn import functional as F 11 | 12 | 13 | def basic_conv(in_planes, out_planes, kernel_size, stride=1, padding=1, groups=1, 14 | with_bn=True, with_relu=True): 15 | """convolution with bn and relu""" 16 | module = [] 17 | has_bias = not with_bn 18 | module.append( 19 | nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, 20 | bias=has_bias) 21 | ) 22 | if with_bn: 23 | module.append(nn.BatchNorm2d(out_planes)) 24 | if with_relu: 25 | module.append(nn.ReLU()) 26 | return nn.Sequential(*module) 27 | 28 | 29 | def depthwise_separable_conv(in_planes, out_planes, kernel_size, stride=1, padding=1, groups=1, 30 | with_bn=True, with_relu=True): 31 | """depthwise separable convolution with bn and relu""" 32 | del groups 33 | 34 | module = [] 35 | module.extend([ 36 | basic_conv(in_planes, in_planes, kernel_size, stride, padding, groups=in_planes, 37 | with_bn=True, with_relu=True), 38 | nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False), 39 | ]) 40 | if with_bn: 41 | module.append(nn.BatchNorm2d(out_planes)) 42 | if with_relu: 43 | module.append(nn.ReLU()) 44 | return nn.Sequential(*module) 45 | 46 | 47 | def stacked_conv(in_planes, out_planes, kernel_size, num_stack, stride=1, padding=1, groups=1, 48 | with_bn=True, with_relu=True, conv_type='basic_conv'): 49 | """stacked convolution with bn and relu""" 50 | if num_stack < 1: 51 | assert ValueError('`num_stack` has to be a positive integer.') 52 | if conv_type == 'basic_conv': 53 | conv = partial(basic_conv, out_planes=out_planes, kernel_size=kernel_size, stride=stride, 54 | padding=padding, groups=groups, with_bn=with_bn, with_relu=with_relu) 55 | elif conv_type == 'depthwise_separable_conv': 56 | conv = partial(depthwise_separable_conv, out_planes=out_planes, kernel_size=kernel_size, stride=stride, 57 | padding=padding, groups=1, with_bn=with_bn, with_relu=with_relu) 58 | else: 59 | raise ValueError('Unknown conv_type: {}'.format(conv_type)) 60 | module = [] 61 | module.append(conv(in_planes=in_planes)) 62 | for n in range(1, num_stack): 63 | module.append(conv(in_planes=out_planes)) 64 | return nn.Sequential(*module) 65 | 66 | 67 | if __name__ == '__main__': 68 | import torch 69 | 70 | model = stacked_conv(4, 2, 3, 3) 71 | print(model) 72 | data = torch.zeros(1, 4, 5, 5) 73 | print(model.forward(data).shape) 74 | -------------------------------------------------------------------------------- /openpifpaf/panoptic_deeplab/decoder/deeplabv3.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # DeepLabV3 decoder. 3 | # Written by Bowen Cheng (bcheng9@illinois.edu) 4 | # ------------------------------------------------------------------------------ 5 | 6 | from collections import OrderedDict 7 | 8 | from torch import nn 9 | 10 | from .aspp import ASPP 11 | 12 | 13 | __all__ = ["DeepLabV3Decoder"] 14 | 15 | 16 | class DeepLabV3Decoder(nn.Module): 17 | def __init__(self, in_channels, feature_key, decoder_channels, atrous_rates, num_classes): 18 | super(DeepLabV3Decoder, self).__init__() 19 | self.aspp = ASPP(in_channels, out_channels=decoder_channels, atrous_rates=atrous_rates) 20 | self.feature_key = feature_key 21 | self.classifier = nn.Sequential( 22 | nn.Conv2d(decoder_channels, decoder_channels, 3, padding=1, bias=False), 23 | nn.BatchNorm2d(decoder_channels), 24 | nn.ReLU(), 25 | nn.Conv2d(decoder_channels, num_classes, 1) 26 | ) 27 | 28 | def set_image_pooling(self, pool_size): 29 | self.aspp.set_image_pooling(pool_size) 30 | 31 | def forward(self, features): 32 | pred = OrderedDict() 33 | res5 = features[self.feature_key] 34 | x = self.aspp(res5) 35 | x = self.classifier(x) 36 | pred['semantic'] = x 37 | return pred 38 | -------------------------------------------------------------------------------- /openpifpaf/panoptic_deeplab/decoder/deeplabv3plus.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------------------ 2 | # DeepLabV3+ decoder. 3 | # Written by Bowen Cheng (bcheng9@illinois.edu) 4 | # ------------------------------------------------------------------------------ 5 | 6 | from collections import OrderedDict 7 | 8 | import torch 9 | from torch import nn 10 | from torch.nn import functional as F 11 | 12 | from .aspp import ASPP 13 | from .conv_module import stacked_conv 14 | 15 | 16 | __all__ = ["DeepLabV3PlusDecoder"] 17 | 18 | 19 | class DeepLabV3PlusDecoder(nn.Module): 20 | def __init__(self, in_channels, feature_key, low_level_channels, low_level_key, low_level_channels_project, 21 | decoder_channels, atrous_rates, num_classes): 22 | super(DeepLabV3PlusDecoder, self).__init__() 23 | self.aspp = ASPP(in_channels, out_channels=decoder_channels, atrous_rates=atrous_rates) 24 | self.feature_key = feature_key 25 | self.low_level_key = low_level_key 26 | # Transform low-level feature 27 | # low_level_channels_project = 48 28 | self.project = nn.Sequential( 29 | nn.Conv2d(low_level_channels, low_level_channels_project, 1, bias=False), 30 | nn.BatchNorm2d(low_level_channels_project), 31 | nn.ReLU() 32 | ) 33 | # Fuse 34 | self.fuse = stacked_conv( 35 | decoder_channels + low_level_channels_project, 36 | decoder_channels, 37 | kernel_size=3, 38 | padding=1, 39 | num_stack=2, 40 | conv_type='depthwise_separable_conv' 41 | ) 42 | self.classifier = nn.Conv2d(decoder_channels, num_classes, 1) 43 | 44 | def set_image_pooling(self, pool_size): 45 | self.aspp.set_image_pooling(pool_size) 46 | 47 | def forward(self, features): 48 | pred = OrderedDict() 49 | l = features[self.low_level_key] 50 | x = features[self.feature_key] 51 | x = self.aspp(x) 52 | # low-level feature 53 | l = self.project(l) 54 | x = F.interpolate(x, size=l.size()[2:], mode='bilinear', align_corners=True) 55 | x = torch.cat((x, l), dim=1) 56 | x = self.fuse(x) 57 | x = self.classifier(x) 58 | pred['semantic'] = x 59 | return pred 60 | -------------------------------------------------------------------------------- /openpifpaf/show/__init__.py: -------------------------------------------------------------------------------- 1 | """Drawing primitives.""" 2 | 3 | from .animation_frame import AnimationFrame 4 | from .canvas import canvas, image_canvas, load_image, white_screen 5 | from .cli import cli, configure 6 | from .fields import arrows, boxes, boxes_wh, circles, margins, quiver 7 | from .painters import AnnotationPainter, CrowdPainter, DetectionPainter, KeypointPainter, KeypointCentPainter 8 | -------------------------------------------------------------------------------- /openpifpaf/show/animation_frame.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | try: 4 | import matplotlib 5 | import matplotlib.animation 6 | import matplotlib.collections 7 | import matplotlib.pyplot as plt 8 | import matplotlib.patches 9 | except ImportError: 10 | matplotlib = None 11 | plt = None 12 | 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class AnimationFrame: 18 | video_fps = 10 19 | video_dpi = 100 20 | 21 | def __init__(self, *, 22 | fig_width=8.0, 23 | fig_init_args=None, 24 | show=False, 25 | video_output=None, 26 | second_visual=False): 27 | self.fig_width = fig_width 28 | self.fig_init_args = fig_init_args or {} 29 | self.show = show 30 | self.video_output = video_output 31 | self.video_writer = None 32 | if self.video_output: 33 | self.video_writer = matplotlib.animation.writers['ffmpeg'](fps=self.video_fps) 34 | 35 | self.second_visual = second_visual 36 | if self.second_visual: 37 | self.fig_width *= 2 38 | 39 | if plt is None: 40 | LOG.error('matplotlib is not installed') 41 | 42 | self.fig = None 43 | self.ax = None 44 | self.ax_second = None 45 | self._skip_frame = False 46 | 47 | LOG.info('video output = %s', video_output) 48 | 49 | @staticmethod 50 | def clean_axis(ax): 51 | xlim = ax.get_xlim() 52 | ylim = ax.get_ylim() 53 | ax.cla() 54 | ax.set_axis_off() 55 | ax.set_xlim(*xlim) 56 | ax.set_ylim(*ylim) 57 | 58 | def skip_frame(self): 59 | self._skip_frame = True 60 | 61 | def iter(self): 62 | video_writer_is_setup = False 63 | try: 64 | while True: 65 | yield (self.ax, self.ax_second) 66 | 67 | if self._skip_frame: 68 | self._skip_frame = False 69 | continue 70 | 71 | # Lazy setup of video writer (needs to be after first yield 72 | # because only that might setup `self.fig`). 73 | if self.video_writer and not video_writer_is_setup: 74 | self.video_writer.setup(self.fig, self.video_output, self.video_dpi) 75 | video_writer_is_setup = True 76 | 77 | if self.show: 78 | plt.pause(0.01) 79 | if self.video_writer: 80 | self.video_writer.grab_frame() 81 | 82 | if self.ax: 83 | self.clean_axis(self.ax) 84 | if self.ax_second: 85 | self.clean_axis(self.ax_second) 86 | 87 | finally: 88 | if self.video_writer: 89 | self.video_writer.finish() 90 | if self.fig: 91 | plt.close(self.fig) 92 | 93 | def frame_init(self, image): 94 | if plt is None: 95 | return None, None 96 | 97 | if 'figsize' not in self.fig_init_args: 98 | self.fig_init_args['figsize'] = ( 99 | self.fig_width, 100 | self.fig_width * image.shape[0] / image.shape[1] 101 | ) 102 | if self.second_visual: 103 | self.fig_init_args['figsize'] = ( 104 | self.fig_init_args['figsize'][0], 105 | self.fig_init_args['figsize'][1] / 2.0, 106 | ) 107 | 108 | self.fig = plt.figure(**self.fig_init_args) 109 | if self.second_visual: 110 | self.ax = plt.Axes(self.fig, [0.0, 0.0, 0.5, 1.0]) 111 | self.ax_second = plt.Axes(self.fig, [0.5, 0.05, 0.45, 0.9]) 112 | self.fig.add_axes(self.ax) 113 | self.fig.add_axes(self.ax_second) 114 | else: 115 | self.ax = plt.Axes(self.fig, [0.0, 0.0, 1.0, 1.0]) 116 | self.ax_second = None 117 | self.fig.add_axes(self.ax) 118 | self.ax.set_axis_off() 119 | # self.ax.set_xlim(0, image.shape[1]) 120 | # self.ax.set_ylim(image.shape[0], 0) 121 | if self.ax_second is not None: 122 | self.ax_second.set_axis_off() 123 | # self.ax_second.set_xlim(0, image.shape[1]) 124 | # self.ax_second.set_ylim(image.shape[0], 0) 125 | 126 | return self.ax, self.ax_second 127 | -------------------------------------------------------------------------------- /openpifpaf/show/canvas.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | import logging 3 | 4 | import numpy as np 5 | from PIL import Image 6 | 7 | try: 8 | import matplotlib.pyplot as plt 9 | except ImportError: 10 | plt = None 11 | 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | 16 | @contextmanager 17 | def canvas(fig_file=None, show=True, dpi=200, nomargin=False, **kwargs): 18 | if 'figsize' not in kwargs: 19 | # kwargs['figsize'] = (15, 8) 20 | kwargs['figsize'] = (10, 6) 21 | 22 | if nomargin: 23 | fig = plt.figure(**kwargs) 24 | ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0]) 25 | fig.add_axes(ax) 26 | else: 27 | fig, ax = plt.subplots(**kwargs) 28 | 29 | yield ax 30 | 31 | fig.set_tight_layout(not nomargin) 32 | if fig_file: 33 | fig.savefig(fig_file, dpi=dpi) # , bbox_inches='tight') 34 | if show: 35 | plt.show() 36 | plt.close(fig) 37 | 38 | 39 | @contextmanager 40 | def image_canvas(image, fig_file=None, show=True, dpi_factor=1.0, fig_width=10.0, **kwargs): 41 | image = np.asarray(image) 42 | if 'figsize' not in kwargs: 43 | kwargs['figsize'] = (fig_width, fig_width * image.shape[0] / image.shape[1]) 44 | 45 | ### to solve the problem with ssh showing 46 | import matplotlib 47 | matplotlib.use('Agg') 48 | 49 | fig = plt.figure(**kwargs) 50 | ax = plt.Axes(fig, [0.0, 0.0, 1.0, 1.0]) 51 | ax.set_axis_off() 52 | ax.set_xlim(0, image.shape[1]) 53 | ax.set_ylim(image.shape[0], 0) 54 | fig.add_axes(ax) 55 | ax.imshow(image) 56 | 57 | yield ax 58 | 59 | if fig_file: 60 | fig.savefig(fig_file, dpi=image.shape[1] / kwargs['figsize'][0] * dpi_factor) 61 | if show: 62 | plt.show() 63 | plt.close(fig) 64 | 65 | 66 | def load_image(path, scale=1.0): 67 | with open(path, 'rb') as f: 68 | image = Image.open(f).convert('RGB') 69 | image = np.asarray(image) * scale / 255.0 70 | return image 71 | 72 | 73 | def white_screen(ax, alpha=0.9): 74 | ax.add_patch( 75 | plt.Rectangle((0, 0), 1, 1, transform=ax.transAxes, alpha=alpha, 76 | facecolor='white') 77 | ) 78 | -------------------------------------------------------------------------------- /openpifpaf/show/cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .painters import KeypointPainter 4 | 5 | LOG = logging.getLogger(__name__) 6 | 7 | 8 | def cli(parser): 9 | group = parser.add_argument_group('show') 10 | group.add_argument('--show-box', default=False, action='store_true') 11 | group.add_argument('--show-joint-scales', default=False, action='store_true') 12 | group.add_argument('--show-joint-confidences', default=False, action='store_true') 13 | group.add_argument('--show-decoding-order', default=False, action='store_true') 14 | group.add_argument('--show-frontier-order', default=False, action='store_true') 15 | group.add_argument('--show-only-decoded-connections', default=False, action='store_true') 16 | 17 | 18 | def configure(args): 19 | KeypointPainter.show_box = args.show_box 20 | KeypointPainter.show_joint_scales = args.show_joint_scales 21 | KeypointPainter.show_joint_confidences = args.show_joint_confidences 22 | KeypointPainter.show_decoding_order = args.show_decoding_order 23 | KeypointPainter.show_frontier_order = args.show_frontier_order 24 | KeypointPainter.show_only_decoded_connections = args.show_only_decoded_connections 25 | -------------------------------------------------------------------------------- /openpifpaf/train_instance_scorer.py: -------------------------------------------------------------------------------- 1 | import json 2 | import random 3 | 4 | import pysparkling 5 | import torch 6 | 7 | from .decoder.instance_scorer import InstanceScorer 8 | from . import show 9 | 10 | DATA_FILE = ('outputs/resnet101block5-pif-paf-edge401-190412-151013.pkl' 11 | '.decodertraindata-edge641-samples0.json') 12 | 13 | 14 | def plot_training_data(train_data, val_data, entry=0, entryname=None): 15 | train_x, train_y = train_data 16 | val_x, val_y = val_data 17 | with show.canvas() as ax: 18 | ax.hist([xx[entry] for xx in train_x[train_y[:, 0] == 1]], 19 | bins=50, alpha=0.3, density=True, color='navy', label='train true') 20 | ax.hist([xx[entry] for xx in train_x[train_y[:, 0] == 0]], 21 | bins=50, alpha=0.3, density=True, color='orange', label='train false') 22 | 23 | ax.hist([xx[entry] for xx in val_x[val_y[:, 0] == 1]], 24 | histtype='step', bins=50, density=True, color='navy', label='val true') 25 | ax.hist([xx[entry] for xx in val_x[val_y[:, 0] == 0]], 26 | histtype='step', bins=50, density=True, color='orange', label='val false') 27 | 28 | if entryname: 29 | ax.set_xlabel(entryname) 30 | ax.legend() 31 | 32 | 33 | def train_val_split_score(data, train_fraction=0.6, balance=True): 34 | xy_list = data.map(lambda d: ([d['score']], [float(d['target'])])).collect() 35 | 36 | if balance: 37 | n_true = sum(1 for x, y in xy_list if y[0] == 1.0) 38 | n_false = sum(1 for x, y in xy_list if y[0] == 0.0) 39 | p_true = min(1.0, n_false / n_true) 40 | p_false = min(1.0, n_true / n_false) 41 | xy_list = [(x, y) for x, y in xy_list 42 | if random.random() < (p_true if y[0] == 1.0 else p_false)] 43 | 44 | n_train = int(train_fraction * len(xy_list)) 45 | 46 | return ( 47 | (torch.tensor([x for x, _ in xy_list[:n_train]]), 48 | torch.tensor([y for _, y in xy_list[:n_train]])), 49 | (torch.tensor([x for x, _ in xy_list[n_train:]]), 50 | torch.tensor([y for _, y in xy_list[n_train:]])), 51 | ) 52 | 53 | 54 | def train_val_split_keypointscores(data, train_fraction=0.6): 55 | xy_list = ( 56 | data 57 | .map(lambda d: ([d['score']] + [xyv[2] for xyv in d['keypoints']] + d['joint_scales'], 58 | [float(d['target'])])) 59 | .collect() 60 | ) 61 | n_train = int(train_fraction * len(xy_list)) 62 | 63 | return ( 64 | (torch.tensor([x for x, _ in xy_list[:n_train]]), 65 | torch.tensor([y for _, y in xy_list[:n_train]])), 66 | (torch.tensor([x for x, _ in xy_list[n_train:]]), 67 | torch.tensor([y for _, y in xy_list[n_train:]])), 68 | ) 69 | 70 | 71 | def train_epoch(model, loader, optimizer): 72 | epoch_loss = 0.0 73 | for x, y in loader: 74 | optimizer.zero_grad() 75 | 76 | y_hat = model(x) 77 | loss = torch.nn.functional.binary_cross_entropy(y_hat, y) 78 | epoch_loss += float(loss.item()) 79 | loss.backward() 80 | optimizer.step() 81 | 82 | return epoch_loss / len(loader) 83 | 84 | def val_epoch(model, loader): 85 | epoch_loss = 0.0 86 | with torch.no_grad(): 87 | for x, y in loader: 88 | y_hat = model(x) 89 | loss = torch.nn.functional.binary_cross_entropy(y_hat, y) 90 | epoch_loss += float(loss.item()) 91 | 92 | return epoch_loss / len(loader) 93 | 94 | 95 | def main(): 96 | sc = pysparkling.Context() 97 | data = sc.textFile(DATA_FILE).map(json.loads).cache() 98 | 99 | train_data_score, val_data_score = train_val_split_score(data) 100 | plot_training_data(train_data_score, val_data_score, entryname='score') 101 | 102 | train_data, val_data = train_val_split_keypointscores(data) 103 | 104 | model = InstanceScorer() 105 | 106 | train_dataset = torch.utils.data.TensorDataset(*train_data) 107 | train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=256, shuffle=True) 108 | 109 | val_dataset = torch.utils.data.TensorDataset(*val_data) 110 | val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=256, shuffle=False) 111 | 112 | optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9) 113 | for epoch_i in range(100): 114 | train_loss = train_epoch(model, train_loader, optimizer) 115 | val_loss = val_epoch(model, val_loader) 116 | print(epoch_i, train_loss, val_loss) 117 | 118 | with torch.no_grad(): 119 | post_train_data = (model(train_data[0]), train_data[1]) 120 | post_val_data = (model(val_data[0]), val_data[1]) 121 | plot_training_data(post_train_data, post_val_data, entryname='optimized score') 122 | torch.save(model, 'instance_scorer.pkl') 123 | 124 | 125 | if __name__ == '__main__': 126 | main() 127 | -------------------------------------------------------------------------------- /openpifpaf/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | """Transform input data.""" 2 | 3 | import torchvision 4 | 5 | from .annotations import AnnotationJitter, NormalizeAnnotations 6 | from .compose import Compose 7 | from .crop import Crop 8 | from .hflip import HFlip 9 | from .image import Blur, ImageTransform, JpegCompression 10 | from .minsize import MinSize 11 | from .multi_scale import MultiScale 12 | from .pad import CenterPad, CenterPadTight, SquarePad 13 | from .preprocess import Preprocess 14 | from .random import DeterministicEqualChoice, RandomApply 15 | from .rotate import RotateBy90 16 | from .scale import RescaleAbsolute, RescaleRelative, ScaleMix 17 | from .unclipped import UnclippedArea, UnclippedSides 18 | from .zoomscale import ZoomScale 19 | from .crop_keemotion import CropKeemotion 20 | 21 | 22 | EVAL_TRANSFORM = Compose([ 23 | NormalizeAnnotations(), 24 | ImageTransform(torchvision.transforms.ToTensor()), 25 | ImageTransform( 26 | torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], 27 | std=[0.229, 0.224, 0.225]), 28 | ), 29 | ]) 30 | 31 | 32 | TRAIN_TRANSFORM = Compose([ 33 | NormalizeAnnotations(), 34 | ImageTransform(torchvision.transforms.ColorJitter( 35 | brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1)), 36 | RandomApply(JpegCompression(), 0.1), # maybe irrelevant for COCO, but good for others 37 | # RandomApply(Blur(), 0.01), # maybe irrelevant for COCO, but good for others 38 | ImageTransform(torchvision.transforms.RandomGrayscale(p=0.01)), 39 | EVAL_TRANSFORM, 40 | ]) 41 | -------------------------------------------------------------------------------- /openpifpaf/transforms/annotations.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from .preprocess import Preprocess 8 | 9 | LOG = logging.getLogger(__name__) 10 | 11 | 12 | class NormalizeAnnotations(Preprocess): 13 | @staticmethod 14 | def normalize_annotations(anns): 15 | anns = copy.deepcopy(anns) 16 | 17 | for ann in anns: 18 | 19 | if 'keypoints' not in ann: 20 | ann['keypoints'] = [] 21 | if 'kp_ball' not in ann: 22 | ann['kp_ball'] = [] 23 | if 'cent' not in ann: 24 | ann['cent'] = [] 25 | 26 | ann['keypoints'] = np.asarray(ann['keypoints'], dtype=np.float32).reshape(-1, 3) 27 | ann['bbox'] = np.asarray(ann['bbox'], dtype=np.float32) 28 | if 'bbox_original' not in ann: 29 | ann['bbox_original'] = np.copy(ann['bbox']) 30 | if 'segmentation' in ann: 31 | del ann['segmentation'] 32 | 33 | if 'kp_ball' in ann: 34 | ann['kp_ball'] = np.asarray(ann['kp_ball'], dtype=np.float32).reshape(-1, 3) 35 | 36 | if 'cent' in ann: 37 | ann['cent'] = np.asarray(ann['cent'], dtype=np.float32).reshape(-1, 3) 38 | 39 | return anns 40 | 41 | 42 | def __call__(self, image, anns, meta): 43 | 44 | anns = self.normalize_annotations(anns) 45 | 46 | if meta is None: 47 | meta = {} 48 | 49 | # fill meta with defaults if not already present 50 | w, h = image.size 51 | meta_from_image = { 52 | 'offset': np.array((0.0, 0.0)), 53 | 'scale': np.array((1.0, 1.0)), 54 | 'rotation': {'angle': 0.0, 'width': None, 'height': None}, 55 | 'valid_area': np.array((0.0, 0.0, w - 1, h - 1)), 56 | 'hflip': False, 57 | 'width_height': np.array((w, h)), 58 | } 59 | 60 | for k, v in meta_from_image.items(): 61 | if k not in meta: 62 | meta[k] = v 63 | 64 | return image, anns, meta 65 | 66 | 67 | class AnnotationJitter(Preprocess): 68 | def __init__(self, epsilon=0.5): 69 | self.epsilon = epsilon 70 | 71 | 72 | def __call__(self, image, anns, meta): 73 | meta = copy.deepcopy(meta) 74 | anns = copy.deepcopy(anns) 75 | 76 | for ann in anns: 77 | keypoints_xy = ann['keypoints'][:, :2] 78 | sym_rnd_kp = (torch.rand(*keypoints_xy.shape).numpy() - 0.5) * 2.0 79 | keypoints_xy += self.epsilon * sym_rnd_kp 80 | 81 | sym_rnd_bbox = (torch.rand((4,)).numpy() - 0.5) * 2.0 82 | ann['bbox'] += 0.5 * self.epsilon * sym_rnd_bbox 83 | 84 | if 'kp_ball' in ann: 85 | ball_xy = ann['kp_ball'][:, :2] 86 | sym_rnd_ball = (torch.rand(*ball_xy.shape).numpy() - 0.5) * 2.0 87 | ball_xy += self.epsilon * sym_rnd_ball 88 | 89 | if 'cent' in ann: 90 | cent_xy = ann['cent'][:, :2] 91 | sym_rnd_cent = (torch.rand(*cent_xy.shape).numpy() - 0.5) * 2.0 92 | cent_xy += self.epsilon * sym_rnd_cent 93 | return image, anns, meta 94 | -------------------------------------------------------------------------------- /openpifpaf/transforms/compose.py: -------------------------------------------------------------------------------- 1 | from .preprocess import Preprocess 2 | import time 3 | 4 | class Compose(Preprocess): 5 | def __init__(self, preprocess_list): 6 | self.preprocess_list = preprocess_list 7 | 8 | def __call__(self, image, anns, meta): 9 | 10 | for i,p in enumerate(self.preprocess_list): 11 | if p is None: 12 | continue 13 | 14 | image, anns, meta = p(image, anns, meta) 15 | 16 | return image, anns, meta 17 | -------------------------------------------------------------------------------- /openpifpaf/transforms/hflip.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | 4 | import numpy as np 5 | import PIL 6 | 7 | from .preprocess import Preprocess 8 | 9 | LOG = logging.getLogger(__name__) 10 | 11 | 12 | class _HorizontalSwap(): 13 | def __init__(self, keypoints, hflip): 14 | self.keypoints = keypoints 15 | self.hflip = hflip 16 | 17 | def __call__(self, keypoints): 18 | target = np.zeros(keypoints.shape) 19 | 20 | for source_i, xyv in enumerate(keypoints): 21 | source_name = self.keypoints[source_i] 22 | target_name = self.hflip.get(source_name) 23 | if target_name: 24 | target_i = self.keypoints.index(target_name) 25 | else: 26 | target_i = source_i 27 | 28 | target[target_i] = xyv 29 | 30 | return target 31 | 32 | 33 | class HFlip(Preprocess): 34 | def __init__(self, keypoints, hflip): 35 | self.swap = _HorizontalSwap(keypoints, hflip) 36 | 37 | 38 | def __call__(self, image, anns, meta): 39 | meta = copy.deepcopy(meta) 40 | anns = copy.deepcopy(anns) 41 | 42 | w, _ = image.size 43 | image = image.transpose(PIL.Image.FLIP_LEFT_RIGHT) 44 | for ann in anns: 45 | ann['keypoints'][:, 0] = -ann['keypoints'][:, 0] - 1.0 + w 46 | if self.swap is not None and not ann['iscrowd']: 47 | ann['keypoints'] = self.swap(ann['keypoints']) 48 | meta['horizontal_swap'] = self.swap 49 | ann['bbox'][0] = -(ann['bbox'][0] + ann['bbox'][2]) - 1.0 + w 50 | 51 | if 'kp_ball' in ann: 52 | ann['kp_ball'][:, 0] = -ann['kp_ball'][:, 0] - 1.0 + w 53 | 54 | if 'cent' in ann: 55 | ann['cent'][:, 0] = -ann['cent'][:, 0] - 1.0 + w 56 | 57 | ann['bmask'] = np.flip(ann['bmask'], axis=1) 58 | 59 | 60 | assert meta['hflip'] is False 61 | meta['hflip'] = True 62 | 63 | meta['valid_area'][0] = -(meta['valid_area'][0] + meta['valid_area'][2]) + w 64 | 65 | return image, anns, meta 66 | -------------------------------------------------------------------------------- /openpifpaf/transforms/image.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | 4 | import numpy as np 5 | import PIL 6 | import scipy 7 | import torch 8 | 9 | from .preprocess import Preprocess 10 | 11 | LOG = logging.getLogger(__name__) 12 | 13 | 14 | 15 | class ImageTransform(Preprocess): 16 | def __init__(self, image_transform): 17 | self.image_transform = image_transform 18 | 19 | def __call__(self, image, anns, meta): 20 | image = self.image_transform(image) 21 | return image, anns, meta 22 | 23 | 24 | class JpegCompression(Preprocess): 25 | def __init__(self, quality=50): 26 | self.quality = quality 27 | 28 | def __call__(self, image, anns, meta): 29 | f = io.BytesIO() 30 | image.save(f, 'jpeg', quality=self.quality) 31 | return PIL.Image.open(f), anns, meta 32 | 33 | 34 | class Blur(Preprocess): 35 | def __init__(self, max_sigma=5.0): 36 | self.max_sigma = max_sigma 37 | 38 | def __call__(self, image, anns, meta): 39 | im_np = np.asarray(image) 40 | sigma = self.max_sigma * float(torch.rand(1).item()) 41 | im_np = scipy.ndimage.filters.gaussian_filter(im_np, sigma=(sigma, sigma, 0)) 42 | return PIL.Image.fromarray(im_np), anns, meta 43 | -------------------------------------------------------------------------------- /openpifpaf/transforms/minsize.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | 4 | from .preprocess import Preprocess 5 | 6 | LOG = logging.getLogger(__name__) 7 | 8 | 9 | class MinSize(Preprocess): 10 | def __init__(self, min_side=1.0): 11 | self.min_side = min_side 12 | 13 | def __call__(self, image, anns, meta): 14 | anns = copy.deepcopy(anns) 15 | for ann in anns: 16 | if ann['bbox'][2] > self.min_side \ 17 | and ann['bbox'][3] > self.min_side: 18 | continue 19 | ann['iscrowd'] = True 20 | 21 | return image, anns, meta 22 | -------------------------------------------------------------------------------- /openpifpaf/transforms/multi_scale.py: -------------------------------------------------------------------------------- 1 | from .preprocess import Preprocess 2 | 3 | 4 | class MultiScale(Preprocess): 5 | def __init__(self, preprocess_list): 6 | """Create lists of preprocesses. 7 | 8 | Must be the most outer preprocess function. 9 | Preprocess_list can contain transforms.Compose() functions. 10 | """ 11 | self.preprocess_list = preprocess_list 12 | 13 | def __call__(self, image, anns, meta): 14 | image_list, anns_list, meta_list = [], [], [] 15 | for p in self.preprocess_list: 16 | this_image, this_anns, this_meta = p(image, anns, meta) 17 | image_list.append(this_image) 18 | anns_list.append(this_anns) 19 | meta_list.append(this_meta) 20 | 21 | return image_list, anns_list, meta_list 22 | -------------------------------------------------------------------------------- /openpifpaf/transforms/pad.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import logging 4 | 5 | import torchvision 6 | 7 | from .preprocess import Preprocess 8 | 9 | import numpy as np 10 | 11 | LOG = logging.getLogger(__name__) 12 | 13 | 14 | class CenterPad(Preprocess): 15 | def __init__(self, target_size): 16 | if isinstance(target_size, int): 17 | target_size = (target_size, target_size) 18 | self.target_size = target_size 19 | 20 | #AMA 21 | def __call__(self, image, anns, meta): 22 | meta = copy.deepcopy(meta) 23 | anns = copy.deepcopy(anns) 24 | 25 | LOG.debug('valid area before pad: %s, image size = %s', meta['valid_area'], image.size) 26 | image, anns, ltrb = self.center_pad(image, anns) 27 | meta['offset'] -= ltrb[:2] 28 | meta['valid_area'][:2] += ltrb[:2] 29 | LOG.debug('valid area after pad: %s, image size = %s', meta['valid_area'], image.size) 30 | 31 | return image, anns, meta 32 | 33 | def center_pad(self, image, anns): 34 | w, h = image.size 35 | 36 | left = int((self.target_size[0] - w) / 2.0) 37 | top = int((self.target_size[1] - h) / 2.0) 38 | if left < 0: 39 | left = 0 40 | if top < 0: 41 | top = 0 42 | 43 | right = self.target_size[0] - w - left 44 | bottom = self.target_size[1] - h - top 45 | if right < 0: 46 | right = 0 47 | if bottom < 0: 48 | bottom = 0 49 | ltrb = (left, top, right, bottom) 50 | LOG.debug('pad with (left, top, right, bottom) %s', ltrb) 51 | 52 | # pad image 53 | image = torchvision.transforms.functional.pad( 54 | image, ltrb, fill=(124, 116, 104)) 55 | 56 | # pad annotations 57 | for ann in anns: 58 | ann['keypoints'][:, 0] += ltrb[0] 59 | ann['keypoints'][:, 1] += ltrb[1] 60 | ann['bbox'][0] += ltrb[0] 61 | ann['bbox'][1] += ltrb[1] 62 | 63 | if 'kp_ball' in ann: 64 | ann['kp_ball'][:, 0] += ltrb[0] 65 | ann['kp_ball'][:, 1] += ltrb[1] 66 | 67 | if 'cent' in ann: 68 | ann['cent'][:, 0] += ltrb[0] 69 | ann['cent'][:, 1] += ltrb[1] 70 | 71 | 72 | ann['bmask'] = np.pad(ann['bmask'], ((top, bottom), (left, right))) 73 | 74 | 75 | 76 | return image, anns, ltrb 77 | 78 | 79 | class CenterPadTight(Preprocess): 80 | def __init__(self, multiple): 81 | self.multiple = multiple 82 | 83 | def __call__(self, image, anns, meta): 84 | meta = copy.deepcopy(meta) 85 | anns = copy.deepcopy(anns) 86 | 87 | LOG.debug('valid area before pad: %s, image size = %s', meta['valid_area'], image.size) 88 | image, anns, ltrb = self.center_pad(image, anns) 89 | meta['offset'] -= ltrb[:2] 90 | meta['valid_area'][:2] += ltrb[:2] 91 | LOG.debug('valid area after pad: %s, image size = %s', meta['valid_area'], image.size) 92 | 93 | 94 | 95 | return image, anns, meta 96 | 97 | def center_pad(self, image, anns): 98 | w, h = image.size 99 | target_width = math.ceil((w - 1) / self.multiple) * self.multiple + 1 100 | target_height = math.ceil((h - 1) / self.multiple) * self.multiple + 1 101 | 102 | left = int((target_width - w) / 2.0) 103 | top = int((target_height - h) / 2.0) 104 | if left < 0: 105 | left = 0 106 | if top < 0: 107 | top = 0 108 | 109 | right = target_width - w - left 110 | bottom = target_height - h - top 111 | if right < 0: 112 | right = 0 113 | if bottom < 0: 114 | bottom = 0 115 | ltrb = (left, top, right, bottom) 116 | LOG.debug('pad with (left, top, right, bottom) %s', ltrb) 117 | 118 | # pad image 119 | image = torchvision.transforms.functional.pad( 120 | image, ltrb, fill=(124, 116, 104)) 121 | 122 | # pad annotations 123 | for ann in anns: 124 | ann['keypoints'][:, 0] += ltrb[0] 125 | ann['keypoints'][:, 1] += ltrb[1] 126 | ann['bbox'][0] += ltrb[0] 127 | ann['bbox'][1] += ltrb[1] 128 | 129 | ann['bmask'] = np.pad(ann['bmask'], ((top, bottom), (left, right))) 130 | 131 | 132 | return image, anns, ltrb 133 | 134 | 135 | class SquarePad(Preprocess): 136 | def __call__(self, image, anns, meta): 137 | center_pad = CenterPad(max(image.size)) 138 | return center_pad(image, anns, meta) 139 | -------------------------------------------------------------------------------- /openpifpaf/transforms/preprocess.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | import copy 3 | import math 4 | import numpy as np 5 | 6 | from ..annotation import AnnotationDet 7 | from . import utils 8 | 9 | 10 | class Preprocess(metaclass=ABCMeta): 11 | @abstractmethod 12 | def __call__(self, image, anns, meta): 13 | """Implementation of preprocess operation.""" 14 | 15 | @staticmethod 16 | def keypoint_sets_inverse(keypoint_sets, meta): 17 | keypoint_sets = keypoint_sets.copy() 18 | 19 | keypoint_sets[:, :, 0] += meta['offset'][0] 20 | keypoint_sets[:, :, 1] += meta['offset'][1] 21 | 22 | keypoint_sets[:, :, 0] = keypoint_sets[:, :, 0] / meta['scale'][0] 23 | keypoint_sets[:, :, 1] = keypoint_sets[:, :, 1] / meta['scale'][1] 24 | 25 | if meta['hflip']: 26 | w = meta['width_height'][0] 27 | keypoint_sets[:, :, 0] = -keypoint_sets[:, :, 0] + (w - 1) 28 | for keypoints in keypoint_sets: 29 | if meta.get('horizontal_swap'): 30 | keypoints[:] = meta['horizontal_swap'](keypoints) 31 | 32 | return keypoint_sets 33 | 34 | @staticmethod 35 | def annotations_inverse(annotations, meta): 36 | annotations = copy.deepcopy(annotations) 37 | 38 | # determine rotation parameters 39 | angle = -meta['rotation']['angle'] 40 | rw = meta['rotation']['width'] 41 | rh = meta['rotation']['height'] 42 | cangle = math.cos(angle / 180.0 * math.pi) 43 | sangle = math.sin(angle / 180.0 * math.pi) 44 | 45 | for ann in annotations: 46 | if isinstance(ann, AnnotationDet): 47 | Preprocess.anndet_inverse(ann, meta) 48 | continue 49 | 50 | # rotation 51 | if angle != 0.0: 52 | xy = ann.data[:, :2] 53 | x_old = xy[:, 0].copy() - (rw - 1)/2 54 | y_old = xy[:, 1].copy() - (rh - 1)/2 55 | xy[:, 0] = (rw - 1)/2 + cangle * x_old + sangle * y_old 56 | xy[:, 1] = (rh - 1)/2 - sangle * x_old + cangle * y_old 57 | 58 | # offset 59 | ann.data[:, 0] += meta['offset'][0] 60 | ann.data[:, 1] += meta['offset'][1] 61 | 62 | # scale 63 | ann.data[:, 0] = ann.data[:, 0] / meta['scale'][0] 64 | ann.data[:, 1] = ann.data[:, 1] / meta['scale'][1] 65 | ann.joint_scales /= meta['scale'][0] 66 | 67 | assert not np.any(np.isnan(ann.data)) 68 | 69 | if meta['hflip']: 70 | w = meta['width_height'][0] 71 | ann.data[:, 0] = -ann.data[:, 0] + (w - 1) 72 | if meta.get('horizontal_swap'): 73 | ann.data[:] = meta['horizontal_swap'](ann.data) 74 | 75 | for _, __, c1, c2 in ann.decoding_order: 76 | c1[:2] += meta['offset'] 77 | c2[:2] += meta['offset'] 78 | 79 | c1[:2] /= meta['scale'] 80 | c2[:2] /= meta['scale'] 81 | 82 | return annotations 83 | 84 | @staticmethod 85 | def anndet_inverse(ann, meta): 86 | angle = -meta['rotation']['angle'] 87 | if angle != 0.0: 88 | rw = meta['rotation']['width'] 89 | rh = meta['rotation']['height'] 90 | ann.bbox = utils.rotate_box(ann.bbox, rw - 1, rh - 1, angle) 91 | 92 | ann.bbox[:2] += meta['offset'] 93 | ann.bbox[:2] /= meta['scale'] 94 | ann.bbox[2:] /= meta['scale'] 95 | -------------------------------------------------------------------------------- /openpifpaf/transforms/random.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import torch 3 | 4 | from .preprocess import Preprocess 5 | 6 | LOG = logging.getLogger(__name__) 7 | 8 | 9 | class RandomApply(Preprocess): 10 | def __init__(self, transform, probability): 11 | self.transform = transform 12 | self.probability = probability 13 | 14 | def __call__(self, image, anns, meta): 15 | if float(torch.rand(1).item()) > self.probability: 16 | return image, anns, meta 17 | return self.transform(image, anns, meta) 18 | 19 | 20 | class DeterministicEqualChoice(Preprocess): 21 | def __init__(self, transforms, salt=0): 22 | self.transforms = transforms 23 | self.salt = salt 24 | 25 | def __call__(self, image, anns, meta): 26 | assert meta['image_id'] > 0 27 | LOG.debug('image id = %d', meta['image_id']) 28 | choice = hash(meta['image_id'] + self.salt) % len(self.transforms) 29 | t = self.transforms[choice] 30 | if t is None: 31 | return image, anns, meta 32 | return t(image, anns, meta) 33 | -------------------------------------------------------------------------------- /openpifpaf/transforms/unclipped.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | 4 | import numpy as np 5 | 6 | from .preprocess import Preprocess 7 | 8 | LOG = logging.getLogger(__name__) 9 | 10 | 11 | class UnclippedSides(Preprocess): 12 | def __init__(self, *, margin=10, clipped_sides_okay=2): 13 | self.margin = margin 14 | self.clipped_sides_okay = clipped_sides_okay 15 | 16 | def __call__(self, image, anns, meta): 17 | anns = copy.deepcopy(anns) 18 | meta_rb = (meta['valid_area'][0] + meta['valid_area'][2], 19 | meta['valid_area'][1] + meta['valid_area'][3]) 20 | for ann in anns: 21 | ann_rb = (ann['bbox'][0] + ann['bbox'][2], 22 | ann['bbox'][1] + ann['bbox'][3]) 23 | clipped_sides = 0 24 | if ann['bbox'][0] - meta['valid_area'][0] < self.margin: 25 | clipped_sides += 1 26 | if ann['bbox'][1] - meta['valid_area'][1] < self.margin: 27 | clipped_sides += 1 28 | if meta_rb[0] - ann_rb[0] < self.margin: 29 | clipped_sides += 1 30 | if meta_rb[1] - ann_rb[1] < self.margin: 31 | clipped_sides += 1 32 | 33 | if clipped_sides <= self.clipped_sides_okay: 34 | continue 35 | ann['iscrowd'] = True 36 | 37 | return image, anns, meta 38 | 39 | 40 | class UnclippedArea(Preprocess): 41 | def __init__(self, *, threshold=0.5): 42 | self.threshold = threshold 43 | 44 | def __call__(self, image, anns, meta): 45 | anns = copy.deepcopy(anns) 46 | for ann in anns: 47 | area_original = np.prod(ann['bbox_original'][2:]) 48 | area_origscale = np.prod(ann['bbox'][2:] / meta['scale']) 49 | LOG.debug('clipped = %.0f, orig = %.0f', area_origscale, area_original) 50 | 51 | if area_original > 0.0 and area_origscale / area_original > self.threshold: 52 | continue 53 | 54 | ann['iscrowd'] = True 55 | 56 | return image, anns, meta 57 | -------------------------------------------------------------------------------- /openpifpaf/transforms/utils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | 5 | def rotate_box(bbox, width, height, angle_degrees): 6 | """Input bounding box is of the form x, y, width, height.""" 7 | 8 | cangle = math.cos(angle_degrees / 180.0 * math.pi) 9 | sangle = math.sin(angle_degrees / 180.0 * math.pi) 10 | 11 | four_corners = np.array([ 12 | [bbox[0], bbox[1]], 13 | [bbox[0] + bbox[2], bbox[1]], 14 | [bbox[0], bbox[1] + bbox[3]], 15 | [bbox[0] + bbox[2], bbox[1] + bbox[3]], 16 | ]) 17 | 18 | x_old = four_corners[:, 0].copy() - width/2 19 | y_old = four_corners[:, 1].copy() - height/2 20 | four_corners[:, 0] = width/2 + cangle * x_old + sangle * y_old 21 | four_corners[:, 1] = height/2 - sangle * x_old + cangle * y_old 22 | 23 | x = np.min(four_corners[:, 0]) 24 | y = np.min(four_corners[:, 1]) 25 | xmax = np.max(four_corners[:, 0]) 26 | ymax = np.max(four_corners[:, 1]) 27 | 28 | return np.array([x, y, xmax - x, ymax - y]) 29 | -------------------------------------------------------------------------------- /openpifpaf/transforms/zoomscale.py: -------------------------------------------------------------------------------- 1 | 2 | import PIL 3 | import copy 4 | 5 | from .preprocess import Preprocess 6 | 7 | class ZoomScale(Preprocess): 8 | def __init__(self): 9 | self.zoom = 1 10 | 11 | def __call__(self, image, anns, meta): 12 | w, h = image.size 13 | target_w, target_h = (w/self.zoom, h/self.zoom) 14 | 15 | top = int((h - target_h)/2) 16 | bottom = int(target_h + (h - target_h)/2) 17 | left = int((w - target_w)/2) 18 | right = int(target_w + (w - target_w)/2) 19 | 20 | # change annotations 21 | anns = copy.deepcopy(anns) 22 | for ann in anns: 23 | ann['bmask'] = ann['bmask'][top:bottom, left:right] 24 | 25 | return image, anns, meta -------------------------------------------------------------------------------- /openpifpaf/utils.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import math 3 | import numpy as np 4 | 5 | 6 | @functools.lru_cache(maxsize=64) 7 | def create_sink(side): 8 | if side == 1: 9 | return np.zeros((2, 1, 1)) 10 | 11 | sink1d = np.linspace((side - 1.0) / 2.0, -(side - 1.0) / 2.0, num=side, dtype=np.float32) 12 | sink = np.stack(( 13 | sink1d.reshape(1, -1).repeat(side, axis=0), 14 | sink1d.reshape(-1, 1).repeat(side, axis=1), 15 | ), axis=0) 16 | return sink 17 | 18 | 19 | def mask_valid_area(intensities, valid_area, *, fill_value=0): 20 | """Mask area. 21 | 22 | Intensities is either a feature map or an image. 23 | """ 24 | if valid_area is None: 25 | return 26 | 27 | if valid_area[1] >= 1.0: 28 | intensities[:, :int(valid_area[1]), :] = fill_value 29 | if valid_area[0] >= 1.0: 30 | intensities[:, :, :int(valid_area[0])] = fill_value 31 | 32 | max_i = int(math.ceil(valid_area[1] + valid_area[3])) + 1 33 | max_j = int(math.ceil(valid_area[0] + valid_area[2])) + 1 34 | if 0 < max_i < intensities.shape[1]: 35 | intensities[:, max_i:, :] = fill_value 36 | if 0 < max_j < intensities.shape[2]: 37 | intensities[:, :, max_j:] = fill_value 38 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/__init__.py: -------------------------------------------------------------------------------- 1 | """Higher level drawing functions.""" 2 | 3 | from .base import BaseVisualizer 4 | from .cif import Cif 5 | from .cifdet import CifDet 6 | from .cifhr import CifHr 7 | from .cli import cli, configure 8 | from .caf import Caf 9 | from .occupancy import Occupancy 10 | from .seeds import Seeds 11 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/base.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | import logging 3 | import numpy as np 4 | 5 | from .. import show 6 | 7 | try: 8 | import matplotlib.cm 9 | import matplotlib.pyplot as plt 10 | from mpl_toolkits.axes_grid1 import make_axes_locatable 11 | 12 | matplotlib.cm.get_cmap('Oranges').set_bad('white', alpha=0.5) 13 | matplotlib.cm.get_cmap('Blues').set_bad('white', alpha=0.5) 14 | matplotlib.cm.get_cmap('Greens').set_bad('white', alpha=0.5) 15 | except ImportError: 16 | plt = None 17 | make_axes_locatable = None 18 | 19 | LOG = logging.getLogger(__name__) 20 | 21 | 22 | class BaseVisualizer: 23 | all_indices = [] 24 | common_ax = None 25 | _image = None 26 | _processed_image = None 27 | 28 | def __init__(self, head_name): 29 | self.head_name = head_name 30 | self._ax = None 31 | 32 | LOG.debug('%s: indices = %s', head_name, self.indices) 33 | 34 | @staticmethod 35 | def image(image): 36 | if image is None: 37 | BaseVisualizer._image = None 38 | return 39 | 40 | BaseVisualizer._image = np.asarray(image) 41 | 42 | @staticmethod 43 | def processed_image(image): 44 | if image is None: 45 | BaseVisualizer._processed_image = None 46 | return 47 | 48 | image = np.moveaxis(np.asarray(image), 0, -1) 49 | image = np.clip(image * 0.25 + 0.5, 0.0, 1.0) 50 | BaseVisualizer._processed_image = image 51 | 52 | @staticmethod 53 | def reset(): 54 | BaseVisualizer._image = None 55 | BaseVisualizer._processed_image = None 56 | 57 | @property 58 | def indices(self): 59 | head_names = self.head_name 60 | if not isinstance(head_names, (tuple, list)): 61 | head_names = (head_names,) 62 | return [f for hn, f in self.all_indices if hn in head_names] 63 | 64 | @staticmethod 65 | def colorbar(ax, colored_element, size='3%', pad=0.05): 66 | divider = make_axes_locatable(ax) 67 | cax = divider.append_axes('right', size=size, pad=pad) 68 | plt.colorbar(colored_element, cax=cax) 69 | 70 | @contextmanager 71 | def image_canvas(self, image, *args, **kwargs): 72 | ax = self._ax or self.common_ax 73 | if ax is not None: 74 | ax.set_axis_off() 75 | ax.imshow(np.asarray(image)) 76 | yield ax 77 | return 78 | 79 | with show.image_canvas(image, *args, **kwargs) as ax: 80 | yield ax 81 | 82 | @contextmanager 83 | def canvas(self, *args, **kwargs): 84 | ax = self._ax or self.common_ax 85 | if ax is not None: 86 | yield ax 87 | return 88 | 89 | with show.canvas(*args, **kwargs) as ax: 90 | yield ax 91 | 92 | @staticmethod 93 | def scale_scalar(field, stride): 94 | field = np.repeat(field, stride, 0) 95 | field = np.repeat(field, stride, 1) 96 | 97 | # center (the result is technically still off by half a pixel) 98 | half_stride = stride // 2 99 | return field[half_stride:-half_stride+1, half_stride:-half_stride+1] 100 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/caf.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | from ..annotation import Annotation 5 | from .. import show 6 | 7 | LOG = logging.getLogger(__name__) 8 | 9 | 10 | class Caf(BaseVisualizer): 11 | show_margin = False 12 | show_background = False 13 | show_confidences = False 14 | show_regressions = False 15 | 16 | def __init__(self, head_name, *, stride=1, keypoints=None, skeleton=None): 17 | super().__init__(head_name) 18 | 19 | self.stride = stride 20 | self.keypoints = keypoints 21 | self.skeleton = skeleton 22 | 23 | self.keypoint_painter = show.KeypointPainter(xy_scale=self.stride) 24 | 25 | def targets(self, field, keypoint_sets): 26 | assert self.keypoints is not None 27 | assert self.skeleton is not None 28 | 29 | annotations = [ 30 | Annotation(keypoints=self.keypoints, skeleton=self.skeleton).set(kps, fixed_score=None) 31 | for kps in keypoint_sets 32 | ] 33 | 34 | self._confidences(field[0]) 35 | self._regressions(field[1], field[2], field[3], field[4], annotations=annotations) 36 | 37 | def predicted(self, field, *, annotations=None): 38 | self._confidences(field[:, 0]) 39 | self._regressions(field[:, 1:3], field[:, 5:7], field[:, 4], field[:, 8], 40 | annotations=annotations, 41 | confidence_fields=field[:, 0], 42 | uv_is_offset=False) 43 | 44 | def _confidences(self, confidences): 45 | if not self.show_confidences: 46 | return 47 | 48 | for f in self.indices: 49 | LOG.debug('%s,%s', 50 | self.keypoints[self.skeleton[f][0] - 1], 51 | self.keypoints[self.skeleton[f][1] - 1]) 52 | 53 | with self.image_canvas(self._processed_image) as ax: 54 | im = ax.imshow(self.scale_scalar(confidences[f], self.stride), 55 | alpha=0.9, vmin=0.0, vmax=1.0, cmap='Blues') 56 | self.colorbar(ax, im) 57 | 58 | def _regressions(self, regression_fields1, regression_fields2, 59 | scale_fields1, scale_fields2, *, 60 | annotations=None, confidence_fields=None, uv_is_offset=True): 61 | if not self.show_regressions: 62 | return 63 | 64 | for f in self.indices: 65 | LOG.debug('%s,%s', 66 | self.keypoints[self.skeleton[f][0] - 1], 67 | self.keypoints[self.skeleton[f][1] - 1]) 68 | confidence_field = confidence_fields[f] if confidence_fields is not None else None 69 | 70 | with self.image_canvas(self._processed_image) as ax: 71 | show.white_screen(ax, alpha=0.5) 72 | if annotations: 73 | self.keypoint_painter.annotations(ax, annotations) 74 | q1 = show.quiver(ax, 75 | regression_fields1[f, :2], 76 | confidence_field=confidence_field, 77 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 78 | cmap='Blues', clim=(0.5, 1.0), width=0.001) 79 | show.quiver(ax, 80 | regression_fields2[f, :2], 81 | confidence_field=confidence_field, 82 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 83 | cmap='Greens', clim=(0.5, 1.0), width=0.001) 84 | show.boxes(ax, scale_fields1[f] / 2.0, 85 | confidence_field=confidence_field, 86 | regression_field=regression_fields1[f, :2], 87 | xy_scale=self.stride, cmap='Blues', fill=False, 88 | regression_field_is_offset=uv_is_offset) 89 | show.boxes(ax, scale_fields2[f] / 2.0, 90 | confidence_field=confidence_field, 91 | regression_field=regression_fields2[f, :2], 92 | xy_scale=self.stride, cmap='Greens', fill=False, 93 | regression_field_is_offset=uv_is_offset) 94 | if self.show_margin: 95 | show.margins(ax, regression_fields1[f, :6], xy_scale=self.stride) 96 | show.margins(ax, regression_fields2[f, :6], xy_scale=self.stride) 97 | 98 | self.colorbar(ax, q1) 99 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/cif.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import imageio 3 | from .base import BaseVisualizer 4 | from ..annotation import Annotation 5 | from .. import show 6 | 7 | LOG = logging.getLogger(__name__) 8 | 9 | 10 | class Cif(BaseVisualizer): 11 | show_margin = False 12 | show_confidences = False 13 | show_regressions = False 14 | show_background = False 15 | 16 | def __init__(self, head_name, *, stride=1, keypoints=None, skeleton=None): 17 | super().__init__(head_name) 18 | 19 | self.stride = stride 20 | self.keypoints = keypoints 21 | self.skeleton = skeleton 22 | 23 | self.keypoint_painter = show.KeypointPainter(xy_scale=self.stride) 24 | 25 | def targets(self, field, keypoint_sets): 26 | assert self.keypoints is not None 27 | #assert self.skeleton is not None 28 | 29 | annotations = [ 30 | Annotation(keypoints=self.keypoints, skeleton=self.skeleton).set(kps, fixed_score=None) 31 | for kps in keypoint_sets 32 | ] 33 | 34 | self._confidences(field[0]) 35 | self._regressions(field[1], field[2], annotations=annotations) 36 | 37 | def predicted(self, field, *, annotations=None): 38 | self._confidences(field[:, 0]) 39 | self._regressions(field[:, 1:3], field[:, 4], 40 | annotations=annotations, 41 | confidence_fields=field[:, 0], 42 | uv_is_offset=False) 43 | 44 | def _confidences(self, confidences): 45 | if not self.show_confidences: 46 | return 47 | 48 | for f in self.indices: 49 | LOG.debug('%s', self.keypoints[f]) 50 | 51 | with self.image_canvas(self._processed_image) as ax: 52 | img = self.scale_scalar(confidences[f], self.stride) 53 | 54 | im = ax.imshow(img, 55 | alpha=0.9, vmin=0.0, vmax=1.0, cmap='Oranges') 56 | self.colorbar(ax, im) 57 | 58 | def _regressions(self, regression_fields, scale_fields, *, 59 | annotations=None, confidence_fields=None, uv_is_offset=True): 60 | if not self.show_regressions: 61 | return 62 | 63 | for f in self.indices: 64 | LOG.debug('%s', self.keypoints[f]) 65 | confidence_field = confidence_fields[f] if confidence_fields is not None else None 66 | 67 | with self.image_canvas(self._processed_image) as ax: 68 | show.white_screen(ax, alpha=0.5) 69 | if annotations: 70 | self.keypoint_painter.annotations(ax, annotations) 71 | q = show.quiver(ax, 72 | regression_fields[f, :2], 73 | confidence_field=confidence_field, 74 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 75 | cmap='Oranges', clim=(0.5, 1.0), width=0.001) 76 | show.boxes(ax, scale_fields[f] / 2.0, 77 | confidence_field=confidence_field, 78 | regression_field=regression_fields[f, :2], 79 | xy_scale=self.stride, cmap='Oranges', fill=False, 80 | regression_field_is_offset=uv_is_offset) 81 | if self.show_margin: 82 | show.margins(ax, regression_fields[f, :6], xy_scale=self.stride) 83 | 84 | self.colorbar(ax, q) 85 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/cifdet.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | from ..annotation import AnnotationDet 5 | from .. import show 6 | 7 | LOG = logging.getLogger(__name__) 8 | 9 | 10 | class CifDet(BaseVisualizer): 11 | show_margin = False 12 | show_confidences = False 13 | show_regressions = False 14 | show_background = False 15 | 16 | def __init__(self, head_name, *, stride=1, categories=None): 17 | super().__init__(head_name) 18 | 19 | self.stride = stride 20 | self.categories = categories 21 | 22 | self.detection_painter = show.DetectionPainter(xy_scale=stride) 23 | 24 | def targets(self, field, detections): 25 | assert self.categories is not None 26 | 27 | annotations = [ 28 | AnnotationDet(self.categories).set(det[0] - 1, None, det[1]) 29 | for det in detections 30 | ] 31 | 32 | self._confidences(field[0]) 33 | self._regressions(field[1], field[2], annotations=annotations) 34 | 35 | def predicted(self, field, *, annotations=None): 36 | self._confidences(field[:, 0]) 37 | self._regressions(field[:, 1:3], field[:, 4:6], 38 | annotations=annotations, 39 | confidence_fields=field[:, 0], 40 | uv_is_offset=False) 41 | 42 | def _confidences(self, confidences): 43 | if not self.show_confidences: 44 | return 45 | 46 | for f in self.indices: 47 | LOG.debug('%s', self.categories[f]) 48 | 49 | with self.image_canvas(self._processed_image) as ax: 50 | im = ax.imshow(self.scale_scalar(confidences[f], self.stride), 51 | alpha=0.9, vmin=0.0, vmax=1.0, cmap='Greens') 52 | self.colorbar(ax, im) 53 | 54 | def _regressions(self, regression_fields, wh_fields, *, 55 | annotations=None, confidence_fields=None, uv_is_offset=True): 56 | if not self.show_regressions: 57 | return 58 | 59 | for f in self.indices: 60 | LOG.debug('%s', self.categories[f]) 61 | confidence_field = confidence_fields[f] if confidence_fields is not None else None 62 | 63 | with self.image_canvas(self._processed_image) as ax: 64 | show.white_screen(ax, alpha=0.5) 65 | if annotations: 66 | self.detection_painter.annotations(ax, annotations, color='gray') 67 | q = show.quiver(ax, 68 | regression_fields[f, :2], 69 | confidence_field=confidence_field, 70 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 71 | cmap='Greens', clim=(0.5, 1.0), width=0.001) 72 | show.boxes_wh(ax, wh_fields[f, 0], wh_fields[f, 1], 73 | confidence_field=confidence_field, 74 | regression_field=regression_fields[f, :2], 75 | xy_scale=self.stride, cmap='Greens', fill=False, 76 | regression_field_is_offset=uv_is_offset) 77 | if self.show_margin: 78 | show.margins(ax, regression_fields[f, :6], xy_scale=self.stride) 79 | 80 | self.colorbar(ax, q) 81 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/cifhr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | 5 | LOG = logging.getLogger(__name__) 6 | 7 | 8 | class CifHr(BaseVisualizer): 9 | show = False 10 | 11 | def __init__(self, *, stride=1, field_names=None): 12 | super().__init__(('cif', 'cifdet')) 13 | 14 | self.stride = stride 15 | self.field_names = field_names 16 | 17 | def predicted(self, fields): 18 | if not self.show: 19 | return 20 | 21 | for f in self.indices: 22 | LOG.debug('%d (field name: %s)', 23 | f, self.field_names[f] if self.field_names else 'unknown') 24 | with self.image_canvas(self._processed_image) as ax: 25 | o = ax.imshow(fields[f], alpha=0.9, vmin=0.0, vmax=1.0, cmap='Oranges') 26 | self.colorbar(ax, o) 27 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/cli.py: -------------------------------------------------------------------------------- 1 | from .base import BaseVisualizer 2 | from .caf import Caf 3 | from .cif import Cif 4 | from .cifdet import CifDet 5 | from .cifhr import CifHr 6 | from .occupancy import Occupancy 7 | from .seeds import Seeds 8 | 9 | 10 | def cli(parser): 11 | group = parser.add_argument_group('pose visualizer') 12 | group.add_argument('--debug-cifhr', default=False, action='store_true') 13 | group.add_argument('--debug-cif-c', default=False, action='store_true') 14 | group.add_argument('--debug-cif-v', default=False, action='store_true') 15 | group.add_argument('--debug-cifdet-c', default=False, action='store_true') 16 | group.add_argument('--debug-cifdet-v', default=False, action='store_true') 17 | group.add_argument('--debug-caf-c', default=False, action='store_true') 18 | group.add_argument('--debug-caf-v', default=False, action='store_true') 19 | 20 | group.add_argument('--debug-indices', default=[], nargs='+', 21 | help=('indices of fields to create debug plots for ' 22 | 'of the form headname:fieldindex, e.g. cif:5')) 23 | 24 | 25 | def enable_all_plots(): 26 | Cif.show_background = True 27 | Cif.show_confidences = True 28 | Cif.show_regressions = True 29 | Caf.show_background = True 30 | Caf.show_confidences = True 31 | Caf.show_regressions = True 32 | CifDet.show_background = True 33 | CifDet.show_confidences = True 34 | CifDet.show_regressions = True 35 | CifHr.show = True 36 | Occupancy.show = True 37 | Seeds.show = True 38 | 39 | 40 | def configure(args): 41 | # configure visualizer 42 | args.debug_indices = [di.partition(':') for di in args.debug_indices] 43 | args.debug_indices = [(di[0], int(di[2])) for di in args.debug_indices] 44 | BaseVisualizer.all_indices = args.debug_indices 45 | 46 | Caf.show_confidences = args.debug_caf_c 47 | Caf.show_regressions = args.debug_caf_v 48 | Cif.show_confidences = args.debug_cif_c 49 | Cif.show_regressions = args.debug_cif_v 50 | CifDet.show_confidences = args.debug_cifdet_c 51 | CifDet.show_regressions = args.debug_cifdet_v 52 | CifHr.show = args.debug_cifhr 53 | 54 | if args.debug_images: 55 | enable_all_plots() 56 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/occupancy.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | 5 | LOG = logging.getLogger(__name__) 6 | 7 | 8 | class Occupancy(BaseVisualizer): 9 | show = False 10 | 11 | def __init__(self, *, field_names=None): 12 | super().__init__('occupancy') 13 | self.field_names = field_names 14 | 15 | def predicted(self, occupancy): 16 | if not self.show: 17 | return 18 | 19 | for f in self.indices: 20 | LOG.debug('%d (field name: %s)', 21 | f, self.field_names[f] if self.field_names else 'unknown') 22 | 23 | # occupancy maps are at a reduced scale wrt the processed image 24 | reduced_image = self._processed_image[::occupancy.reduction, ::occupancy.reduction] 25 | 26 | with self.image_canvas(reduced_image) as ax: 27 | occ = occupancy.occupancy[f].copy() 28 | occ[occ > 0] = 1.0 29 | ax.imshow(occ, alpha=0.7) 30 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/seeds.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | from .. import show 5 | 6 | LOG = logging.getLogger(__name__) 7 | 8 | 9 | class Seeds(BaseVisualizer): 10 | show = False 11 | show_confidences = False 12 | 13 | def __init__(self, *, stride=1): 14 | super().__init__('seeds') 15 | self.stride = stride 16 | 17 | def predicted(self, seeds): 18 | """Seeds are: confidence, field_index, x, y, ...""" 19 | if not self.show: 20 | return 21 | 22 | field_indices = {s[1] for s in seeds} 23 | 24 | with self.image_canvas(self._processed_image) as ax: 25 | show.white_screen(ax) 26 | for f in field_indices: 27 | x = [s[2] * self.stride for s in seeds if s[1] == f] 28 | y = [s[3] * self.stride for s in seeds if s[1] == f] 29 | ax.plot(x, y, 'o') 30 | if self.show_confidences: 31 | c = [s[0] for s in seeds if s[1] == f] 32 | for xx, yy, cc in zip(x, y, c): 33 | ax.text(xx, yy, '{:.2f}'.format(cc)) 34 | -------------------------------------------------------------------------------- /openpifpaf/visualizer/seg.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .base import BaseVisualizer 4 | from ..annotation import Annotation 5 | from .. import show 6 | 7 | LOG = logging.getLogger(__name__) 8 | 9 | ### AMA 10 | class Seg(BaseVisualizer): 11 | show_margin = False 12 | show_background = False 13 | show_confidences = False 14 | show_regressions = False 15 | 16 | def __init__(self, head_name, *, stride=1, keypoints=None, skeleton=None): 17 | super().__init__(head_name) 18 | 19 | self.stride = stride 20 | self.keypoints = keypoints 21 | self.skeleton = skeleton 22 | 23 | self.keypoint_painter = show.KeypointPainter(xy_scale=self.stride) 24 | 25 | def targets(self, field, keypoint_sets): 26 | assert self.keypoints is not None 27 | assert self.skeleton is not None 28 | 29 | annotations = [ 30 | Annotation(keypoints=self.keypoints, skeleton=self.skeleton).set(kps, fixed_score=None) 31 | for kps in keypoint_sets 32 | ] 33 | 34 | self._confidences(field[0]) 35 | self._regressions(field[1], field[2], field[3], field[4], annotations=annotations) 36 | 37 | def predicted(self, field, *, annotations=None): 38 | self._confidences(field[:, 0]) 39 | self._regressions(field[:, 1:3], field[:, 5:7], field[:, 4], field[:, 8], 40 | annotations=annotations, 41 | confidence_fields=field[:, 0], 42 | uv_is_offset=False) 43 | 44 | def _confidences(self, confidences): 45 | if not self.show_confidences: 46 | return 47 | 48 | for f in self.indices: 49 | LOG.debug('%s,%s', 50 | self.keypoints[self.skeleton[f][0] - 1], 51 | self.keypoints[self.skeleton[f][1] - 1]) 52 | 53 | with self.image_canvas(self._processed_image) as ax: 54 | im = ax.imshow(self.scale_scalar(confidences[f], self.stride), 55 | alpha=0.9, vmin=0.0, vmax=1.0, cmap='Blues') 56 | self.colorbar(ax, im) 57 | 58 | def _regressions(self, regression_fields1, regression_fields2, 59 | scale_fields1, scale_fields2, *, 60 | annotations=None, confidence_fields=None, uv_is_offset=True): 61 | if not self.show_regressions: 62 | return 63 | 64 | for f in self.indices: 65 | LOG.debug('%s,%s', 66 | self.keypoints[self.skeleton[f][0] - 1], 67 | self.keypoints[self.skeleton[f][1] - 1]) 68 | confidence_field = confidence_fields[f] if confidence_fields is not None else None 69 | 70 | with self.image_canvas(self._processed_image) as ax: 71 | show.white_screen(ax, alpha=0.5) 72 | if annotations: 73 | self.keypoint_painter.annotations(ax, annotations) 74 | q1 = show.quiver(ax, 75 | regression_fields1[f, :2], 76 | confidence_field=confidence_field, 77 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 78 | cmap='Blues', clim=(0.5, 1.0), width=0.001) 79 | show.quiver(ax, 80 | regression_fields2[f, :2], 81 | confidence_field=confidence_field, 82 | xy_scale=self.stride, uv_is_offset=uv_is_offset, 83 | cmap='Greens', clim=(0.5, 1.0), width=0.001) 84 | show.boxes(ax, scale_fields1[f] / 2.0, 85 | confidence_field=confidence_field, 86 | regression_field=regression_fields1[f, :2], 87 | xy_scale=self.stride, cmap='Blues', fill=False, 88 | regression_field_is_offset=uv_is_offset) 89 | show.boxes(ax, scale_fields2[f] / 2.0, 90 | confidence_field=confidence_field, 91 | regression_field=regression_fields2[f, :2], 92 | xy_scale=self.stride, cmap='Greens', fill=False, 93 | regression_field_is_offset=uv_is_offset) 94 | if self.show_margin: 95 | show.margins(ax, regression_fields1[f, :6], xy_scale=self.stride) 96 | show.margins(ax, regression_fields2[f, :6], xy_scale=self.stride) 97 | 98 | self.colorbar(ax, q1) 99 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | -------------------------------------------------------------------------------- /run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ENVDIR=${ENVDIR:-`dirname $0`/env} 4 | 5 | source $ENVDIR/bin/activate 6 | if [[ $# -eq 0 ]] ; then 7 | $BASH --noprofile --norc -i 8 | else 9 | "$@" 10 | fi 11 | -------------------------------------------------------------------------------- /setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | _ENVDIR=${ENVDIR} 3 | ENVDIR=${ENVDIR:-`dirname $0`/env} 4 | 5 | git submodule update --init --recursive 6 | python -m venv $ENVDIR 7 | source $ENVDIR/bin/activate 8 | pip install -r requirements.txt 9 | if [[ -z $_ENVDIR ]] ; then 10 | pip install nbstripout 11 | nbstripout --install 12 | fi 13 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [options] 2 | setup_requires = 3 | numpy 4 | cython 5 | 6 | [tool:pytest] 7 | norecursedirs = checkpoints data-mscoco demo dist docs outputs .git .eggs 8 | 9 | [versioneer] 10 | VCS = git 11 | style = pep440 12 | versionfile_source = openpifpaf/_version.py 13 | # versionfile_build = openpifpaf/_version.py 14 | tag_prefix = v 15 | #parentdir_prefix = 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools.extension import Extension 3 | 4 | try: 5 | from Cython.Build import cythonize 6 | except ImportError: 7 | cythonize = None 8 | try: 9 | import numpy 10 | except ImportError: 11 | numpy = None 12 | 13 | import versioneer 14 | 15 | 16 | class NumpyIncludePath(): 17 | """Lazy import of numpy to get include path.""" 18 | @staticmethod 19 | def __str__(): 20 | import numpy 21 | return numpy.get_include() 22 | 23 | 24 | if cythonize is not None and numpy is not None: 25 | EXTENSIONS = cythonize([Extension('openpifpaf.functional', 26 | ['openpifpaf/functional.pyx'], 27 | include_dirs=[numpy.get_include()]), 28 | ], 29 | annotate=True, 30 | compiler_directives={'language_level': 3}) 31 | else: 32 | EXTENSIONS = [Extension('openpifpaf.functional', 33 | ['openpifpaf/functional.pyx'], 34 | include_dirs=[NumpyIncludePath()])] 35 | 36 | 37 | setup( 38 | name='openpifpaf', 39 | version=versioneer.get_version(), 40 | cmdclass=versioneer.get_cmdclass(), 41 | packages=[ 42 | 'openpifpaf', 43 | 'openpifpaf.datasets', 44 | 'openpifpaf.decoder', 45 | 'openpifpaf.decoder.generator', 46 | 'openpifpaf.encoder', 47 | 'openpifpaf.network', 48 | 'openpifpaf.show', 49 | 'openpifpaf.transforms', 50 | 'openpifpaf.visualizer', 51 | ], 52 | license='GNU AGPLv3', 53 | description='PifPaf: Composite Fields for Human Pose Estimation', 54 | long_description=open('README.md', encoding='utf-8').read(), 55 | long_description_content_type='text/markdown', 56 | author='Sven Kreiss', 57 | author_email='research@svenkreiss.com', 58 | url='https://github.com/vita-epfl/openpifpaf', 59 | ext_modules=EXTENSIONS, 60 | zip_safe=False, 61 | 62 | python_requires='>=3.6', 63 | install_requires=[ 64 | 'numpy>=1.16', 65 | 'pysparkling', # for log analysis 66 | 'python-json-logger', 67 | 'scipy', 68 | 'torch>=1.3.1', 69 | 'torchvision>=0.4', 70 | 'pillow', 71 | 'dataclasses; python_version<"3.7"', 72 | ], 73 | extras_require={ 74 | 'dev': [ 75 | 'flameprof', 76 | 'jupyter-book>=0.7.4', 77 | 'matplotlib', 78 | 'nbdime', 79 | 'nbstripout', 80 | ], 81 | 'onnx': [ 82 | 'onnx', 83 | 'onnx-simplifier>=0.2.9', 84 | ], 85 | 'test': [ 86 | 'nbval', 87 | 'onnx', 88 | 'onnx-simplifier>=0.2.9', 89 | 'pylint', 90 | 'pytest', 91 | 'opencv-python', 92 | 'thop', 93 | ], 94 | 'train': [ 95 | 'matplotlib', # required by pycocotools 96 | 'pycocotools', # pre-install cython (currently incompatible with numpy 1.18 or above) 97 | ], 98 | }, 99 | ) 100 | -------------------------------------------------------------------------------- /tests/coco/images/puppy_dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ispgroupucl/DeepSportLab/67b03dd7e8b012de6309adf3fce2694046ccbdad/tests/coco/images/puppy_dog.jpg -------------------------------------------------------------------------------- /tests/coco/train1.json: -------------------------------------------------------------------------------- 1 | { 2 | "images": [{ 3 | "file_name": "puppy_dog.jpg", 4 | "id": 1234 5 | }], 6 | "categories": [ 7 | { 8 | "supercategory": "person", 9 | "id": 1, 10 | "name": "person", 11 | "keypoints": [ 12 | "nose","left_eye","right_eye","left_ear","right_ear", 13 | "left_shoulder","right_shoulder","left_elbow","right_elbow", 14 | "left_wrist","right_wrist","left_hip","right_hip", 15 | "left_knee","right_knee","left_ankle","right_ankle" 16 | ], 17 | "skeleton": [ 18 | [16,14],[14,12],[17,15],[15,13],[12,13],[6,12],[7,13],[6,7], 19 | [6,8],[7,9],[8,10],[9,11],[2,3],[1,2],[1,3],[2,4],[3,5],[4,6],[5,7] 20 | ] 21 | } 22 | ], 23 | "annotations": [ 24 | { 25 | "num_keypoints": 17, 26 | "iscrowd": 0, 27 | "keypoints": [ 28 | 100, 100, 2, 29 | 100, 100, 2, 30 | 100, 100, 2, 31 | 100, 100, 2, 32 | 100, 100, 2, 33 | 100, 100, 2, 34 | 100, 100, 2, 35 | 100, 100, 2, 36 | 100, 100, 2, 37 | 100, 100, 2, 38 | 100, 100, 2, 39 | 100, 100, 2, 40 | 100, 100, 2, 41 | 100, 100, 2, 42 | 100, 100, 2, 43 | 100, 100, 2, 44 | 100, 100, 2 45 | ], 46 | "image_id": 1234, 47 | "bbox": [80, 80, 40, 40], 48 | "category_id": 1, 49 | "id": 1235 50 | } 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /tests/test_clis.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | import pytest 7 | 8 | 9 | PYTHON = 'python3' if sys.platform != 'win32' else 'python' 10 | 11 | 12 | @pytest.mark.parametrize( 13 | 'batch_size,with_debug,with_dense', 14 | [(1, False, False), (2, False, False), (1, True, False), (1, False, True)]) 15 | def test_predict(batch_size, with_debug, with_dense, tmpdir): 16 | """Test predict cli. 17 | 18 | with_debug makes sure that debugging works in this environment. 19 | For example, at some point, --debug required matplotlib which was unintentional. 20 | """ 21 | 22 | if batch_size > 1 and sys.platform.startswith('win'): 23 | pytest.skip('multiprocess decoding not supported on windows') 24 | 25 | cmd = [ 26 | PYTHON, '-m', 'openpifpaf.predict', 27 | '--checkpoint=shufflenetv2k16w', 28 | '--batch-size={}'.format(batch_size), 29 | '--loader-workers=0', 30 | '--json-output', str(tmpdir), 31 | '--long-edge=181', 32 | 'docs/coco/000000081988.jpg', 33 | ] 34 | if with_debug: 35 | cmd.append('--debug') 36 | if with_dense: 37 | cmd.append('--dense-connections') 38 | 39 | subprocess.run(cmd, check=True) 40 | assert os.path.exists(os.path.join(tmpdir, '000000081988.jpg.predictions.json')) 41 | 42 | 43 | def test_predict_realistic_resolution(tmpdir): 44 | """Test predict cli at realistic resolution.""" 45 | 46 | cmd = [ 47 | PYTHON, '-m', 'openpifpaf.predict', 48 | '--checkpoint=shufflenetv2k16w', 49 | '--batch-size=1', 50 | '--loader-workers=0', 51 | '--json-output', str(tmpdir), 52 | '--long-edge=641', 53 | 'docs/coco/000000081988.jpg', 54 | ] 55 | subprocess.run(cmd, check=True) 56 | 57 | out_file = os.path.join(tmpdir, '000000081988.jpg.predictions.json') 58 | assert os.path.exists(out_file) 59 | 60 | with open(out_file, 'r') as f: 61 | predictions = json.load(f) 62 | 63 | assert len(predictions) == 5 64 | 65 | 66 | @pytest.mark.skipif(sys.platform == 'win32', reason='does not run on windows') 67 | @pytest.mark.parametrize('with_debug', [False, True]) 68 | def test_video(with_debug, tmpdir): 69 | cmd = [ 70 | PYTHON, '-m', 'openpifpaf.video', 71 | '--checkpoint=shufflenetv2k16w', 72 | '--source=docs/coco/000000081988.jpg', 73 | '--json-output={}'.format(os.path.join(tmpdir, 'video.json')), 74 | ] 75 | if with_debug: 76 | cmd.append('--debug') 77 | 78 | subprocess.run(cmd, check=True) 79 | assert os.path.exists(os.path.join(tmpdir, 'video.json')) 80 | -------------------------------------------------------------------------------- /tests/test_forward.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | import openpifpaf 4 | 5 | 6 | def test_forward(): 7 | openpifpaf.network.heads.CompositeFieldFused.quad = 0 8 | model, _ = openpifpaf.network.factory( 9 | base_name='resnet18', 10 | head_names=['cif', 'caf', 'caf25'], 11 | pretrained=False, 12 | ) 13 | 14 | dummy_image_batch = torch.zeros((1, 3, 241, 321)) 15 | cif, caf = model(dummy_image_batch) 16 | assert cif.shape == (1, 17, 5, 16, 21) 17 | assert caf.shape == (1, 19, 9, 16, 21) 18 | 19 | 20 | def test_forward_dense(): 21 | openpifpaf.network.heads.CompositeFieldFused.quad = 0 22 | model, _ = openpifpaf.network.factory( 23 | base_name='resnet18', 24 | head_names=['cif', 'caf', 'caf25'], 25 | dense_connections=True, 26 | pretrained=False, 27 | ) 28 | 29 | dummy_image_batch = torch.zeros((1, 3, 241, 321)) 30 | cif, caf = model(dummy_image_batch) 31 | assert cif.shape == (1, 17, 5, 16, 21) 32 | assert caf.shape == (1, 19 + 25, 9, 16, 21) 33 | 34 | 35 | def test_forward_headquad(): 36 | openpifpaf.network.heads.CompositeFieldFused.quad = 1 37 | model, _ = openpifpaf.network.factory( 38 | base_name='resnet18', 39 | head_names=['cif', 'caf', 'caf25'], 40 | pretrained=False, 41 | ) 42 | 43 | dummy_image_batch = torch.zeros((1, 3, 241, 321)) 44 | cif, caf = model(dummy_image_batch) 45 | assert cif.shape == (1, 17, 5, 31, 41) 46 | assert caf.shape == (1, 19, 9, 31, 41) 47 | -------------------------------------------------------------------------------- /tests/test_help.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import sys 3 | 4 | import pytest 5 | 6 | from openpifpaf import __version__ 7 | 8 | 9 | PYTHON = 'python3' if sys.platform != 'win32' else 'python' 10 | 11 | 12 | MODULE_NAMES = [ 13 | 'predict', 14 | 'train', 15 | 'logs', 16 | 'eval_coco', 17 | 'export_onnx', 18 | 'migrate', 19 | 'count_ops', 20 | 'benchmark', 21 | ] 22 | 23 | 24 | if sys.platform != 'win32': 25 | MODULE_NAMES.append('video') 26 | 27 | 28 | @pytest.mark.parametrize('module_name', MODULE_NAMES) 29 | def test_help(module_name): 30 | help_text = subprocess.check_output([ 31 | PYTHON, '-m', 'openpifpaf.{}'.format(module_name), 32 | '--help', 33 | ]) 34 | 35 | assert len(help_text) > 10 36 | 37 | 38 | @pytest.mark.parametrize('module_name', MODULE_NAMES) 39 | def test_version(module_name): 40 | output = subprocess.check_output([ 41 | PYTHON, '-m', 'openpifpaf.{}'.format(module_name), 42 | '--version', 43 | ]) 44 | cli_version = output.decode().strip().replace('.dirty', '') 45 | 46 | assert cli_version == 'OpenPifPaf {}'.format(__version__.replace('.dirty', '')) 47 | 48 | 49 | @pytest.mark.parametrize('module_name', MODULE_NAMES) 50 | def test_usage(module_name): 51 | output = subprocess.check_output([ 52 | PYTHON, '-m', 'openpifpaf.{}'.format(module_name), 53 | '--help', 54 | ]) 55 | 56 | assert output.decode().startswith('usage: python3 -m openpifpaf.{}'.format(module_name)) 57 | -------------------------------------------------------------------------------- /tests/test_image_scale.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL.Image 3 | import pytest 4 | import scipy.ndimage 5 | 6 | 7 | @pytest.mark.parametrize('resample', [PIL.Image.BILINEAR, PIL.Image.BICUBIC]) 8 | @pytest.mark.xfail 9 | def test_pil_resize(resample): 10 | d_in = np.array([[0, 10, 20, 30, 40, 50]], dtype=np.uint8) 11 | image = PIL.Image.fromarray(d_in, mode='L') 12 | 13 | w, _ = image.size 14 | target_w = (w - 1) * 2 + 1 15 | image = image.resize((target_w, 1), resample=resample) 16 | 17 | d_out = np.asarray(image) 18 | print(d_out) 19 | assert np.all(d_in == d_out[0, ::2]) 20 | 21 | 22 | @pytest.mark.parametrize('order', [0, 1, 2, 3]) 23 | def test_scipy_zoom(order): 24 | d_in = np.array([[0, 10, 20, 30, 40, 50]], dtype=np.uint8) 25 | 26 | w = d_in.shape[1] 27 | target_w = (w - 1) * 2 + 1 28 | d_out = scipy.ndimage.zoom(d_in, (1, target_w / w), order=order) 29 | 30 | print(d_out) 31 | assert np.all(d_in == d_out[0, ::2]) 32 | -------------------------------------------------------------------------------- /tests/test_input_processing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import torchvision 4 | 5 | 6 | def test_resize(): 7 | np_image = np.zeros((5, 5)) 8 | np_image[2, 2] = 1.0 9 | 10 | image = PIL.Image.fromarray(np_image) 11 | image = torchvision.transforms.functional.resize(image, (10, 10)) 12 | 13 | np_result = np.asarray(image) 14 | assert np.all(np_result[:5] == np_result[:4:-1]) # symmetric 15 | assert np.all(np_result[:, :5] == np_result[:, :4:-1]) # symmetric 16 | 17 | 18 | def test_resize_bicubic(): 19 | np_image = np.zeros((5, 5)) 20 | np_image[2, 2] = 1.0 21 | 22 | image = PIL.Image.fromarray(np_image) 23 | image = torchvision.transforms.functional.resize(image, (10, 10), PIL.Image.BICUBIC) 24 | 25 | np_result = np.asarray(image) 26 | assert np.all(np_result[:5] == np_result[:4:-1]) # symmetric 27 | assert np.all(np_result[:, :5] == np_result[:, :4:-1]) # symmetric 28 | -------------------------------------------------------------------------------- /tests/test_localization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import openpifpaf.network.nets 4 | import openpifpaf.utils 5 | from openpifpaf.datasets.constants import COCO_KEYPOINTS, COCO_PERSON_SKELETON 6 | 7 | 8 | def localize(x): 9 | openpifpaf.network.heads.CompositeFieldFused.quad = 0 10 | 11 | black = torch.zeros((3, 321, 321)) 12 | im = black.clone() 13 | im[:, 0, x] = 1000.0 14 | 15 | model, _ = openpifpaf.network.factory( 16 | base_name='resnet18', 17 | head_names=['cif', 'caf'], 18 | pretrained=False) 19 | model.eval() 20 | 21 | decode = openpifpaf.decoder.CifCaf( 22 | openpifpaf.decoder.FieldConfig(), 23 | keypoints=COCO_KEYPOINTS, skeleton=COCO_PERSON_SKELETON) 24 | cif_ref = decode.fields_batch(model, torch.unsqueeze(black, 0))[0][0] 25 | cif = decode.fields_batch(model, torch.unsqueeze(im, 0))[0][0] 26 | 27 | # intensity only, first field, first row 28 | cif_ref = cif_ref[0][0][0] 29 | cif = cif[0][0][0] 30 | assert len(cif_ref) == 21 # (321 - 1) / 16 + 1 31 | assert len(cif) == len(cif_ref) 32 | 33 | active_cif = np.nonzero(cif_ref - cif) 34 | return active_cif[0].tolist() 35 | 36 | 37 | def test_pixel_to_field_left(): 38 | assert localize(0) == [0, 1, 2, 3, 4, 5, 6] 39 | 40 | 41 | def test_pixel_to_field_center(): 42 | assert localize(160) == [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] 43 | 44 | 45 | def test_pixel_to_field_right(): 46 | assert localize(320) == [14, 15, 16, 17, 18, 19, 20] 47 | -------------------------------------------------------------------------------- /tests/test_multiprocessing.py: -------------------------------------------------------------------------------- 1 | """Tests configuration system for multiprocessing. 2 | 3 | Multiprocessing using 'fork' works in our configuration system but 4 | has been shown to be problematic on Mac-based systems with threading in the 5 | main process. 6 | Multiprocessing with 'spawn' does not work for configuration 7 | (new default for Python 3.8 on Mac). 8 | 9 | Ref: 10 | * https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods 11 | """ 12 | 13 | import multiprocessing 14 | import sys 15 | 16 | import pytest 17 | 18 | 19 | class Data: 20 | d = None 21 | 22 | 23 | def get_data(_): 24 | return Data.d 25 | 26 | 27 | def test_class_attr(): 28 | if sys.platform.startswith('win'): 29 | pytest.skip('multiprocessing not supported on windows') 30 | 31 | Data.d = 0.5 32 | 33 | multiprocessing_context = multiprocessing.get_context('fork') 34 | worker_pool = multiprocessing_context.Pool(2) 35 | result = worker_pool.starmap(get_data, [(0.0,), (1.0,)]) 36 | assert result == [0.5, 0.5] 37 | -------------------------------------------------------------------------------- /tests/test_network.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openpifpaf 4 | 5 | 6 | def test_local_checkpoint(): 7 | # make sure model is cached 8 | _, __ = openpifpaf.network.factory(checkpoint='shufflenetv2k16w') 9 | 10 | local_path = openpifpaf.network.local_checkpoint_path('shufflenetv2k16w') 11 | assert local_path is not None 12 | assert os.path.exists(local_path) 13 | -------------------------------------------------------------------------------- /tests/test_onnx_export.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | import torch 4 | 5 | import openpifpaf 6 | import openpifpaf.export_onnx 7 | 8 | 9 | @pytest.mark.skipif(not torch.__version__.startswith('1.5'), reason='only PyTorch 1.5') 10 | def test_onnx_exportable(tmpdir): 11 | outfile = str(tmpdir.join('openpifpaf-shufflenetv2k16w.onnx')) 12 | assert not os.path.exists(outfile) 13 | 14 | model, _ = openpifpaf.network.factory( 15 | base_name='shufflenetv2k16w', 16 | head_names=['cif', 'caf', 'caf25'], 17 | pretrained=False, 18 | ) 19 | openpifpaf.export_onnx.apply(model, outfile, verbose=False) 20 | assert os.path.exists(outfile) 21 | openpifpaf.export_onnx.check(outfile) 22 | 23 | openpifpaf.export_onnx.polish(outfile, outfile + '.polished') 24 | assert os.path.exists(outfile + '.polished') 25 | 26 | openpifpaf.export_onnx.simplify(outfile, outfile + '.simplified') 27 | assert os.path.exists(outfile + '.simplified') 28 | -------------------------------------------------------------------------------- /tests/test_scale_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | import openpifpaf 4 | 5 | 6 | def test_relative_scale_loss(): 7 | loss = openpifpaf.network.losses.ScaleLoss(1.0) 8 | x = torch.ones((4,)).log() 9 | t = torch.ones((4,)) 10 | loss_values = loss(x, t) 11 | assert loss_values.numpy().tolist() == [0, 0, 0, 0] 12 | 13 | 14 | def test_relative_scale_loss_masked(): 15 | loss = openpifpaf.network.losses.ScaleLoss(1.0) 16 | x = torch.ones((4,)).log() 17 | t = torch.ones((4,)) 18 | 19 | mask = x > 1.0 20 | x = torch.masked_select(x, mask) 21 | t = torch.masked_select(t, mask) 22 | 23 | loss_values = loss(x, t) 24 | assert loss_values.sum().numpy() == 0.0 25 | -------------------------------------------------------------------------------- /tests/test_train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import pytest 4 | 5 | 6 | TRAIN_COMMAND = [ 7 | 'python3', '-m', 'openpifpaf.train', 8 | '--lr=1e-3', 9 | '--momentum=0.9', 10 | '--epochs=1', 11 | '--batch-size=1', 12 | '--basenet=resnet18', '--no-pretrain', 13 | '--head-quad=1', 14 | '--headnets', 'cif', 'caf', 'caf25', 15 | '--square-edge=161', 16 | '--cocokp-train-annotations', 'tests/coco/train1.json', 17 | '--coco-train-image-dir', 'tests/coco/images/', 18 | '--cocokp-val-annotations', 'tests/coco/train1.json', 19 | '--coco-val-image-dir', 'tests/coco/images/', 20 | ] 21 | 22 | 23 | PREDICT_COMMAND = [ 24 | 'python3', '-m', 'openpifpaf.predict', 25 | 'tests/coco/images/puppy_dog.jpg', 26 | ] 27 | 28 | 29 | @pytest.mark.skipif(os.getenv('PIFPAFTRAINING') != '1', reason='env PIFPAFTRAINING is not set') 30 | def test_train(tmp_path): 31 | # train a model 32 | train_cmd = TRAIN_COMMAND + ['--out={}'.format(os.path.join(tmp_path, 'train_test.pkl'))] 33 | print(' '.join(train_cmd)) 34 | subprocess.run(train_cmd, check=True, capture_output=True) 35 | print(os.listdir(tmp_path)) 36 | 37 | # find the trained model checkpoint 38 | final_model = next(iter(f for f in os.listdir(tmp_path) if f.endswith('.pkl'))) 39 | 40 | # run a prediction with that model 41 | predict_cmd = PREDICT_COMMAND + [ 42 | '--checkpoint={}'.format(os.path.join(tmp_path, final_model)), 43 | '--json-output={}'.format(tmp_path), 44 | ] 45 | print(' '.join(predict_cmd)) 46 | subprocess.run(predict_cmd, check=True, capture_output=True) 47 | print(os.listdir(tmp_path)) 48 | 49 | assert 'puppy_dog.jpg.predictions.json' in os.listdir(tmp_path) 50 | -------------------------------------------------------------------------------- /tests/test_transforms.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import PIL 3 | import pytest 4 | 5 | from openpifpaf import transforms 6 | 7 | 8 | def apply_transform(im_np, anns, transform=None): 9 | im = PIL.Image.fromarray(im_np) 10 | 11 | transform_list = [transforms.NormalizeAnnotations()] 12 | if transform is not None: 13 | transform_list.append(transform) 14 | 15 | im_transformed, anns_transformed, meta = transforms.Compose(transform_list)(im, anns, None) 16 | im_transformed_np = np.asarray(im_transformed) 17 | 18 | return im_transformed_np, anns_transformed, meta 19 | 20 | 21 | def single_pixel_transform(x, y, transform, image_wh=(13, 11)): 22 | im = np.zeros((image_wh[1], image_wh[0], 3), dtype=np.uint8) 23 | im[y, x, :] = 255 24 | 25 | anns = [{ 26 | 'keypoints': [(x, y, 2.0)], 27 | 'bbox': [0.0, 0.0, image_wh[0], image_wh[1]], 28 | }] 29 | 30 | im_transformed, anns_transformed, _ = apply_transform(im, anns, transform) 31 | 32 | image_yx = np.unravel_index( 33 | np.argmax(im_transformed[:, :, 0]), 34 | shape=im_transformed[:, :, 0].shape, 35 | ) 36 | 37 | return ( 38 | [image_yx[1], image_yx[0]], 39 | # pylint: disable=unsubscriptable-object 40 | anns_transformed[0]['keypoints'][0][:2].tolist(), 41 | ) 42 | 43 | 44 | def test_rescale_absolute(x=5, y=5): 45 | image_xy, keypoint_xy = single_pixel_transform( 46 | x, y, transforms.RescaleAbsolute(7), image_wh=(11, 11)) 47 | print(image_xy, keypoint_xy) 48 | assert image_xy == keypoint_xy 49 | 50 | 51 | def test_crop(x=4, y=6): 52 | image_xy, keypoint_xy = single_pixel_transform(x, y, transforms.Crop(7), (9, 11)) 53 | print(image_xy, keypoint_xy) 54 | assert image_xy == keypoint_xy 55 | 56 | 57 | def test_pad(x=4, y=6): 58 | image_xy, keypoint_xy = single_pixel_transform(x, y, transforms.CenterPad(17)) 59 | print(image_xy, keypoint_xy) 60 | assert image_xy == keypoint_xy 61 | 62 | 63 | @pytest.mark.parametrize('x', range(10)) 64 | def test_rotateby90(x, y=6): 65 | transform = transforms.Compose([ 66 | transforms.SquarePad(), 67 | transforms.RotateBy90(), 68 | ]) 69 | image_xy, keypoint_xy = single_pixel_transform(x, y, transform) 70 | print(image_xy, keypoint_xy) 71 | assert image_xy == pytest.approx(keypoint_xy) 72 | -------------------------------------------------------------------------------- /visdrone_debug: -------------------------------------------------------------------------------- 1 | /data/george-data/visdrone_debug --------------------------------------------------------------------------------