├── .github └── workflows │ ├── deploy.yml │ ├── pre_release_suffix.py │ └── test.yml ├── .gitignore ├── COPYING ├── README.md ├── docs ├── backends.rst ├── capsule_class.rst ├── conf.py ├── creating_a_capsule.rst ├── file_structure.rst ├── index.rst ├── inputs_and_outputs.rst ├── introduction.rst ├── options.rst ├── requirements.txt ├── runtime_environment.rst └── stream_state.rst ├── tests ├── __init__.py ├── conftest.py ├── dependencies.py ├── test_batch_executor.py ├── test_loading.py ├── test_modifiers.py ├── test_node_definition.py └── test_packaged_capsules.py ├── tools ├── Dockerfile.build-env ├── Dockerfile.ubuntu ├── README.md ├── __init__.py ├── compile.py ├── openvisioncapsule_tools │ ├── __init__.py │ ├── bulk_accuracy │ │ ├── bulk_test_main.py │ │ ├── capsule_accuracy_common.py │ │ ├── capsule_accuracy_event_loop.py │ │ ├── capsule_accuracy_report.py │ │ ├── capsule_mgmt_basic.py │ │ ├── capsule_mgmt_local.py │ │ └── capsule_mgmt_remote.py │ ├── bulk_testing.sh │ ├── capsule_benchmark │ │ ├── README.md │ │ ├── args.py │ │ ├── benchmarking.py │ │ ├── capsules.py │ │ ├── main.py │ │ ├── output.py │ │ ├── requirements.txt │ │ └── workers.py │ ├── capsule_classifier_accuracy │ │ ├── capsule_classifier_accuracy.py │ │ └── detect_videos.py │ ├── capsule_infer │ │ ├── __init__.py │ │ ├── capsule_infer.py │ │ └── capsule_inference.py │ ├── capsule_packaging │ │ ├── __init__.py │ │ └── capsule_packaging.py │ ├── cli.py │ ├── command_utils.py │ ├── options.json │ ├── print_module_info.py │ ├── print_utils.py │ └── translations │ │ └── portal.en.yml ├── poetry.lock └── pyproject.toml ├── vcap ├── examples │ ├── classifier_gait_example │ │ ├── README.md │ │ ├── backend.py │ │ ├── capsule.py │ │ ├── config.py │ │ ├── dataset_metadata.json │ │ └── meta.conf │ └── detector_person_example │ │ ├── README.md │ │ ├── backend.py │ │ ├── capsule.py │ │ ├── dataset_metadata.json │ │ └── meta.conf ├── setup.py └── vcap │ ├── __init__.py │ ├── backend.py │ ├── batch_executor.py │ ├── caching.py │ ├── capsule.py │ ├── deprecation.py │ ├── detection_node.py │ ├── device_mapping.py │ ├── loading │ ├── capsule_loading.py │ ├── crypto_utils.py │ ├── errors.py │ ├── import_hacks.py │ └── vcap_packaging.py │ ├── modifiers.py │ ├── node_description.py │ ├── options.py │ ├── stream_state.py │ ├── testing │ ├── __init__.py │ ├── capsule_loading.py │ ├── input_output_validation.py │ └── thread_validation.py │ └── version.py └── vcap_utils ├── setup.py └── vcap_utils ├── __init__.py ├── algorithms ├── __init__.py ├── distances.py ├── iou_cost_matrix.py ├── linear_assignment.py └── nonmax_suppression.py ├── backends ├── __init__.py ├── backend_rpc_process.py ├── base_encoder.py ├── base_openvino.py ├── base_tensorflow.py ├── crowd_density.py ├── depth.py ├── load_utils.py ├── openface_encoder.py ├── predictions.py ├── segmentation.py ├── tf_image_classification.py └── tf_object_detection.py └── version.py /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Publish packages 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ['3.6'] 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | with: 17 | # Include tags in the checkout 18 | fetch-depth: 0 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Get Pip cache directory 24 | id: pip-cache 25 | run: | 26 | echo "::set-output name=dir::$(pip cache dir)" 27 | - uses: actions/cache@v1 28 | id: cache 29 | with: 30 | path: ${{ steps.pip-cache.outputs.dir }} 31 | key: ${{ runner.os }}-${{ hashFiles('**/setup.py') }}-pip-cache 32 | restore-keys: | 33 | ${{ runner.os }}-pip-cache 34 | - name: Install dependencies 35 | run: | 36 | pip install --upgrade setuptools wheel twine 37 | - name: Publish vcap 38 | working-directory: ./vcap 39 | env: 40 | TWINE_USERNAME: __token__ 41 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 42 | run: | 43 | PRE_RELEASE_SUFFIX=$(python3 ../.github/workflows/pre_release_suffix.py) \ 44 | python3 setup.py sdist bdist_wheel 45 | python3 -m twine upload dist/* 46 | - name: Publish vcap-utils 47 | working-directory: ./vcap_utils 48 | env: 49 | TWINE_USERNAME: __token__ 50 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 51 | run: | 52 | PRE_RELEASE_SUFFIX=$(python3 ../.github/workflows/pre_release_suffix.py) \ 53 | python3 setup.py sdist bdist_wheel 54 | python3 -m twine upload dist/* 55 | -------------------------------------------------------------------------------- /.github/workflows/pre_release_suffix.py: -------------------------------------------------------------------------------- 1 | """Prints a suffix that will be appended to the end of the version strings 2 | for the vcap and vcap-utils packages. 3 | 4 | For example, if there have been 3 commits since the last tagged release and the 5 | current version string in the setup.py is '0.1.3', the suffix will be '.dev3'. 6 | The resulting package version will then become '0.1.3.dev3'. 7 | 8 | For commits that have a tag, this script will print nothing. That way, the 9 | resulting package will have no pre-release suffix. 10 | 11 | Ideally, we would be using setuptools_scm to manage all of this for us. 12 | Unfortunately, setuptools_scm doesn't work for repositories with multiple 13 | packages. See: https://github.com/pypa/setuptools_scm/issues/357 14 | """ 15 | 16 | import subprocess 17 | import re 18 | import sys 19 | 20 | TAG_PATTERN = r"^v\d+\.\d+\.+\d+-?(\d+)?-?(.+)?$" 21 | 22 | result = subprocess.run(["git", "describe", "--tags"], 23 | check=True, stdout=subprocess.PIPE, encoding="utf-8") 24 | 25 | match = re.match(TAG_PATTERN, result.stdout) 26 | if match is None: 27 | print(f"Could not match tag: '{result.stdout}'", file=sys.stderr) 28 | sys.exit(1) 29 | 30 | if match.group(1) is not None: 31 | print(f".dev{match.group(1)}") 32 | else: 33 | print("") 34 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | pull_request: 5 | branches: [ master ] 6 | push: 7 | 8 | 9 | jobs: 10 | test: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [3.6] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | with: 20 | # Include tags in the checkout 21 | fetch-depth: 0 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Get Pip cache directory 27 | id: pip-cache 28 | run: | 29 | echo "::set-output name=dir::$(pip cache dir)" 30 | - uses: actions/cache@v1 31 | id: cache 32 | with: 33 | path: ${{ steps.pip-cache.outputs.dir }} 34 | key: ${{ runner.os }}-${{ hashFiles('**/setup.py') }}-pip-cache 35 | restore-keys: | 36 | ${{ runner.os }}-pip-cache 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip install flake8 pytest 41 | pip install -e "./vcap[easy,tests]" 42 | pip install -e "./vcap_utils[tests]" 43 | - name: Lint with flake8 44 | run: | 45 | # stop the build if there are Python syntax errors or undefined names 46 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 47 | # exit-zero treats all errors as warnings 48 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=79 --statistics 49 | - name: Test with pytest 50 | run: | 51 | pytest 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.cap 2 | *.pb 3 | *.jpg 4 | *.png 5 | 6 | __pycache__/ 7 | *.egg-info/ 8 | .idea 9 | .eggs/ 10 | _build/ 11 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (C) 2020, OpenCV Foundation, all rights reserved. 2 | Copyright (C) 2020, Dilili Labs, all rights reserved. 3 | Third party copyrights are property of their respective owners. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its contributors 16 | may be used to endorse or promote products derived from this software without 17 | specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenVisionCapsules 2 | 3 | [![Documentation Status](https://readthedocs.org/projects/openvisioncapsules/badge/?version=latest)](https://openvisioncapsules.readthedocs.io/en/latest/?badge=latest) 4 | 5 | This repository contains the OpenVisionCapsules SDK, a set of Python libraries 6 | for encapsulating machine learning and computer vision algorithms for 7 | intelligent video analytics. 8 | 9 | Encapsulating an algorithm allows it to be deployed as a single, self-describing 10 | file that inputs and outputs data in a standard format. This makes deployment 11 | and integration significantly easier than starting with a model file or a 12 | snippet of source code. Capsules are descriptive of their input and output 13 | requirements, allowing OpenVisionCapsules to route data between capsules 14 | automatically. 15 | 16 | This project is split into two packages, `vcap` and `vcap-utils`. `vcap` 17 | contains the necessary facilities to create and encapsulate an algorithm. 18 | `vcap-utils` contains a set of utilities that make encapsulating algorithms of 19 | certain types easier. 20 | 21 | # Project Status 22 | 23 | OpenVisionCapsules is in a developer preview phase. We're looking for developer 24 | feedback before reaching a stable 1.0 release. If you find any bugs or have 25 | suggestions, please open an issue. 26 | 27 | # Getting Started 28 | 29 | Take a look at the [documentation here][docs]. 30 | 31 | A couple example capsules are available under `vcap/examples`, demonstrating 32 | how to create classifier and detector capsules from TensorFlow models. 33 | 34 | # Installation 35 | 36 | To install OpenVisionCapsules locally, clone the repository and run the 37 | following commands to install the `vcap` and `vcap-utils` packages in the 38 | current environment. 39 | 40 | 44 | ``` 45 | pip3 install -e ./vcap 46 | pip3 install -e ./vcap_utils 47 | ``` 48 | 49 | # Examples 50 | 51 | To make use of the example capsules in the `vcap/examples/` directory, make 52 | sure to run the tests with pytest (from the root of the repo). The tests 53 | download all the necessary models and images, including the models for the 54 | example capsules. 55 | 56 | Make sure vcap & vcap-utils installation is done before tests, 57 | 58 | ``` 59 | pytest -v -x . 60 | ``` 61 | 62 | A repository of open source capsules can be found [here][capsule_zoo]. 63 | 64 | [docs]: https://openvisioncapsules.readthedocs.io/en/latest/ 65 | [capsule_zoo]: https://github.com/aotuai/capsule_zoo 66 | 67 | capsule inference test, 68 | ``` 69 | python3 tools/openvisioncapsule_tools/capsule_infer/capsule_infer.py --capsule ../capsule-zoo/capsules/detector_person_vehicle_bike_openvino --images tests/test_images/two_people.jpg 70 | ``` 71 | -------------------------------------------------------------------------------- /docs/backends.rst: -------------------------------------------------------------------------------- 1 | .. _`Backends`: 2 | 3 | ######## 4 | Backends 5 | ######## 6 | 7 | Introduction 8 | ============ 9 | 10 | A backend is what provides the low-level analysis on a video frame. For machine 11 | learning, this is the place where the frame would be fed into the model and the 12 | results would be returned. Every capsule must define a backend class that 13 | subclasses the BaseBackend class. 14 | 15 | The application will create an instance of the backend class for each device 16 | string returned by the capsule's device mapper. 17 | 18 | Required Methods 19 | ================ 20 | 21 | All backends must subclass the BaseBackend abstract base class, meaning that 22 | there are a couple methods that the backend must implement. 23 | 24 | .. autoclass:: vcap.BaseBackend 25 | :members: process_frame, close 26 | 27 | Batching Methods 28 | ================ 29 | 30 | Batching refers to the process of collecting more than one video frame into a 31 | "batch" and sending them all out for processing at once. Certain algorithms see 32 | performance improvements when batching is used, because doing so decreases the 33 | amount of round-trips the video frames take between devices. 34 | 35 | If you wish to use batching in your capsule, you may call the ``send_to_batch`` 36 | method in ``process_frame`` instead of doing analysis in that method directly. 37 | The ``send_to_batch`` method sends the input to a ``BatchExecutor`` which collects 38 | inference requests for this capsule from different streams. Then, the 39 | ``BatchExecutor`` routinely calls your backend's ``batch_predict`` method with a 40 | list of the collected inputs. As a result, users of ``send_to_batch`` must 41 | override the ``batch_predict`` method in addition to the other required methods. 42 | 43 | The ``send_to_batch`` method is asynchronous. Instead of immediately returning 44 | analysis results, it returns a ``concurrent.futures.Future`` where the result will be provided. 45 | Simple batching capsules may call ``send_to_batch``, then immediately call 46 | ``result`` to block for the result. 47 | 48 | .. code-block:: python 49 | 50 | result = self.send_to_batch(frame).result() 51 | 52 | An argument of any type may be provided to ``send_to_batch``, as the argument 53 | will be passed in a list to ``batch_predict`` without modification. In many 54 | cases only the video frame needs to be provided, but additional metadata may be 55 | included as necessary to fit your algorithm's needs. 56 | 57 | .. autoclass:: vcap.BaseBackend 58 | :members: batch_predict 59 | 60 | -------------------------------------------------------------------------------- /docs/capsule_class.rst: -------------------------------------------------------------------------------- 1 | ################# 2 | The Capsule Class 3 | ################# 4 | 5 | Introduction 6 | ============ 7 | 8 | The Capsule class provides information to the application about what a capsule 9 | is and how it should be run. Every capsule defines a Capsule class that extends 10 | BaseCapsule in its ``capsule.py`` file. 11 | 12 | .. code-block:: python 13 | 14 | from vcap import ( 15 | BaseCapsule, 16 | NodeDescription, 17 | options 18 | ) 19 | 20 | class Capsule(BaseCapsule): 21 | name = "detector_person" 22 | version = 1 23 | stream_state = StreamState 24 | input_type = NodeDescription(size=NodeDescription.Size.NONE), 25 | output_type = NodeDescription( 26 | size=NodeDescription.Size.ALL, 27 | detections=["person"]) 28 | backend_loader = backend_loader_func 29 | options = { 30 | "threshold": options.FloatOption( 31 | default=0.5, 32 | min_val=0.1, 33 | max_val=1.0) 34 | } 35 | 36 | .. autoclass:: vcap.BaseCapsule 37 | :members: 38 | :exclude-members: backends_lock, backends, get_state, close, clean_up, process_frame 39 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | 3 | # -- Path setup -------------------------------------------------------------- 4 | 5 | # If extensions (or modules to document with autodoc) are in another directory, 6 | # add these directories to sys.path here. If the directory is relative to the 7 | # documentation root, use os.path.abspath to make it absolute, like shown here. 8 | # 9 | import os 10 | import sys 11 | sys.path.insert(0, os.path.abspath('../vcap')) 12 | sys.path.insert(0, os.path.abspath('../vcap_utils')) 13 | 14 | 15 | # -- Project information ----------------------------------------------------- 16 | 17 | project = 'OpenVisionCapsules' 18 | copyright = '2020, OpenCV Foundation and Dilili Labs' 19 | author = 'Aotu' 20 | 21 | add_module_names = False 22 | 23 | 24 | # -- General configuration --------------------------------------------------- 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be 27 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 28 | # ones. 29 | extensions = [ 30 | "sphinx.ext.autodoc", 31 | "sphinx_rtd_theme", 32 | ] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # List of patterns, relative to source directory, that match files and 38 | # directories to ignore when looking for source files. 39 | # This pattern also affects html_static_path and html_extra_path. 40 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 41 | 42 | autoclass_content = "both" 43 | 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | html_theme = "sphinx_rtd_theme" 47 | 48 | # Mock out any third-party imports 49 | autodoc_mock_imports = ["numpy", "tensorflow", "cv2", "Cryptodome"] 50 | -------------------------------------------------------------------------------- /docs/creating_a_capsule.rst: -------------------------------------------------------------------------------- 1 | ################## 2 | Creating a Capsule 3 | ################## 4 | 5 | For application developers, OpenVisionCapsules provides a function to package 6 | up unpackaged capsules. The optional ``key`` field encrypts the capsule with 7 | AES. 8 | 9 | .. code-block:: python 10 | 11 | from vcap import package_capsule 12 | 13 | package_capsule(Path("detector_person"), 14 | Path("capsules", "detector_person.cap"), 15 | key="[AES Key]") 16 | 17 | For capsule developers for an application, it is the job of the application to 18 | provide a way to package capsules. Please see the documentation for the 19 | application you are using for more information. 20 | 21 | Creating an Object Detector Capsule with Supervisely 22 | ==================================================== 23 | 24 | If you’ve trained tensorflow-object-detection-API object detector using 25 | Supervisely, you can follow the following steps to deploy your model as a 26 | capsule: 27 | 28 | Set up the TF Object Detection API 29 | ---------------------------------- 30 | 31 | First, set up the Tensorflow Object Detection API on your machine by cloning 32 | the ``tensorflow/models`` repository and following the object detection API 33 | installation instructions. Make sure the tests pass before continuing- 34 | otherwise, you might have forgotten to set up certain environment variables! 35 | 36 | Freeze your trained Supervisely model 37 | ------------------------------------- 38 | 39 | Next, you must download your trained Supervisely model and extract it. Inside 40 | you should see the following directory structure: 41 | 42 | .. code-block:: 43 | 44 | 45 | ├── config.json 46 | ├── model.config 47 | └── model_weights 48 | ├── checkpoint 49 | ├── model.ckpt.data-00000-of-00001 50 | ├── model.ckpt.index 51 | └── model.ckpt.meta 52 | 53 | Now, you must simply freeze the model to get the ``frozen_inference_graph.pb``. 54 | To do that, run ``models/research/object_detection/export_inference_graph.py`` 55 | script inside of your downloaded model directory. 56 | 57 | .. code-block:: bash 58 | 59 | python PATH/TO/export_inference_graph.py \ 60 | --input_type image_tensor \ 61 | --pipeline_config_path model.config \ 62 | --trained_checkpoint_prefix model_weights/model.ckpt \ 63 | --output_directory . 64 | 65 | There should now be a frozen_inference_graph.pb in the current directory. This 66 | is the model file that has been optimized for inference, and is much more 67 | portable for production use. 68 | -------------------------------------------------------------------------------- /docs/file_structure.rst: -------------------------------------------------------------------------------- 1 | ############## 2 | File Structure 3 | ############## 4 | 5 | All capsules start off their life as unpackaged capsules. An unpackaged capsule 6 | is simply a directory containing all files that will be packaged into the 7 | capsule. This directory must contain, at minimum, a meta.conf file and a 8 | capsule.py file. 9 | 10 | .. code-block:: 11 | 12 | detector_person 13 | ├── meta.conf 14 | └── capsule.py 15 | 16 | The meta.conf file is a simple configuration file which specifies the major and 17 | minor version of OpenVisionCapsules that this capsule requires. Applications use 18 | this information to decide if your capsule is compatible with the version of 19 | OpenVisionCapsules the application uses. A capsule with a compatibility version 20 | of 0.1 are expected to be compatible with applications that use 21 | OpenVisionCapsules version 0.1 through 0.x, but not 1.x or 2.x. 22 | 23 | .. code-block:: ini 24 | 25 | [about] 26 | api_compatibility_version = 0.1 27 | 28 | 29 | The capsule.py file is the meat of the capsule. It contains the actual behavior 30 | of the capsule. We will talk more about the contents of this file in a later 31 | section. 32 | 33 | If your capsule uses other files for its operation, like a model file, it should 34 | be included in this directory as well. All files in the capsule's directory 35 | will be included and made accessible once it's packaged. 36 | 37 | .. code-block:: 38 | 39 | person_detection_capsule 40 | ├── meta.conf 41 | ├── capsule.py 42 | └── frozen_inference_graph.pb 43 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ####################### 2 | OpenVisionCapsules Docs 3 | ####################### 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | introduction 9 | file_structure 10 | runtime_environment 11 | capsule_class 12 | backends 13 | inputs_and_outputs 14 | options 15 | creating_a_capsule 16 | stream_state 17 | 18 | This is a guide on how to encapsulate an algorithm using OpenVisionCapsules. 19 | Capsules are discrete components that define new ways to analyze video streams. 20 | 21 | This guide discusses the Open Vision Capsule system generally, not any specific 22 | information on how to write a Capsule of a certain type or with certain 23 | technology. Example capsules are available under ``vcap/examples`` that show 24 | how to encapsulate models from various popular machine learning frameworks. 25 | -------------------------------------------------------------------------------- /docs/inputs_and_outputs.rst: -------------------------------------------------------------------------------- 1 | .. _`Inputs and Outputs`: 2 | 3 | ################## 4 | Inputs and Outputs 5 | ################## 6 | 7 | Introduction 8 | ============ 9 | 10 | Capsules are defined by the data they take as input and the information they 11 | give as output. Applications use this information to connect capsules to each 12 | other and schedule their execution. These inputs and outputs are `defined` 13 | by NodeDescription objects and `realized` by DetectionNode objects. 14 | 15 | .. autoclass:: vcap.DetectionNode 16 | 17 | .. autoclass:: vcap.NodeDescription 18 | 19 | Examples 20 | ======== 21 | 22 | detections 23 | ---------- 24 | 25 | A capsule that can encode cars or trucks would use a NodeDescription like this 26 | as its ``input_type``: 27 | 28 | .. code-block:: python 29 | 30 | NodeDescription(detections=["car", "truck"]) 31 | 32 | A capsule that can detect people and dogs would use a NodeDescription like this 33 | as its ``output_type``: 34 | 35 | .. code-block:: python 36 | 37 | NodeDescription(detections=["person", "dog"]) 38 | 39 | attributes 40 | ---------- 41 | 42 | A capsule that operates on detections that have been classified for gender use 43 | a NodeDescription like this as its ``input_type``: 44 | 45 | .. code-block:: python 46 | 47 | NodeDescription( 48 | attributes={ 49 | "gender": ["male", "female"], 50 | "color": ["red", "blue", "green"] 51 | }) 52 | 53 | A capsule that can classify people’s gender as either male or female would have 54 | the following NodeDescription as its ``output_type``: 55 | 56 | .. code-block:: python 57 | 58 | NodeDescription( 59 | detections=["person"], 60 | attributes={ 61 | "gender": ["male", "female"] 62 | }) 63 | 64 | encoded 65 | ------- 66 | 67 | A capsule that operates on detections of cars that have been encoded use a 68 | NodeDescription like this as its ``input_type``: 69 | 70 | .. code-block:: python 71 | 72 | NodeDescription( 73 | detections=["car"], 74 | encoded=True) 75 | 76 | A capsule that encodes people would use a NodeDescription like this as its 77 | ``output_type``: 78 | 79 | .. code-block:: python 80 | 81 | NodeDescription( 82 | detections=["person"], 83 | encoded=True) 84 | 85 | tracked 86 | ------- 87 | 88 | A capsule that operates on person detections that have been tracked would use a 89 | NodeDescription like this as its ``input_type``. 90 | 91 | .. code-block:: python 92 | 93 | NodeDescription( 94 | detections=["person"], 95 | tracked=True) 96 | 97 | A capsule that tracks people would use a NodeDescription like this as its 98 | ``output_type``: 99 | 100 | .. code-block:: python 101 | 102 | NodeDescription( 103 | detections=["person"], 104 | tracked=True) 105 | 106 | extra_data 107 | ---------- 108 | 109 | A capsule that operates on people detections with a "process_extra_fast" 110 | ``extra_data`` field would use a NodeDescription like this as its 111 | ``input_type``: 112 | 113 | .. code-block:: python 114 | 115 | NodeDescription( 116 | detections=["person"], 117 | extra_data=["process_extra_fast"]) 118 | 119 | A capsule that adds an "is_special" ``extra_data`` field to its person-detected 120 | output would use a NodeDescription like this as its ``output_type``: 121 | 122 | .. code-block:: python 123 | 124 | NodeDescription( 125 | detections=["person"], 126 | extra_data=["is_special"]) 127 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | ############ 2 | Introduction 3 | ############ 4 | 5 | What is a Capsule? 6 | ------------------ 7 | 8 | A capsule is a single file with a ``.cap`` file extension. It contains the 9 | code, metadata, model files, and any other files the capsule needs to operate. 10 | 11 | Capsules take a frame and information from other capsules as input, run some 12 | kind of analysis, and provide metadata about the frame as output. For example, 13 | a person detection capsule would take a frame as input and output person 14 | detections in that frame. A gender classifier capsule would take this frame and 15 | each person detection as input, and output a male or female classification for 16 | that detection. 17 | 18 | Capsules provide metadata describing these inputs and outputs alongside other 19 | information on the capsule. Applications that are compatible OpenVisionCapsules 20 | use this metadata to know when to run the capsule and with what input. 21 | -------------------------------------------------------------------------------- /docs/options.rst: -------------------------------------------------------------------------------- 1 | .. _`Options`: 2 | 3 | ####### 4 | Options 5 | ####### 6 | 7 | Introduction 8 | ============ 9 | 10 | Capsules can provide runtime configuration options that change the way the 11 | capsule operates. These options will appear on the client and can also be 12 | changed in the UI. Options have a type and constraints that define what values 13 | are valid. 14 | 15 | .. autoclass:: vcap.FloatOption 16 | 17 | .. autoclass:: vcap.IntOption 18 | 19 | .. autoclass:: vcap.EnumOption 20 | 21 | .. autoclass:: vcap.BoolOption 22 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx==3.1.0 2 | -------------------------------------------------------------------------------- /docs/runtime_environment.rst: -------------------------------------------------------------------------------- 1 | ################### 2 | Runtime Environment 3 | ################### 4 | 5 | Loading 6 | ======= 7 | 8 | When a capsule is loaded, the ``capsule.py`` file is imported as a module and 9 | an instance of the ``Capsule`` class defined in that module is instantiated. 10 | Then, for each compatible device, an instance of the capsule's ``Backend`` 11 | class is created using the provided ``backend_loader`` function. 12 | 13 | Importing 14 | ========= 15 | 16 | Capsules have access to a number of helpful libraries, including: 17 | 18 | - The entirety of the Python standard library 19 | - Numpy (``import numpy``) 20 | - OpenCV (``import cv2``) 21 | - Tensorflow (``import tensorflow``) 22 | - Scikit Learn (``import sklearn``) 23 | - OpenVino (``import openvino``) 24 | 25 | Applications may provide more libraries in addition to these. Please see that 26 | application's documentation for more information. 27 | 28 | Importing From Other Files in the Capsule 29 | ----------------------------------------- 30 | 31 | In order to allow for more complex capsules that have code reuse within them, 32 | capsules may consist of multiple Python files. These files are made available 33 | through relative imports. 34 | 35 | For example, with the following directory structure: 36 | 37 | .. code-block:: 38 | 39 | capsule_dir/ 40 | ├── capsule.py 41 | ├── backend.py 42 | └── utils/ 43 | ├── img_utils.py 44 | └── ml_utils.py 45 | 46 | The ``capsule.py`` file may import the other Python files like so: 47 | 48 | .. code-block:: python 49 | 50 | from . import backend 51 | from .utils import img_utils, ml_utils 52 | 53 | Note that non-relative imports to these files will `not` work: 54 | 55 | .. code-block:: python 56 | 57 | import backend 58 | from utils import img_utils, ml_utils 59 | 60 | Limiting GPU memory Growth 61 | ----------------------------------------- 62 | 63 | By default, OpenVisionCapsules maps all available memory of all visible CUDA configured GPUs. 64 | To prevent this, use the following Environment flag while using Tensorflow. 65 | 66 | .. code-block:: python 67 | 68 | TF_FORCE_GPU_ALLOW_GROWTH=True 69 | 70 | - This Environment variable is only applicable to Tensorflow. 71 | 72 | For proper reference, visit Tensorflow: https://www.tensorflow.org/guide/gpu#limiting_gpu_memory_growth 73 | -------------------------------------------------------------------------------- /docs/stream_state.rst: -------------------------------------------------------------------------------- 1 | .. _`Stream State`: 2 | 3 | ############ 4 | Stream State 5 | ############ 6 | 7 | 8 | It is sometimes desirable to carry state throughout the lifetime of an entire 9 | video stream, rather than on a frame-by-frame basis. This is where StreamState 10 | comes in. 11 | 12 | If the ``stream_state`` field of the capsule's Capsule class is set, the 13 | ``process_frame`` method for your capsule's backend will be passed an instance 14 | of the provided class. Any state that should exist for the duration of the 15 | videostream may be saved here. 16 | 17 | This is commonly used by capsules that track objects between video frames. 18 | Information on previous detections can be stored in the StreamState object and 19 | read when new detections are found. This can also be useful for result 20 | smoothing, for caching frames (think RNN models), and many other use cases. 21 | 22 | A capsule's StreamState class does not need to implement any methods and has 23 | no functional purpose outside of the capsule. 24 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencv/open_vision_capsules/c6c6a9c19682d811211b7baffd116555703a0ba7/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import urllib.request 3 | 4 | import pytest 5 | 6 | from vcap.testing import verify_all_threads_closed 7 | 8 | _TEST_IMAGE_DIR = Path("tests/test_images") 9 | _S3_BASE_URL = "https://open-vision-capsules.s3-us-west-1.amazonaws.com" \ 10 | "/test-dependencies/{}/{}" 11 | 12 | 13 | @pytest.fixture(autouse=True, scope="session") 14 | def model_dependencies(): 15 | dependencies = { 16 | "models": { 17 | "classification_gait_model.pb": 18 | Path("vcap/examples/classifier_gait_example/"), 19 | "ssd_mobilenet_v1_coco.pb": 20 | Path("vcap/examples/detector_person_example") 21 | }, 22 | "images": [ 23 | "no_people.jpg", 24 | "one_person.jpg", 25 | "two_people.jpg" 26 | ] 27 | } 28 | 29 | # Get models 30 | for model_name, model_path in dependencies["models"].items(): 31 | filepath = model_path / model_name 32 | if not filepath.exists(): 33 | s3_url = _S3_BASE_URL.format("models", model_name) 34 | urllib.request.urlretrieve(s3_url, filepath) 35 | 36 | # Get images 37 | _TEST_IMAGE_DIR.mkdir(exist_ok=True) 38 | for image_name in dependencies["images"]: 39 | filepath = _TEST_IMAGE_DIR / image_name 40 | if not filepath.exists(): 41 | s3_url = _S3_BASE_URL.format("images", image_name) 42 | urllib.request.urlretrieve(s3_url, filepath) 43 | 44 | 45 | @pytest.fixture(autouse=True, scope="session") 46 | def all_test_setup_teardown(): 47 | """Runs before and after the entire testing session""" 48 | 49 | yield 50 | 51 | # Any teardown for all tests goes here 52 | verify_all_threads_closed() 53 | 54 | 55 | @pytest.fixture(autouse=True) 56 | def every_test_setup_teardown(): 57 | """Runs before and after each test""" 58 | # Any setup for each test goes here 59 | yield 60 | 61 | # Any teardown for each tests goes here 62 | verify_all_threads_closed() 63 | -------------------------------------------------------------------------------- /tests/dependencies.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | TEST_IMAGES = Path("tests/test_images") 4 | 5 | ONE_PERSON_IMAGE = TEST_IMAGES / "one_person.jpg" 6 | 7 | TWO_PEOPLE_IMAGE = TEST_IMAGES / "two_people.jpg" 8 | 9 | NO_PEOPLE_IMAGE = TEST_IMAGES / "no_people.jpg" 10 | 11 | ALL_IMAGE_PATHS = [ONE_PERSON_IMAGE, 12 | TWO_PEOPLE_IMAGE, 13 | NO_PEOPLE_IMAGE] 14 | -------------------------------------------------------------------------------- /tests/test_batch_executor.py: -------------------------------------------------------------------------------- 1 | import random 2 | from concurrent.futures import Future 3 | from typing import Any, Generator, List, Tuple 4 | 5 | import pytest 6 | 7 | from vcap.batch_executor import BatchExecutor, _Request 8 | 9 | 10 | @pytest.fixture() 11 | def batch_executor(): 12 | """To use this fixture, replace batch_executor.batch_fn with your own 13 | batch function.""" 14 | 15 | def batch_fn(inputs): 16 | raise NotImplemented 17 | 18 | batch_executor = BatchExecutor(batch_fn=batch_fn) 19 | yield batch_executor 20 | batch_executor.close() 21 | 22 | 23 | def batch_fn_base(inputs: List[int], raises: bool) \ 24 | -> Generator[Any, None, None]: 25 | """Process results and yield them as they are processed 26 | 27 | This function is to be used as a base for other test cases for batch_fn 28 | variants. 29 | 30 | :param inputs: A list of inputs 31 | :param raises: If True, raises an error on the 5th input. If False, 32 | no exception will be raised. 33 | """ 34 | for i in inputs: 35 | if i == 5 and raises: 36 | # This occurs on the 5th input if raises=True. 37 | # This is used to test BatchExecutor's handling of exceptions 38 | raise RuntimeError("Oh no, a batch_fn error has occurred!") 39 | yield i * 100 40 | 41 | 42 | def batch_fn_returns_generator(inputs: List[int]) \ 43 | -> Generator[Any, None, None]: 44 | return (o for o in batch_fn_base(inputs, raises=False)) 45 | 46 | 47 | def batch_fn_returns_generator_raises(inputs: List[int]) \ 48 | -> Generator[Any, None, None]: 49 | return (o for o in batch_fn_base(inputs, raises=True)) 50 | 51 | 52 | def batch_fn_returns_list(inputs: List[int]) -> List[Any]: 53 | """Process results and yield them at the end, as a list.""" 54 | return list(batch_fn_base(inputs, raises=False)) 55 | 56 | 57 | def batch_fn_returns_list_raises(inputs: List[int]) -> List[Any]: 58 | return list(batch_fn_base(inputs, raises=True)) 59 | 60 | 61 | @pytest.mark.parametrize( 62 | argnames=["batch_fn", "expect_partial_results"], 63 | argvalues=[ 64 | (batch_fn_returns_generator_raises, True), 65 | (batch_fn_returns_list_raises, False) 66 | ] 67 | ) 68 | def test_exceptions_during_batch_fn( 69 | batch_executor, batch_fn, expect_partial_results): 70 | """Test that BatchExecutor catches exceptions that occur in the batch_fn 71 | and propagates them through the requests Future objects. 72 | 73 | If an exception occurs after processing some of the batch, the expectation 74 | is that the unprocessed inputs of the batch will get an exception 75 | set (expect_partial_results=True). If the exception happens before 76 | receiving any results, all future objects should have exceptions set. 77 | """ 78 | batch_executor.batch_fn = batch_fn 79 | request_batch = [ 80 | _Request( 81 | future=Future(), 82 | input_data=i) 83 | for i in range(10) 84 | ] 85 | batch_executor._on_requests_ready(request_batch) 86 | for i, request in enumerate(request_batch): 87 | if expect_partial_results and i < 5: 88 | result = request.future.result(timeout=5) 89 | assert result == request.input_data * 100, \ 90 | "The result for this future doesn't match the input that " \ 91 | "was supposed to have been routed to it!" 92 | else: 93 | with pytest.raises(RuntimeError): 94 | request.future.result(timeout=5) 95 | 96 | 97 | @pytest.mark.parametrize( 98 | argnames=["batch_fn"], 99 | argvalues=[ 100 | (batch_fn_returns_generator,), 101 | (batch_fn_returns_list,) 102 | ] 103 | ) 104 | def test_relevant_input_outputs_match(batch_executor, batch_fn): 105 | """Test the output for any given input is routed to the correct 106 | Future object. """ 107 | batch_executor.batch_fn = batch_fn 108 | 109 | # Submit input values in a random order 110 | request_inputs = list(range(10000)) 111 | random.seed("vcap? More like vgood") 112 | random.shuffle(request_inputs) 113 | 114 | # Submit inputs to the BatchExecutor and keep track of their futures 115 | inputs_and_futures: List[Tuple[int, Future]] = [] 116 | for input_data in request_inputs: 117 | future = batch_executor.submit(input_data) 118 | inputs_and_futures.append((input_data, future)) 119 | 120 | # Verify that all outputs are the expected ones for their respective input 121 | for input_data, future in inputs_and_futures: 122 | result = future.result(timeout=5) 123 | assert result == input_data * 100, \ 124 | "The result for this future doesn't match the input that " \ 125 | "was supposed to have been routed to it!" 126 | 127 | assert batch_executor.total_imgs_in_pipeline == 0 128 | -------------------------------------------------------------------------------- /tests/test_loading.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | from zipfile import ZipFile 4 | from typing import Tuple 5 | 6 | from vcap import BaseCapsule, package_capsule, CAPSULE_EXTENSION 7 | from vcap.loading.capsule_loading import capsule_module_name 8 | from vcap.testing.capsule_loading import load_capsule_with_one_device 9 | 10 | 11 | def test_load_capsule_from_memory(): 12 | """Test that a capsule can be loaded from memory, without any source code 13 | available. 14 | """ 15 | capsule_path = Path("vcap", "examples", "classifier_gait_example") 16 | packaged_capsule_path = (capsule_path 17 | .with_name(capsule_path.stem) 18 | .with_suffix(CAPSULE_EXTENSION)) 19 | package_capsule(capsule_path, packaged_capsule_path) 20 | 21 | capsule: BaseCapsule = load_capsule_with_one_device( 22 | packaged_capsule_path, 23 | from_memory=True) 24 | 25 | try: 26 | assert capsule.name == "classifier_gait_example" 27 | finally: 28 | capsule.close() 29 | 30 | 31 | def test_load_duplicate_capsule(): 32 | """Tests that the capsule's modules are not re-imported when a duplicate 33 | capsule is loaded. 34 | 35 | This caching behavior isn't necessarily critical for correct behavior, but 36 | this test ensures our understanding of Python's import system holds. 37 | """ 38 | capsule_path = Path("vcap", "examples", "classifier_gait_example") 39 | capsule_copy_1, packaged_capsule_path = _package_and_load_capsule( 40 | capsule_path) 41 | 42 | # Modify the capsule's module 43 | module_copy_1 = _get_capsule_module(packaged_capsule_path) 44 | assert module_copy_1 is not None 45 | module_copy_1.some_attribute = "Hello" 46 | 47 | # Load the same capsule again 48 | load_capsule_with_one_device( 49 | packaged_capsule_path, 50 | inference_mode=False) 51 | 52 | # Test that the module modification still exists after the re-load 53 | module_copy_2 = _get_capsule_module(packaged_capsule_path) 54 | assert module_copy_2 is not None 55 | assert module_copy_2.some_attribute == "Hello" 56 | 57 | 58 | def test_load_modified_capsule(): 59 | """Tests that the capsule's modules are re-imported when a modified version 60 | of a capsule is loaded. 61 | 62 | This ensures that code changes in the revised capsule will be reflected 63 | when in use. 64 | """ 65 | capsule_path = Path("vcap", "examples", "classifier_gait_example") 66 | capsule_revision_1, packaged_capsule_path = _package_and_load_capsule( 67 | capsule_path) 68 | 69 | # Modify the first revision capsule's module 70 | module_revision_1 = _get_capsule_module(packaged_capsule_path) 71 | assert module_revision_1 is not None 72 | module_revision_1.some_attribute = "Hello" 73 | 74 | # Modify the capsule file 75 | with ZipFile(packaged_capsule_path, "a") as new_capsule_zip: 76 | new_capsule_zip.writestr("random_file.txt", 77 | "I'm here to mess up your capsule!") 78 | 79 | # Upload a second revision of the capsule 80 | load_capsule_with_one_device( 81 | packaged_capsule_path, 82 | inference_mode=False) 83 | 84 | # Test that the module modification from the first revision is not 85 | # reflected in the second revision 86 | module_revision_2 = _get_capsule_module(packaged_capsule_path) 87 | assert module_revision_2 is not None 88 | assert not hasattr(module_revision_2, "some_attribute") 89 | 90 | 91 | def _get_capsule_module(path: Path): 92 | module_name = capsule_module_name(path.read_bytes()) 93 | return sys.modules[module_name] 94 | 95 | 96 | def _package_and_load_capsule(path: Path) -> Tuple[BaseCapsule, Path]: 97 | packaged_capsule_path = (path 98 | .with_name(path.stem) 99 | .with_suffix(CAPSULE_EXTENSION)) 100 | package_capsule(path, packaged_capsule_path) 101 | 102 | capsule: BaseCapsule = load_capsule_with_one_device( 103 | packaged_capsule_path, 104 | # These tests are just for loading, not running inference 105 | inference_mode=False) 106 | 107 | return capsule, packaged_capsule_path 108 | -------------------------------------------------------------------------------- /tests/test_node_definition.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from vcap import NodeDescription, DetectionNode 7 | 8 | DIFFERENCE_CASES = [ 9 | # Test the null case 10 | (NodeDescription(size=NodeDescription.Size.NONE), 11 | NodeDescription(size=NodeDescription.Size.NONE), 12 | NodeDescription(size=NodeDescription.Size.NONE), 13 | NodeDescription(size=NodeDescription.Size.NONE)), 14 | 15 | # Test encoding differences 16 | (NodeDescription( 17 | size=NodeDescription.Size.SINGLE, 18 | encoded=False), 19 | NodeDescription( 20 | size=NodeDescription.Size.SINGLE, 21 | encoded=True), 22 | NodeDescription( 23 | size=NodeDescription.Size.SINGLE, 24 | encoded=True), 25 | NodeDescription( 26 | size=NodeDescription.Size.SINGLE)), 27 | 28 | # Test detection differences and Size differences 29 | (NodeDescription( 30 | size=NodeDescription.Size.SINGLE, 31 | detections=["car"]), 32 | NodeDescription( 33 | size=NodeDescription.Size.ALL, 34 | detections=["car", "person"]), 35 | NodeDescription( 36 | size=NodeDescription.Size.ALL, 37 | detections=["person"]), 38 | NodeDescription( 39 | size=NodeDescription.Size.SINGLE)), 40 | 41 | # Test attribute differences 42 | (NodeDescription( 43 | size=NodeDescription.Size.SINGLE, 44 | attributes={"Gait": ["walking", "running"]}), 45 | NodeDescription( 46 | size=NodeDescription.Size.SINGLE, 47 | attributes={"Gait": ["walking", "running"], 48 | "Speeding": ["yeah", "naw"]}), 49 | NodeDescription( 50 | size=NodeDescription.Size.SINGLE, 51 | attributes={"Speeding": ["yeah", "naw"]}), 52 | NodeDescription( 53 | size=NodeDescription.Size.SINGLE)), 54 | 55 | # Test extra_data differences 56 | (NodeDescription( 57 | size=NodeDescription.Size.SINGLE, 58 | extra_data=["behavior_confidence"]), 59 | NodeDescription( 60 | size=NodeDescription.Size.SINGLE, 61 | extra_data=["behavior_confidence", "det_score"]), 62 | NodeDescription( 63 | size=NodeDescription.Size.SINGLE, 64 | extra_data=["det_score"]), 65 | NodeDescription( 66 | size=NodeDescription.Size.SINGLE)), 67 | 68 | # Test tracked differences 69 | (NodeDescription( 70 | size=NodeDescription.Size.SINGLE, 71 | tracked=False), 72 | NodeDescription( 73 | size=NodeDescription.Size.SINGLE, 74 | tracked=True), 75 | NodeDescription( 76 | size=NodeDescription.Size.SINGLE, 77 | tracked=True), 78 | NodeDescription( 79 | size=NodeDescription.Size.SINGLE)) 80 | ] 81 | 82 | 83 | @pytest.mark.parametrize(('desc1', 'desc2', 'diff_1_2', 'diff_2_1'), 84 | DIFFERENCE_CASES) 85 | def test_node_description_difference(desc1, desc2, diff_1_2, diff_2_1): 86 | """Test comparing two node descriptions""" 87 | assert desc1.difference(desc2) == diff_1_2 88 | assert desc2.difference(desc1) == diff_2_1 89 | 90 | 91 | DESCRIPTION_CASES = [ 92 | (DetectionNode( 93 | name="person", 94 | coords=[[0, 0]] * 4), 95 | True, False, False, False, False, False, False), 96 | 97 | (DetectionNode( 98 | name="person", 99 | coords=[[0, 0]] * 4, 100 | encoding=np.array([1])), 101 | True, False, False, True, False, False, False), 102 | 103 | (DetectionNode( 104 | name="hair", 105 | coords=[[0, 0]] * 4, 106 | attributes={"Gender": "boy"}), 107 | False, False, True, False, False, False, False), 108 | 109 | (DetectionNode( 110 | name="cat", 111 | coords=[[0, 0]] * 4, 112 | attributes={"Uniform": "Police", "Gender": "girl"}, 113 | encoding=np.ndarray([1, 2, 3, 4, 5])), 114 | False, False, True, False, True, True, False), 115 | 116 | (DetectionNode( 117 | name="person", 118 | coords=[[0, 0]] * 4, 119 | attributes={"more": "irrelevant"}, 120 | encoding=np.ndarray([1, 2, 3, 4, 5]), 121 | extra_data={"behavior_confidence": 0.9991999}), 122 | True, True, False, True, False, False, False), 123 | 124 | (DetectionNode( 125 | name="person", 126 | coords=[[0, 0]] * 4, 127 | track_id=uuid4()), 128 | True, False, False, False, False, False, True) 129 | ] 130 | 131 | 132 | @pytest.mark.parametrize(('det_node', 133 | 'nd1', 'nd2', 'nd3', 'nd4', 'nd5', 'nd6', 'nd7'), 134 | DESCRIPTION_CASES) 135 | def test_detection_node_descriptions(det_node, 136 | nd1, nd2, nd3, nd4, nd5, nd6, nd7): 137 | """Test that DetectionNodes can accurately generate a node_description 138 | for themselves""" 139 | 140 | node_desc_1 = NodeDescription( 141 | size=NodeDescription.Size.SINGLE, 142 | detections=["person", "dog"]) 143 | node_desc_2 = NodeDescription( 144 | size=NodeDescription.Size.SINGLE, 145 | extra_data=["behavior_confidence"]) 146 | node_desc_3 = NodeDescription( 147 | size=NodeDescription.Size.SINGLE, 148 | attributes={"Gender": ["boy", "girl"]}) 149 | node_desc_4 = NodeDescription( 150 | size=NodeDescription.Size.SINGLE, 151 | detections=["person"], 152 | encoded=True) 153 | node_desc_5 = NodeDescription( 154 | size=NodeDescription.Size.SINGLE, 155 | detections=["hair", "cat"], 156 | attributes={"Uniform": ["Police", "Worker", "Civilian"]}, 157 | encoded=True) 158 | node_desc_6 = NodeDescription( 159 | size=NodeDescription.Size.SINGLE, 160 | attributes={"Uniform": ["Police", "Worker", "Civilian"], 161 | "Gender": ["boy", "girl"]}) 162 | node_desc_7 = NodeDescription( 163 | size=NodeDescription.Size.SINGLE, 164 | tracked=True) 165 | 166 | assert NodeDescription(size=NodeDescription.Size.SINGLE).describes(det_node) 167 | assert node_desc_1.describes(det_node) == nd1 168 | assert node_desc_2.describes(det_node) == nd2 169 | assert node_desc_3.describes(det_node) == nd3 170 | assert node_desc_4.describes(det_node) == nd4 171 | assert node_desc_5.describes(det_node) == nd5 172 | assert node_desc_6.describes(det_node) == nd6 173 | assert node_desc_7.describes(det_node) == nd7 174 | 175 | 176 | def test_describes_error(): 177 | # Test that a ValueError gets raised when a DetectionNode has an attribute 178 | # with values that are not described by the NodeDescription 179 | node_desc = NodeDescription( 180 | size=NodeDescription.Size.SINGLE, 181 | attributes={"Gender": ["boy", "girl"]}) 182 | det_node = DetectionNode( 183 | name="irrelevant", 184 | coords=[[0, 0]] * 4, 185 | attributes={"Gender": "NOT EXISTENT VALUE"} 186 | ) 187 | with pytest.raises(ValueError): 188 | node_desc.describes(det_node) 189 | -------------------------------------------------------------------------------- /tests/test_packaged_capsules.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from vcap.testing.input_output_validation import perform_capsule_tests 4 | 5 | from .dependencies import ALL_IMAGE_PATHS 6 | 7 | 8 | def test_classifier_gait_example(): 9 | """Verify that the gait classifier can be packaged and used.""" 10 | perform_capsule_tests( 11 | Path("vcap", "examples", "classifier_gait_example"), 12 | ALL_IMAGE_PATHS) 13 | 14 | 15 | def test_detector_person_example(): 16 | """Verify that the person detector can be packaged and used.""" 17 | perform_capsule_tests( 18 | Path("vcap", "examples", "detector_person_example"), 19 | ALL_IMAGE_PATHS) 20 | -------------------------------------------------------------------------------- /tools/Dockerfile.build-env: -------------------------------------------------------------------------------- 1 | ARG UBUNTU_VERSION=24.04 2 | ARG PYTHON_VERSION=3 3 | 4 | FROM ubuntu:${UBUNTU_VERSION} 5 | 6 | ARG PYTHON_VERSION 7 | ENV PYTHON_VERSION=${PYTHON_VERSION} 8 | 9 | ENV DEBIAN_FRONTEND=noninteractive 10 | ENV PATH="/root/.local/bin:$PATH" 11 | 12 | RUN apt-get update \ 13 | && apt-get install -y --no-install-recommends \ 14 | python${PYTHON_VERSION} \ 15 | python${PYTHON_VERSION}-venv \ 16 | python${PYTHON_VERSION}-pip \ 17 | python${PYTHON_VERSION}-dev \ 18 | curl \ 19 | build-essential \ 20 | gcc \ 21 | g++ \ 22 | libssl-dev \ 23 | libffi-dev \ 24 | && apt-get clean \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | # Create a virtual environment 28 | RUN python${PYTHON_VERSION} -m venv /opt/venv 29 | ENV PATH="/opt/venv/bin:$PATH" 30 | 31 | ENV GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 32 | ENV GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 33 | 34 | # Upgrade pip and install setuptools and wheel within the virtual environment 35 | RUN pip install --upgrade pip setuptools wheel 36 | 37 | # Install Poetry 38 | RUN curl -sSL https://install.python-poetry.org | python - 39 | 40 | WORKDIR /open_vision_capsules/tools 41 | 42 | # Copy just the pyproject.toml and poetry.lock files 43 | COPY tools/pyproject.toml tools/poetry.lock* . 44 | 45 | # Configure Poetry to use the virtual environment 46 | RUN poetry config virtualenvs.create false 47 | RUN poetry config virtualenvs.path /opt/venv 48 | 49 | # Install project dependencies 50 | RUN poetry lock 51 | # RUN pip install \ 52 | # opencv_contrib_python=="4.10.0.84" \ 53 | # opencv_contrib_python_headless=="4.10.0.84" 54 | RUN poetry install --no-interaction --no-ansi --no-root 55 | -------------------------------------------------------------------------------- /tools/Dockerfile.ubuntu: -------------------------------------------------------------------------------- 1 | ARG UBUNTU_VERSION=24.04 2 | ARG PYTHON_VERSION=3 3 | 4 | FROM ubuntu:${UBUNTU_VERSION} AS brainframe_ubuntu 5 | 6 | ARG PYTHON_VERSION 7 | ENV PYTHON_VERSION=${PYTHON_VERSION} 8 | 9 | ENV DEBIAN_FRONTEND=noninteractive 10 | ENV PATH="/root/.local/bin:$PATH" 11 | 12 | RUN apt-get update \ 13 | && apt-get install -y --no-install-recommends \ 14 | python${PYTHON_VERSION} \ 15 | python${PYTHON_VERSION}-venv \ 16 | python${PYTHON_VERSION}-pip \ 17 | python${PYTHON_VERSION}-dev \ 18 | curl \ 19 | build-essential \ 20 | libgl1-mesa-glx \ 21 | libglib2.0-0 \ 22 | && apt-get clean \ 23 | && rm -rf /var/lib/apt/lists/* 24 | 25 | # Create a virtual environment 26 | RUN python${PYTHON_VERSION} -m venv /opt/venv 27 | ENV PATH="/opt/venv/bin:$PATH" 28 | 29 | # Upgrade pip and install setuptools and wheel within the virtual environment 30 | RUN pip install --upgrade pip setuptools wheel 31 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # How to Build 2 | 3 | ```shell 4 | python3 compile.py build_ext build_binaries 5 | ``` 6 | Clean up a build, 7 | 8 | ```shell 9 | python3 compile.py clean 10 | ``` 11 | 12 | # Capsule Inference 13 | 14 | This tool is used during development to quickly and easily run inference 15 | with a capsule. 16 | 17 | ## Usage 18 | Make sure you are running in an environment with `vcap` and `vcap-utils` 19 | installed. 20 | 21 | Running is simple. Point the script to the capsule of choice, and list one 22 | or more images to run inference on. For example: 23 | ```shell 24 | python3 capsule_infer.py --capsule my-detector-capsule/ --images img.png 25 | img2.png 26 | ``` 27 | 28 | You can also point `--images` to a path that contains images. 29 | 30 | # Capsule Classifier accuracy 31 | 32 | This tool is used during development to quickly and easily run accuracy benchmark 33 | with a capsule. The output is a set of benchmark reports. 34 | 35 | ## Usage 36 | Make sure you are running in an environment with `vcap` and `vcap-utils` 37 | installed. 38 | 39 | Running is simple. Point the script to the capsule of choice, and list one 40 | or more images to run inference on. For example: 41 | ```shell 42 | python3 capsule_classifier_accuracy.py --capsule --images-true --images-false --data attribute= true_threshold=0.0 false_threshold=0.0 detection= 43 | ``` 44 | 45 | Optional image categories are as follows, 46 | 47 | `--images`, `--images-true`, `--images-false` 48 | 49 | # Build the module 50 | Build the environment first, 51 | ```shell 52 | DOCKER_BUILDKIT=1 docker build --file tools/Dockerfile.build-env --build-arg UBUNTU_VERSION=20.04 --tag open_vision_capsule_env --no-cache . 53 | ``` 54 | Then build the module 55 | ```shell 56 | docker run -it --rm -v ./tools:/open_vision_capsules/tools -w /open_vision_capsules/tools open_vision_capsule_env:latest bash -c "poetry build" 57 | ``` 58 | Build an Ubuntu image for testing the module 59 | ```shell 60 | DOCKER_BUILDKIT=1 docker build --file tools/Dockerfile.ubuntu --build-arg UBUNTU_VERSION=20.04 --tag ubuntu-2004 . 61 | ``` 62 | Run the Ubuntu image for testing the module 63 | ```shell 64 | docker run -it --rm -v ./tools:/open_vision_capsules/tools -w /open_vision_capsules/ ubuntu-2004 bash 65 | ``` 66 | Then 67 | ```shell 68 | pip install tools/dist/openvisioncapsule_tools-0.3.8-py3-none-any.whl 69 | openvisioncapsule-tools 70 | ``` 71 | 72 | 73 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencv/open_vision_capsules/c6c6a9c19682d811211b7baffd116555703a0ba7/tools/__init__.py -------------------------------------------------------------------------------- /tools/compile.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021 Dilili Labs, Inc. All rights reserved. Contains Dilili Labs Proprietary Information. RESTRICTED COMPUTER SOFTWARE. LIMITED RIGHTS DATA. 3 | # 4 | # This script compiles a Python app using Cython. 5 | 6 | import multiprocessing 7 | import re 8 | import subprocess 9 | import sysconfig 10 | from distutils.cmd import Command 11 | from distutils.command.build_ext import build_ext 12 | from pathlib import Path 13 | 14 | from Cython.Build import cythonize 15 | from Cython.Compiler import Options as CompilerOptions 16 | from setuptools import setup 17 | from setuptools.extension import Extension 18 | 19 | THREAD_COUNT = multiprocessing.cpu_count() * 2 20 | 21 | BUILD_PATH = Path("build/") 22 | PYTHON_VER = "python3.8" 23 | 24 | BFAPP_PY_DIRS = [ 25 | "capsule_infer", 26 | "capsule_classifier_accuracy", 27 | ] 28 | 29 | # Used to exclude various Python files from Cythonization. If the path is a 30 | # file, then that file will not by Cythonized. If the path is a directory, then 31 | # everything in that directory will not be Cythonized. 32 | cythonize_excludes = [ 33 | # These aren't runtime files 34 | r"tests/.*", 35 | ] 36 | 37 | 38 | def is_excluded(path): 39 | """ 40 | :param path: The path to check 41 | :return: True if the file should be excluded from build-related actions 42 | """ 43 | for exclude in cythonize_excludes: 44 | if re.match(exclude, str(path)): 45 | return True 46 | 47 | return False 48 | 49 | 50 | class BuildBinariesCommand(Command): 51 | """Builds a Python App run files into a binary.""" 52 | 53 | description = "build the main scripts into binaries" 54 | user_options = [] 55 | 56 | def __init__(self, dist, executable_paths): 57 | self._executable_paths = executable_paths 58 | super().__init__(dist) 59 | 60 | def initialize_options(self): 61 | pass 62 | 63 | def finalize_options(self): 64 | pass 65 | 66 | def run(self): 67 | for path in self._executable_paths: 68 | self._build_binary(path) 69 | 70 | @staticmethod 71 | def _build_binary(source: Path): 72 | c_file = source.parent / (source.stem + ".c") 73 | exe_file = source.with_suffix("") 74 | 75 | cython_command = f"python3 -m cython --embed -3 -o {c_file} {source}" 76 | subprocess.run(cython_command, shell=True, check=True) 77 | 78 | gcc_command = ( 79 | f"gcc -Os " 80 | f"-I {sysconfig.get_path('include')} " 81 | f"-o {exe_file} " 82 | f"{c_file} " 83 | f"-l{PYTHON_VER} -lpthread -lm -lutil -ldl " 84 | ) 85 | subprocess.run(gcc_command, shell=True, check=True) 86 | 87 | 88 | class BuildBinariesCommand(BuildBinariesCommand): 89 | def __init__(self, dist): 90 | binaries = [] 91 | for BFAPP_PY_DIR in BFAPP_PY_DIRS: 92 | binaries += list(Path(BFAPP_PY_DIR).glob(BFAPP_PY_DIR + ".py")) 93 | super().__init__(dist, binaries) 94 | 95 | 96 | class CleanCommand(Command): 97 | """Cleans the resulting build files from the project directory.""" 98 | 99 | description = "delete output files from the build" 100 | user_options = [] 101 | 102 | def initialize_options(self): 103 | pass 104 | 105 | def finalize_options(self): 106 | pass 107 | 108 | def run(self): 109 | clean_paths = (Path(BFAPP_PY_DIR) for BFAPP_PY_DIR in BFAPP_PY_DIRS) 110 | clean_extensions = ["*.so", "*.c", "*.html"] + BFAPP_PY_DIRS 111 | 112 | for clean_path in clean_paths: 113 | for clean_extension in clean_extensions: 114 | for clean_file in clean_path.rglob(clean_extension): 115 | if not is_excluded(clean_file): 116 | print(f"delete {clean_file}") 117 | clean_file.unlink() 118 | 119 | # Delete the output files from creating the executables 120 | build_folder_files = list(BUILD_PATH.glob("**/**/*")) 121 | 122 | for path in build_folder_files: # executable_paths: 123 | if path.is_file(): 124 | print(f"delete {path}") 125 | path.unlink() 126 | 127 | 128 | class BuildExtParallel(build_ext): 129 | def __init__(self, dist, extension_path): 130 | self._extension_path = extension_path 131 | 132 | dist.ext_modules = cythonize( 133 | self._find_extension_modules(), 134 | compiler_directives={ 135 | "language_level": "3", 136 | }, 137 | annotate=False, 138 | nthreads=THREAD_COUNT, 139 | ) 140 | super().__init__(dist) 141 | 142 | def finalize_options(self): 143 | super().finalize_options() 144 | if self.parallel is None: 145 | self.parallel = THREAD_COUNT 146 | 147 | def _find_extension_modules(self): 148 | extension_modules = [] 149 | for extension_path in self._extension_path: 150 | for py_file in extension_path.rglob("*.py"): 151 | if not is_excluded(py_file): 152 | mod_path = str(py_file).replace("/", ".").replace(".py", "") 153 | extension = Extension( 154 | # In past releases, -O3 causes segfaults after ~4-6 hours of use. 155 | # If changed, make sure to run 24h to confirm no stability regressions. 156 | mod_path, 157 | [str(py_file)], 158 | extra_compile_args=["-O1"], 159 | ) 160 | extension_modules.append(extension) 161 | return extension_modules 162 | 163 | 164 | class BuildExtParallel(BuildExtParallel): 165 | def __init__(self, dist): 166 | paths = [] 167 | paths += (Path(BFAPP_PY_DIR) for BFAPP_PY_DIR in BFAPP_PY_DIRS) 168 | super().__init__(dist, paths) 169 | 170 | 171 | """Information on Compiler Options: 172 | https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html 173 | """ 174 | CompilerOptions.error_on_unknown_names = True 175 | CompilerOptions.error_on_uninitialized = True 176 | 177 | setup( 178 | name="open_vision_capsule_tools_cython", 179 | cmdclass={ 180 | "build_ext": BuildExtParallel, 181 | "build_binaries": BuildBinariesCommand, 182 | "clean": CleanCommand, 183 | }, 184 | ) 185 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencv/open_vision_capsules/c6c6a9c19682d811211b7baffd116555703a0ba7/tools/openvisioncapsule_tools/__init__.py -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/bulk_test_main.py: -------------------------------------------------------------------------------- 1 | import glob 2 | from io import BytesIO 3 | from pathlib import Path 4 | import base64 5 | 6 | import cv2 7 | import numpy as np 8 | from PIL import Image 9 | 10 | from tools.bulk_accuracy.capsule_accuracy_common import ( 11 | SampleType, 12 | AccuracyProcessor, 13 | ) 14 | from tools.bulk_accuracy.capsule_accuracy_report import output_detection_result_csv, sum_classified_data 15 | from tools.bulk_accuracy.capsule_mgmt_local import LocalCapsuleManagement, load_local_capsule 16 | from tools.bulk_accuracy.capsule_mgmt_remote import RemoteCapsuleManagement 17 | 18 | 19 | def test_safety_string_cell_video_file(): 20 | input_path = "/home/leefr/Downloads/shanxi/test.mp4" 21 | capsule_names = ["detector_driver_and_special_vehicle_fast", "detector_cell", "tracker_vehicle_iou", "encoder_person_openvino", "tracker_person"] 22 | capsule_mgmt = RemoteCapsuleManagement(capsule_names) 23 | accuracyProcessor = AccuracyProcessor( 24 | capsule_mgmt, 25 | SampleType.video_file, 26 | input_path, 27 | 1, 28 | show_rendered_image=True, 29 | ) 30 | accuracyProcessor.run() 31 | output_detection_result_csv("/home/leefr/temp/analysis-data-cell", accuracyProcessor.detected_detail_data) 32 | 33 | 34 | def test_safety_string_cell_images(): 35 | input_path = "/home/leefr/Downloads/shanxi/images" 36 | capsule_names = ["detector_driver_and_special_vehicle_fast", "detector_cell", "tracker_vehicle_iou", "encoder_person_openvino", "tracker_person"] 37 | capsule_mgmt = RemoteCapsuleManagement(capsule_names) 38 | accuracyProcessor = AccuracyProcessor( 39 | capsule_mgmt, 40 | SampleType.image, 41 | input_path, 42 | 1, 43 | show_rendered_image=True, 44 | ) 45 | accuracyProcessor.run() 46 | output_detection_result_csv("/home/leefr/temp/analysis-data-cell", accuracyProcessor.detected_detail_data) 47 | 48 | 49 | def test_classifier_video_file(): 50 | # capsule_names = ["detector_person_openvino", "classifier_phoning_openvino"] 51 | # capsule_mgmt = RemoteCapsuleManagement(["classifier_phoning_openvino"]) 52 | # input_path = "/home/leefr/Downloads/phoning" 53 | input_path = "/home/leefr/Downloads/20220524HBPhoning/test" 54 | 55 | capsule_mgmt = LocalCapsuleManagement() 56 | analyze_files = [ 57 | Path(p).resolve() for p in glob.iglob(input_path + "/**/*.mp4", recursive=True) 58 | ] 59 | 60 | detected_data = [] 61 | video_file_num = 1 62 | for video_file in analyze_files: 63 | accuracyProcessor = AccuracyProcessor( 64 | capsule_mgmt, 65 | SampleType.video_file, 66 | video_file, 67 | video_file_num, 68 | show_rendered_image=True, 69 | filter_class_names=["person"], 70 | ) 71 | # accuracyProcessor.start() 72 | # accuracyProcessor.join() 73 | accuracyProcessor.run() 74 | detected_data.extend(accuracyProcessor.detected_detail_data) 75 | video_file_num += 1 76 | output_detection_result_csv("/home/leefr/temp/analysis-data", detected_data) 77 | 78 | 79 | def test_classifier_images(): 80 | # options = { 81 | # "true threshold": 0.0, 82 | # "false threshold": 0.0, 83 | # } 84 | capsule_mgmt = LocalCapsuleManagement(only_classified=True) 85 | # capsule_names = ["detector_person_administration", "classifier_phoning_openvino"] 86 | 87 | accuracyProcessor = AccuracyProcessor( 88 | capsule_mgmt, 89 | SampleType.image, 90 | "/home/leefr/capsules-test/pictures/test-data/phone", 91 | 1, 92 | show_rendered_image=False, 93 | filter_class_names=["person"], 94 | ) 95 | accuracyProcessor.run() 96 | detected_data = accuracyProcessor.detected_detail_data 97 | output_detection_result_csv("/home/leefr/temp/analysis-data-images", detected_data) 98 | sum_classified_data(detected_data) 99 | 100 | 101 | def test_package_capsule(): 102 | packaged_capsule_path = "/home/leefr/brainframe/pharmacy/private/attach_original_image1.2.cap" 103 | unpackaged_capsule_path = "/home/leefr/brainframe/pharmacy/private/attach_original_image" 104 | capsule_path = Path(packaged_capsule_path) 105 | if capsule_path.exists(): 106 | capsule_path.unlink() 107 | load_local_capsule( 108 | packaged_capsule_path=packaged_capsule_path, 109 | unpackaged_capsule_path=unpackaged_capsule_path, 110 | ) 111 | 112 | 113 | def main(): 114 | test_package_capsule() 115 | # test_classifier_video_file() 116 | # test_classifier_images() 117 | # test_safety_string_cell_video_file() 118 | # test_safety_string_cell_images() 119 | 120 | 121 | if __name__ == "__main__": 122 | main() 123 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/capsule_accuracy_event_loop.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from copy import deepcopy 3 | 4 | import cv2 5 | from brainframe.api import BrainFrameAPI 6 | from brainframe.api.bf_codecs import image_utils 7 | 8 | 9 | def get_positions(bbox): 10 | p1, p2 = bbox[0], bbox[2] 11 | x1, x2 = p1[0], p2[0] 12 | y1, y2 = p1[1], p2[1] 13 | return x1, x2, y1, y2 14 | 15 | 16 | def get_first_item(tup4): 17 | a, b, c, d = tup4 18 | return c 19 | 20 | 21 | class VideoEventLoop: 22 | def __init__(self, bf_server_url): 23 | self.exit_flag = False 24 | self.api: BrainFrameAPI = BrainFrameAPI(bf_server_url) 25 | print("Connecting to Brainframe Server...") 26 | self.api.wait_for_server_initialization() 27 | self.stream_processors = {} 28 | print("Connected to Brainframe Server...") 29 | 30 | def video_event_loop(self): 31 | for zone_status_packet in self.api.get_zone_status_stream(timeout=5): 32 | if self.exit_flag: 33 | return 34 | for stream_id, zone_statuses in zone_status_packet.items(): 35 | if stream_id not in self.stream_processors: 36 | stream_processor = StreamProcessor(stream_id) 37 | self.stream_processors[stream_id] = stream_processor 38 | else: 39 | stream_processor = self.stream_processors[stream_id] 40 | stream_processor.process_zone_statuses(zone_statuses) 41 | if stream_processor.latest_rendered_img is not None: 42 | # cv2.waitKey(1) 43 | # cv2.imshow("demo", stream_processor.latest_rendered_img) 44 | pass 45 | 46 | 47 | class StreamProcessor: 48 | backend_color = (255, 255, 255) 49 | frontend_color = (0, 0, 0) 50 | 51 | def __init__(self, stream_id): 52 | self.stream_id = stream_id 53 | self.latest_original_img = None 54 | self.latest_detections = None 55 | self.latest_rendered_img = None 56 | self.latest_detection_info = {} 57 | self.detection_number = 0 58 | 59 | def process_zone_statuses(self, zone_statuses): 60 | self.detection_number = 0 61 | if zone_statuses is None: 62 | return 63 | for zone_status_name in zone_statuses: 64 | zone_status = zone_statuses[zone_status_name] 65 | if zone_status_name == "Screen": 66 | self.pickup_image_from_detections(zone_status) 67 | detections = zone_status.within 68 | self.render_image(zone_status_name, detections) 69 | 70 | def pickup_image_from_detections(self, zone_status): 71 | attach_original_image_detection = None 72 | idx = -1 73 | for detection in zone_status.within: 74 | idx += 1 75 | if detection.class_name == "attach_original_image": 76 | attach_original_image_detection = detection 77 | break 78 | if attach_original_image_detection is not None: 79 | del zone_status.within[idx] 80 | if attach_original_image_detection is not None and "img" in attach_original_image_detection.extra_data: 81 | img_encoded = attach_original_image_detection.extra_data["img"] 82 | img_bytes = base64.b64decode(img_encoded.encode("ascii")) 83 | self.latest_original_img = image_utils.decode(img_bytes) 84 | self.latest_rendered_img = deepcopy(self.latest_original_img) 85 | 86 | def render_image(self, zone_status_name, detections): 87 | self.latest_rendered_img = deepcopy(self.latest_original_img) 88 | print(f"set to original image") 89 | detections = sorted(detections, key=lambda e: get_first_item(get_positions(e.bbox))) 90 | lines = [] 91 | for detection in detections: 92 | self.detection_number += 1 93 | bbox = detection.bbox 94 | x1, x2, y1, y2 = get_positions(bbox) 95 | size = f"{abs(x1 - x2)}X{abs(y1 - y2)}" 96 | 97 | lines.append("") 98 | lines.append(f"{self.detection_number}") 99 | lines.append(f"{detection.class_name}: {size}") 100 | for key in detection.attributes: 101 | lines.append(f"{key}: {detection.attributes[key]}") 102 | 103 | for key in detection.extra_data: 104 | lines.append(f"{key}: {detection.extra_data[key]}") 105 | 106 | print(f"draw detection: {self.detection_number}") 107 | cv2.rectangle(self.latest_rendered_img, (x1, y1), (x2, y2), StreamProcessor.backend_color, 1) 108 | # cv2.rectangle(self.latest_rendered_img, (x1, y1), (x1 + 50, y1 + 50), StreamProcessor.backend_color, -1) 109 | cv2.circle(self.latest_rendered_img, (x1 + 25, y1 + 25), 25, StreamProcessor.backend_color, -1) 110 | cv2.putText( 111 | img=self.latest_rendered_img, 112 | text=f"{self.detection_number}", 113 | fontFace=cv2.FONT_HERSHEY_SIMPLEX, 114 | fontScale=1.2, 115 | org=(x1 + 15, y1 + 40), 116 | color=StreamProcessor.frontend_color, 117 | thickness=2, 118 | ) 119 | self.latest_detection_info[zone_status_name] = lines 120 | if self.latest_rendered_img is not None: 121 | cv2.waitKey(1) 122 | cv2.imshow("rendered", self.latest_rendered_img) 123 | print(f"show image") 124 | # timestamp = datetime.datetime.now() 125 | # cv2.imwrite(f"/tmp/{timestamp}.jpg", self.latest_rendered_img) 126 | if self.latest_original_img is not None: 127 | cv2.waitKey(1) 128 | cv2.imshow("original", self.latest_original_img) 129 | 130 | def get_rendered_image(self, height=None, width=None): 131 | pass 132 | 133 | 134 | def main(): 135 | video_event_loop = VideoEventLoop("http://localhost") 136 | video_event_loop.video_event_loop() 137 | 138 | 139 | if __name__ == "__main__": 140 | main() 141 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/capsule_accuracy_report.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | from pathlib import Path 5 | 6 | from tools.bulk_accuracy.capsule_accuracy_common import DetectionResult 7 | 8 | 9 | def write_to_file(output_directory, lines): 10 | output_file_path = Path(output_directory, f'analysis_detection_result_{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.csv') 11 | output_file_path.parent.mkdir(parents=True, exist_ok=True) 12 | with open(str(output_file_path), "w") as fp: 13 | fp.writelines(lines) 14 | 15 | 16 | def output_detection_result_csv(output_directory, data_list): 17 | title = ( 18 | f"sample_path,offset_time,img_id,classified_type,confidence_value{os.linesep}" 19 | ) 20 | lines = [title] 21 | 22 | for e in data_list: 23 | dd: DetectionResult = e 24 | line = f"{dd.sample_path},{dd.offset_time},{dd.img_id},{dd.classified_type},{dd.confidence_value}{os.linesep}" 25 | lines.append(line) 26 | 27 | write_to_file(output_directory, lines) 28 | 29 | 30 | def output_detection_result_json(output_directory, data_list): 31 | lines = [] 32 | for e in data_list: 33 | dd: DetectionResult = e 34 | line = dd.to_json_str() 35 | lines.append(f"{line}{os.linesep}") 36 | 37 | write_to_file(output_directory, lines) 38 | 39 | 40 | def load_detection_result_json(input_json_file): 41 | data_list = [] 42 | with open(input_json_file, "r") as fp: 43 | lines = fp.readlines() 44 | for line in lines: 45 | json_obj = json.loads(line) 46 | data_list.append(DetectionResult.from_json(json_obj)) 47 | return data_list 48 | 49 | 50 | def load_detection_result_csv(input_csv_file): 51 | data_list = [] 52 | with open(input_csv_file, "r") as fp: 53 | lines = fp.readlines() 54 | for idx in range(1, len(lines)): 55 | line = lines[idx] 56 | data_list.append(DetectionResult.from_csv(line)) 57 | return data_list 58 | 59 | 60 | def sum_classified_data(detected_data_list): 61 | class GroupedDetectionResult: 62 | def __init__(self, classified_type, threshold_confidence_value): 63 | self.classified_type = classified_type 64 | self.threshold_confidence_value = threshold_confidence_value 65 | self.num_target_yes = 0 66 | self.num_target_no = 0 67 | self.num_unknown = 0 68 | 69 | def accumulate(self, rst: DetectionResult): 70 | if rst.classified_type == self.classified_type: 71 | if rst.confidence_value >= self.threshold_confidence_value: 72 | self.num_target_yes += 1 73 | else: 74 | self.num_unknown += 1 75 | else: 76 | self.num_target_no += 1 77 | 78 | def to_string(self): 79 | return f"{self.classified_type}\t{format(self.threshold_confidence_value, '.2f')}\t{self.num_target_yes}\t{self.num_unknown}\t{self.num_target_no}" 80 | 81 | all_true, all_false = [], [] 82 | for threshold in range(20): 83 | sub_sum_true, sub_sum_false = GroupedDetectionResult(True, 0.05 * threshold), GroupedDetectionResult(False, 0.05 * threshold) 84 | for detection in detected_data_list: 85 | sub_sum_true.accumulate(detection) 86 | sub_sum_false.accumulate(detection) 87 | all_true.append(sub_sum_true) 88 | all_false.append(sub_sum_false) 89 | 90 | print("True=>") 91 | for e in all_true: 92 | print(e.to_string()) 93 | 94 | print("False=>") 95 | for e in all_false: 96 | print(e.to_string()) 97 | 98 | 99 | if __name__ == "__main__": 100 | data = load_detection_result_csv("/home/leefr/temp/analysis-data/analysis_data_20220623005435.csv") 101 | sum_classified_data(data) 102 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/capsule_mgmt_basic.py: -------------------------------------------------------------------------------- 1 | class BasicCapsuleManagement: 2 | def process_image(self, frame): 3 | pass 4 | 5 | def get_positions(self, bbox): 6 | pass 7 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/capsule_mgmt_local.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from copy import deepcopy 3 | from pathlib import Path 4 | 5 | from vcap import DetectionNode 6 | from vcap.loading.capsule_loading import load_capsule 7 | from vcap.loading.packaging import package_capsule 8 | 9 | from tools.bulk_accuracy.capsule_mgmt_basic import BasicCapsuleManagement 10 | 11 | 12 | class LocalCapsuleManagement(BasicCapsuleManagement): 13 | def __init__(self, only_classified=False): 14 | self.only_classified = only_classified 15 | self.classifier_name = "phoning" 16 | self.detect_person, self.detect_person_options = load_local_capsule( 17 | "/home/leefr/brainframe/pharmacy/private/detector_80_objects.cap", 18 | "/home/leefr/brainframe/pharmacy/private/detector_80_objects" 19 | ) 20 | self.classified_phoning, self.classified_phoning_options = load_local_capsule( 21 | "/home/leefr/brainframe/pharmacy/private/classifier_phoning_factory_openvino3.1.cap", 22 | "/home/leefr/brainframe/pharmacy/private/classifier_phoning_factory_openvino", 23 | { 24 | "true threshold": 0.0, 25 | "false threshold": 0.0, 26 | } 27 | ) 28 | 29 | def process_image(self, frame): 30 | if self.only_classified: 31 | h_orig, w_orig = frame.shape[:-1] 32 | points = [[0, 0], [w_orig, h_orig]] 33 | detection = DetectionNode( 34 | name=self.classifier_name, 35 | coords=points, 36 | attributes={}, 37 | extra_data={} 38 | ) 39 | 40 | self.classified_phoning.process_frame( 41 | frame=frame, 42 | detection_node=detection, 43 | options=self.classified_phoning_options, 44 | state=self.classified_phoning.stream_state(), 45 | ) 46 | return [detection] 47 | else: 48 | detections = self.detect_person.process_frame( 49 | frame=frame, 50 | detection_node=None, 51 | options=self.detect_person_options, 52 | state=self.detect_person.stream_state(), 53 | ) 54 | for detection in detections: 55 | self.classified_phoning.process_frame( 56 | frame=frame, 57 | detection_node=detection, 58 | options=self.classified_phoning_options, 59 | state=self.classified_phoning.stream_state(), 60 | ) 61 | return detections 62 | 63 | def get_positions(self, bbox): 64 | return bbox.x1, bbox.x2, bbox.y1, bbox.y2 65 | 66 | 67 | def load_local_capsule(packaged_capsule_path, unpackaged_capsule_path, options=None): 68 | if not Path(packaged_capsule_path).exists(): 69 | package_capsule(Path(unpackaged_capsule_path), Path(packaged_capsule_path)) 70 | 71 | capsule = load_capsule(path=packaged_capsule_path) 72 | capsule_options = deepcopy(capsule.default_options) 73 | 74 | if options is not None: 75 | for key in options: 76 | val = options[key] 77 | capsule_options[key] = val 78 | return capsule, capsule_options 79 | 80 | 81 | class LocalCapsule: 82 | def __init__(self, packaged_capsule_path, unpackaged_capsule_path, options=None): 83 | self.packaged_capsule_path = packaged_capsule_path 84 | self.unpackaged_capsule_path = unpackaged_capsule_path 85 | self.options = options 86 | self.capsule = None 87 | self.capsule_options = None 88 | self.initial_capsule() 89 | 90 | def initial_capsule(self, ): 91 | if not Path(self.packaged_capsule_path).exists(): 92 | package_capsule(Path(self.unpackaged_capsule_path), Path(self.packaged_capsule_path)) 93 | 94 | self.capsule = load_capsule(path=self.packaged_capsule_path) 95 | self.capsule_options = deepcopy(self.capsule.default_options) 96 | 97 | if self.options is not None: 98 | for key in self.options: 99 | val = self.options[key] 100 | self.capsule_options[key] = val 101 | 102 | def process_image(self, frame, detections=None): 103 | if frame is None: 104 | return None 105 | if detections is None: 106 | detections = self.capsule.process_frame( 107 | frame=frame, 108 | detection_node=None, 109 | options=self.capsule_options, 110 | state=self.capsule.stream_state, 111 | ) 112 | else: 113 | for detection in detections: 114 | self.capsule.process_frame( 115 | frame=frame, 116 | detection_node=detection, 117 | options=self.capsule_options, 118 | state=self.capsule.stream_state, 119 | ) 120 | return detections 121 | 122 | 123 | if __name__ == "__main__": 124 | # /home/leefr/capsules-test/capsules/detector_person_openvino.cap 125 | img = cv2.imread("/home/leefr/Pictures/test.jpg") 126 | local_capsule_mgmt = LocalCapsuleManagement() 127 | classified_detections = local_capsule_mgmt.process_image(img) 128 | 129 | print(classified_detections) 130 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_accuracy/capsule_mgmt_remote.py: -------------------------------------------------------------------------------- 1 | from brainframe.api import BrainFrameAPI 2 | 3 | from tools.bulk_accuracy.capsule_mgmt_basic import BasicCapsuleManagement 4 | 5 | 6 | class RemoteCapsuleManagement(BasicCapsuleManagement): 7 | def __init__(self, capsule_names, options=None): 8 | self.capsule_names = capsule_names 9 | self.options = options 10 | self.previous_capsule_options = {} 11 | self.api = None 12 | self.initial_global_capsule_options() 13 | 14 | def initial_global_capsule_options(self): 15 | bf_server_url = "http://localhost:80" 16 | self.api: BrainFrameAPI = BrainFrameAPI(bf_server_url) 17 | print("Connecting to Brainframe Server...") 18 | self.api.wait_for_server_initialization() 19 | if self.options is not None: 20 | for capsule_name in self.capsule_names: 21 | current_options = self.api.get_capsule_option_vals( 22 | capsule_name=capsule_name 23 | ) 24 | self.previous_capsule_options[capsule_name] = current_options 25 | self.api.set_capsule_option_vals( 26 | capsule_name=capsule_name, option_vals=self.options 27 | ) 28 | 29 | def recover_global_capsule_options(self): 30 | for capsule_name in self.previous_capsule_options: 31 | previous_options = self.previous_capsule_options[capsule_name] 32 | self.api.set_capsule_option_vals( 33 | capsule_name=capsule_name, option_vals=previous_options 34 | ) 35 | 36 | def process_image(self, frame, detections=None): 37 | detections = self.api.process_image(frame, self.capsule_names, {}) 38 | return detections 39 | 40 | def get_positions(self, bbox): 41 | p1, p2 = bbox[0], bbox[2] 42 | x1, x2 = p1[0], p2[0] 43 | y1, y2 = p1[1], p2[1] 44 | return x1, x2, y1, y2 45 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/bulk_testing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PROG_PREFIX=~/brainframe 4 | DATA_PREFIX=~/dataset/dataset-2022-06-03 5 | CAPSULE=$PROG_PREFIX/pharmacy/private/classifier_phoning_factory_openvino 6 | 7 | IMAGE_TRUE_DIR=val_phone 8 | IMAGE_FALSE_DIR=val_nophone 9 | ATTRIBUTE=phoning 10 | DETECTION=person 11 | 12 | CAPSULE_INFER_PATH=$PROG_PREFIX/open_vision_capsules/tools/capsule_infer 13 | CMD="python3 $PROG_PREFIX/open_vision_capsules/tools/capsule_classifier_accuracy/capsule_classifier_accuracy.py" 14 | # CMD=capsule_classifier_accuracy_v0.3 15 | 16 | BASIC_ARGS="--capsule $CAPSULE --images-true $DATA_PREFIX/$IMAGE_TRUE_DIR --images-false $DATA_PREFIX/$IMAGE_FALSE_DIR --nowait --data attribute=$ATTRIBUTE detection=$DETECTION " 17 | 18 | # Basic while loop 19 | i=3 20 | while [ $i -le 9 ] 21 | do 22 | j=0 23 | while [ $j -le 9 ] 24 | do 25 | OPTIONS="{\"true threshold\": 0.${i}, \"false threshold\": 0.${j}}" 26 | ARGS="$BASIC_ARGS true_threshold=0.$i false_threshold=0.$j" 27 | echo $OPTIONS > options.json 28 | echo $OPTIONS 29 | echo $CMD $ARGS 30 | export PYTHONPATH=$PYTHONPATH:$CAPSULE_INFER_PATH 31 | $CMD $ARGS 32 | ((j++)) 33 | done 34 | ((i++)) 35 | done 36 | echo All done 37 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/README.md: -------------------------------------------------------------------------------- 1 | # Capsule Benchmark 2 | This tool is used to produce benchmark result graphs for one or more capsules. 3 | 4 | The tool will benchmark the capsule at different levels of parallelism. 5 | Parallelism is defined as how many concurrent requests are being sent to the 6 | capsule at any given time. For example, a parallelism of 3 is equivalent to 7 | 3 threads pushing requests to the capsule as fast as they can be completed. 8 | 9 | The reason that parallelism is a focus on this benchmark is because many deep 10 | learning algorithms benefit from 'batching' requests. That is, sending multiple 11 | images or inputs to the GPU / other devices concurrently. 12 | 13 | ## Usage 14 | 15 | ### Install Dependencies 16 | First, install the `vcap` and `vcap_utils` libraries. Then run: 17 | ``` 18 | pip3 install -r requirements.txt 19 | ``` 20 | 21 | ### Running the script 22 | 23 | Store one or more unpackaged capsules in a directory, in this example called 24 | `capsules`. Then run the script and point it to the directory with the 25 | capsule(s). 26 | ``` 27 | cd tools/capsule_benchmark 28 | python3 main.py --capsule-dir /path/to/capsules/ 29 | ``` 30 | 31 | The parallelism (x axis) can be changed by varying the `--parallelism`. 32 | By default, it runs tests with 1, 2, 3, 5, and 10 parallelism. This can be 33 | changed, for example, `--parallelism 1 10 100` will run 3 tests with 1, 10, 34 | and 100 as the parallelism parameters. 35 | 36 | The number of samples can also be adjusted. `--num-samples 100` will run 100 37 | samples per test. 38 | 39 | After running, an `output.html` graph will be in the current directory, 40 | also configurable via `--output-graph`. 41 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | 4 | 5 | def parse_args() -> argparse.Namespace: 6 | parser = argparse.ArgumentParser( 7 | description='Benchmark capsule parallelism') 8 | 9 | # noinspection PyTypeChecker 10 | parser.add_argument('-c', '--capsule-dir', type=Path, required=True, 11 | help="Directory of unpackaged capsules to be tested") 12 | parser.add_argument('-w', '--parallelism', type=int, nargs="+", 13 | default=[1, 2, 3, 5, 10], 14 | help="Number of threads to parallelize to. In bash " 15 | "you can use {0..10} or `seq 0 2 10` to avoid " 16 | "having to list a lot of values.") 17 | parser.add_argument('-s', '--num-samples', type=int, default=100, 18 | help="Number of samples to run for each capsule") 19 | 20 | # noinspection PyTypeChecker 21 | parser.add_argument('-f', '--output-csv', type=Path, default=None, 22 | help="If specified, results will be written to the " 23 | "provided path in .csv format") 24 | # noinspection PyTypeChecker 25 | parser.add_argument('-g', '--output-graph', type=Path, 26 | default="output.html", 27 | help="Results will be rendered to a graph " 28 | "and saved at the provided path as HTML." 29 | "Example: output.html") 30 | 31 | return parser.parse_args() 32 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/benchmarking.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from pathlib import Path 3 | from typing import Any, Callable, Iterable, List, NamedTuple, Optional 4 | 5 | import numpy as np 6 | from tqdm import tqdm 7 | from vcap import BaseCapsule, NodeDescription 8 | from vcap.testing.input_output_validation import make_detection_node 9 | 10 | from capsules import CapsuleDir 11 | from workers import CapsuleThreadPool 12 | 13 | 14 | class BenchmarkSuite: 15 | class Result(NamedTuple): 16 | capsule_name: str 17 | num_workers: int 18 | num_samples: int 19 | fps: float 20 | 21 | def __init__(self, capsule_dir: Path, num_workers: Iterable[int], 22 | image_func: Optional[Callable[[], np.ndarray]] = None): 23 | """ 24 | :param capsule_dir: Directory containing unpackaged capsules to test 25 | :param num_workers: Iterable containing the different num_worker values 26 | to test 27 | :param image_func: Function that returns an image. Can return the same 28 | image over and over, or different images 29 | """ 30 | 31 | self.capsule_dir = CapsuleDir(capsule_dir) 32 | self.num_workers = num_workers 33 | 34 | # Generate a random image to run the benchmark on. 35 | # Generating an image for each process_frame has a big overhead, 36 | # limiting speed to < 100 FPS. 37 | self.rng = np.random.RandomState(1337) 38 | self.image = self.rng.randint(0, 255, (1920, 1080, 3), 39 | dtype=np.uint8) 40 | self.image.flags.writeable = False 41 | 42 | self.capsule_dir.package_capsules() 43 | 44 | def test(self, num_samples: int) -> List[Result]: 45 | results: List[self.Result] = [] 46 | 47 | total_tests = len(self.capsule_dir) * len(list(self.num_workers)) 48 | with tqdm(total=total_tests) as progress_bar: 49 | for capsule in self.capsule_dir: 50 | for num_workers in self.num_workers: 51 | worker_pool = CapsuleThreadPool(num_workers) 52 | 53 | duration = self.perform_test(capsule, worker_pool, 54 | num_samples) 55 | result = self.Result( 56 | capsule_name=capsule.name, 57 | num_workers=num_workers, 58 | num_samples=num_samples, 59 | fps=num_samples / duration.total_seconds() 60 | ) 61 | results.append(result) 62 | 63 | progress_bar.update(1) 64 | worker_pool.shutdown() 65 | 66 | capsule.close() 67 | 68 | return results 69 | 70 | def generate_input_kwargs(self, capsule): 71 | if capsule.input_type.size is NodeDescription.Size.NONE: 72 | input_node = None 73 | else: 74 | input_node = make_detection_node(self.image.shape, capsule.input_type) 75 | 76 | # Set node size to cover entire frame 77 | height, width, _ = self.image.shape 78 | input_node.coords = [[0, 0], 79 | [width, 0], 80 | [width, height], 81 | [0, height]] 82 | 83 | if capsule.input_type.size is NodeDescription.Size.ALL: 84 | input_node = [input_node] 85 | 86 | return {"frame": self.image, 87 | "detection_node": input_node, 88 | "options": capsule.default_options, 89 | "state": capsule.stream_state()} 90 | 91 | def perform_test(self, capsule: BaseCapsule, 92 | worker_pool: CapsuleThreadPool, num_samples: int) \ 93 | -> datetime.timedelta: 94 | 95 | # Warm things up, such as getting model on GPU if capsule uses it 96 | warmup_results = worker_pool.map( 97 | lambda kwargs: capsule.process_frame(**kwargs), 98 | [self.generate_input_kwargs(capsule) for _ in range(50)] 99 | ) 100 | for _ in warmup_results: 101 | pass 102 | 103 | # Generate test args before starting the test, so that the 104 | # benchmark is purely just for the capsule 105 | test_inputs = [self.generate_input_kwargs(capsule) 106 | for _ in range(num_samples)] 107 | 108 | # Begin the benchmark 109 | start_time = datetime.datetime.now() 110 | results = worker_pool.map( 111 | lambda kwargs: capsule.process_frame(**kwargs), 112 | test_inputs) 113 | 114 | for _ in results: 115 | pass 116 | 117 | end_time = datetime.datetime.now() 118 | duration = end_time - start_time 119 | 120 | return duration 121 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/capsules.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Iterator, Optional 3 | 4 | from vcap import BaseCapsule 5 | from vcap.loading.capsule_loading import load_capsule 6 | from vcap.loading.packaging import CAPSULE_EXTENSION, package_capsule 7 | 8 | 9 | class CapsuleDir: 10 | 11 | def __init__(self, capsule_dir: Path): 12 | self.capsule_dir = capsule_dir 13 | 14 | def package_capsules(self, output_dir: Optional[Path] = None): 15 | unpackaged_capsules = (path for path in self.capsule_dir.iterdir() 16 | if path.is_dir()) 17 | 18 | output_dir = self.capsule_dir if output_dir is None else output_dir 19 | 20 | for unpackaged_capsule in unpackaged_capsules: 21 | capsule_name = unpackaged_capsule \ 22 | .with_suffix(CAPSULE_EXTENSION) \ 23 | .name 24 | 25 | packaged_capsule_path = output_dir / capsule_name 26 | 27 | package_capsule(unpackaged_capsule, packaged_capsule_path) 28 | 29 | def __iter__(self) -> Iterator[BaseCapsule]: 30 | """Iterates through directory, returning loaded capsules""" 31 | capsule_files = self.capsule_dir.glob(f"*{CAPSULE_EXTENSION}") 32 | 33 | for capsule_file in capsule_files: 34 | capsule = load_capsule(capsule_file) 35 | yield capsule 36 | 37 | def __len__(self) -> int: 38 | """Count of (packaged) capsules in directory""" 39 | capsule_files = self.capsule_dir.glob(f"*{CAPSULE_EXTENSION}") 40 | return len(list(capsule_files)) 41 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/main.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | import output 4 | from args import parse_args 5 | from benchmarking import BenchmarkSuite 6 | 7 | 8 | def main(): 9 | args = parse_args() 10 | 11 | testing_suite = BenchmarkSuite(args.capsule_dir, args.parallelism) 12 | results = testing_suite.test(args.num_samples) 13 | 14 | df = pd.DataFrame.from_records(results, 15 | columns=BenchmarkSuite.Result._fields) 16 | df.sort_values(by=list(df.columns), inplace=True, ignore_index=True) 17 | 18 | output.generate_output( 19 | output=df, 20 | csv_path=args.output_csv, 21 | graph_path=args.output_graph 22 | ) 23 | 24 | 25 | if __name__ == '__main__': 26 | main() 27 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/output.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Optional 3 | 4 | import altair as alt 5 | import pandas as pd 6 | 7 | 8 | def generate_output(output: pd.DataFrame, *, 9 | csv_path: Optional[Path] = None, 10 | graph_path: Optional[Path] = None): 11 | if csv_path is not None: 12 | generate_csv_output(output, csv_path) 13 | if graph_path is not None: 14 | generate_graph_output(output, graph_path) 15 | 16 | 17 | def generate_csv_output(output: pd.DataFrame, csv_path: Path): 18 | output.to_csv(csv_path) 19 | 20 | 21 | def generate_graph_output(output: pd.DataFrame, graph_path: Path): 22 | num_workers = alt.X('num_workers:O', 23 | axis=alt.Axis(title="Parallelization")) 24 | fps = alt.Y('fps:Q', axis=alt.Axis(title="FPS")) 25 | capsules = alt.Color('capsule_name:N', legend=alt.Legend(title="Capsule")) 26 | 27 | graph: alt.Chart = alt.Chart(output).mark_line().encode( 28 | x=num_workers, 29 | y=fps, 30 | color=capsules, 31 | ) 32 | 33 | graph: alt.Chart = graph.properties( 34 | width=600, 35 | height=600 36 | ) 37 | 38 | graph.save(str(graph_path)) 39 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/requirements.txt: -------------------------------------------------------------------------------- 1 | pandas==1.0.5 2 | altair==4.1.0 3 | tqdm==4.66.3 4 | mock==4.0.2 -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_benchmark/workers.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures.thread import ThreadPoolExecutor 2 | from time import sleep 3 | 4 | 5 | class CapsuleThreadPool(ThreadPoolExecutor): 6 | 7 | def __init__(self, num_workers: int): 8 | super().__init__(max_workers=num_workers) 9 | 10 | self.num_workers = num_workers 11 | 12 | self._warm_workers() 13 | 14 | def _warm_workers(self): 15 | """Initialize the worker pool before starting the test""" 16 | 17 | def waste_time(_): 18 | sleep(0.25) 19 | 20 | for _ in self.map(waste_time, range(self.num_workers)): 21 | pass 22 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_classifier_accuracy/detect_videos.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | from argparse import ArgumentParser 4 | from copy import deepcopy 5 | from pathlib import Path 6 | # from PIL import Image 7 | import cv2 8 | import glob 9 | 10 | from datetime import datetime 11 | import logging 12 | from vcap import ( 13 | options, 14 | common_detector_options, 15 | load_capsule, FloatOption 16 | ) 17 | from tools.capsule_infer.capsule_infer import ( 18 | read_options, 19 | capsule_options_and_key, 20 | capsule_inference, 21 | parse_images, 22 | capsule_infer_add_args, 23 | parse_capsule_info) 24 | 25 | Path("../logs").mkdir(parents=True, exist_ok=True) 26 | appendix = datetime.now().strftime('%Y%d%m%H%M%S') 27 | logging.basicConfig(filename='../logs/brainframe' + appendix + '.log', 28 | format='%(asctime)s %(levelname)-8s %(message)s', 29 | datefmt='%Y-%d-%m %H:%M:%S', 30 | level=logging.INFO) 31 | 32 | DETECTOR_PERSON_CAPSULE_PATH = "/home/leefr/capsules-test/capsules/detector_person_openvino.cap" 33 | CLASSIFIER_PHONING_CAPSULE_PATH = '/home/leefr/capsules-test/capsules/classifier_phoning_factory_openvino.cap' 34 | 35 | capsule_detector = load_capsule(path=DETECTOR_PERSON_CAPSULE_PATH) 36 | options_capsule_detector = deepcopy(capsule_detector.default_options) 37 | options_capsule_detector["threshold"] = 0.5 38 | 39 | capsule_classifier = load_capsule(path=CLASSIFIER_PHONING_CAPSULE_PATH) 40 | options_capsule_classifier = deepcopy(capsule_classifier.default_options) 41 | options_capsule_classifier["threshold"] = 0.5 42 | 43 | img_id = 1 44 | 45 | 46 | def split_video(video_file_path, dest_file_path): 47 | cap = cv2.VideoCapture(str(video_file_path)) 48 | while cap.isOpened(): 49 | _, img = cap.read() 50 | if img is None: 51 | print("WAS NONE") 52 | break 53 | 54 | detect_persons(img, dest_file_path) 55 | 56 | 57 | def detect_persons(img, dest_file_path): 58 | all_detections = capsule_detector.process_frame( 59 | frame=img, 60 | detection_node=None, 61 | options=options_capsule_detector.default_options, 62 | state=options_capsule_detector.stream_state() 63 | ) 64 | people_detections = [d for d in all_detections 65 | if d.class_name == "person"] 66 | 67 | classify_persons(img, people_detections, dest_file_path) 68 | 69 | 70 | def classify_persons(img_origin, people_detections, dest_file_path): 71 | global img_id 72 | 73 | for people in people_detections: 74 | bbox = people.bbox 75 | x0, y0, x1, y1 = bbox.x1, bbox.y1, bbox.x2, bbox.y2 76 | # img_object = img_origin.crop((x0, y0, x1, y1)) 77 | img_object = img_origin[y0:y1, x0:x1] 78 | if img_object is None: 79 | print(f"Failed to crop object: {people}") 80 | continue 81 | 82 | class_of_object = capsule_classifier.process_frame( 83 | frame=img_object, 84 | detection_node=None, 85 | options=options_capsule_classifier.default_options, 86 | state=options_capsule_classifier.stream_state() 87 | ) 88 | 89 | img_object_path = os.path.join(dest_file_path, "{:06d}.jpg".format(img_id)) 90 | img_id += 1 91 | 92 | try: 93 | # img_object.save(img_object_path) 94 | cv2.imwrite(img_object_path, img_object) 95 | except Exception as e: 96 | print(f"Failed to save object: {img_object_path} {e}") 97 | 98 | 99 | if __name__ == "__main__": 100 | parser = ArgumentParser( 101 | description="A helpful tool for running inference and generate accuracy benchmarking report on a capsule." 102 | ) 103 | 104 | # args = parser.parse_args() 105 | # 106 | # cmdline = read_cmdline() 107 | 108 | # construct the argument parser and parse the arguments 109 | # parser = argparse.ArgumentParser() 110 | parser.add_argument("-c", "--capsule", required=True, 111 | help="Capsule path") 112 | parser.add_argument("-v", "--source", required=True, 113 | help="Either a video file or a video directory " 114 | "to process") 115 | parser.add_argument("-d", "--dest", required=True, 116 | help="A directory to store the processed images") 117 | parser.add_argument("-w", "--num_workers", type=int, default=10, 118 | help="How many videos to parse at a time") 119 | args = parser.parse_args() 120 | print("Starting Converted with parameters", args) 121 | 122 | analyze_files = [] 123 | path = Path(args.source).resolve() 124 | if path.is_dir(): 125 | analyze_files = [Path(p).resolve() for p in glob.iglob(str(path) + "/**/*.*", recursive=True)] 126 | elif path.is_file(): 127 | analyze_files = [path] 128 | 129 | Path(args.dest).mkdir(parents=True, exist_ok=True) 130 | 131 | # Verify that all the video files exist 132 | analyze_files = [str(file.resolve()) for file in analyze_files if 133 | Path(file).exists()] 134 | 135 | for src_file in analyze_files: 136 | split_video(src_file, args.dest) 137 | # print(f"Finished: {src_file}") 138 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_infer/__init__.py: -------------------------------------------------------------------------------- 1 | from .capsule_inference import capsule_inference_main 2 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_infer/capsule_inference.py: -------------------------------------------------------------------------------- 1 | from openvisioncapsule_tools.command_utils import command, subcommand_parse_args, by_name 2 | from .capsule_infer import capsule_infer_add_args, capsule_infer 3 | 4 | @command("capsule_inference") 5 | def capsule_inference_main(): 6 | parser = capsule_infer_add_args() 7 | args = subcommand_parse_args(parser) 8 | capsule_infer(args) 9 | 10 | 11 | if __name__ == "__main__": 12 | by_name["capsule_inference"]() 13 | 14 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_packaging/__init__.py: -------------------------------------------------------------------------------- 1 | from .capsule_packaging import capsule_packaging_main 2 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/capsule_packaging/capsule_packaging.py: -------------------------------------------------------------------------------- 1 | from vcap.loading.vcap_packaging import packaging, packaging_parse_args 2 | from openvisioncapsule_tools.command_utils import command, subcommand_parse_args, by_name 3 | 4 | 5 | @command("capsule_packaging") 6 | def capsule_packaging_main(): 7 | parser = packaging_parse_args() 8 | args = subcommand_parse_args(parser) 9 | rtn = packaging(args.capsule_dir, args.capsule_file, args.capsule_key) 10 | if rtn == False: 11 | parser.print_help() 12 | 13 | 14 | if __name__ == "__main__": 15 | by_name["capsule_packaging"]() 16 | 17 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['BF_LOG_PRINT'] = 'TRUE' 3 | 4 | import signal 5 | import sys 6 | from argparse import ArgumentParser 7 | 8 | import i18n 9 | import yaml 10 | from openvisioncapsule_tools import print_utils, command_utils 11 | from openvisioncapsule_tools.capsule_packaging import capsule_packaging 12 | from openvisioncapsule_tools.capsule_infer import capsule_infer 13 | 14 | 15 | def add_translations(translations, prefix=""): 16 | for key, value in translations.items(): 17 | full_key = f"{prefix}.{key}" if prefix else key 18 | if isinstance(value, dict): 19 | add_translations(value, full_key) 20 | else: 21 | i18n.add_translation(full_key, value) 22 | 23 | 24 | def cli_main(): 25 | i18n.load_path.append(_TRANSLATIONS_PATH) 26 | 27 | # Load translations manually 28 | translation_file = os.path.join(_TRANSLATIONS_PATH, "portal.en.yml") 29 | if os.path.exists(translation_file): 30 | with open(translation_file, "r") as file: 31 | translations = yaml.safe_load(file) 32 | add_translations(translations) 33 | 34 | parser = ArgumentParser( 35 | description=i18n.t("en.portal.description"), usage=i18n.t("en.portal.usage") 36 | ) 37 | 38 | parser.add_argument( 39 | "command", default=None, nargs="?", help=i18n.t("en.portal.command-help") 40 | ) 41 | 42 | args = parser.parse_args(sys.argv[1:2]) 43 | 44 | if args.command is None: 45 | parser.print_help() 46 | elif args.command in command_utils.by_name: 47 | command = command_utils.by_name[args.command] 48 | command() 49 | else: 50 | error_message = i18n.t("portal.unknown-command") 51 | error_message = error_message.format(command=args.command) 52 | print_utils.print_color( 53 | error_message, color=print_utils.Color.RED, file=sys.stderr 54 | ) 55 | parser.print_help() 56 | 57 | 58 | _TRANSLATIONS_PATH = os.path.join(os.path.dirname(__file__), "translations") 59 | 60 | if __name__ == "__main__": 61 | cli_main() 62 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/command_utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from argparse import ArgumentParser 3 | 4 | by_name = {} 5 | """A dict that maps command names to their corresponding function""" 6 | 7 | 8 | def command(name): 9 | """A decorator that associates command functions to their command name.""" 10 | 11 | def wrapper(function): 12 | by_name[name] = function 13 | 14 | return wrapper 15 | 16 | 17 | def subcommand_parse_args(parser: ArgumentParser): 18 | arg_list = sys.argv[2:] 19 | args = parser.parse_args(arg_list) 20 | 21 | # Run in non-interactive mode if any flags were provided 22 | if len(arg_list) > 0: 23 | args.noninteractive = True 24 | 25 | return args 26 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/options.json: -------------------------------------------------------------------------------- 1 | { 2 | "true threshold": 0.9, 3 | "false threshold": 0.9 4 | } -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/print_module_info.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import sys 3 | import os 4 | import subprocess 5 | from pkg_resources import get_distribution, DistributionNotFound 6 | 7 | def get_openvino_version(openvino_path): 8 | # Method 1: Try to use openvino.runtime 9 | try: 10 | from openvino.runtime import get_version 11 | return get_version() 12 | except ImportError: 13 | pass 14 | 15 | # Method 2: Try to use openvino.inference_engine 16 | try: 17 | from openvino.inference_engine import get_version 18 | return get_version() 19 | except ImportError: 20 | pass 21 | 22 | # Method 3: Check for version.txt in various locations 23 | version_files = [ 24 | os.path.join(openvino_path, 'deployment_tools', 'model_optimizer', 'version.txt'), 25 | os.path.join(openvino_path, 'version.txt'), 26 | os.path.join(openvino_path, '..', 'version.txt'), 27 | ] 28 | for version_file in version_files: 29 | if os.path.exists(version_file): 30 | with open(version_file, 'r') as f: 31 | return f.read().strip() 32 | 33 | # Method 4: Try to run the command-line version check 34 | try: 35 | result = subprocess.run(['mo', '--version'], capture_output=True, text=True) 36 | if result.returncode == 0: 37 | return result.stdout.strip() 38 | except FileNotFoundError: 39 | pass 40 | 41 | return "Version not found" 42 | 43 | def print_module_info(module_name): 44 | try: 45 | # Try to get distribution info 46 | distribution = get_distribution(module_name) 47 | ver = distribution.version 48 | location = distribution.location 49 | except DistributionNotFound: 50 | # If distribution is not found, try to import the module directly 51 | try: 52 | module = importlib.import_module(module_name) 53 | if module_name == 'openvino': 54 | # Special handling for OpenVINO 55 | openvino_path = os.path.dirname(os.path.dirname(os.path.dirname(module.__file__))) 56 | ver = get_openvino_version(openvino_path) 57 | else: 58 | ver = getattr(module, '__version__', 'Unknown') 59 | location = getattr(module, '__file__', 'Unknown location') 60 | except ImportError: 61 | ver = 'Not found' 62 | location = 'ImportError' 63 | except AttributeError: 64 | ver = 'Version not found' 65 | location = getattr(module, '__file__', 'Unknown location') 66 | except Exception as e: 67 | ver = 'Error' 68 | location = str(e) 69 | 70 | print(f'{module_name}:') 71 | print(f' Version: {ver}') 72 | print(f' Location: {location}') 73 | 74 | # If the module is already imported, print its actual location in sys.modules 75 | if module_name in sys.modules: 76 | print(f' Imported from: {sys.modules[module_name].__file__}') 77 | print() 78 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/print_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import readline 3 | import sys 4 | from enum import Enum 5 | from pathlib import Path 6 | 7 | import i18n 8 | 9 | 10 | class Color(Enum): 11 | """ANSI escape codes representing colors in the terminal theme.""" 12 | 13 | MAGENTA = "\033[95m" 14 | BLUE = "\033[94m" 15 | GREEN = "\033[92m" 16 | YELLOW = "\033[93m" 17 | RED = "\033[91m" 18 | END = "\033[0m" 19 | BOLD = "\033[1m" 20 | UNDERLINE = "\033[4m" 21 | 22 | 23 | def print_color(message, color: Color, **kwargs) -> None: 24 | color = _check_no_color(color) 25 | print(f"{color.value}{message}{Color.END.value}", **kwargs) 26 | 27 | 28 | def _check_no_color(color: Color) -> Color: 29 | """Turns off color if the user wants. 30 | See: https://no-color.org/ 31 | """ 32 | if "NO_COLOR" in os.environ: 33 | return Color.END 34 | return color 35 | 36 | -------------------------------------------------------------------------------- /tools/openvisioncapsule_tools/translations/portal.en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | portal: 3 | description: "visioncapsule-tools is a group of commands written in Python. They 4 | help to download, package and test VisionCapsules. To see options of each 5 | command, run \"openvisioncapsule-tools {command} --help\"." 6 | usage: >- 7 | openvisioncapsule-tools [] 8 | 9 | commands: 10 | capsule_packaging 11 | capsule_inference 12 | 13 | Examples: 14 | openvisioncapsule-tools capsule_packaging 15 | 16 | command-help: "The openvisioncapsule-tools command to run" 17 | unknown-command: "Unknown command: {command}" 18 | interrupted: "The operation was interrupted" 19 | -------------------------------------------------------------------------------- /tools/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "openvisioncapsule-tools" 3 | version = "0.3.8" 4 | description = "A tool to help users to package, test OpenVisionCapsules" 5 | authors = ["Stephen Li "] 6 | license = "BSD license" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.8" 10 | PyYAML = "^6.0.2" 11 | # altair = "*" # "5.4.1" 12 | Cython = "0.29.30" 13 | matplotlib = "3.5.1" 14 | numpy = ">=1.19.2,<1.20.0" 15 | opencv_contrib_python = "*" 16 | opencv_contrib_python_headless = "*" 17 | # opencv_python = "4.5.5.62" 18 | openvino = "2024.1.0" 19 | pandas = "1.1.5" 20 | Pillow = "7.0.0" 21 | python_i18n = "0.3.9" 22 | setuptools = "73.0.1" 23 | tools = "0.1.9" 24 | tqdm = "4.54.1" 25 | vcap = "0.3.8.1" 26 | 27 | [tool.poetry.scripts] 28 | openvisioncapsule-tools = 'openvisioncapsule_tools.cli:cli_main' 29 | 30 | [tool.poetry.dev-dependencies] 31 | black = "^21.7b0" 32 | isort = "^5.9.2" 33 | 34 | [build-system] 35 | requires = ["poetry-core>=1.0.0"] 36 | build-backend = "poetry.core.masonry.api" 37 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/README.md: -------------------------------------------------------------------------------- 1 | A simple example of a classifier that wraps around an Inception Resnet v2 50 2 | model trained in COCO. It is used to classify a person's gait. 3 | 4 | Model: inception_resnet_v2_50 5 | 6 | Dataset: Coco 7 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/backend.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from typing import Dict 3 | 4 | from . import config 5 | from vcap import ( 6 | Crop, 7 | DETECTION_NODE_TYPE, 8 | OPTION_TYPE, 9 | BaseStreamState) 10 | from vcap_utils.backends import TFImageClassifier 11 | 12 | 13 | class Backend(TFImageClassifier): 14 | def process_frame(self, frame: np.ndarray, 15 | detection_node: DETECTION_NODE_TYPE, 16 | options: Dict[str, OPTION_TYPE], 17 | state: BaseStreamState) -> DETECTION_NODE_TYPE: 18 | crop = (Crop 19 | .from_detection(detection_node) 20 | .pad_percent(top=10, bottom=10, left=10, right=10) 21 | .apply(frame)) 22 | 23 | prediction = self.send_to_batch(crop).result() 24 | 25 | detection_node.attributes[config.category] = prediction.name 26 | detection_node.extra_data[config.extra_data] = prediction.confidence 27 | return detection_node 28 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/capsule.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from vcap import BaseCapsule, NodeDescription, DeviceMapper, BaseBackend 4 | 5 | from . import config 6 | from .backend import Backend 7 | 8 | 9 | class Capsule(BaseCapsule): 10 | name = "classifier_gait_example" 11 | version = 1 12 | device_mapper = DeviceMapper.map_to_all_gpus() 13 | input_type = NodeDescription( 14 | size=NodeDescription.Size.SINGLE, 15 | detections=["person"]) 16 | output_type = NodeDescription( 17 | size=NodeDescription.Size.SINGLE, 18 | detections=["person"], 19 | attributes={config.category: config.values}, 20 | extra_data=[config.extra_data]) 21 | options = {} 22 | 23 | @staticmethod 24 | def backend_loader(capsule_files: Dict[str, bytes], device: str) \ 25 | -> BaseBackend: 26 | 27 | # Real capsules do not need to do this check. This is only to provide 28 | # a warning for this example because the model is not included in the 29 | # repo. 30 | model_filename = "classification_gait_model.pb" 31 | try: 32 | model_file = capsule_files[model_filename] 33 | except KeyError as exc: 34 | message = f"Model [{model_filename}] not found. Did you make " \ 35 | f"sure to run tests? Example models files are not " \ 36 | f"stored directly in the repo, but are downloaded " \ 37 | f"when tests are run." 38 | raise FileNotFoundError(message) from exc 39 | 40 | return Backend(model_bytes=model_file, 41 | metadata_bytes=capsule_files["dataset_metadata.json"], 42 | model_name="inception_resnet_v2", 43 | device=device) 44 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/config.py: -------------------------------------------------------------------------------- 1 | extra_data = "gait_confidence" 2 | category = "Gait" 3 | values = [ 4 | "Unknown Gait", 5 | "Running", 6 | "Walking" 7 | ] 8 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/dataset_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "label_map": { 3 | "0": "Unknown Gait", 4 | "1": "Running", 5 | "2": "Walking" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /vcap/examples/classifier_gait_example/meta.conf: -------------------------------------------------------------------------------- 1 | [about] 2 | api_compatibility_version = 0.3 3 | -------------------------------------------------------------------------------- /vcap/examples/detector_person_example/README.md: -------------------------------------------------------------------------------- 1 | A simple example of a detector that wraps around a MobileNet model trained on 2 | COCO. It is used to detect people in frames. 3 | 4 | Runtime options are provided to adjust the confidence threshold for detections 5 | and to set a maximum frame size when processing. 6 | 7 | Model: ssd_mobilenet_v1_coco 8 | 9 | Dataset: Coco 10 | 11 | From Tensorflow model zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md 12 | 13 | ## Model File License 14 | 15 | Copyright 2017 The TensorFlow Authors. All Rights Reserved. 16 | 17 | Licensed under the Apache License, Version 2.0 (the "License"); 18 | you may not use this file except in compliance with the License. 19 | You may obtain a copy of the License at 20 | 21 | http://www.apache.org/licenses/LICENSE-2.0 22 | 23 | Unless required by applicable law or agreed to in writing, software 24 | distributed under the License is distributed on an "AS IS" BASIS, 25 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 26 | See the License for the specific language governing permissions and 27 | limitations under the License. 28 | 29 | -------------------------------------------------------------------------------- /vcap/examples/detector_person_example/backend.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | import numpy as np 4 | 5 | from vcap import ( 6 | Clamp, 7 | DetectionNode, 8 | rect_to_coords, 9 | DETECTION_NODE_TYPE, 10 | OPTION_TYPE, 11 | BaseStreamState) 12 | from vcap_utils.backends import TFObjectDetector 13 | 14 | 15 | class Backend(TFObjectDetector): 16 | def process_frame(self, frame: np.ndarray, 17 | detection_node: DETECTION_NODE_TYPE, 18 | options: Dict[str, OPTION_TYPE], 19 | state: BaseStreamState): 20 | if options["scale_frame"]: 21 | max_frame_side_length = options["scale_frame_max_side_length"] 22 | clamp = Clamp(frame=frame, 23 | max_width=max_frame_side_length, 24 | max_height=max_frame_side_length) 25 | frame = clamp.apply() 26 | 27 | predictions = self.send_to_batch(frame).result() 28 | 29 | results = [] 30 | 31 | # Convert all predictions with the required confidence that are people 32 | # to DetectionNodes 33 | for pred in predictions: 34 | if pred.confidence >= options["detection_threshold"] \ 35 | and pred.name == "person": 36 | results.append(DetectionNode( 37 | name=pred.name, 38 | coords=rect_to_coords(pred.rect))) 39 | 40 | # If we scaled the frame down earlier before processing, we need to 41 | # scale detections back up to match the original frame size 42 | if options["scale_frame"]: 43 | clamp.scale_detection_nodes(results) 44 | 45 | return results 46 | -------------------------------------------------------------------------------- /vcap/examples/detector_person_example/capsule.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from vcap import BaseBackend, BaseCapsule, DeviceMapper, NodeDescription, \ 4 | FloatOption, BoolOption, IntOption 5 | 6 | from .backend import Backend 7 | 8 | 9 | class Capsule(BaseCapsule): 10 | name = "detector_person_example" 11 | version = 1 12 | device_mapper = DeviceMapper.map_to_all_gpus() 13 | # This capsule takes no input from other capsules 14 | input_type = NodeDescription(size=NodeDescription.Size.NONE) 15 | # This capsule produces DetectionNodes of people 16 | output_type = NodeDescription( 17 | size=NodeDescription.Size.ALL, 18 | detections=["person"]) 19 | options = { 20 | "detection_threshold": FloatOption( 21 | description="The confidence threshold for the model. A higher " 22 | "value means fewer detections", 23 | default=0.5, 24 | min_val=0.1, 25 | max_val=1.0), 26 | "scale_frame": BoolOption( 27 | description="If true, the frame width and height will be clamped " 28 | "to the value of scale_frame_max_side_length, " 29 | "preserving aspect ratio", 30 | default=False), 31 | 32 | "scale_frame_max_side_length": IntOption( 33 | description="The width or height to scale frames down to " 34 | "if scale_frames is True", 35 | default=2000, 36 | min_val=200, 37 | max_val=4000) 38 | } 39 | 40 | @staticmethod 41 | def backend_loader(capsule_files: Dict[str, bytes], device: str) \ 42 | -> BaseBackend: 43 | 44 | # Real capsules do not need to do this check. This is only to provide 45 | # a warning for this example because the model is not included in the 46 | # repo. 47 | model_filename = "ssd_mobilenet_v1_coco.pb" 48 | try: 49 | model_file = capsule_files[model_filename] 50 | except KeyError as exc: 51 | message = f"Model [{model_filename}] not found. Did you make " \ 52 | f"sure to run tests? Example models files are not " \ 53 | f"stored directly in the repo, but are downloaded " \ 54 | f"when tests are run." 55 | raise FileNotFoundError(message) from exc 56 | 57 | return Backend(model_bytes=model_file, 58 | metadata_bytes=capsule_files["dataset_metadata.json"], 59 | device=device) 60 | -------------------------------------------------------------------------------- /vcap/examples/detector_person_example/dataset_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "label_map": { 3 | "1": "person", 4 | "2": "bicycle", 5 | "3": "car", 6 | "4": "motorcycle", 7 | "5": "airplane", 8 | "6": "bus", 9 | "7": "train", 10 | "8": "truck", 11 | "9": "boat", 12 | "10": "traffic light", 13 | "11": "fire hydrant", 14 | "13": "stop sign", 15 | "14": "parking meter", 16 | "15": "bench", 17 | "16": "bird", 18 | "17": "cat", 19 | "18": "dog", 20 | "19": "horse", 21 | "20": "sheep", 22 | "21": "cow", 23 | "22": "elephant", 24 | "23": "bear", 25 | "24": "zebra", 26 | "25": "giraffe", 27 | "27": "backpack", 28 | "28": "umbrella", 29 | "31": "handbag", 30 | "32": "tie", 31 | "33": "suitcase", 32 | "34": "frisbee", 33 | "35": "skis", 34 | "36": "snowboard", 35 | "37": "sports ball", 36 | "38": "kite", 37 | "39": "baseball bat", 38 | "40": "baseball glove", 39 | "41": "skateboard", 40 | "42": "surfboard", 41 | "43": "tennis racket", 42 | "44": "bottle", 43 | "46": "wine glass", 44 | "47": "cup", 45 | "48": "fork", 46 | "49": "knife", 47 | "50": "spoon", 48 | "51": "bowl", 49 | "52": "banana", 50 | "53": "apple", 51 | "54": "sandwich", 52 | "55": "orange", 53 | "56": "broccoli", 54 | "57": "carrot", 55 | "58": "hot dog", 56 | "59": "pizza", 57 | "60": "donut", 58 | "61": "cake", 59 | "62": "chair", 60 | "63": "couch", 61 | "64": "potted plant", 62 | "65": "bed", 63 | "67": "dining table", 64 | "70": "toilet", 65 | "72": "tv", 66 | "73": "laptop", 67 | "74": "mouse", 68 | "75": "remote", 69 | "76": "keyboard", 70 | "77": "cell phone", 71 | "78": "microwave", 72 | "79": "oven", 73 | "80": "toaster", 74 | "81": "sink", 75 | "82": "refrigerator", 76 | "84": "book", 77 | "85": "clock", 78 | "86": "vase", 79 | "87": "scissors", 80 | "88": "teddy bear", 81 | "89": "hair drier", 82 | "90": "toothbrush" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /vcap/examples/detector_person_example/meta.conf: -------------------------------------------------------------------------------- 1 | [about] 2 | api_compatibility_version = 0.3 3 | -------------------------------------------------------------------------------- /vcap/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | from pathlib import Path 4 | 5 | from setuptools import setup, find_namespace_packages 6 | 7 | # Get package version/metadata 8 | about = {} 9 | exec(Path("vcap/version.py").read_text(), about) 10 | 11 | test_packages = ["pytest", "mock"] 12 | 13 | PRE_RELEASE_SUFFIX = os.environ.get("PRE_RELEASE_SUFFIX", "") 14 | 15 | setup( 16 | name='vcap', 17 | description="A library for creating OpenVisionCapsules in Python", 18 | author="Aotu.ai", 19 | packages=find_namespace_packages(include=["vcap*"]), 20 | version=about["__version__"] + PRE_RELEASE_SUFFIX, 21 | 22 | install_requires=[ 23 | "pycryptodomex==3.9.7", 24 | "scipy==1.4.1", 25 | "scikit-learn==0.22.2", 26 | "numpy>=1.16,<2", 27 | "tensorflow~=2.5.0", 28 | ], 29 | extras_require={ 30 | "tests": test_packages, 31 | "easy": ["opencv-python-headless~=4.1.2"], 32 | }, 33 | tests_require=test_packages, 34 | classifiers=[ 35 | "Programming Language :: Python :: 3", 36 | "License :: OSI Approved :: BSD License", 37 | "Operating System :: OS Independent", 38 | ], 39 | python_requires='>=3.6', 40 | ) 41 | -------------------------------------------------------------------------------- /vcap/vcap/__init__.py: -------------------------------------------------------------------------------- 1 | from .version import __version__ 2 | from .deprecation import deprecated 3 | from .capsule import BaseCapsule 4 | from .stream_state import BaseStreamState 5 | from .backend import BaseBackend 6 | from .detection_node import DetectionNode, rect_to_coords, BoundingBox 7 | from .node_description import NodeDescription, DETECTION_NODE_TYPE 8 | from .device_mapping import DeviceMapper 9 | from .loading.capsule_loading import load_capsule_from_bytes, load_capsule 10 | from .modifiers import Crop, Clamp, Resize, SizeFilter 11 | from .options import ( 12 | FloatOption, 13 | EnumOption, 14 | IntOption, 15 | BoolOption, 16 | Option, 17 | common_detector_options, 18 | OPTION_TYPE 19 | ) 20 | from .loading.vcap_packaging import ( 21 | CAPSULE_EXTENSION, 22 | package_capsule 23 | ) 24 | from .loading.errors import ( 25 | CapsuleLoadError, 26 | InvalidCapsuleError, 27 | IncompatibleCapsuleError, 28 | ) 29 | -------------------------------------------------------------------------------- /vcap/vcap/backend.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from concurrent.futures import Future 3 | from typing import Any, Dict, List 4 | 5 | import numpy as np 6 | 7 | from vcap.batch_executor import BatchExecutor 8 | from vcap.node_description import DETECTION_NODE_TYPE 9 | from vcap.options import OPTION_TYPE 10 | from vcap.stream_state import BaseStreamState 11 | 12 | 13 | class BaseBackend(abc.ABC): 14 | """An object that provides low-level prediction functionality for batches 15 | of frames. 16 | """ 17 | 18 | def __init__(self): 19 | self._batch_executor = BatchExecutor(self.batch_predict) 20 | 21 | def send_to_batch(self, input_data: Any) -> Future: 22 | """Sends the given object to the batch_predict method for processing. 23 | This call does not block. Instead, the result will be provided on the 24 | returned Future. The batch_predict method must be overridden on the 25 | backend this method is being called on. 26 | 27 | :param input_data: The input object to send to batch_predict 28 | :return: A Future where results will be stored 29 | """ 30 | return self._batch_executor.submit(input_data) 31 | 32 | @property 33 | def workload(self) -> float: 34 | """Returns a unit representing the amount of 'work' being processed 35 | This value is comparable only by backends of the same capsule, and 36 | is intended to give the scheduler the ability to pick the least busy 37 | backend. 38 | """ 39 | return self._batch_executor.total_imgs_in_pipeline 40 | 41 | @abc.abstractmethod 42 | def process_frame(self, 43 | frame: np.ndarray, 44 | detection_node: DETECTION_NODE_TYPE, 45 | options: Dict[str, OPTION_TYPE], 46 | state: BaseStreamState) \ 47 | -> DETECTION_NODE_TYPE: 48 | """A method that does the pre-processing, inference, and postprocessing 49 | work for a frame. 50 | 51 | If the capsule uses an algorithm that benefits from batching, 52 | this method may call ``self.send_to_batch``, which will asynchronously 53 | send work out for batching. Doing so requires that the 54 | ``batch_predict`` method is overridden. 55 | 56 | :param frame: A numpy array representing a frame. It is of shape 57 | (height, width, num_channels) and the frames come in BGR order. 58 | :param detection_node: The detection_node type as specified by the 59 | ``input_type`` 60 | :param options: A dictionary of key (string) value pairs. The key is 61 | the name of a capsule option, and the value is its configured value 62 | at the time of processing. Capsule options are specified using the 63 | ``options`` field in the Capsule class. 64 | :param state: This will be a StreamState object of the type specified 65 | by the ``stream_state`` attribute on the Capsule class. If no 66 | StreamState object was specified, a simple BaseStreamState object 67 | will be passed in. The StreamState will be the same object for all 68 | frames in the same video stream. 69 | """ 70 | 71 | def batch_predict(self, input_data_list: List[Any]) -> List[Any]: 72 | """This method takes in a batch as input and provides a list of result 73 | objects of any type as output. What the result objects are will depend 74 | on the algorithm being defined, but the number of prediction objects 75 | returned _must_ match the number of video frames provided as input. 76 | 77 | :param input_data_list: A list of objects. Whatever the model requires 78 | for each frame. 79 | """ 80 | raise NotImplementedError( 81 | "Attempt to do batch prediction on a Backend that does not have " 82 | "the batch_predict method defined. Did you call send_to_batch on " 83 | "a backend that does not override batch_predict?") 84 | 85 | def close(self) -> None: 86 | """De-initializes the backend. This is called when the capsule is being 87 | unloaded. This method should be overridden by any Backend that needs 88 | to release resources or close other threads. 89 | 90 | The backend will stop receiving frames before this method is 91 | called, and will not receive frames again. 92 | """ 93 | self._batch_executor.close() 94 | -------------------------------------------------------------------------------- /vcap/vcap/batch_executor.py: -------------------------------------------------------------------------------- 1 | import queue 2 | from concurrent.futures import Future 3 | from queue import Queue 4 | from threading import Thread 5 | from typing import Any, Callable, Iterable, List, NamedTuple, Optional 6 | 7 | from vcap import deprecated 8 | 9 | 10 | class _Request(NamedTuple): 11 | """Used by BatchExecutor to keep track of requests and their respective 12 | future objects. 13 | """ 14 | 15 | future: Future 16 | """The Future object for the BatchExecutor to return the output""" 17 | 18 | input_data: Any 19 | """A unit of input data expected by the batch_fn.""" 20 | 21 | 22 | class BatchExecutor: 23 | """Feeds jobs into batch_fn in batches, returns results through Futures. 24 | 25 | This class simplifies centralizing work from a multitude of sources and 26 | running that work in a batched predict function, then returning that 27 | work to the respective Futures. 28 | """ 29 | 30 | def __init__(self, 31 | batch_fn: Callable[[List[Any]], Iterable[Any]], 32 | max_batch_size=40, 33 | num_workers: int = 1): 34 | """Initialize a new BatchExecutor 35 | 36 | :param batch_fn: A function that takes in a list of inputs and iterates 37 | the outputs in the same order as the inputs. 38 | :param max_batch_size: The maximum length of list to feed to batch_fn 39 | :param num_workers: How many workers should be calling batch_fn 40 | """ 41 | self.batch_fn = batch_fn 42 | self.max_batch_size = max_batch_size 43 | self._request_queue: Queue[_Request] = Queue() 44 | self.workers = [Thread(target=self._worker, 45 | daemon=True, 46 | name="BatchExecutorThread") 47 | for _ in range(num_workers)] 48 | 49 | # The number of images currently in the work queue or being processed 50 | self._num_imgs_being_processed: int = 0 51 | 52 | self._running: bool = True 53 | 54 | for worker in self.workers: 55 | worker.start() 56 | 57 | @property 58 | def total_imgs_in_pipeline(self) -> int: 59 | return self._request_queue.qsize() + self._num_imgs_being_processed 60 | 61 | def submit(self, input_data: Any, future: Future = None) -> Future: 62 | """Submits a job and returns a Future that will be fulfilled later.""" 63 | future = future or Future() 64 | 65 | self._request_queue.put(_Request( 66 | future=future, 67 | input_data=input_data)) 68 | return future 69 | 70 | def _on_requests_ready(self, batch: List[_Request]) -> None: 71 | """Push inputs through the given prediction backend 72 | 73 | :param batch: A list of requests to work on 74 | """ 75 | # Extract the futures from the requests 76 | inputs: List[Any] = [req.input_data for req in batch] 77 | futures: List[Future] = [req.future for req in batch] 78 | 79 | # Route the results to each request 80 | try: 81 | for prediction in self.batch_fn(inputs): 82 | # Popping the futures ensures that if an error occurs, only 83 | # the futures that haven't had a result set will have 84 | # set_exception called 85 | futures.pop(0).set_result(prediction) 86 | except BaseException as exc: 87 | # Catch exceptions and pass them to the futures, similar to the 88 | # ThreadPoolExecutor implementation: 89 | # https://github.com/python/cpython/blob/91e93794/Lib/concurrent/futures/thread.py#L51 90 | for future in futures: 91 | future.set_exception(exc) 92 | 93 | def _worker(self): 94 | self._running = True 95 | 96 | while self._running: 97 | # Get a new batch 98 | batch = self._get_next_batch() 99 | 100 | # If no batch was able to be retrieved, restart the loop 101 | if batch is None: 102 | continue 103 | 104 | # Check to make sure the thread isn't trying to end 105 | if not self._running: 106 | break 107 | 108 | self._num_imgs_being_processed += len(batch) 109 | self._on_requests_ready(batch) 110 | self._num_imgs_being_processed -= len(batch) 111 | 112 | self._running = False 113 | 114 | def _get_next_batch(self) -> Optional[List[_Request]]: 115 | """A helper function to help make the main thread loop more readable 116 | :returns: A non-empty list of collected items, or None if the worker is 117 | no longer running (i.e. self._continue == False) 118 | """ 119 | batch: List[_Request] = [] 120 | while len(batch) < self.max_batch_size: 121 | # Check if there's a new request 122 | try: 123 | # Try to get a new request. Have a timeout to check if closing 124 | new_request = self._request_queue.get(timeout=.1) 125 | except queue.Empty: 126 | # If the thread is being requested to close, exit early 127 | if not self._running: 128 | return None 129 | 130 | # Wait for requests again 131 | continue 132 | 133 | batch.append(new_request) 134 | 135 | # If the request queue is now empty, let worker run everything in 136 | # the batch 137 | if self._request_queue.empty(): 138 | break 139 | 140 | return batch 141 | 142 | def close(self) -> None: 143 | """Stop the BatchExecutor gracefully.""" 144 | self._running = False 145 | for worker in self.workers: 146 | worker.join() 147 | -------------------------------------------------------------------------------- /vcap/vcap/caching.py: -------------------------------------------------------------------------------- 1 | def cache(method): 2 | """Caches the result of a method call. Caches are specific to the instance 3 | of the object that this method is for. 4 | """ 5 | 6 | def on_call(self, *args, **kwargs): 7 | name = method.__name__ 8 | try: 9 | return self._cache[name] 10 | except AttributeError: 11 | # Create the cache if necessary 12 | self._cache = {} 13 | except KeyError: 14 | # Handled below 15 | pass 16 | 17 | val = method(self, *args, **kwargs) 18 | self._cache[name] = val 19 | return val 20 | 21 | return on_call 22 | -------------------------------------------------------------------------------- /vcap/vcap/deprecation.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from typing import Optional 3 | from pkg_resources import parse_version 4 | 5 | import functools 6 | 7 | from . import __version__ 8 | 9 | 10 | def deprecated(message: str = "", 11 | remove_in: Optional[str] = None, 12 | current_version=__version__): 13 | """Mark a function as deprecated 14 | :param message: Extra details to be added to the warning. For example, 15 | the details may point users to a replacement method. 16 | :param remove_in: The version when the decorated method will be removed. 17 | The default is None, specifying that the function is not currently planned 18 | to be removed. 19 | :param current_version: Defaults to the vcaps current version. 20 | """ 21 | 22 | def wrapper(function): 23 | warning_msg = f"'{function.__qualname__}' is deprecated " 24 | warning_msg += (f"and is scheduled to be removed in {remove_in}. " 25 | if remove_in is not None else ". ") 26 | warning_msg += str(message) 27 | 28 | # Only show the DeprecationWarning on the first call 29 | warnings.simplefilter("once", DeprecationWarning, append=True) 30 | 31 | @functools.wraps(function) 32 | def inner(*args, **kwargs): 33 | if remove_in is not None and \ 34 | parse_version(current_version) >= parse_version(remove_in): 35 | raise DeprecationWarning(warning_msg) 36 | else: 37 | warnings.warn(warning_msg, category=DeprecationWarning) 38 | return function(*args, **kwargs) 39 | 40 | return inner 41 | 42 | return wrapper 43 | -------------------------------------------------------------------------------- /vcap/vcap/detection_node.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict, Optional, Union 2 | from uuid import UUID 3 | 4 | import numpy as np 5 | 6 | from .caching import cache 7 | 8 | 9 | class BoundingBox: 10 | def __init__(self, x1, y1, x2, y2): 11 | self.x1 = x1 12 | self.y1 = y1 13 | self.x2 = x2 14 | self.y2 = y2 15 | 16 | @property 17 | def size(self): 18 | """ 19 | :return: The width and height of the bounding box 20 | """ 21 | return self.x2 - self.x1, self.y2 - self.y1 22 | 23 | @property 24 | def rect(self): 25 | """Get the bounding box in 'rect' format, as (x1, y1, x2, y2)""" 26 | return self.x1, self.y1, self.x2, self.y2 27 | 28 | @property 29 | def xywh(self): 30 | """Return the bounding box in 'xywh' format, as 31 | (x1, y1, width, height)""" 32 | return (self.x1, self.y1, 33 | self.x2 - self.x1, 34 | self.y2 - self.y1) 35 | 36 | @property 37 | def center(self): 38 | return (self.x1 + (self.x2 - self.x1) / 2, 39 | self.y1 + (self.y2 - self.y1) / 2) 40 | 41 | @property 42 | def height(self): 43 | return self.y2 - self.y1 44 | 45 | @property 46 | def width(self): 47 | return self.x2 - self.x1 48 | 49 | def __eq__(self, other): 50 | return other.__dict__ == self.__dict__ 51 | 52 | 53 | class DetectionNode: 54 | """Capsules use DetectionNode objects to communicate results to other 55 | capsules and the application itself. A DetectionNode contains information 56 | on a detection in the current frame. Capsules that detect objects in a 57 | frame create new DetectionNodes. Capsules that discover attributes about 58 | detections add data to existing DetectionNodes. 59 | """ 60 | 61 | def __init__(self, *, name: str, 62 | coords: List[List[Union[int, float]]], 63 | attributes: Dict[str, str] = None, 64 | children: List['DetectionNode'] = None, 65 | encoding: Optional[np.ndarray] = None, 66 | track_id: Optional[UUID] = None, 67 | extra_data: Dict[str, object] = None): 68 | """ 69 | :param name: The detection class name. This describes what the 70 | detection is. A detection of a person would have a name="person". 71 | :param coords: A list of coordinates defining the detection as a 72 | polygon in-frame. Comes in the format ``[[x,y], [x,y]...]``. 73 | :param attributes: A key-value store where the key is the type of 74 | attribute being described and the value is the attribute's value. 75 | For instance, a capsule that detects gender might add a "gender" 76 | key to this dict, with a value of either "masculine" or "feminine". 77 | :param children: Child DetectionNodes that are a "part" of the parent, 78 | for instance, a head DetectionNode might be a child of a person 79 | DetectionNode 80 | :param encoding: An array of float values that represent an encoding of 81 | the detection. This can be used to recognize specific instances of 82 | a class. For instance, given a picture of person’s face, the 83 | encoding of that face and the encodings of future faces can be 84 | compared to find that person in the future. 85 | :param track_id: If this object is tracked, this is the unique 86 | identifier for this detection node that ties it to other detection 87 | nodes in future and past frames within the same stream. 88 | :param extra_data: A dict of miscellaneous data. This data is provided 89 | directly to clients without modification, so it’s a good way to 90 | pass extra information from a capsule to other applications. 91 | """ 92 | self.class_name = name 93 | self.coords = coords 94 | self.attributes = attributes if attributes else {} 95 | self.encoding = encoding 96 | self.track_id = track_id 97 | self.children = children if children else [] 98 | self.extra_data = extra_data if extra_data else {} 99 | 100 | self._bbox = None 101 | 102 | def scale(self, scale_amount_x: float, scale_amount_y: float): 103 | """Scales the detection coordinates of the tree by the given scales. 104 | 105 | :param scale_amount_x: The amount to scale x by 106 | :param scale_amount_y: The amount to scale y by 107 | """ 108 | for i, c in enumerate(self.coords): 109 | self.coords[i] = [round(c[0] * scale_amount_x), 110 | round(c[1] * scale_amount_y)] 111 | self._bbox = None 112 | 113 | if self.children is not None: 114 | for child in self.children: 115 | child.scale(scale_amount_x, scale_amount_y) 116 | 117 | def offset(self, offset_x: int, offset_y: int): 118 | """Offsets the detection coordinates of the tree by the given offsets. 119 | 120 | :param offset_x: The amount to offset x by 121 | :param offset_y: The amount to offset y by 122 | """ 123 | for i, c in enumerate(self.coords): 124 | self.coords[i] = [c[0] - offset_x, c[1] - offset_y] 125 | self._bbox = None 126 | 127 | if self.children is not None: 128 | for child in self.children: 129 | child.offset(offset_x, offset_y) 130 | 131 | @property 132 | @cache 133 | def all_attributes(self): 134 | """Return all attributes including child attributes""" 135 | 136 | def get_attrs(child): 137 | attributes = child.attributes 138 | for child in child.children: 139 | attributes.update(get_attrs(child)) 140 | return attributes 141 | 142 | return get_attrs(self) 143 | 144 | def __repr__(self): 145 | rep = { 146 | "class_name": self.class_name, 147 | "track_id": self.track_id, 148 | "attributes": self.attributes, 149 | "extra_data": self.extra_data, 150 | "coords": [(round(x), round(y)) for x, y in self.coords], 151 | "encoding": "Encoded" if self.encoding is not None else None 152 | } 153 | 154 | return str(rep) 155 | 156 | @property 157 | def bbox(self) -> BoundingBox: 158 | if self._bbox is None: 159 | self._bbox = self._make_bbox() 160 | return self._bbox 161 | 162 | def _make_bbox(self): 163 | """ 164 | :return: Create a fully containing bounding box of the detection 165 | polygon 166 | """ 167 | sorted_x = sorted([c[0] for c in self.coords]) 168 | sorted_y = sorted([c[1] for c in self.coords]) 169 | return BoundingBox(sorted_x[0], sorted_y[0], 170 | sorted_x[-1], sorted_y[-1]) 171 | 172 | 173 | def rect_to_coords(rect): 174 | """Converts a rect in the [x1, y1, x2, y2] format to coordinates.""" 175 | return [[rect[0], rect[1]], 176 | [rect[2], rect[1]], 177 | [rect[2], rect[3]], 178 | [rect[0], rect[3]]] 179 | -------------------------------------------------------------------------------- /vcap/vcap/device_mapping.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from threading import RLock 4 | from typing import Callable, List 5 | 6 | import tensorflow as tf 7 | from tensorflow.python.client import device_lib 8 | 9 | _devices = None 10 | _devices_lock = RLock() 11 | 12 | 13 | def get_all_devices() -> List[str]: 14 | """Returns a list of devices in the computer. Example: 15 | ['CPU:0', 'XLA_GPU:0', 'XLA_CPU:0', 'GPU:0', 'GPU:1', 'GPU:2', 'GPU:3'] 16 | """ 17 | global _devices, _devices_lock 18 | 19 | # Initialize the device list if necessary 20 | with _devices_lock: 21 | if _devices is None: 22 | # Use the TF_FORCE_GPU_ALLOW_GROWTH=true environment variable to 23 | # force allow tensorflow to take up GPU memory dynamically, instead 24 | # of allocating 100% of memory at first model-load. 25 | # 26 | # TODO: Use tf.config.list_physical_devices in TF 2.1 27 | 28 | with tf.compat.v1.Session(): 29 | all_devices = device_lib.list_local_devices() 30 | 31 | # Get the device names and remove duplicates, just in case... 32 | tf_discovered_devices = { 33 | d.name.replace("/device:", "") 34 | for d in all_devices 35 | } 36 | 37 | # Discover devices using OpenVINO 38 | try: 39 | 40 | from openvino.runtime import Core 41 | 42 | ie = Core() 43 | 44 | openvino_discovered_devices = { 45 | d for d in ie.available_devices 46 | if not d.lower().startswith("cpu") 47 | } 48 | except ModuleNotFoundError: 49 | logging.warning("OpenVINO library not found. " 50 | "OpenVINO devices will not be discovered. ") 51 | openvino_discovered_devices = set() 52 | _devices = list(tf_discovered_devices 53 | | openvino_discovered_devices) 54 | return _devices 55 | 56 | 57 | class DeviceMapper: 58 | def __init__(self, filter_func: Callable[[List[str]], List[str]]): 59 | """The filter will take in a list of devices formatted as 60 | ["CPU:0", "CPU:1", "GPU:0", "GPU:1"], etc and output a filtered list of 61 | devices. 62 | """ 63 | self.filter_func = filter_func 64 | 65 | @staticmethod 66 | def map_to_all_gpus(cpu_fallback=True) -> 'DeviceMapper': 67 | def filter_func(devices): 68 | gpu_devices = [d for d in devices if d.startswith("GPU:")] 69 | if not gpu_devices and cpu_fallback: 70 | return ["CPU:0"] 71 | return gpu_devices 72 | 73 | return DeviceMapper(filter_func=filter_func) 74 | 75 | @staticmethod 76 | def map_to_single_cpu() -> 'DeviceMapper': 77 | def filter_func(devices): 78 | return [next(d for d in devices if d.startswith("CPU:"))] 79 | 80 | return DeviceMapper(filter_func=filter_func) 81 | 82 | @staticmethod 83 | def map_to_openvino_devices(): 84 | """Intelligently load capsules onto available OpenVINO-compatible 85 | devices. 86 | 87 | Since support for OpenVINO devices is experimental, there is a 88 | temporary environment variable being added to whitelist devices 89 | specifically. This variable will be deprecated and removed after a 90 | short testing period. 91 | 92 | The device "CPU" is _always_ allowed and always loaded onto and cannot 93 | be excluded. 94 | 95 | Here are the cases: 96 | ['CPU:0', 'HDDL', ...] => ["MULTI:CPU,HDDL"] 97 | 98 | ['CPU:0'] => ["CPU"] 99 | Always load onto CPU. 100 | """ 101 | 102 | def filter_func(devices): 103 | 104 | devices_by_priority = os.environ.get( 105 | "OPENVINO_DEVICE_PRIORITY", 106 | "CPU,HDDL").split(",") 107 | load_to_devices = [] 108 | 109 | for device in devices_by_priority: 110 | for existing_device in devices: 111 | if existing_device.lower().startswith(device.lower()): 112 | load_to_devices.append(device) 113 | 114 | if len(load_to_devices) > 1: 115 | return ["MULTI:" + ",".join(load_to_devices)] 116 | else: 117 | return load_to_devices 118 | 119 | return DeviceMapper(filter_func=filter_func) 120 | -------------------------------------------------------------------------------- /vcap/vcap/loading/crypto_utils.py: -------------------------------------------------------------------------------- 1 | from typing import BinaryIO 2 | 3 | from Cryptodome.Cipher import AES 4 | 5 | 6 | def encrypt(password: str, data: bytes, dest: BinaryIO) -> None: 7 | """Encrypt the given source to dest. 8 | 9 | :param password: The password to encrypt with 10 | :param data: Data to encrypt 11 | :param dest: A file-like object to write results to 12 | """ 13 | cipher = AES.new(password.encode('utf-8'), AES.MODE_EAX) 14 | ciphertext, tag = cipher.encrypt_and_digest(data) 15 | 16 | [dest.write(x) for x in (cipher.nonce, tag, ciphertext)] 17 | 18 | 19 | def decrypt(password: str, data: bytes) -> bytes: 20 | """Decrypt the given bytes. 21 | 22 | :param password: The password to decrypt with 23 | :param data: Data to decrypt 24 | :return: The resulting bytes 25 | """ 26 | nonce = data[:16] 27 | tag = data[16:32] 28 | ciphertext = data[32:] 29 | 30 | cipher = AES.new(password.encode("utf-8"), AES.MODE_EAX, nonce) 31 | return cipher.decrypt_and_verify(ciphertext, tag) 32 | -------------------------------------------------------------------------------- /vcap/vcap/loading/errors.py: -------------------------------------------------------------------------------- 1 | class CapsuleLoadError(Exception): 2 | """Raised when there was an error loading a capsule.""" 3 | 4 | def __init__(self, message, capsule_name=None): 5 | self.capsule_name = capsule_name 6 | if capsule_name is not None: 7 | message = f"Capsule {capsule_name}: {message}" 8 | 9 | super().__init__(message) 10 | 11 | 12 | class IncompatibleCapsuleError(CapsuleLoadError): 13 | """This is thrown when a capsule is of an incompatible API version for this 14 | version of the library. 15 | """ 16 | 17 | 18 | class InvalidCapsuleError(CapsuleLoadError): 19 | """This is thrown when a capsule has the correct API compatibility version, 20 | but it still doesn't have the correct information on it. 21 | """ 22 | -------------------------------------------------------------------------------- /vcap/vcap/loading/import_hacks.py: -------------------------------------------------------------------------------- 1 | """Code which changes the way Python processes imports. Allows for capsules to 2 | import Python modules and packages within the capsule itself. 3 | """ 4 | from importlib.abc import Loader, MetaPathFinder 5 | from importlib.machinery import ModuleSpec 6 | import os 7 | from pathlib import Path 8 | from zipfile import ZipFile 9 | 10 | 11 | class ZipFinder(MetaPathFinder): 12 | """An import finder that allows modules and packages inside a zip file to be 13 | imported. 14 | """ 15 | 16 | def __init__(self, zip_file: ZipFile, 17 | capsule_dir_path: Path, 18 | root_package_name: str): 19 | """ 20 | :param zip_file: The ZipFile loaded in memory 21 | :param capsule_dir_path: 22 | The path to the directory where the development version of the 23 | capsule is stored. For example, if the capsule being loaded is in 24 | ../capsules/my_capsule_name.py 25 | 26 | Then capsule_dir_path would be 27 | ../capsules/my_capsule_name/ 28 | 29 | The reason for this is so that debugging can work within capsules. 30 | When a capsule is executed, it is "compiled" to this path, so that 31 | debug information still works (along with breakpoints!) 32 | :param root_package_name: The name of the root package that this zip 33 | file provides 34 | """ 35 | self._zip_file = zip_file 36 | self._capsule_dir_path = capsule_dir_path 37 | self._capsule_dir_name = self._capsule_dir_path.name 38 | self._root_package_name = root_package_name 39 | 40 | def find_spec(self, fullname, _path=None, _target=None): 41 | if not self._in_capsule(fullname): 42 | # If the capsule directory name is not in the fullname, it's 43 | # not our job to import it 44 | return None 45 | 46 | if fullname == self._root_package_name: 47 | # If the capsule root directory is being loaded, return a 48 | # modulespec 49 | return ModuleSpec( 50 | name=fullname, 51 | loader=None, 52 | is_package=True) 53 | 54 | # Get rid of the capsule name prefix 55 | pruned_fullname = _remove_capsule_name(self._capsule_dir_name, 56 | fullname) 57 | package_path = _package_fullname_to_path(pruned_fullname) 58 | module_path = _module_fullname_to_path(pruned_fullname) 59 | 60 | if package_path in self._zip_file.namelist(): 61 | # If a directory exists with a name that matches the import, we 62 | # assume it is a package import. 63 | return ModuleSpec(name=fullname, 64 | loader=None, 65 | is_package=True) 66 | elif module_path in self._zip_file.namelist(): 67 | # If a .py file exists with a name that matches the import, we 68 | # assume it is a module import 69 | module_file_path = self._capsule_dir_path / module_path 70 | loader = ZipModuleLoader(zip_file=self._zip_file, 71 | module_file_path=module_file_path, 72 | capsule_dir_name=self._capsule_dir_name) 73 | return ModuleSpec(name=fullname, 74 | loader=loader) 75 | 76 | raise ImportError(f"Problem while importing {fullname}") 77 | 78 | def _in_capsule(self, fullname): 79 | parts = fullname.split(".") 80 | if parts[0] != self._root_package_name: 81 | return False 82 | return True 83 | 84 | 85 | class ZipModuleLoader(Loader): 86 | """Loads modules from a zip file.""" 87 | 88 | def __init__(self, zip_file: ZipFile, 89 | module_file_path: Path, 90 | capsule_dir_name: str): 91 | """ 92 | :param zip_file: The ZipFile loaded in memory 93 | :param module_file_path: The path to where the python file would be 94 | in the filesystem if it was being run directly as opposed to in a 95 | *.cap zip. 96 | """ 97 | self._zip_file = zip_file 98 | self._module_file_path = module_file_path 99 | self._capsule_dir_name = capsule_dir_name 100 | 101 | def create_module(self, spec): 102 | return None 103 | 104 | def exec_module(self, module): 105 | pruned_name = _remove_capsule_name( 106 | capsule_name=self._capsule_dir_name, 107 | fullname=module.__name__) 108 | zip_path = _module_fullname_to_path(pruned_name) 109 | 110 | # Extract the code from the zip 111 | code = self._zip_file.read(zip_path) 112 | 113 | # Compile code with the file path set first, so that debugging and 114 | # tracebacks work in development (they reference a file) 115 | # also breakpoints work in IDE's, which is quite helpful. 116 | compiled = compile(code, self._module_file_path, "exec") 117 | exec(compiled, module.__dict__) 118 | return module 119 | 120 | 121 | def _package_fullname_to_path(fullname): 122 | """Converts a package's fullname to a file path that should be the package's 123 | directory. 124 | 125 | :param fullname: The fullname of a package, like package_a.package_b 126 | :return: A derived filepath, like package_a/package_b 127 | """ 128 | return fullname.replace(".", os.sep) + os.sep 129 | 130 | 131 | def _module_fullname_to_path(fullname): 132 | """Converts a module's fullname to a file path that should be the module's 133 | Python file. 134 | 135 | :param fullname: The fullname of a module, like package_a.my_module 136 | :return: A derived filepath, like package_a/my_module.py 137 | """ 138 | return fullname.replace(".", os.sep) + ".py" 139 | 140 | 141 | def _remove_capsule_name(capsule_name, fullname): 142 | """Remove "capsule_name" from capsule_name.some_module.some_module2 143 | Since the files in the zip file won't have "capsule_name" in the paths 144 | """ 145 | parts = fullname.split(".") 146 | return ".".join(parts[1:]) 147 | -------------------------------------------------------------------------------- /vcap/vcap/loading/vcap_packaging.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | from io import BytesIO 4 | from pathlib import Path 5 | from zipfile import ZipFile 6 | from typing import Optional, Union 7 | 8 | from vcap.loading.crypto_utils import encrypt, decrypt 9 | import os 10 | 11 | CAPSULE_EXTENSION = ".cap" 12 | """The file extension that capsule files use.""" 13 | 14 | CAPSULE_FILE_NAME = "capsule.py" 15 | """The name of the main file in the capsule.""" 16 | 17 | META_FILE_NAME = "meta.conf" 18 | """The name of the meta file that contains information such as the API 19 | compatibility version of this capsule""" 20 | 21 | 22 | class UnpackagedCapsuleError(Exception): 23 | pass 24 | 25 | 26 | def package_capsule(unpackaged_dir: Path, output_file: Path, key=None): 27 | """Packages an unpackaged capsule up into a zip and applies encryption. 28 | 29 | :param unpackaged_dir: The directory to package up 30 | :param output_file: The name and location to save the packaged capsule 31 | :param key: An AES key to encrypt the capsule with 32 | """ 33 | # Check to make sure there is a capsule.py 34 | if len(list(unpackaged_dir.glob(CAPSULE_FILE_NAME))) == 0: 35 | raise UnpackagedCapsuleError( 36 | f"Unpackaged capsule {unpackaged_dir} is missing a " 37 | f"{CAPSULE_FILE_NAME}") 38 | # Check to make sure there's a meta.conf 39 | if len(list(unpackaged_dir.glob(META_FILE_NAME))) == 0: 40 | raise UnpackagedCapsuleError( 41 | f"Unpackaged capsule {unpackaged_dir} is missing a " 42 | f"{META_FILE_NAME}") 43 | 44 | saved_zip_bytes = BytesIO() 45 | with ZipFile(saved_zip_bytes, "w") as capsule_file: 46 | for f in unpackaged_dir.glob("**/*"): 47 | relative_name = f.relative_to(unpackaged_dir) 48 | capsule_file.write(f, relative_name) 49 | 50 | with output_file.open("wb") as output: 51 | # Move the read cursor to the start of the zip data 52 | saved_zip_bytes.seek(0) 53 | if key is None: 54 | output.write(saved_zip_bytes.read()) 55 | else: 56 | encrypt(key, saved_zip_bytes.read(), output) 57 | 58 | 59 | def unpackage_capsule(capsule_file: Union[str, Path], 60 | unpackage_to: Union[str, Path], 61 | key: Optional[str] = None): 62 | """Unpackagea capsule from the given bytes. 63 | 64 | :param path: The path to the capsule file 65 | :param key: The AES key to decrypt the capsule with, or None if the capsule 66 | is not encrypted 67 | :return: None 68 | """ 69 | path = Path(capsule_file) 70 | 71 | data=path.read_bytes() 72 | 73 | if key is not None: 74 | # Decrypt the capsule into its original form, a zip file 75 | data = decrypt(key, data) 76 | 77 | file_like = BytesIO(data) 78 | 79 | with ZipFile(file_like, "r") as capsule_file: 80 | os.makedirs(unpackage_to, exist_ok=True) 81 | for name in capsule_file.namelist(): 82 | 83 | output_path = Path(unpackage_to + '/' + name) 84 | print(output_path) 85 | 86 | if name.endswith('/'): 87 | os.makedirs(output_path, exist_ok=True) 88 | else: 89 | output_file = capsule_file.read(name) 90 | with output_path.open("wb") as output: 91 | output.write(output_file) 92 | 93 | 94 | def packaging_parse_args(): 95 | parser = ArgumentParser(description='Package capsules and unpackage a capsule file') 96 | parser.add_argument( 97 | "--capsule-dir", 98 | type=Path, 99 | required=False, 100 | help="The parent directory with capsule directories for packaging" 101 | ) 102 | parser.add_argument( 103 | "--capsule-file", 104 | type=Path, 105 | required=False, 106 | help="The capsule for unpackaging" 107 | ) 108 | parser.add_argument( 109 | "--capsule-key", 110 | type=str, 111 | required=False, 112 | help="The encrption key for packaging or unpackaging" 113 | ) 114 | return parser 115 | 116 | 117 | def packaging(capsule_dir, capsule_file, capsule_key): 118 | if capsule_dir is not None: 119 | for path in capsule_dir.iterdir(): 120 | if path.is_dir(): 121 | package_capsule( 122 | unpackaged_dir=path, 123 | output_file=path.with_suffix(CAPSULE_EXTENSION), 124 | key=capsule_key, 125 | ) 126 | elif capsule_file is not None: 127 | capsule_filepath, file_ext = os.path.splitext(capsule_file) 128 | if file_ext == CAPSULE_EXTENSION: 129 | unpackage_capsule( 130 | capsule_file = capsule_file, 131 | unpackage_to = capsule_filepath, 132 | key = capsule_key 133 | ) 134 | else: 135 | return False 136 | else: 137 | return False 138 | 139 | return True 140 | 141 | 142 | def packaging_main(): 143 | parser = packaging_parse_args() 144 | args = parser.parse_args() 145 | rtn = packaging(args.capsule_dir, args.capsule_file, args.capsule_key) 146 | if rtn == False: 147 | parser.print_help() 148 | 149 | 150 | if __name__ == "__main__": 151 | packaging_main() 152 | -------------------------------------------------------------------------------- /vcap/vcap/options.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import List, Optional, Union 3 | 4 | 5 | OPTION_TYPE = Union[int, float, bool, str] 6 | 7 | 8 | def _check_number_range(value, min_val, max_val): 9 | error = ValueError(f"Value {value} is not in " 10 | f"{min_val} <= x <= {max_val}") 11 | 12 | if min_val is not None and value < min_val: 13 | raise error 14 | if max_val is not None and value > max_val: 15 | raise error 16 | 17 | 18 | class Option(ABC): 19 | """A class that all options subclass. Added mostly for documentation an 20 | type hinting purposes. 21 | """ 22 | 23 | def __init__(self, default, description): 24 | self.default = default 25 | self.description: Optional[str] = description 26 | self.check(default) 27 | 28 | @abstractmethod 29 | def check(self, value): 30 | """Checks the given value to see if it fits this option. 31 | 32 | :param value: The value to check 33 | :raises ValueError: If the value does not fit the option 34 | """ 35 | pass 36 | 37 | 38 | class FloatOption(Option): 39 | """A capsule option that holds a floating point value with defined 40 | boundaries. 41 | """ 42 | 43 | def __init__(self, *, default: float, 44 | min_val: Optional[float], 45 | max_val: Optional[float], 46 | description: Optional[str] = None): 47 | """ 48 | :param default: The default value of this option 49 | :param min_val: The minimum allowed value for this option, inclusive, 50 | or None for no lower limit 51 | :param max_val: The maximum allowed value for this option, inclusive, 52 | or None for no upper limit 53 | :param description: The description for this option 54 | """ 55 | self.min_val = min_val 56 | self.max_val = max_val 57 | 58 | super().__init__(default=default, description=description) 59 | 60 | def check(self, value): 61 | if not isinstance(value, float): 62 | raise ValueError(f"Expecting type float, got {type(value)}") 63 | _check_number_range(value, self.min_val, self.max_val) 64 | 65 | 66 | class IntOption(Option): 67 | """A capsule option that holds an integer value. 68 | """ 69 | 70 | def __init__(self, *, default: int, 71 | min_val: Optional[int], 72 | max_val: Optional[int], 73 | description: Optional[str] = None): 74 | """ 75 | :param default: The default value of this option 76 | :param min_val: The minimum allowed value for this option, inclusive, 77 | or None for no lower limit 78 | :param max_val: The maximum allowed value for this option, inclusive, 79 | or None for no upper limit 80 | :param description: The description for this option 81 | """ 82 | self.min_val = min_val 83 | self.max_val = max_val 84 | 85 | super().__init__(default=default, description=description) 86 | 87 | def check(self, value): 88 | if not isinstance(value, int): 89 | raise ValueError(f"Expecting type int, got {type(value)}") 90 | _check_number_range(value, self.min_val, self.max_val) 91 | 92 | 93 | class EnumOption(Option): 94 | """A capsule option that holds a choice from a discrete set of string 95 | values. 96 | """ 97 | 98 | def __init__(self, *, default: str, choices: List[str], 99 | description: Optional[str] = None): 100 | """ 101 | :param default: The default value of this option 102 | :param choices: A list of all valid values for this option 103 | :param description: The description for this option 104 | """ 105 | assert len(choices) > 0 106 | 107 | self.choices = choices 108 | 109 | super().__init__(default=default, description=description) 110 | 111 | def check(self, value): 112 | if not isinstance(value, str): 113 | raise ValueError(f"Expecting type float, got {type(value)}") 114 | 115 | if value not in self.choices: 116 | raise ValueError(f"Value {value} is not one of {self.choices}") 117 | 118 | 119 | class BoolOption(Option): 120 | """A capsule option that holds an boolean value.""" 121 | 122 | def __init__(self, *, default: bool, 123 | description: Optional[str] = None): 124 | """ 125 | :param default: The default value of this option 126 | :param description: The description for this option 127 | """ 128 | super().__init__(default=default, description=description) 129 | 130 | def check(self, value): 131 | if not isinstance(value, bool): 132 | raise ValueError(f"Expecting type bool, got {type(value)}") 133 | 134 | 135 | def check_option_values(capsule, option_vals): 136 | """Checks the given option values against the capsule's available options to 137 | make sure they are of the right name, type, and fit within the constraints. 138 | """ 139 | for name, val in option_vals.items(): 140 | if name not in capsule.options: 141 | raise ValueError(f"'{name}' is not a valid option for this capsule") 142 | option = capsule.options[name] 143 | try: 144 | option.check(val) 145 | except ValueError as e: 146 | raise ValueError(f"For option {name}: {e}") 147 | 148 | 149 | """Options that detector capsules tend to have 150 | to avoid breaking API too often, make these options be short, with minimal 151 | keys. Try to add a name that is descriptive to the use case of the capsule. 152 | 153 | If an existing one is changed, it is the responsibility of the person 154 | changing to verify that all capsules have been updated. This is considered 155 | a breaking capsule API change if the key is changed. 156 | """ 157 | 158 | common_detector_options = { 159 | "threshold": FloatOption( 160 | default=0.5, 161 | min_val=0.0, 162 | max_val=1.0, 163 | description="The confidence threshold for the detector to return a " 164 | "detection for an object. Lower => more detections, " 165 | "higher means fewer but more accurate detections."), 166 | } 167 | -------------------------------------------------------------------------------- /vcap/vcap/stream_state.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | 3 | 4 | class BaseStreamState(ABC): 5 | """Filler class in case we want to add abstract methods in the future, 6 | such as maybe requiring a method to clear/reset/copy StreamStates...""" 7 | pass 8 | -------------------------------------------------------------------------------- /vcap/vcap/testing/__init__.py: -------------------------------------------------------------------------------- 1 | from .capsule_loading import load_capsule_with_one_device 2 | from .input_output_validation import perform_capsule_tests 3 | from .thread_validation import verify_all_threads_closed 4 | -------------------------------------------------------------------------------- /vcap/vcap/testing/capsule_loading.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import mock 4 | 5 | from vcap import BaseCapsule 6 | from vcap.loading.capsule_loading import load_capsule_from_bytes, load_capsule 7 | 8 | 9 | def load_capsule_with_one_device(packaged_capsule_path: Path, 10 | from_memory: bool = False, 11 | inference_mode: bool = True) -> BaseCapsule: 12 | """ 13 | Load the capsule, but patch out the DeviceMapper so that it never returns 14 | multiple devices. 15 | 16 | Essentially, this disables capsules from loading backends onto multiple 17 | devices, for example GPUs. This is for the purpose of speeding up test 18 | setup and teardown. 19 | """ 20 | 21 | def mock_init(self, filter_func): 22 | # Don't call the superclass init so that self.filter_func isn't 23 | # overridden 24 | self.filter_func = lambda all_devices: [filter_func(all_devices)[0]] 25 | 26 | with mock.patch('vcap.device_mapping.DeviceMapper.__init__', 27 | mock_init): 28 | if from_memory: 29 | data = packaged_capsule_path.read_bytes() 30 | capsule: BaseCapsule = load_capsule_from_bytes( 31 | data=data, 32 | inference_mode=inference_mode) 33 | else: 34 | capsule: BaseCapsule = load_capsule(packaged_capsule_path, 35 | inference_mode=inference_mode) 36 | 37 | return capsule 38 | -------------------------------------------------------------------------------- /vcap/vcap/testing/thread_validation.py: -------------------------------------------------------------------------------- 1 | import re 2 | import threading 3 | from typing import List, Optional 4 | 5 | 6 | def verify_all_threads_closed(allowable_threads: Optional[List[str]] = None): 7 | """A convenient function to throw an error if all threads are not closed. 8 | 9 | :param allowable_threads: A list of regular expressions that match to 10 | thread names that are allowed to stay open 11 | """ 12 | if allowable_threads is None: 13 | allowable_threads = [] 14 | allowable_threads += [ 15 | r"pydevd\.Writer", 16 | r"pydevd\.Reader", 17 | r"pydevd\.CommandThread", 18 | r"profiler\.Reader", 19 | r"MainThread", 20 | r"ThreadPoolExecutor-\d+_\d+" 21 | ] 22 | 23 | open_threads = [] 24 | 25 | for thread in threading.enumerate(): 26 | matched = False 27 | for name_pattern in allowable_threads: 28 | if re.match(name_pattern, thread.name): 29 | matched = True 30 | break 31 | 32 | if not matched: 33 | open_threads.append(thread) 34 | 35 | if len(open_threads) != 0: 36 | raise EnvironmentError( 37 | "Not all threads were shut down! Currently running threads: " + 38 | str(open_threads)) 39 | -------------------------------------------------------------------------------- /vcap/vcap/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.3.8.1" 2 | -------------------------------------------------------------------------------- /vcap_utils/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | from pathlib import Path 4 | 5 | from setuptools import setup, find_namespace_packages 6 | 7 | # Get package version/metadata 8 | about = {} 9 | exec(Path("vcap_utils/version.py").read_text(), about) 10 | 11 | test_packages = ["pytest", "mock"] 12 | 13 | PRE_RELEASE_SUFFIX = os.environ.get("PRE_RELEASE_SUFFIX", "") 14 | 15 | setup( 16 | name='vcap-utils', 17 | description="Utilities for creating OpenVisionCapsules easily in Python", 18 | author="Aotu.ai", 19 | packages=find_namespace_packages(include=["vcap_utils*"]), 20 | version=about["__version__"] + PRE_RELEASE_SUFFIX, 21 | 22 | install_requires=[ 23 | "vcap", 24 | ], 25 | 26 | extras_require={ 27 | "tests": test_packages, 28 | }, 29 | tests_require=test_packages, 30 | classifiers=[ 31 | "Programming Language :: Python :: 3", 32 | "License :: OSI Approved :: BSD License", 33 | "Operating System :: OS Independent", 34 | ], 35 | python_requires='>=3.6', 36 | ) 37 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .version import __version__ 2 | from .backends import * 3 | from .algorithms import * 4 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | from .iou_cost_matrix import detection_area, detection_intersection, \ 2 | detection_iou, bbox_intersection, bbox_iou, iou_cost_matrix 3 | from .linear_assignment import linear_assignment 4 | from .nonmax_suppression import non_max_suppression 5 | from .distances import cosine_distance, euclidian_distance 6 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/algorithms/distances.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def cosine_distance(encoding_to_compare: np.ndarray, encodings: np.ndarray, 5 | is_normalized: bool = False): 6 | """Return the cosine distance of 'encoding_to_compare' to every encoding 7 | in 'encodings'""" 8 | if not is_normalized: 9 | # normalize arrays to unit length vectors (length 1) 10 | e_normed = np.linalg.norm(encodings, axis=1, keepdims=True) 11 | encodings = np.asarray(encodings) / e_normed 12 | 13 | etc_normed = np.linalg.norm(encoding_to_compare, axis=0, keepdims=True) 14 | encoding_to_compare = np.asarray(encoding_to_compare) / etc_normed 15 | return 1. - np.dot(encodings, encoding_to_compare.T) 16 | 17 | 18 | def euclidian_distance(encoding_to_compare: np.ndarray, encodings: np.ndarray): 19 | """ 20 | Given a list of face encodings, compare them to a known face encoding and 21 | get a euclidean distance for each comparison face. The distance tells you 22 | how similar the faces are. 23 | 24 | :param encoding_to_compare: A face encoding to compare against 25 | :param encodings: List of face encodings to compare 26 | :return: A numpy ndarray with the distance for each face in the same order 27 | as the 'faces' array 28 | """ 29 | if len(encodings) == 0: 30 | return np.empty(0) 31 | assert encodings.shape[-1] == encoding_to_compare.shape[-1], \ 32 | "Encoding shapes are incorrect!" 33 | return np.linalg.norm(encodings - encoding_to_compare, axis=1) 34 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/algorithms/iou_cost_matrix.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union, Optional 2 | 3 | import numpy as np 4 | from vcap.detection_node import DetectionNode 5 | 6 | Num = Union[int, float] 7 | 8 | 9 | def detection_area(detection: DetectionNode): 10 | bbox = detection.bbox.xywh 11 | return bbox_area(bbox) 12 | 13 | 14 | def detection_intersection(detection: DetectionNode, 15 | candidates: List[DetectionNode]) -> np.ndarray: 16 | candidate_bboxes = [candidate.bbox.xywh for candidate in candidates] 17 | bbox = detection.bbox.xywh 18 | return bbox_intersection(bbox, candidate_bboxes) 19 | 20 | 21 | def detection_iou(detection: DetectionNode, 22 | candidates: List[DetectionNode]) -> np.ndarray: 23 | """Computer intersection over union. 24 | 25 | Parameters 26 | ---------- 27 | bbox : DetectionNode 28 | candidates : List of DetectionNodes 29 | 30 | Returns 31 | ------- 32 | ndarray 33 | The intersection over union in [0, 1] between the `bbox` and each 34 | candidate. A higher score means a larger fraction of the `bbox` is 35 | occluded by the candidate. 36 | 37 | """ 38 | candidate_bboxes = [candidate.bbox.xywh for candidate in candidates] 39 | bbox = detection.bbox.xywh 40 | return bbox_iou(bbox, candidate_bboxes) 41 | 42 | 43 | def bbox_area(bbox: List[Num]): 44 | bbox = np.array(bbox) 45 | area_bbox = bbox[2:].prod() 46 | 47 | return area_bbox 48 | 49 | 50 | def bbox_intersection(bbox: List[Num], candidate_bboxes: List[List[Num]]) \ 51 | -> np.ndarray: 52 | candidate_bboxes = np.array(candidate_bboxes) 53 | bbox = np.array(bbox) 54 | 55 | bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:] 56 | candidates_tl = candidate_bboxes[:, :2] 57 | candidates_br = candidate_bboxes[:, :2] + candidate_bboxes[:, 2:] 58 | 59 | tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis], 60 | np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]] 61 | br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis], 62 | np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]] 63 | wh = np.maximum(0., br - tl) 64 | 65 | area_intersection = wh.prod(axis=1) 66 | 67 | return area_intersection 68 | 69 | 70 | def bbox_iou(bbox: List[Num], candidate_bboxes: List[List[Num]]) -> np.ndarray: 71 | """ 72 | :param bbox: Bounding box in the format [x, y, width, height] 73 | :param candidate_bboxes: A list of bounding boxes in the bbox format 74 | :return: 75 | """ 76 | candidate_bboxes = np.array(candidate_bboxes) 77 | bbox = np.array(bbox) 78 | 79 | area_bbox = bbox_area(bbox) 80 | area_candidates = candidate_bboxes[:, 2:].prod(axis=1) 81 | 82 | area_intersection = bbox_intersection(bbox, candidate_bboxes) 83 | area_union = area_bbox + area_candidates - area_intersection 84 | 85 | iou = area_intersection / area_union 86 | 87 | return iou 88 | 89 | 90 | def iou_cost_matrix( 91 | detections_a: List[DetectionNode], 92 | detections_b: List[DetectionNode], 93 | detections_a_indices: Optional[List[int]] = None, 94 | detections_b_indices: Optional[List[int]] = None) -> np.ndarray: 95 | """An intersection over union distance metric. 96 | 97 | Parameters 98 | ---------- 99 | detections_a : List[vcap.DetectionNode] 100 | A list of detections. 101 | detections_b : List[vcap.DetectionNode] 102 | A list of detections. 103 | detections_a_indices : Optional[List[int]] 104 | A list of indices to detections that should be matched. Defaults to 105 | all `detections`. 106 | detections_b_indices : Optional[List[int]] 107 | A list of indices to detections that should be matched. Defaults 108 | to all `detections`. 109 | 110 | Returns 111 | ------- 112 | ndarray 113 | Returns a cost matrix of shape 114 | len(detections_a_indices), len(detections_b_indices) where entry (i, j) is 115 | `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`. 116 | 117 | """ 118 | 119 | if detections_a_indices is None: 120 | detections_a_indices = np.arange(len(detections_a)) 121 | if detections_b_indices is None: 122 | detections_b_indices = np.arange(len(detections_b)) 123 | 124 | cost_matrix = np.zeros( 125 | (len(detections_a_indices), len(detections_b_indices))) 126 | for row, track_idx in enumerate(detections_a_indices): 127 | detection_from_a = detections_a[track_idx] 128 | candidate_detections = [detections_b[index] 129 | for index in detections_b_indices] 130 | cost_matrix[row, :] = 1. - detection_iou(detection_from_a, 131 | candidate_detections) 132 | return cost_matrix 133 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/algorithms/nonmax_suppression.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def non_max_suppression(detection_nodes, max_bbox_overlap, scores=None): 5 | """Suppress overlapping detections. 6 | 7 | Original code from [1]_ has been adapted to include confidence score. 8 | 9 | .. [1] http://www.pyimagesearch.com/2015/02/16/ 10 | faster-non-maximum-suppression-python/ 11 | 12 | Examples 13 | -------- 14 | 15 | >>> boxes = [d.roi for d in detections] 16 | >>> scores = [d.confidence for d in detections] 17 | >>> indices = non_max_suppression(boxes, max_bbox_overlap, scores) 18 | >>> detections = [detections[i] for i in indices] 19 | 20 | Parameters 21 | ---------- 22 | boxes : ndarray 23 | Array of ROIs (x, y, width, height). 24 | max_bbox_overlap : float 25 | ROIs that overlap more than this values are suppressed. 26 | scores : Optional[array_like] 27 | Detector confidence score. 28 | 29 | Returns 30 | ------- 31 | List[int] 32 | Returns indices of detections that have survived non-maxima suppression. 33 | 34 | """ 35 | boxes = np.array([d.bbox.xywh for d in detection_nodes]) 36 | 37 | if len(detection_nodes) != 0: 38 | if 'detection_confidence' in detection_nodes[0].extra_data: 39 | detection_confidence = 'detection_confidence' 40 | elif 'confidence' in detection_nodes[0].extra_data: 41 | detection_confidence = 'confidence' 42 | else: 43 | detection_confidence = None 44 | else: 45 | detection_confidence = None 46 | 47 | if detection_confidence is not None: 48 | scores = np.array([d.extra_data[detection_confidence] 49 | for d in detection_nodes]) 50 | 51 | if len(boxes) == 0: 52 | return [] 53 | 54 | boxes = boxes.astype(np.float) 55 | pick = [] 56 | 57 | x1 = boxes[:, 0] 58 | y1 = boxes[:, 1] 59 | x2 = boxes[:, 2] + boxes[:, 0] 60 | y2 = boxes[:, 3] + boxes[:, 1] 61 | 62 | area = (x2 - x1 + 1) * (y2 - y1 + 1) 63 | if scores is not None: 64 | idxs = np.argsort(scores) 65 | else: 66 | idxs = np.argsort(y2) 67 | 68 | while len(idxs) > 0: 69 | last = len(idxs) - 1 70 | i = idxs[last] 71 | pick.append(i) 72 | 73 | xx1 = np.maximum(x1[i], x1[idxs[:last]]) 74 | yy1 = np.maximum(y1[i], y1[idxs[:last]]) 75 | xx2 = np.minimum(x2[i], x2[idxs[:last]]) 76 | yy2 = np.minimum(y2[i], y2[idxs[:last]]) 77 | 78 | w = np.maximum(0, xx2 - xx1 + 1) 79 | h = np.maximum(0, yy2 - yy1 + 1) 80 | 81 | overlap = (w * h) / area[idxs[:last]] 82 | 83 | idxs = np.delete( 84 | idxs, np.concatenate( 85 | ([last], np.where(overlap > max_bbox_overlap)[0]))) 86 | 87 | return [detection_nodes[i] for i in pick] 88 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_tensorflow import BaseTFBackend 2 | from .base_openvino import BaseOpenVINOBackend 3 | from .tf_object_detection import TFObjectDetector 4 | from .tf_image_classification import TFImageClassifier 5 | from .crowd_density import CrowdDensityCounter 6 | from .depth import DepthPredictor 7 | from .segmentation import Segmenter 8 | from .openface_encoder import OpenFaceEncoder 9 | from .base_encoder import BaseEncoderBackend 10 | from .backend_rpc_process import BackendRpcProcess 11 | from .load_utils import parse_dataset_metadata_bytes, parse_tf_model_bytes 12 | from .predictions import ( 13 | EncodingPrediction, 14 | SegmentationPrediction, 15 | ClassificationPrediction, 16 | DepthPrediction, 17 | DensityPrediction, 18 | DetectionPrediction, 19 | ) 20 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/base_encoder.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import numpy as np 4 | 5 | from vcap.backend import BaseBackend 6 | 7 | 8 | class BaseEncoderBackend(BaseBackend): 9 | # TODO: Remove the concept of Backends needing a 'distances' function 10 | @abc.abstractmethod 11 | def distances(self, encoding_to_compare: np.ndarray, 12 | encodings: np.ndarray) -> np.ndarray: 13 | """ 14 | :param encoding_to_compare: An object encoding to compare 15 | :param encodings: List of object encodings to compare against 16 | :return: A numpy ndarray with the distance for each face in the same 17 | order as the 'encodings_to_compare' array 18 | """ 19 | pass 20 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/base_tensorflow.py: -------------------------------------------------------------------------------- 1 | from vcap.backend import BaseBackend 2 | 3 | 4 | class BaseTFBackend(BaseBackend): 5 | def close(self): 6 | super().close() 7 | self.session.close() 8 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/crowd_density.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | from .load_utils import parse_tf_model_bytes 6 | from .predictions import DensityPrediction 7 | from .base_tensorflow import BaseTFBackend 8 | 9 | 10 | class CrowdDensityCounter(BaseTFBackend): 11 | """Runs object detection using a given model, meaning that it takes an 12 | image as input and returns bounding boxes with classifications as output. 13 | """ 14 | 15 | def __init__(self, model_bytes, 16 | device: str=None, 17 | session_config: tf.compat.v1.ConfigProto=None): 18 | """ 19 | :param model_bytes: Model file data, likely a loaded *.pb file 20 | :param device: The device to run the model on 21 | :param session_config: Model configuration options 22 | """ 23 | super().__init__() 24 | self.graph, self.session = parse_tf_model_bytes(model_bytes, 25 | device, 26 | session_config) 27 | 28 | self.input = self.graph.get_tensor_by_name("input:0") 29 | self.output = self.graph.get_tensor_by_name("Threshold_32/Relu:0") 30 | 31 | def batch_predict(self, img_bgr) -> DensityPrediction: 32 | """Takes a numpy BGR image of the format that OpenCV gives, and returns 33 | predicted labels in the form of a list of Detection objects 34 | from predictions.py 35 | 36 | :img_bgr: An OpenCV image in BGR format (the default) 37 | :return: List of labels 38 | """ 39 | preprocessed = preprocess(img_bgr) 40 | 41 | # Run the network 42 | feed = {self.input: preprocessed} 43 | density_map = self.session.run([self.output], feed) 44 | density_map = np.squeeze(density_map) 45 | 46 | return DensityPrediction(density_map) 47 | 48 | 49 | def preprocess(img_bgr): 50 | # Convert the image to grayscale if it wasn't already 51 | if len(img_bgr.shape) == 3: 52 | gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY) 53 | gray = gray.astype(np.float32, copy=False) 54 | elif len(img_bgr.shape) == 2: 55 | gray = img_bgr 56 | else: 57 | raise RuntimeError(f"Unexpected image shape: {img_bgr.shape}") 58 | 59 | h = gray.shape[0] 60 | w = gray.shape[1] 61 | h_1 = int((h / 4) * 4) 62 | w_1 = int((w / 4) * 4) 63 | small_gray = cv2.resize(gray, (w_1, h_1)) 64 | reshaped_small_gray_img = small_gray.reshape( 65 | (1, 1, small_gray.shape[0], small_gray.shape[1])) 66 | return reshaped_small_gray_img -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/depth.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | from .base_tensorflow import BaseTFBackend 6 | from .predictions import DepthPrediction 7 | from .load_utils import parse_tf_model_bytes 8 | 9 | 10 | class DepthPredictor(BaseTFBackend): 11 | """ 12 | Loads a model and uses it to run depth prediction, meaning that it takes 13 | an image as input and returns a matrix the size of the input image where 14 | the value of each 'pixel' corresponds to the pixel's distance from the 15 | camera in meters. 16 | 17 | Does not support batch prediction TODO: Is this true? 18 | """ 19 | 20 | def __init__(self, model_bytes, 21 | device: str=None, 22 | session_config: tf.compat.v1.ConfigProto=None): 23 | """ 24 | :param model_bytes: Model file data, likely a loaded *.pb file 25 | :param device: The device to run the model on 26 | :param session_config: Model configuration options 27 | """ 28 | super().__init__() 29 | self.graph, self.session = parse_tf_model_bytes(model_bytes, 30 | device, 31 | session_config) 32 | 33 | # Pull all the necessary inputs from the network 34 | self.input_image = self.graph.get_tensor_by_name("input:0") 35 | 36 | # Pull the necessary attributes that we want to get from the network 37 | # after running it 38 | self.segmented_tensor = self.graph.get_tensor_by_name( 39 | "output_prediction:0") 40 | 41 | def batch_predict(self, img_bgr) -> DepthPrediction: 42 | """ 43 | Takes numpy BGR images of the format that OpenCV gives, and returns 44 | predicted labels in the form of a list of DetectionPrediction 45 | objects from predictions.py 46 | 47 | :param img_bgr: A numpy BGR image from OpenCV 48 | :return: List of list of DetectionPrediction objects 49 | """ 50 | img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB) 51 | feed = {self.input_image: img_rgb} 52 | segmented_image = self.session.run(self.segmented_tensor, 53 | feed_dict=feed) 54 | 55 | out = np.squeeze(segmented_image) 56 | 57 | return DepthPrediction(out) 58 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/load_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import tensorflow as tf 4 | 5 | 6 | def parse_tf_model_bytes(model_bytes, 7 | device: str = None, 8 | session_config: tf.compat.v1.ConfigProto = None): 9 | """ 10 | 11 | :param model_bytes: The bytes of the model to load 12 | :param device_id: The device that this model should be loaded onto 13 | :param session_config: Configuration options for multiple monitors 14 | :return: 15 | """ 16 | 17 | # Load the model as a graph 18 | detection_graph = tf.Graph() 19 | with detection_graph.as_default(): 20 | # Load a (frozen) Tensorflow model from memory 21 | graph_def = tf.compat.v1.GraphDef() 22 | graph_def.ParseFromString(model_bytes) 23 | 24 | with tf.device(device): 25 | tf.import_graph_def(graph_def, 26 | input_map=None, 27 | return_elements=None, 28 | producer_op_list=None, 29 | name='') 30 | 31 | if session_config is None: 32 | session_config = tf.compat.v1.ConfigProto() 33 | 34 | if device is not None: 35 | # allow_soft_placement lets us remap GPU only ops to GPU, and doesn't 36 | # crash for non-gpu only ops (it will place those on CPU, instead) 37 | session_config.allow_soft_placement = True 38 | 39 | # Create a session for later use 40 | persistent_sess = tf.compat.v1.Session(graph=detection_graph, 41 | config=session_config) 42 | 43 | return detection_graph, persistent_sess 44 | 45 | 46 | def parse_dataset_metadata_bytes(dataset_metadata_bytes): 47 | """ Loads the label_map from the dataset_metadata bytes """ 48 | dataset_metadata = json.loads(dataset_metadata_bytes.decode("utf-8")) 49 | 50 | # Convert the keys of the label map to integer values, instead of strings 51 | label_map = dataset_metadata["label_map"] 52 | label_map = {int(k): v for k, v in label_map.items()} 53 | return label_map 54 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/openface_encoder.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from typing import List 3 | 4 | import cv2 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | from .base_encoder import BaseEncoderBackend 9 | from .base_tensorflow import BaseTFBackend 10 | from .load_utils import parse_tf_model_bytes 11 | from .predictions import EncodingPrediction 12 | from vcap_utils.algorithms import euclidian_distance 13 | 14 | 15 | class OpenFaceEncoder(BaseEncoderBackend, BaseTFBackend): 16 | """Encode faces (or whatever the model is capable of encoding) 17 | using this module. It will easily load and run what is necessary. 18 | 19 | Specifically meant to work with models trained from this repository: 20 | https://github.com/davidsandberg/facenet 21 | """ 22 | 23 | def __init__(self, model_bytes, model_name, 24 | device: str = None, 25 | session_config: tf.compat.v1.ConfigProto = None): 26 | """ 27 | :param model_bytes: Model file bytes, a loaded *.pb file 28 | :param model_name: The name of the model in order to load correct 29 | input/output tensor node names 30 | :param device: The device to run the model on 31 | :param session_config: Model configuration options 32 | """ 33 | super().__init__() 34 | 35 | assert model_name in model_to_ops_map.keys(), \ 36 | "The model name must be from here: " + str(model_to_ops_map.keys()) 37 | 38 | self.config = model_to_ops_map[model_name] 39 | self._preprocess = self.config.preprocess 40 | self.graph, self.session = parse_tf_model_bytes(model_bytes, 41 | device, 42 | session_config) 43 | 44 | # Pull all the necessary inputs from the network 45 | self.image_tensor = self.graph.get_tensor_by_name( 46 | self.config.input_node) 47 | self.embedding = self.graph.get_tensor_by_name( 48 | self.config.embedding) 49 | self.phase_train_placeholder = self.graph.get_tensor_by_name( 50 | 'phase_train:0') 51 | 52 | def batch_predict(self, imgs_bgr) -> List[EncodingPrediction]: 53 | imgs_rgb = [self._preprocess(img) for img in imgs_bgr] 54 | 55 | # Batch predict for the list of images 56 | feed = {self.image_tensor: imgs_rgb, 57 | self.phase_train_placeholder: False} 58 | embeddings = self.session.run(self.embedding, feed_dict=feed) 59 | preds = [EncodingPrediction(e, self.config.distance) 60 | for e in embeddings] 61 | return preds 62 | 63 | def distances(self, encoding_to_compare, encodings): 64 | return self.config.distance(encoding_to_compare, encodings) 65 | 66 | 67 | def _preprocess_vggface2_center_loss(img): 68 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 69 | img = cv2.resize(img, vggface2_center_loss.img_size) 70 | 71 | # Pre-whiten the image 72 | mean = np.mean(img) 73 | std = np.std(img) 74 | std_adj = np.maximum(std, 1.0 / np.sqrt(img.size)) 75 | img = np.multiply(np.subtract(img, mean), 1 / std_adj) 76 | 77 | return img 78 | 79 | 80 | ModelConfig = namedtuple("ModelConfig", 81 | ["input_node", "embedding", "img_size", 82 | "preprocess", "distance"]) 83 | 84 | vggface2_center_loss = ModelConfig(input_node="batch_join:0", 85 | embedding="embeddings:0", 86 | img_size=(160, 160), 87 | preprocess=_preprocess_vggface2_center_loss, 88 | distance=euclidian_distance) 89 | model_to_ops_map = {"vggface2_center_loss": vggface2_center_loss} 90 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/predictions.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, List, Union 2 | import random 3 | 4 | import cv2 5 | import numpy as np 6 | 7 | from vcap.caching import cache 8 | 9 | 10 | class DetectionPrediction: 11 | def __init__(self, class_name: str, 12 | class_id: int, 13 | rect: List[Union[int, float]], 14 | confidence: float): 15 | """An object detection label. Basically a bounding box classifying 16 | some portion of an image. 17 | 18 | :param class_name: The class name that was predicted 19 | (Price_Tag, Shelf_Empty, Person) 20 | :param class_id: The class number used by the network 21 | :param rect: [xmin, ymin, xmax, ymax] format 22 | :param confidence: The confidence in percent from 0-1, representing 23 | how confident the network is with the prediction 24 | """ 25 | self._color = None 26 | 27 | # The full rect, (x1, y1, x2, y2) format 28 | self.rect = tuple(rect) 29 | 30 | self.name = str(class_name) 31 | self.confidence = float(confidence) 32 | self.class_num = int(class_id) 33 | 34 | assert 0. <= self.confidence <= 1, "Confidence must be between 0 and 1!" 35 | 36 | def __str__(self): 37 | return (str(self.name) + " " + 38 | str(self.rect) + " " + 39 | str(round(self.confidence, 2))) 40 | 41 | def __repr__(self): 42 | return self.__str__() 43 | 44 | @property 45 | def p1(self): 46 | """ Top Left of the rect""" 47 | return tuple(self.rect[:2]) 48 | 49 | @property 50 | def p2(self): 51 | """ Top Right of the rect""" 52 | return tuple(self.rect[2:]) 53 | 54 | @property 55 | def color(self): 56 | if self._color is None: 57 | # Generate a color based on the class name 58 | rand_seed = random.Random(self.name) 59 | self._color = tuple([rand_seed.randint(0, 255) for i in range(3)]) 60 | 61 | return self._color 62 | 63 | 64 | class ClassificationPrediction: 65 | def __init__(self, 66 | class_scores: np.ndarray, # floats 67 | class_names: List[str]): 68 | """A classification of an entire image. 69 | 70 | :param class_scores: A list of scores for each class 71 | :param class_names: A list of class names, same length as class scores. 72 | Each index corresponds to an index in class_scores 73 | """ 74 | self.class_scores = class_scores 75 | self.class_names = class_names 76 | 77 | def __iter__(self): 78 | """Iterate over class_name and class_score""" 79 | for name, score in zip(self.class_names, self.class_scores): 80 | # Ensure that no numpy floats get returned (causing bad errors) 81 | yield str(name), float(score) 82 | 83 | def __str__(self): 84 | return f"{self.name} {self.confidence:.2f}" 85 | 86 | def __repr__(self): 87 | return self.__str__() 88 | 89 | @property 90 | @cache 91 | def name(self) -> str: 92 | """The class name that was predicted with the highest confidence""" 93 | return self.class_names[self.class_num] 94 | 95 | @property 96 | @cache 97 | def class_num(self) -> int: 98 | """The class number used by the network for the class with the highest 99 | confidence""" 100 | return int(np.argmax(self.class_scores)) 101 | 102 | @property 103 | @cache 104 | def confidence(self) -> float: 105 | """The confidence in percent from 0-1, representing how confident the 106 | network is with its highest confidence prediction""" 107 | return float(self.class_scores[self.class_num]) 108 | 109 | 110 | class EncodingPrediction: 111 | def __init__(self, encoding_vector: np.ndarray, 112 | distance_func: Callable): 113 | """ 114 | 115 | :param encoding_vector: The encoding prediction 116 | :param distance_func: A function with args (encoding, other_encodings) 117 | returns the distance from one encoding to all the other_encodings in 118 | """ 119 | self.vector = encoding_vector 120 | self.distance = distance_func 121 | 122 | 123 | class DensityPrediction: 124 | def __init__(self, density_map): 125 | self.map = density_map 126 | self.count = np.sum(density_map) 127 | 128 | def __str__(self): 129 | return str(self.count) 130 | 131 | def __repr__(self): 132 | return self.__str__() 133 | 134 | def resized_map(self, new_size): 135 | """ 136 | Returns a resized density map, where the sum of each value is still the 137 | same count 138 | :param new_size: (new width, new height)""" 139 | 140 | new_map = cv2.resize(self.map.copy(), new_size) 141 | cur_count = np.sum(new_map) 142 | 143 | # Avoid dividing by zero 144 | if cur_count == 0: 145 | return new_map 146 | 147 | scale = self.count / cur_count 148 | new_map *= scale 149 | return new_map 150 | 151 | 152 | class SegmentationPrediction: 153 | def __init__(self, segmentation, label_map): 154 | """ 155 | :param segmentation: 2d segmented image where the value is the label num 156 | :param label_map: Formatted as {"1" : {"label" : "chair", "color" : [12, 53, 100]}} 157 | """ 158 | self.segmentation = segmentation 159 | self.label_map = label_map 160 | 161 | def colored(self): 162 | """Convert segmented image to RGB image using label_map""" 163 | 164 | color_img = np.zeros( 165 | (self.segmentation.shape[0], self.segmentation.shape[1], 3), 166 | dtype=np.uint8) 167 | 168 | # Replace segment values with color pixels using label_map values 169 | for label_num, label in self.label_map.items(): 170 | color_img[self.segmentation == int(label_num)] = np.array( 171 | label["color"], dtype=np.uint8) 172 | 173 | return color_img 174 | 175 | 176 | class DepthPrediction: 177 | def __init__(self, depth_prediction): 178 | """ 179 | :param depth_prediction: 2d image where the value is the depth 180 | """ 181 | self.depth_prediction = depth_prediction 182 | 183 | def normalized(self): 184 | """Convert segmented image to RGB image using label_map""" 185 | 186 | # Scale results to max at 255 for image display 187 | max_distance = np.max(self.depth_prediction) 188 | pred = 255 * self.depth_prediction // max_distance 189 | 190 | # Convert results to uint8 191 | pred = pred.astype(np.uint8, copy=True) 192 | 193 | # Do fancy coloring 194 | pred = cv2.applyColorMap(pred, cv2.COLORMAP_JET) 195 | 196 | return pred 197 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/segmentation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | from .base_tensorflow import BaseTFBackend 5 | from .load_utils import parse_dataset_metadata_bytes, parse_tf_model_bytes 6 | from .predictions import SegmentationPrediction 7 | 8 | 9 | class Segmenter(BaseTFBackend): 10 | """Loads a model and uses it to run image segmentation, meaning that it 11 | takes an image as input and returns a matrix the size of the input image 12 | where the value of each 'pixel' corresponds to an image segment type. 13 | 14 | Does not support batch prediction TODO: Is this true? 15 | """ 16 | 17 | def __init__(self, model_bytes, metadata_bytes, 18 | device: str = None, 19 | session_config: tf.compat.v1.ConfigProto = None): 20 | """ 21 | :param model_bytes: Model file data, likely a loaded *.pb file 22 | :param metadata_bytes: The dataset metadata file data, likely named 23 | "dataset_metadata.json" 24 | :param device: The device to run the model on 25 | :param session_config: Model configuration options 26 | """ 27 | super().__init__() 28 | 29 | self.graph, self.session = parse_tf_model_bytes(model_bytes, 30 | device, 31 | session_config) 32 | self.label_map = parse_dataset_metadata_bytes(metadata_bytes) 33 | 34 | # Pull all the necessary inputs from the network 35 | self.input_image = self.graph.get_tensor_by_name("input") 36 | 37 | # Pull the necessary attributes that we want to get from the network 38 | # after running it 39 | self.segmented_tensor = self.graph.get_tensor_by_name( 40 | "output_prediction") 41 | 42 | def batch_predict(self, img_rgb) -> SegmentationPrediction: 43 | """Takes numpy BGR images of the format that OpenCV gives, and returns 44 | predicted labels in the form of a list of DetectionPrediction objects 45 | from predictions.py. 46 | 47 | :imgs: An OpenCV image 48 | :param bgr: image is bgr format (bgr=True, default) or rgb (bgr=False) 49 | :return: List of list of DetectionPrediction objects 50 | """ 51 | # TODO: The story between RGB and BGR here is confusing 52 | # This method should take a BGR image and convert it 53 | # Batch predict for the list of images 54 | feed = {self.input_image: img_rgb} 55 | segmented_image = self.session.run(self.segmented_tensor, 56 | feed_dict=feed) 57 | 58 | out = np.squeeze(segmented_image) 59 | 60 | return SegmentationPrediction(out, self.label_map) 61 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/tf_image_classification.py: -------------------------------------------------------------------------------- 1 | """This module is made for running image classification inference easily on 2 | various Slim models 3 | """ 4 | import cv2 5 | import numpy as np 6 | import tensorflow as tf 7 | from collections import namedtuple 8 | from typing import List 9 | 10 | from . import load_utils 11 | from .base_tensorflow import BaseTFBackend 12 | from .predictions import ClassificationPrediction 13 | 14 | 15 | class TFImageClassifier(BaseTFBackend): 16 | def __init__(self, model_bytes, metadata_bytes, model_name, 17 | device: str = None, 18 | session_config: tf.compat.v1.ConfigProto = None): 19 | """ 20 | :param model_bytes: Loaded model data, likely from a *.pb file 21 | :param metadata_bytes: Loaded dataset metadata, likely from a file 22 | named "dataset_metadata.json" 23 | :param model_name: Currently supported model names are listed in the 24 | model_to_ops_map variable. 25 | :param device: The device to run the model on 26 | :param session_config: Model configuration options 27 | """ 28 | super().__init__() 29 | assert model_name in model_to_ops_map.keys(), \ 30 | "The model name must be from here: " + str(model_to_ops_map.keys()) 31 | 32 | self.config = model_to_ops_map[model_name] 33 | self.graph, self.session = load_utils.parse_tf_model_bytes( 34 | model_bytes, device, session_config) 35 | self.label_map = load_utils.parse_dataset_metadata_bytes( 36 | metadata_bytes) 37 | self.class_names = tuple( 38 | class_name for key, class_name in 39 | sorted(self.label_map.items(), key=lambda i: i[0])) 40 | 41 | # Create the input node to the graph, with preprocessing built-in 42 | with self.graph.as_default(): 43 | # Create a new input node for images of various sizes 44 | self.input_node = tf.compat.v1.placeholder( 45 | dtype=tf.float32, 46 | shape=[None, self.config.img_size, self.config.img_size, 3]) 47 | 48 | # Create the preprocessing node 49 | preprocessed = self.config.preprocess(self.input_node) 50 | 51 | # Connect the new input to the loaded graphs input 52 | self.output_node, = tf.import_graph_def( 53 | self.graph.as_graph_def(), 54 | input_map={self.config.input_node: preprocessed}, 55 | return_elements=[self.config.output_node]) 56 | 57 | # Tensorflow models are known to process the first batch of frames 58 | # slowly due to some lazy initialization logic. Send a fake image to 59 | # force the model to be fully loaded. 60 | fake_img = np.zeros((50, 50, 3), dtype=np.uint8) 61 | self.batch_predict([fake_img]) 62 | 63 | def batch_predict(self, imgs_bgr) -> List[ClassificationPrediction]: 64 | """ 65 | :param imgs_bgr: A list of images [img1, img2] in BGR format 66 | """ 67 | 68 | # Preprocess each image 69 | imgs_bgr_resized = [_resize_and_pad(img, self.config.img_size) 70 | for img in imgs_bgr] 71 | 72 | # Batch predict for the list of images 73 | feed = {self.input_node: imgs_bgr_resized} 74 | outs = self.session.run(self.output_node, 75 | feed_dict=feed) 76 | 77 | # Create the predictions for return 78 | predictions = [ClassificationPrediction( 79 | class_scores=out, class_names=self.class_names) 80 | for out in outs 81 | ] 82 | 83 | return predictions 84 | 85 | 86 | def _preprocess_inception_resnet_v2(img): 87 | # BGR to RGB conversion 88 | img = img[..., ::-1] 89 | 90 | # Normalize the image between -1 and 1 900-1000 FPS 91 | img = (2.0 / 255.0) * img - 1.0 92 | return img 93 | 94 | 95 | def _preprocess_resnet_v2_50(img): 96 | # BGR to RGB conversion 97 | img = img[..., ::-1] 98 | return img 99 | 100 | 101 | def _resize_and_pad(img, desired_size): 102 | """ 103 | Resize an image to the desired width and height 104 | :param img: 105 | :param desired_size: 106 | :return: 107 | """ 108 | old_size = img.shape[:2] # old_size is in (height, width) format 109 | 110 | ratio = float(desired_size) / max(old_size) 111 | new_size = tuple([int(x * ratio) for x in old_size]) 112 | 113 | if new_size[0] == 0: 114 | new_size = (new_size[0] + 1, new_size[1]) 115 | 116 | if new_size[1] == 0: 117 | new_size = (new_size[0], new_size[1] + 1) 118 | 119 | # New_size should be in (width, height) format 120 | im = cv2.resize(img, (new_size[1], new_size[0])) 121 | 122 | delta_w = desired_size - new_size[1] 123 | delta_h = desired_size - new_size[0] 124 | top, bottom = delta_h // 2, delta_h - (delta_h // 2) 125 | left, right = delta_w // 2, delta_w - (delta_w // 2) 126 | 127 | color = [0, 0, 0] 128 | img = cv2.copyMakeBorder(im, top, bottom, left, right, 129 | cv2.BORDER_CONSTANT, 130 | value=color) 131 | return img 132 | 133 | 134 | ModelConfig = namedtuple("ModelConfig", 135 | ["input_node", "output_node", "img_size", 136 | "preprocess"]) 137 | resnet_v2_50 = \ 138 | ModelConfig(input_node="resnet_v2_50/Pad:0", 139 | output_node="resnet_v2_50/predictions/Reshape_1:0", 140 | img_size=230, 141 | preprocess=_preprocess_resnet_v2_50) 142 | inception_resnet_v2 = \ 143 | ModelConfig(input_node="input:0", 144 | output_node="InceptionResnetV2/Logits/Predictions:0", 145 | img_size=299, 146 | preprocess=_preprocess_inception_resnet_v2) 147 | 148 | inception_v4 = \ 149 | ModelConfig(input_node="input:0", 150 | output_node="InceptionV4/Logits/Predictions:0", 151 | img_size=299, 152 | preprocess=_preprocess_inception_resnet_v2) 153 | 154 | mobilenet_v1 = \ 155 | ModelConfig(input_node="input:0", 156 | output_node="MobilenetV1/Predictions/Reshape_1:0", 157 | img_size=224, 158 | preprocess=_preprocess_inception_resnet_v2) 159 | 160 | model_to_ops_map = {"resnet_v2_50": resnet_v2_50, 161 | "inception_resnet_v2": inception_resnet_v2, 162 | "mobilenet_v1": mobilenet_v1, 163 | "inception_v4": inception_v4} 164 | """ 165 | models_to_ops_map keeps track of information for different model names, so 166 | that the Brain interface can work without the user needing to remember these 167 | specific node names. 168 | """ 169 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/backends/tf_object_detection.py: -------------------------------------------------------------------------------- 1 | from itertools import groupby 2 | from typing import List 3 | 4 | import numpy as np 5 | import tensorflow as tf 6 | 7 | from .base_tensorflow import BaseTFBackend 8 | from .load_utils import parse_dataset_metadata_bytes, parse_tf_model_bytes 9 | from .predictions import DetectionPrediction 10 | 11 | 12 | class TFObjectDetector(BaseTFBackend): 13 | """Loads a TensorFlow model and uses it to run object detection, meaning 14 | that it takes an image as input and returns bounding boxes with 15 | classifications as output. 16 | """ 17 | 18 | def __init__(self, model_bytes, metadata_bytes, 19 | confidence_thresh=0.05, 20 | device: str = None, 21 | session_config: tf.compat.v1.ConfigProto = None): 22 | """ 23 | :param model_bytes: Model file data, likely a loaded *.pb file 24 | :param metadata_bytes: The dataset metadata file data, likely named 25 | "dataset_metadata.json" 26 | :param device: The device to run the model on 27 | :param confidence_thresh: The required confidence threshold to filter 28 | predictions by. Must be between 0 and 1. There is often an 29 | additional level of confidence thresholding in the capsule, but 30 | this is here to avoid creating an unreasonable amount of objects 31 | for each prediction. 32 | :param session_config: Model configuration options 33 | """ 34 | super().__init__() 35 | 36 | assert 0.0 < confidence_thresh < 1.0, \ 37 | "Confidence_thresh must be a number between 0 and 1." 38 | 39 | self.min_confidence = confidence_thresh 40 | self.graph, self.session = parse_tf_model_bytes(model_bytes, 41 | device, 42 | session_config) 43 | self.label_map = parse_dataset_metadata_bytes(metadata_bytes) 44 | 45 | # Pull all the necessary inputs from the network 46 | self.image_tensor = self.graph.get_tensor_by_name('image_tensor:0') 47 | 48 | # Pull the necessary attributes that we want to get from the network 49 | # after running it 50 | self.boxes_tensor = self.graph.get_tensor_by_name('detection_boxes:0') 51 | self.scores_tensor = self.graph.get_tensor_by_name('detection_scores:0') 52 | self.classes_tensor = self.graph.get_tensor_by_name( 53 | 'detection_classes:0') 54 | 55 | # Tensorflow models are known to process the first batch of frames 56 | # slowly due to some lazy initialization logic. Send a fake image to 57 | # force the model to be fully loaded. 58 | fake_img = np.zeros((50, 50, 3), dtype=np.uint8) 59 | self.batch_predict([fake_img]) 60 | 61 | def batch_predict(self, imgs_bgr: List[np.ndarray]) -> List[List[DetectionPrediction]]: 62 | """Takes a list of numpy BGR images of the format that OpenCV gives, and returns 63 | predicted labels in the form of a list of DetectionPrediction objects 64 | from predictions.py 65 | 66 | The code in this function is dedictatd to intelligently sorting 67 | images by their size, and then batching them. Tensorflow can't batch 68 | inputs of different sizes, so this is something this function takes 69 | care of. 70 | 71 | The hardest part is making sure that the outputs are re-ordered back 72 | to their original input order. 73 | 74 | :imgs_bgr: A list of OpenCV images in BGR format (the default) 75 | :return: List of list of DetectionPrediction objects 76 | """ 77 | # TODO: This function desperately needs testing, it's easily broken 78 | # Preprocess each image 79 | imgs_rgb = [img[..., ::-1] for img in imgs_bgr] 80 | 81 | # Sort images by their size before batching and running inference 82 | original_indexes = sorted(range(len(imgs_rgb)), 83 | key=lambda index: imgs_rgb[index].shape) 84 | sorted_imgs = sorted(imgs_rgb, key=lambda img: img.shape) 85 | 86 | # Run batches of images, grouped by their size, and get the results 87 | results = [] 88 | for index, group in groupby(sorted_imgs, key=lambda img: img.shape): 89 | # Run inference on each group of equally sized images 90 | batch_results = self._process_batch(list(group)) 91 | results += batch_results 92 | 93 | # Reorder the results back to the original order of the images 94 | reordered_results = sorted(enumerate(results), 95 | key=lambda index_img: original_indexes[ 96 | index_img[0]]) 97 | reordered_results = [img for _, img in reordered_results] 98 | 99 | return reordered_results 100 | 101 | def _process_batch(self, imgs_rgb: List[np.ndarray]): 102 | """This function will ONLY WORK if imgs_bgr contains a list of images 103 | of the same size (resolution, depth). It returns the results of 104 | imgs_rgb in their original error.""" 105 | assert all([imgs_rgb[0].shape == img.shape for img in imgs_rgb]), \ 106 | "The resolution/depth of all images in this function must be equal!" 107 | 108 | # Batch predict for the list of images 109 | feed = {self.image_tensor: imgs_rgb} 110 | all_boxes, all_scores, all_classes = self.session.run( 111 | [self.boxes_tensor, self.scores_tensor, self.classes_tensor], 112 | feed_dict=feed) 113 | 114 | # Convert output to DetectionPrediction format 115 | all_preds = [] 116 | for img_id in range(len(imgs_rgb)): 117 | # Format the data to a single long array of boxes, scores, and 118 | # classes. Each index corresponds with each other. 119 | boxes = np.squeeze(all_boxes[img_id]) 120 | scores = np.squeeze(all_scores[img_id]) 121 | classes = np.squeeze(all_classes[img_id]) 122 | 123 | labels = self._postprocess_output(imgs_rgb[img_id], 124 | boxes=boxes, 125 | scores=scores, 126 | classes=classes) 127 | 128 | all_preds.append(labels) 129 | 130 | return all_preds 131 | 132 | def _postprocess_output(self, image, boxes, scores, classes): 133 | # Prepare the prediction output 134 | h, w, _ = image.shape 135 | labels = [] 136 | 137 | for i, score in enumerate(scores): 138 | if score < self.min_confidence: 139 | continue 140 | 141 | # Convert the tensorflow format to 'rect' format of 142 | # [x1, y1, x2, y2] pixels on frame 143 | rect = [int(round(boxes[i][1] * w, 0)), 144 | int(round(boxes[i][0] * h, 0)), 145 | int(round(boxes[i][3] * w, 0)), 146 | int(round(boxes[i][2] * h, 0))] 147 | 148 | lbl = DetectionPrediction( 149 | str(self.label_map[int(classes[i])]), 150 | classes[i], 151 | rect, 152 | scores[i]) 153 | labels.append(lbl) 154 | return labels 155 | -------------------------------------------------------------------------------- /vcap_utils/vcap_utils/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.3.8.1" 2 | --------------------------------------------------------------------------------