├── .clang-format ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── Bugs.md │ ├── Feature-request.md │ └── documentation.md └── workflows │ ├── build-and-test.yml │ ├── deploy.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── .readthedocs.yaml ├── CHANGELOG.md ├── CMakeLists.txt ├── LICENSE ├── MANIFEST.in ├── README.md ├── cmake ├── CCache.cmake ├── Compiler.cmake ├── FindEigen.cmake ├── FindFmtLib.cmake ├── FindLz4.cmake ├── FindOcean.cmake ├── FindPortAudio.cmake ├── FindRapidjsonLib.cmake ├── FindTurboJpeg.cmake ├── FindZstd.cmake ├── FindxxHash.cmake ├── LibrariesSetup.cmake ├── Options.cmake ├── Platform.cmake └── ReadMe.md ├── csrc ├── Pybind11.cpp ├── VrsBindings.cpp ├── VrsBindings.h ├── reader │ ├── AsyncVRSReader.cpp │ ├── AsyncVRSReader.h │ ├── FactoryHelper.hpp │ ├── FilteredFileReader.cpp │ ├── FilteredFileReader.h │ ├── MultiVRSReader.cpp │ ├── MultiVRSReader.h │ ├── Reader.cpp │ ├── Reader.h │ ├── VRSReader.cpp │ ├── VRSReader.h │ └── VRSReaderBase.h ├── utils │ ├── PyBuffer.cpp │ ├── PyBuffer.h │ ├── PyExceptions.cpp │ ├── PyExceptions.h │ ├── PyFileSpec.cpp │ ├── PyFileSpec.h │ ├── PyRecord.cpp │ ├── PyRecord.h │ ├── PyUtils.cpp │ ├── PyUtils.h │ ├── Utils.cpp │ └── Utils.h └── writer │ ├── PyDataPiece.h │ ├── PyRecordable.cpp │ ├── PyRecordable.h │ ├── StreamFactory.cpp │ ├── StreamFactory.h │ ├── VRSWriter.cpp │ ├── VRSWriter.h │ ├── Writer.cpp │ ├── Writer.h │ └── datalayouts │ ├── SampleDataLayout.cpp │ └── SampleDataLayout.h ├── docs ├── Makefile ├── requirements.txt └── source │ ├── conf.py │ ├── index.rst │ └── modules │ ├── base.rst │ ├── filter.rst │ ├── index.rst │ ├── reader.rst │ ├── record.rst │ ├── slice.rst │ └── utils.rst ├── pixi.toml ├── pyvrs ├── __init__.py ├── base.py ├── datalayout.py ├── filter.py ├── reader.py ├── record.py ├── slice.py ├── utils.py └── writer.py ├── release_utils.py ├── scripts ├── build-macos-arm64-package.sh ├── install-macos-deps.sh └── install-manylinux-deps.sh ├── setup.py ├── test ├── pyvrs_async_reader_test.py ├── pyvrs_reader_test.py ├── pyvrs_utils_test.py └── test_data │ └── synthetic.vrs └── version.txt /.clang-format: -------------------------------------------------------------------------------- 1 | # The primary clang-format config file. 2 | # - AllowShortBlocksOnASingleLine: Empty 3 | --- 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: AlwaysBreak 6 | AlignConsecutiveMacros: false 7 | AlignConsecutiveAssignments: false 8 | AlignConsecutiveBitFields: false 9 | AlignConsecutiveDeclarations: false 10 | AlignEscapedNewlines: Left 11 | AlignOperands: DontAlign 12 | AlignTrailingComments: false 13 | AllowAllArgumentsOnNextLine: true 14 | AllowAllConstructorInitializersOnNextLine: true 15 | AllowAllParametersOfDeclarationOnNextLine: false 16 | AllowShortEnumsOnASingleLine: true 17 | AllowShortBlocksOnASingleLine: Never 18 | AllowShortCaseLabelsOnASingleLine: false 19 | AllowShortFunctionsOnASingleLine: Empty 20 | AllowShortLambdasOnASingleLine: All 21 | AllowShortIfStatementsOnASingleLine: Never 22 | AllowShortLoopsOnASingleLine: false 23 | AlwaysBreakAfterReturnType: None 24 | AlwaysBreakBeforeMultilineStrings: true 25 | AlwaysBreakTemplateDeclarations: Yes 26 | BinPackArguments: false 27 | BinPackParameters: false 28 | BreakBeforeBinaryOperators: None 29 | BreakBeforeBraces: Attach 30 | BreakInheritanceList: BeforeColon 31 | BreakBeforeTernaryOperators: true 32 | BreakConstructorInitializers: BeforeColon 33 | BreakAfterJavaFieldAnnotations: false 34 | BreakStringLiterals: false 35 | ColumnLimit: 100 36 | CommentPragmas: '^ IWYU pragma:' 37 | CompactNamespaces: false 38 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 39 | ConstructorInitializerIndentWidth: 4 40 | ContinuationIndentWidth: 4 41 | Cpp11BracedListStyle: true 42 | DeriveLineEnding: true 43 | DerivePointerAlignment: false 44 | DisableFormat: false 45 | FixNamespaceComments: true 46 | ForEachMacros: 47 | - FOR_EACH 48 | - FOR_EACH_R 49 | - FOR_EACH_RANGE 50 | IncludeBlocks: Preserve 51 | IncludeCategories: 52 | - Regex: '^<.*\.h(pp)?>' 53 | Priority: 1 54 | - Regex: '^<.*' 55 | Priority: 2 56 | - Regex: '.*' 57 | Priority: 3 58 | IndentCaseLabels: true 59 | IndentCaseBlocks: false 60 | IndentGotoLabels: true 61 | IndentPPDirectives: None 62 | IndentExternBlock: AfterExternBlock 63 | IndentWidth: 2 64 | IndentWrappedFunctionNames: false 65 | InsertTrailingCommas: None 66 | JavaScriptQuotes: Leave 67 | JavaScriptWrapImports: true 68 | KeepEmptyLinesAtTheStartOfBlocks: false 69 | MacroBlockBegin: '' 70 | MacroBlockEnd: '' 71 | MaxEmptyLinesToKeep: 1 72 | NamespaceIndentation: None 73 | ObjCBinPackProtocolList: Auto 74 | ObjCBlockIndentWidth: 2 75 | ObjCBreakBeforeNestedBlockParam: true 76 | ObjCSpaceAfterProperty: false 77 | ObjCSpaceBeforeProtocolList: false 78 | PenaltyBreakAssignment: 2 79 | PenaltyBreakBeforeFirstCallParameter: 1 80 | PenaltyBreakComment: 300 81 | PenaltyBreakFirstLessLess: 120 82 | PenaltyBreakString: 1000 83 | PenaltyBreakTemplateDeclaration: 10 84 | PenaltyExcessCharacter: 1000000 85 | PenaltyReturnTypeOnItsOwnLine: 200 86 | PointerAlignment: Left 87 | ReflowComments: true 88 | SortIncludes: true 89 | SortUsingDeclarations: true 90 | SpaceAfterCStyleCast: false 91 | SpaceAfterLogicalNot: false 92 | SpaceAfterTemplateKeyword: true 93 | SpaceBeforeAssignmentOperators: true 94 | SpaceBeforeCpp11BracedList: false 95 | SpaceBeforeCtorInitializerColon: true 96 | SpaceBeforeInheritanceColon: true 97 | SpaceBeforeParens: ControlStatements 98 | SpaceBeforeRangeBasedForLoopColon: true 99 | SpaceInEmptyBlock: false 100 | SpaceInEmptyParentheses: false 101 | SpacesBeforeTrailingComments: 1 102 | SpacesInAngles: false 103 | SpacesInConditionalStatement: false 104 | SpacesInContainerLiterals: true 105 | SpacesInCStyleCastParentheses: false 106 | SpacesInParentheses: false 107 | SpacesInSquareBrackets: false 108 | SpaceBeforeSquareBrackets: false 109 | Standard: Latest 110 | TabWidth: 8 111 | UseCRLF: false 112 | UseTab: Never 113 | ... 114 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | This Code of Conduct also applies outside the project spaces when there is a 56 | reasonable belief that an individual's behavior may have a negative impact on 57 | the project or its community. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported by contacting the project team at . All 63 | complaints will be reviewed and investigated and will result in a response that 64 | is deemed necessary and appropriate to the circumstances. The project team is 65 | obligated to maintain confidentiality with regard to the reporter of an incident. 66 | Further details of specific enforcement policies may be posted separately. 67 | 68 | Project maintainers who do not follow or enforce the Code of Conduct in good 69 | faith may face temporary or permanent repercussions as determined by other 70 | members of the project's leadership. 71 | 72 | ## Attribution 73 | 74 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 75 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 76 | 77 | [homepage]: https://www.contributor-covenant.org 78 | 79 | For answers to common questions about this code of conduct, see 80 | https://www.contributor-covenant.org/faq 81 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to vrs 2 | 3 | We want to make contributing to this project as easy and transparent as 4 | possible. 5 | 6 | ## Pull Requests 7 | 8 | We actively welcome your pull requests. 9 | 10 | 1. Fork the repo and create your branch from `main`. 11 | 2. If you've added code that should be tested, add tests. 12 | 3. If you've changed APIs, update the documentation. 13 | 4. Ensure the test suite passes. 14 | 5. Make sure your code lints. 15 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 16 | 17 | ## Contributor License Agreement ("CLA") 18 | 19 | In order to accept your pull request, we need you to submit a CLA. You only need 20 | to do this once to work on any of Facebook's open source projects. 21 | 22 | Complete your CLA here: 23 | 24 | ## Issues 25 | 26 | We use GitHub issues to track public bugs. Please ensure your description is 27 | clear and has sufficient instructions to be able to reproduce the issue. 28 | 29 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 30 | disclosure of security bugs. In those cases, please go through the process 31 | outlined on that page and do not file a public issue. 32 | 33 | ## License 34 | 35 | By contributing to vrs, you agree that your contributions will be licensed under 36 | the LICENSE file in the root directory of this source tree. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Bugs.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bugs" 3 | about: ' Report bugs in pyvrs' 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | --- 8 | 9 | ## Instructions To Reproduce the 🐛 Bug: 10 | 11 | 1. Full runnable code or full changes you made: 12 | 13 | ``` 14 | If making changes to the project itself, please use output of the following command: 15 | git rev-parse HEAD; git diff 16 | 17 | 18 | ``` 19 | 20 | 2. What exact command you run: 21 | 3. **Full logs** or other relevant observations: 22 | 23 | ``` 24 | 25 | ``` 26 | 27 | 4. please simplify the steps as much as possible so they do not require 28 | additional resources to run, such as a private dataset. 29 | 30 | ## Expected behavior: 31 | 32 | If there are no obvious error in "what you observed" provided above, please tell 33 | us the expected behavior. 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680Feature Request" 3 | about: Suggest an improvement or new feature 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | --- 8 | 9 | --- 10 | 11 | ## 🚀 Feature 12 | 13 | A clear and concise description of the feature proposal. 14 | 15 | ## Motivation & Examples 16 | 17 | Tell us why the feature is useful. 18 | 19 | Describe what the feature would look like, if it is implemented. Best 20 | demonstrated using **code examples** in addition to words. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation 3 | about: Report a documentation issue 4 | title: '' 5 | labels: documentation 6 | assignees: '' 7 | --- 8 | -------------------------------------------------------------------------------- /.github/workflows/build-and-test.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | name: Build pyvrs on ${{ matrix.os }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | os: [ubuntu-latest, macOS-latest] 16 | fail-fast: false 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | with: 22 | submodules: recursive 23 | - name: Set up Python 24 | uses: actions/setup-python@v5 25 | with: 26 | python-version: "3.9" 27 | 28 | - name: Install dependencies 29 | run: | 30 | if [ "$RUNNER_OS" == "Linux" ]; then 31 | sudo apt-get update -y 32 | sudo apt-get install -o Acquire::Retries=5 \ 33 | cmake git ninja-build libgtest-dev libfmt-dev \ 34 | libjpeg-dev libturbojpeg-dev libpng-dev \ 35 | liblz4-dev libzstd-dev libxxhash-dev \ 36 | libboost-system-dev libboost-filesystem-dev libboost-thread-dev libboost-chrono-dev libboost-date-time-dev \ 37 | portaudio19-dev libopus-dev 38 | python -m pip install -U pip 39 | python -m pip install pybind11[global] 40 | 41 | elif [ "$RUNNER_OS" == "macOS" ]; then 42 | brew install cmake git ninja googletest glog fmt \ 43 | jpeg-turbo libpng \ 44 | lz4 zstd xxhash \ 45 | boost \ 46 | portaudio opus pybind11 47 | 48 | else 49 | echo "$RUNNER_OS not supported" 50 | exit 1 51 | fi 52 | python -m pip install -U pip 53 | python -m pip install numpy typing dataclasses pytest parameterized Pillow 54 | python -m pip install cibuildwheel==2.17.0 55 | 56 | - name: Build wheels for CPython 57 | run: | 58 | python -m cibuildwheel --output-dir dist 59 | env: 60 | CIBW_ARCHS_LINUX: x86_64 61 | CIBW_ARCHS_MACOS: "x86_64 arm64" 62 | CIBW_BUILD: "cp39-*64 cp310-*64 cp311-*64 cp312-*64" 63 | CIBW_BEFORE_BUILD_LINUX: bash scripts/install-manylinux-deps.sh 64 | CIBW_BEFORE_BUILD_MACOS: bash scripts/install-macos-deps.sh 65 | CIBW_SKIP: "*-manylinux_i686 *musllinux*" 66 | 67 | - name: Build and Test Python 68 | shell: bash 69 | run: | 70 | pip install -e . 71 | python -m pytest test 72 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deployment 2 | 3 | on: 4 | # this workflow can only be manually triggered for now. 5 | workflow_dispatch: 6 | inputs: 7 | deploy: 8 | description: 'Where to deploy the artifacts? Only build (build), deploy to test PyPI (test), or deploy to PyPI (prod).' 9 | required: true 10 | type: choice 11 | default: 'test' 12 | options: 13 | - build 14 | - test 15 | - prod 16 | 17 | jobs: 18 | create-sdist: 19 | if: github.repository == 'facebookresearch/pyvrs' 20 | runs-on: ubuntu-latest 21 | name: Create source Distribution 22 | steps: 23 | - uses: actions/checkout@v4 24 | with: 25 | submodules: recursive 26 | - name: Set up Python 27 | uses: actions/setup-python@v5 28 | with: 29 | python-version: '3.9' 30 | - name: Upgrade pip 31 | run: | 32 | python3 -m pip install --upgrade pip 33 | - name: Create Source Distribution 34 | run: | 35 | python3 -m pip install setuptools wheel twine 36 | python3 setup.py sdist 37 | 38 | - uses: actions/upload-artifact@v2 39 | with: 40 | name: python-package-distributions 41 | path: dist/*.tar.gz 42 | 43 | build-wheels: 44 | if: github.repository == 'facebookresearch/pyvrs' 45 | name: Build wheels on ${{ matrix.os }} 46 | runs-on: ${{ matrix.os }} 47 | strategy: 48 | matrix: 49 | os: [macos-latest, ubuntu-latest] 50 | steps: 51 | - uses: actions/checkout@v4 52 | with: 53 | submodules: recursive 54 | - name: Set up Python 55 | uses: actions/setup-python@v5 56 | with: 57 | python-version: '3.9' 58 | - name: Upgrade pip 59 | run: | 60 | python3 -m pip install --upgrade pip 61 | 62 | - name: Install cibuildwheel 63 | run: | 64 | python3 -m pip install cibuildwheel==2.17.0 65 | 66 | - name: Build wheels for CPython 67 | run: | 68 | python3 -m cibuildwheel --output-dir dist 69 | env: 70 | CIBW_ARCHS_LINUX: x86_64 71 | CIBW_ARCHS_MACOS: "x86_64 arm64" 72 | CIBW_BUILD: "cp39-*64 cp310-*64 cp311-*64 cp312-*64" 73 | CIBW_BEFORE_BUILD_LINUX: bash scripts/install-manylinux-deps.sh 74 | CIBW_BEFORE_BUILD_MACOS: bash scripts/install-macos-deps.sh 75 | CIBW_SKIP: "*-manylinux_i686 *musllinux*" 76 | - uses: actions/upload-artifact@v2 77 | with: 78 | name: python-package-distributions 79 | path: dist 80 | 81 | publish-to-pypi: 82 | runs-on: ubuntu-latest 83 | needs: 84 | - build-wheels 85 | - create-sdist 86 | steps: 87 | - name: Download wheels from previous jobs 88 | # by default this will download all artifacts 89 | uses: actions/download-artifact@v3 90 | with: 91 | name: python-package-distributions 92 | # PyPI publish action uploads everything under dist/* by default 93 | path: dist 94 | 95 | - name: Display the list of artifacts 96 | run: ls -R dist 97 | 98 | - name: Publish to Test PyPI 99 | if: github.event.inputs.deploy == 'test' 100 | uses: pypa/gh-action-pypi-publish@v1.4.2 101 | with: 102 | user: __token__ 103 | password: ${{ secrets.TEST_PYPI_TOKEN }} 104 | repository_url: https://test.pypi.org/legacy/ 105 | skip_existing: true 106 | verbose: true 107 | 108 | - name: Publish to PyPI 109 | if: github.event.inputs.deploy == 'prod' && startsWith(github.ref, 'refs/tags') 110 | uses: pypa/gh-action-pypi-publish@v1.4.2 111 | with: 112 | user: __token__ 113 | password: ${{ secrets.PYPI_TOKEN }} 114 | verbose: true 115 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: pyvrs Release 2 | 3 | on: 4 | # Temporary disable this job until pyvrs becomes public. 5 | # This job will keep failing unless the repo is public 6 | workflow_dispatch: 7 | 8 | jobs: 9 | get-current-version: 10 | if: github.repository == 'facebookresearch/pyvrs' 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: checkout-repo-content 14 | uses: actions/checkout@v4 15 | with: 16 | submodules: recursive 17 | 18 | - name: setup-python 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: 3.9 22 | 23 | - name: get current version and tag 24 | id: get-current-version-and-tag 25 | run: | 26 | output=$(python3 release_utils.py --get-current-version) 27 | echo $output 28 | current_version=$(echo $output | awk '{print $1}') 29 | current_tag=$(echo $output | awk '{print $2}') 30 | echo "current version is $current_version" 31 | echo "current tag is $current_tag" 32 | echo ::set-output name=version::$current_version 33 | echo ::set-output name=tag::$current_tag 34 | outputs: 35 | current_version: ${{ steps.get-current-version-and-tag.outputs.version }} 36 | current_tag: ${{ steps.get-current-version-and-tag.outputs.tag }} 37 | 38 | create-release-if-not-exist: 39 | if: github.repository == 'facebookresearch/pyvrs' 40 | runs-on: ubuntu-latest 41 | needs: 42 | - get-current-version 43 | steps: 44 | - id: get-latest-tag 45 | uses: pozetroninc/github-action-get-latest-release@master 46 | with: 47 | repository: ${{ github.repository }} 48 | - name: checkout-repo-content 49 | uses: actions/checkout@v4 50 | 51 | - name: Create release 52 | if: needs.get-current-version.outputs.current_tag != steps.get-latest-tag.outputs.release 53 | uses: ncipollo/release-action@v1 54 | with: 55 | tag: '${{ needs.get-current-version.outputs.current_tag }}' 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | CMakeLists.txt.user 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pybind11"] 2 | path = csrc/pybind11 3 | url = https://github.com/pybind/pybind11.git 4 | [submodule "vrs"] 5 | path = csrc/vrs 6 | url = https://github.com/facebookresearch/vrs.git 7 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | submodules: 9 | include: all 10 | recursive: true 11 | 12 | # Set the version of Python and other tools you might need 13 | build: 14 | os: ubuntu-latest 15 | apt_packages: 16 | - cmake 17 | - ninja-build 18 | - ccache 19 | - libgtest-dev 20 | - libfmt-dev 21 | - libjpeg-dev 22 | - libturbojpeg-dev 23 | - libpng-dev 24 | - liblz4-dev 25 | - libzstd-dev 26 | - libxxhash-dev 27 | - libboost-system-dev 28 | - libboost-filesystem-dev 29 | - libboost-thread-dev 30 | - libboost-chrono-dev 31 | - libboost-date-time-dev 32 | - portaudio19-dev 33 | 34 | tools: 35 | python: "3.9" 36 | 37 | # Build documentation in the docs/ directory with Sphinx 38 | sphinx: 39 | configuration: docs/source/conf.py 40 | 41 | # Optionally declare the Python requirements required to build your docs 42 | python: 43 | install: 44 | - requirements: docs/requirements.txt 45 | - method: pip 46 | path: . 47 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # pyvrs Open Source Project Change Log 2 | 3 | This file captures the main changes made to the pyvrs open source project since 4 | its creation. None of the changes made before pyvrs was open sourced are meant 5 | to be covered. 6 | 7 | # Version 1.2.2 (Feb 26, 2025) 8 | 9 | - Async handler fixes. 10 | - Misc changes. 11 | 12 | # Version 1.2.1 (May 15, 2024) 13 | 14 | - Misc changes. 15 | 16 | # Version 1.2.0 (February 6, 2024) 17 | 18 | - Misc changes. 19 | 20 | # Version 1.1.0 (August 4, 2023) 21 | 22 | - Updated to match the last VRS version bump. 23 | - VRS moved from using Cereal's internal copy of rapidjson, to using rapidjson 24 | directly from GitHub. 25 | - Updated build scripts accordingly. 26 | - Support for Ubuntu-Latest 27 | 28 | # Version 1.0.4 (Mar 7, 2023) 29 | 30 | - Initial private release. 31 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cmake_minimum_required(VERSION 3.16) 16 | 17 | set(VRS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/csrc/vrs") 18 | set(OSS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/csrc/vrs") 19 | 20 | # To customize where external dependencies are stored 21 | if ("${EXTERNAL_DEPENDENCIES_DIR}" STREQUAL "") 22 | set(EXTERNAL_DEPENDENCIES_DIR "${CMAKE_BINARY_DIR}/external") 23 | endif() 24 | file(MAKE_DIRECTORY "${EXTERNAL_DEPENDENCIES_DIR}") 25 | message(STATUS "External dependencies at ${EXTERNAL_DEPENDENCIES_DIR}") 26 | 27 | # Initialize key cmake settings before the project is configured 28 | set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) 29 | include(${CMAKE_MODULE_PATH}/Platform.cmake) 30 | include(${CMAKE_MODULE_PATH}/Compiler.cmake) 31 | include(${CMAKE_MODULE_PATH}/Options.cmake) 32 | 33 | # --- Version numbers for generated libraries --- 34 | file(READ "version.txt" VRS_DEFAULT_VERSION) 35 | # Strip trailing newline 36 | string(REGEX REPLACE "\n$" "" VRS_DEFAULT_VERSION "${VRS_DEFAULT_VERSION}") 37 | if ("${VRS_DEFAULT_VERSION}x" STREQUAL "x") 38 | message(FATAL_ERROR "Could not get version number from file 'version.txt'") 39 | endif() 40 | 41 | # Declare the project, make cmake chose/test the compiler, etc 42 | project(vrsbindings 43 | VERSION ${VRS_DEFAULT_VERSION} 44 | DESCRIPTION "Meta VRS File Format Project" 45 | LANGUAGES CXX 46 | ) 47 | message(STATUS "CMAKE_CXX_COMPILER: ${CMAKE_CXX_COMPILER}") 48 | message(STATUS "CMAKE_CXX_COMPILER_VERSION: ${CMAKE_CXX_COMPILER_VERSION}") 49 | 50 | # If available, use ccache 51 | include(${CMAKE_MODULE_PATH}/CCache.cmake) 52 | # Library setup 53 | include(${CMAKE_MODULE_PATH}/LibrariesSetup.cmake) 54 | 55 | # Include the libraries 56 | add_subdirectory(csrc/vrs/vrs) 57 | add_subdirectory(csrc/pybind11) 58 | 59 | # Keep functionality to reader only for now 60 | file (GLOB_RECURSE CSRCS 61 | "csrc/Pybind11.cpp" 62 | "csrc/VrsBindings.cpp" 63 | "csrc/VrsBindings.h" 64 | "csrc/reader/*.cpp" 65 | "csrc/reader/*.h" 66 | "csrc/reader/*.hpp" 67 | "csrc/utils/*.cpp" 68 | "csrc/utils/*.h" 69 | "csrc/writer/*.cpp" 70 | "csrc/writer/*.h" 71 | "csrc/writer/datalayouts/*.cpp" 72 | "csrc/writer/datalayouts/*.h" 73 | ) 74 | 75 | # Remove fb-only source files (if there are still any lying around) 76 | file (GLOB_RECURSE FB_CSRCS "csrc/*/*_fb\.*") 77 | if (FB_CSRCS) 78 | list(REMOVE_ITEM CSRCS ${FB_CSRCS}) 79 | endif() 80 | 81 | pybind11_add_module(vrsbindings ${CSRCS}) 82 | 83 | target_include_directories(vrsbindings PUBLIC csrc) 84 | target_link_libraries(vrsbindings 85 | PUBLIC 86 | vrslib 87 | vrs_utils 88 | ) 89 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in 2 | include version.txt 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is pyvrs? 2 | 3 | pyvrs is a Python interface for C++ library [VRS](https://github.com/facebookresearch/vrs) using [pybind11](https://github.com/pybind/pybind11). 4 | 5 | # Documentation 6 | 7 | See [API documentation](https://pyvrs.readthedocs.io/en/latest/) 8 | 9 | # Installation 10 | ## Install released builds 11 | pypi package is built with [this Github Action](https://github.com/facebookresearch/pyvrs/blob/main/.github/workflows/deploy.yml) manually. 12 | ``` 13 | pip install vrs 14 | ``` 15 | 16 | :warning: Note: Work on the Windows version of the PyPI package is currently in progress and will be completed soon. In the meantime, please build the package from the source. Further details can be found in the section below. 17 | 18 | ## From source 19 | 20 | ### Mac and Linux: 21 | ``` 22 | # Build locally 23 | git clone --recursive https://github.com/facebookresearch/pyvrs.git 24 | cd pyvrs 25 | # if you are updating an existing checkout 26 | git submodule sync --recursive 27 | git submodule update --init --recursive 28 | 29 | # Install VRS dependencies: https://github.com/facebookresearch/vrs#instructions-macos-and-ubuntu-and-container 30 | 31 | python -m pip install -e . 32 | ``` 33 | 34 | ### Windows (via pixi): 35 | 36 | ``` 37 | # Download pyvrs 38 | git clone https://github.com/facebookresearch/pyvrs.git 39 | cd pyvrs 40 | git submodule sync --recursive 41 | git submodule update --init --recursive 42 | 43 | # Install pixi (details can be found: https://pixi.sh/latest/#__tabbed_1_2) 44 | iwr -useb https://pixi.sh/install.ps1 | iex 45 | # Install VRS dependencies: 46 | pixi run install_pyvrs 47 | 48 | # Start pixi shell to execute your python scripts 49 | pixi shell 50 | python yourPythonScript.py 51 | ``` 52 | 53 | # Contributing 54 | 55 | We welcome contributions! See [CONTRIBUTING](CONTRIBUTING.md) for details on how 56 | to get started, and our [code of conduct](CODE_OF_CONDUCT.md). 57 | 58 | # License 59 | 60 | VRS is released under the [Apache 2.0 license](LICENSE). 61 | -------------------------------------------------------------------------------- /cmake/CCache.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | if (NOT BUILD_ON_WINDOWS) 16 | find_program(ccache_path ccache) 17 | if (ccache_path) 18 | message(STATUS "CCACHE enabled") 19 | set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${ccache_path}) 20 | set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${ccache_path}) 21 | endif (ccache_path) 22 | endif (NOT BUILD_ON_WINDOWS) 23 | -------------------------------------------------------------------------------- /cmake/Compiler.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | set(CMAKE_CXX_STANDARD 17) 16 | set(CMAKE_CXX_STANDARD_REQUIRED ON) #...is required... 17 | set(CMAKE_CXX_EXTENSIONS OFF) #...without compiler extensions like gnu++11 18 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) #...is required to build pyvrs 19 | -------------------------------------------------------------------------------- /cmake/FindEigen.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ## FindEigen.cmake 16 | # A CMake module to always fetch and configure Eigen3 v3.4.90 via FetchContent, 17 | # ignoring any system-installed versions. 18 | # Place this file in your project's cmake/ directory and add that directory to CMAKE_MODULE_PATH. 19 | 20 | # Usage in your top-level CMakeLists.txt: 21 | # 22 | # list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") 23 | # find_package(Eigen3 REQUIRED) 24 | # target_link_libraries(your_target PRIVATE Eigen3::Eigen) 25 | 26 | # Prevent multiple inclusions 27 | if(DEFINED FIND_EIGEN3_MODULE) 28 | return() 29 | endif() 30 | set(FIND_EIGEN3_MODULE TRUE) 31 | 32 | # If Eigen3::Eigen already exists (alias or imported), assume provided and skip fetch 33 | if(TARGET Eigen3::Eigen) 34 | message(STATUS "Eigen3::Eigen target already exists, skipping FetchContent.") 35 | set(Eigen3_FOUND TRUE) 36 | return() 37 | endif() 38 | 39 | # Fetch Eigen3 v3.4.90 40 | include(FetchContent) 41 | message(STATUS "Fetching Eigen3 v3.4.90 from Git via FetchContent...") 42 | FetchContent_Declare( 43 | eigen 44 | GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git 45 | GIT_TAG 19cacd3ecb9dab73c2dd7bc39d9193e06ba92bdd # v3.4.90 46 | ) 47 | FetchContent_MakeAvailable(eigen) 48 | 49 | # Create an imported interface target for Eigen3 only if not pre-existing 50 | #add_library(Eigen3::Eigen INTERFACE IMPORTED GLOBAL) 51 | # Mark imported target to avoid being treated as alias 52 | #set_target_properties(Eigen3::Eigen PROPERTIES 53 | # IMPORTED_GLOBAL TRUE 54 | #) 55 | 56 | #target_include_directories(Eigen3::Eigen INTERFACE 57 | # $ 58 | # $ 59 | #) 60 | 61 | # Mark Eigen3 as found and set version 62 | set(Eigen3_FOUND TRUE) 63 | set(Eigen3_VERSION 3.4.90 CACHE STRING "Eigen3 version fetched by FetchContent") 64 | -------------------------------------------------------------------------------- /cmake/FindFmtLib.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | find_package(fmt) 16 | 17 | if (NOT TARGET fmt::fmt) 18 | find_path(Fmt_INCLUDE_DIR fmt/core.h) 19 | mark_as_advanced(Fmt_INCLUDE_DIR) 20 | 21 | find_library(Fmt_LIBRARY NAMES fmt fmtd) 22 | mark_as_advanced(Fmt_LIBRARY) 23 | 24 | include(FindPackageHandleStandardArgs) 25 | find_package_handle_standard_args( 26 | fmt 27 | DEFAULT_MSG 28 | Fmt_LIBRARY Fmt_INCLUDE_DIR 29 | ) 30 | set(Fmt_LIBRARIES ${Fmt_LIBRARY}) 31 | set(Fmt_INCLUDE_DIRS ${Fmt_INCLUDE_DIR}) 32 | add_library(fmt::fmt UNKNOWN IMPORTED) 33 | set_target_properties( 34 | fmt::fmt PROPERTIES 35 | INTERFACE_INCLUDE_DIRECTORIES "${Fmt_INCLUDE_DIR}" 36 | ) 37 | set_target_properties( 38 | fmt::fmt PROPERTIES 39 | IMPORTED_LINK_INTERFACE_LANGUAGES "C" 40 | IMPORTED_LOCATION "${Fmt_LIBRARY}" 41 | ) 42 | endif() 43 | -------------------------------------------------------------------------------- /cmake/FindLz4.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find Lz4 16 | # Find the Lz4 compression library and includes 17 | # 18 | # Lz4_INCLUDE_DIRS - where to find Lz4.h, etc. 19 | # Lz4_LIBRARIES - List of libraries when using Lz4. 20 | # Lz4_FOUND - True if Lz4 found. 21 | 22 | find_path(Lz4_INCLUDE_DIRS NAMES lz4.h) 23 | find_library(Lz4_LIBRARIES NAMES lz4 liblz4) 24 | mark_as_advanced(Lz4_LIBRARIES Lz4_INCLUDE_DIRS) 25 | 26 | include(FindPackageHandleStandardArgs) 27 | find_package_handle_standard_args(Lz4 DEFAULT_MSG Lz4_LIBRARIES Lz4_INCLUDE_DIRS) 28 | 29 | if (Lz4_FOUND AND NOT (TARGET Lz4::Lz4)) 30 | add_library(Lz4::Lz4 UNKNOWN IMPORTED) 31 | set_target_properties(Lz4::Lz4 32 | PROPERTIES 33 | IMPORTED_LOCATION ${Lz4_LIBRARIES} 34 | INTERFACE_INCLUDE_DIRECTORIES ${Lz4_INCLUDE_DIRS} 35 | ) 36 | endif() 37 | -------------------------------------------------------------------------------- /cmake/FindOcean.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # FindOcean.cmake — fetch, build, and install Facebook Ocean as static libraries 16 | # Place this in a folder on CMAKE_MODULE_PATH (e.g. cmake/modules) 17 | 18 | # Prevent multiple inclusion 19 | if(Ocean_FOUND) 20 | return() 21 | endif() 22 | 23 | # -------------------------- 24 | # 0) Standard install dirs 25 | include(GNUInstallDirs) 26 | 27 | # -------------------------- 28 | # 1) Fetch & build Ocean 29 | include(FetchContent) 30 | FetchContent_Declare( 31 | ocean 32 | GIT_REPOSITORY https://github.com/facebookresearch/ocean.git 33 | GIT_TAG origin/main 34 | ) 35 | set(OCEAN_BUILD_MINIMAL ON) 36 | set(OCEAN_BUILD_TEST OFF) 37 | message(STATUS "⤷ FetchContent: pulling and building Facebook Ocean…") 38 | FetchContent_MakeAvailable(ocean) 39 | 40 | # -------------------------- 41 | # 2) Normalize component include dirs to avoid build-tree paths 42 | foreach(_lib IN ITEMS ocean_base ocean_cv ocean_math) 43 | # Clear any pre-existing include dirs 44 | set_target_properties(${_lib} PROPERTIES 45 | INTERFACE_INCLUDE_DIRECTORIES "" 46 | ) 47 | # Add proper GENERATOR_EXPR-based includes 48 | target_include_directories(${_lib} INTERFACE 49 | $ 50 | $ 51 | ) 52 | endforeach() 53 | 54 | # -------------------------- 55 | # 3) Bundle into a single INTERFACE target 56 | add_library(ocean INTERFACE) 57 | target_link_libraries(ocean INTERFACE 58 | ocean_base 59 | ocean_cv 60 | ocean_math 61 | ) 62 | add_library(Ocean::Ocean ALIAS ocean) 63 | 64 | # -------------------------- 65 | # 4) Installation 66 | install( 67 | TARGETS 68 | ocean_base 69 | ocean_cv 70 | ocean_math 71 | ocean 72 | EXPORT OceanTargets 73 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 74 | INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} 75 | ) 76 | install( 77 | DIRECTORY ${ocean_SOURCE_DIR}/impl/ 78 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} 79 | FILES_MATCHING PATTERN "*.h" 80 | ) 81 | install( 82 | EXPORT OceanTargets 83 | FILE FindOceanTargets.cmake 84 | NAMESPACE Ocean:: 85 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Ocean 86 | ) 87 | 88 | # -------------------------- 89 | # 5) In-tree find_package support 90 | set(Ocean_FOUND TRUE) 91 | set(Ocean_LIBRARIES Ocean::Ocean) 92 | set(Ocean_INCLUDE_DIRS "${ocean_SOURCE_DIR}/impl") 93 | include(FindPackageHandleStandardArgs) 94 | find_package_handle_standard_args(Ocean 95 | REQUIRED_VARS Ocean_LIBRARIES 96 | ) 97 | -------------------------------------------------------------------------------- /cmake/FindPortAudio.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find PortAudio 16 | # Find the PortAudio compression library and includes 17 | # 18 | # PortAudio_INCLUDE_DIRS - where to find PortAudio.h, etc. 19 | # PortAudio_LIBRARIES - List of libraries when using PortAudio. 20 | # PortAudio_FOUND - True if PortAudio found. 21 | 22 | find_path(PortAudio_INCLUDE_DIRS NAMES portaudio.h) 23 | find_library(PortAudio_LIBRARIES NAMES portaudio) 24 | mark_as_advanced(PortAudio_LIBRARIES PortAudio_INCLUDE_DIRS) 25 | 26 | include(FindPackageHandleStandardArgs) 27 | find_package_handle_standard_args(PortAudio DEFAULT_MSG PortAudio_LIBRARIES PortAudio_INCLUDE_DIRS) 28 | 29 | if (PortAudio_FOUND AND NOT (TARGET PortAudio::PortAudio)) 30 | add_library(PortAudio::PortAudio UNKNOWN IMPORTED) 31 | set_target_properties(PortAudio::PortAudio 32 | PROPERTIES 33 | IMPORTED_LOCATION ${PortAudio_LIBRARIES} 34 | INTERFACE_INCLUDE_DIRECTORIES ${PortAudio_INCLUDE_DIRS} 35 | ) 36 | endif() 37 | -------------------------------------------------------------------------------- /cmake/FindRapidjsonLib.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The package version of rapidjson provided by apt on Linux or brew on Mac date from August 2016, 16 | # despite being the last official version (v1.1.0), and don't include important fixes. 17 | # So we get the code from GitHub at a known working state. 18 | find_path(RAPIDJSON_INSTALL NAMES rapidjson/rapidjson.h) 19 | if (RAPIDJSON_INSTALL) 20 | message(WARNING "Found RapidJson at ${RAPIDJSON_INSTALL} It might be used instead " 21 | "of VRS' version, and cause builds errors because the code is too old.") 22 | endif() 23 | 24 | set(RAPIDJSON_DIR "${EXTERNAL_DEPENDENCIES_DIR}/rapidjson") 25 | set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_DIR}/include") 26 | set(RAPIDJSON_INCLUDE_FILE "${RAPIDJSON_INCLUDE_DIR}/rapidjson/rapidjson.h") 27 | 28 | if (NOT EXISTS "${RAPIDJSON_INCLUDE_FILE}") 29 | execute_process(COMMAND git clone https://github.com/Tencent/rapidjson.git "${RAPIDJSON_DIR}") 30 | endif() 31 | execute_process(COMMAND cd "${RAPIDJSON_DIR}" && git -f checkout a95e013b97ca6523f32da23f5095fcc9dd6067e5) 32 | if (NOT EXISTS "${RAPIDJSON_INCLUDE_FILE}") 33 | message(FATAL_ERROR "Could not setup rapidjson external dependency at ${RAPIDJSON_DIR}") 34 | endif() 35 | 36 | add_library(rapidjson::rapidjson INTERFACE IMPORTED) 37 | set_target_properties( 38 | rapidjson::rapidjson PROPERTIES 39 | INTERFACE_INCLUDE_DIRECTORIES "${RAPIDJSON_INCLUDE_DIR}" 40 | ) 41 | add_dependencies(rapidjson::rapidjson rapidjson) 42 | 43 | message(STATUS "Found RapidJson: ${RAPIDJSON_DIR}") 44 | -------------------------------------------------------------------------------- /cmake/FindTurboJpeg.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find libjpeg-turbo 16 | # Find the libjpeg-turbo compression library and includes 17 | # 18 | # TurboJpeg_INCLUDE_DIRS - where to find turbojpeg.h, etc. 19 | # TurboJpeg_LIBRARIES - List of libraries when using TurboJpeg. 20 | # TurboJpeg_FOUND - True if TurboJpeg found. 21 | 22 | find_path(TurboJpeg_INCLUDE_DIRS NAMES turbojpeg.h) 23 | find_library(TurboJpeg_LIBRARIES NAMES libturbojpeg turbojpeg) 24 | mark_as_advanced(TurboJpeg_LIBRARIES TurboJpeg_INCLUDE_DIRS) 25 | 26 | include(FindPackageHandleStandardArgs) 27 | find_package_handle_standard_args(TurboJpeg DEFAULT_MSG TurboJpeg_LIBRARIES TurboJpeg_INCLUDE_DIRS) 28 | 29 | if (TurboJpeg_FOUND AND NOT (TARGET TurboJpeg::TurboJpeg)) 30 | add_library(TurboJpeg::TurboJpeg UNKNOWN IMPORTED) 31 | set_target_properties(TurboJpeg::TurboJpeg 32 | PROPERTIES 33 | IMPORTED_LOCATION ${TurboJpeg_LIBRARIES} 34 | INTERFACE_INCLUDE_DIRECTORIES ${TurboJpeg_INCLUDE_DIRS} 35 | ) 36 | endif() 37 | -------------------------------------------------------------------------------- /cmake/FindZstd.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # - Find Zstd 16 | # Find the Zstd compression library and includes 17 | # 18 | # Zstd_INCLUDE_DIRS - where to find Zstd.h, etc. 19 | # Zstd_LIBRARIES - List of libraries when using Zstd. 20 | # Zstd_FOUND - True if Zstd found. 21 | 22 | find_path(Zstd_INCLUDE_DIRS NAMES zstd.h) 23 | find_library(Zstd_LIBRARIES NAMES zstd) 24 | mark_as_advanced(Zstd_LIBRARIES Zstd_INCLUDE_DIRS) 25 | 26 | include(FindPackageHandleStandardArgs) 27 | find_package_handle_standard_args(Zstd DEFAULT_MSG Zstd_LIBRARIES Zstd_INCLUDE_DIRS) 28 | 29 | if (Zstd_FOUND AND NOT (TARGET Zstd::Zstd)) 30 | add_library(Zstd::Zstd UNKNOWN IMPORTED) 31 | set_target_properties(Zstd::Zstd 32 | PROPERTIES 33 | IMPORTED_LOCATION ${Zstd_LIBRARIES} 34 | INTERFACE_INCLUDE_DIRECTORIES ${Zstd_INCLUDE_DIRS} 35 | ) 36 | endif() 37 | -------------------------------------------------------------------------------- /cmake/FindxxHash.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | find_path(xxHash_INCLUDE_DIRS 16 | NAME xxhash.h 17 | PATH_SUFFIXES include 18 | ) 19 | find_library(xxHash_LIBRARIES 20 | NAMES xxhash xxHash 21 | PATH_SUFFIXES lib 22 | ) 23 | mark_as_advanced(xxHash_LIBRARIES xxHash_INCLUDE_DIRS) 24 | 25 | include(FindPackageHandleStandardArgs) 26 | find_package_handle_standard_args(xxHash DEFAULT_MSG xxHash_LIBRARIES xxHash_INCLUDE_DIRS) 27 | 28 | if (xxHash_FOUND AND NOT (TARGET xxHash::xxHash)) 29 | add_library(xxHash::xxHash UNKNOWN IMPORTED) 30 | set_target_properties(xxHash::xxHash 31 | PROPERTIES 32 | IMPORTED_LOCATION ${xxHash_LIBRARIES} 33 | INTERFACE_INCLUDE_DIRECTORIES ${xxHash_INCLUDE_DIRS} 34 | ) 35 | endif() 36 | -------------------------------------------------------------------------------- /cmake/LibrariesSetup.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # for those who have a custom homebrew installation 16 | if (EXISTS "$ENV{HOME}/homebrew") 17 | list(APPEND CMAKE_FIND_ROOT_PATH "$ENV{HOME}/homebrew") 18 | endif() 19 | 20 | find_package(Boost REQUIRED 21 | COMPONENTS 22 | filesystem 23 | chrono 24 | date_time 25 | system 26 | thread 27 | ) 28 | find_package(Eigen REQUIRED) 29 | find_package(FmtLib REQUIRED) 30 | find_package(RapidjsonLib REQUIRED) 31 | find_package(Lz4 REQUIRED) 32 | find_package(Zstd REQUIRED) 33 | find_package(xxHash REQUIRED) 34 | find_package(PNG REQUIRED) 35 | find_package(JPEG REQUIRED) 36 | find_package(TurboJpeg REQUIRED) 37 | find_package(Ocean REQUIRED) 38 | 39 | # Setup unit test infra, but only if unit tests are enabled 40 | if (UNIT_TESTS) 41 | enable_testing() 42 | find_package(GTest REQUIRED) 43 | endif() 44 | -------------------------------------------------------------------------------- /cmake/Options.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | if ("${CMAKE_BUILD_TYPE}" STREQUAL "") 16 | set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Type of build: Debug or Release." FORCE) 17 | endif() 18 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release") 19 | 20 | set(UNIT_TESTS ON CACHE BOOL "Disable unit tests.") 21 | -------------------------------------------------------------------------------- /cmake/Platform.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Support external Android toolchain file 16 | if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") 17 | set(ANDROID true) 18 | endif() 19 | 20 | # Cache the ANDROID flag 21 | set(ANDROID "${ANDROID}" CACHE BOOL "Build for Android. (DO NOT EDIT).") 22 | mark_as_advanced(FORCE ANDROID) # Hide this flag in ccmake & qtcreator: it can't be toogled from the UI 23 | 24 | # Cache the IOS flag 25 | set(IOS "${IOS}" CACHE BOOL "Build for iOS. (DO NOT EDIT).") 26 | mark_as_advanced(FORCE IOS) # Hide this flag in ccmake & qtcreator: it can't be toogled from the UI 27 | 28 | if (${CMAKE_HOST_WIN32}) 29 | set(BUILD_ON_WINDOWS true) 30 | set(BUILD_ON_NAME Windows) 31 | if (ANDROID) 32 | set(TARGET_ANDROID true) 33 | message(FATAL_ERROR "Targetting Android from Windows not supported") 34 | else() 35 | set(TARGET_WINDOWS true) 36 | endif() 37 | 38 | elseif (${CMAKE_HOST_UNIX}) 39 | if (${CMAKE_HOST_APPLE}) 40 | set(BUILD_ON_MACOS true) 41 | set(BUILD_ON_NAME MacOS) 42 | if (ANDROID) 43 | set(TARGET_ANDROID true) 44 | else() 45 | set(TARGET_MACOS true) 46 | endif() 47 | 48 | else() 49 | set(BUILD_ON_LINUX true) 50 | set(BUILD_ON_NAME Linux) 51 | if (ANDROID) 52 | set(TARGET_ANDROID true) 53 | else() 54 | set(TARGET_LINUX true) 55 | endif() 56 | endif() 57 | 58 | else() 59 | message(FATAL_ERROR "Build platform not recognized.") 60 | endif() 61 | 62 | if (TARGET_ANDROID) 63 | set(TARGET_NAME Android) 64 | else() 65 | set(TARGET_NAME "${BUILD_ON_NAME}") 66 | endif() 67 | -------------------------------------------------------------------------------- /cmake/ReadMe.md: -------------------------------------------------------------------------------- 1 | # What is in this folder? 2 | 3 | The files in this folder are complementary cmake files required by the cmake 4 | system to find additional libraries VRS depend on, as well as define useful 5 | definitions and helper cmake definitions. 6 | -------------------------------------------------------------------------------- /csrc/Pybind11.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Includes needed for bindings (including marshalling STL containers) 18 | #include 19 | 20 | #include 21 | 22 | #include "VrsBindings.h" 23 | #include "reader/Reader.h" 24 | #include "utils/Utils.h" 25 | 26 | #if IS_VRS_FB_INTERNAL() 27 | #include "archive/Archive.h" 28 | #include "fb/FbInternal.h" 29 | #include "fb/dataset_snapshot/PyDatasetSnapshot.h" 30 | #include "filter/Filter.h" // Disable filter internally until AsyncImageFilter is reworked. 31 | #include "writer/Writer.h" 32 | #endif 33 | 34 | #ifndef PYBIND_MODULE_NAME 35 | #define PYBIND_MODULE_NAME vrsbindings 36 | #endif 37 | 38 | namespace py = pybind11; 39 | 40 | PYBIND11_MODULE(PYBIND_MODULE_NAME, m) { 41 | m.doc() = R"pbdoc( 42 | Python bindings for VRS 43 | --------------------------------- 44 | .. currentmodule:: vrsbindings 45 | .. autosummary:: 46 | :toctree: _generate 47 | )pbdoc"; 48 | 49 | // NOTE: the order of includes matters for FB-internal python stubs generation 50 | #if IS_VRS_FB_INTERNAL() 51 | // Register some very basic types used in FB bindings 52 | pyvrs::pybind_fbinternal_basics(m); 53 | #endif 54 | 55 | // Register submodules. 56 | pyvrs::pybind_utils(m); 57 | 58 | #if IS_VRS_FB_INTERNAL() 59 | pyvrs::pybind_fbinternal(m); 60 | #endif 61 | pyvrs::pybind_reader(m); 62 | #if IS_VRS_FB_INTERNAL() 63 | pyvrs::pybind_filter(m); // Disable filter internally until AsyncImageFilter is reworked. 64 | pyvrs::pybind_writer(m); 65 | pyvrs::pybind_archive(m); 66 | pyvrs::pybind_dataset_snapshot(m); 67 | #endif 68 | 69 | #ifdef VERSION_INFO 70 | m.attr("__version__") = VERSION_INFO; 71 | #else 72 | m.attr("__version__") = "dev"; 73 | #endif 74 | } 75 | -------------------------------------------------------------------------------- /csrc/VrsBindings.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "VrsBindings.h" 18 | 19 | #include 20 | 21 | namespace pyvrs { 22 | 23 | #if IS_VRS_OSS_CODE() 24 | void initVrsBindings(const char* clientName) {} 25 | #endif 26 | 27 | } // namespace pyvrs 28 | -------------------------------------------------------------------------------- /csrc/VrsBindings.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #define PY_SSIZE_T_CLEAN 19 | #include 20 | 21 | namespace pyvrs { 22 | /// Initialize the VRS Python bindings. 23 | /// This function only performs initialization once, upon its first call. 24 | void initVrsBindings(const char* clientName = nullptr); 25 | } // namespace pyvrs 26 | -------------------------------------------------------------------------------- /csrc/reader/AsyncVRSReader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | #include "MultiVRSReader.h" 32 | #include "VRSReader.h" 33 | 34 | namespace pyvrs { 35 | 36 | using namespace vrs; 37 | using namespace std; 38 | 39 | /// \brief The base class for asynchronous job 40 | /// This class captures the asyncio's event loop, and creates a future for that loop so that we can 41 | /// later call loop.call_soon_threadsafe(future.set_result()) to set that future's result. 42 | class AsyncJob { 43 | public: 44 | AsyncJob() 45 | : loop_{py::module_::import("asyncio").attr("get_running_loop")()}, 46 | future_{loop_.attr("create_future")()} {} 47 | AsyncJob(const AsyncJob& other) = delete; 48 | virtual ~AsyncJob() = default; 49 | 50 | AsyncJob& operator=(const AsyncJob&) = delete; 51 | 52 | virtual void performJob(VRSReaderBase& reader) = 0; 53 | 54 | py::object await() { 55 | return future_.attr("__await__")(); 56 | } 57 | 58 | protected: 59 | py::object loop_; 60 | py::object future_; 61 | }; 62 | 63 | /// \brief Asynchronous job class for reading a record 64 | class AsyncReadJob : public AsyncJob { 65 | public: 66 | explicit AsyncReadJob(uint32_t index) : index_(index) {} 67 | 68 | void performJob(VRSReaderBase& reader) override; 69 | 70 | private: 71 | uint32_t index_; 72 | }; 73 | 74 | using AsyncJobQueue = JobQueue>; 75 | class AsyncReadHandler; 76 | 77 | /// \brief Python awaitable record 78 | /// This class only exposes __await__ method to Python which does the following: 79 | /// - Creates an AsyncReadJob object that: 80 | /// 1 - captures asyncio.events.get_running_loop, 81 | /// 2 - creates a future for that loop via loop.create_future(), 82 | /// 3 - captures the record index 83 | /// - Send a job to AsyncReader's background thread 84 | /// - Call future.__await__() and Python side waits until set_result will be called by AsyncReader 85 | class AwaitableRecord { 86 | public: 87 | AwaitableRecord(uint32_t index, AsyncReadHandler& readHandler); 88 | AwaitableRecord(const AwaitableRecord& other); 89 | 90 | py::object await() const; 91 | 92 | private: 93 | uint32_t index_; 94 | AsyncReadHandler& readHandler_; 95 | }; 96 | 97 | /// \brief Helper class to manage the background async thread 98 | class AsyncReadHandler { 99 | public: 100 | AsyncReadHandler(VRSReaderBase& reader) 101 | : reader_{reader}, asyncThread_(&AsyncReadHandler::asyncThreadActivity, this) {} 102 | 103 | VRSReaderBase& getReader() const { 104 | return reader_; 105 | } 106 | AsyncJobQueue& getQueue() { 107 | return workerQueue_; 108 | } 109 | 110 | void asyncThreadActivity(); 111 | void cleanup(); 112 | 113 | private: 114 | VRSReaderBase& reader_; 115 | AsyncJobQueue workerQueue_; 116 | atomic shouldEndAsyncThread_ = false; 117 | thread asyncThread_; 118 | }; 119 | 120 | /// \brief The async VRSReader class 121 | /// This class extends VRSReader and adds asynchronous APIs to read records. 122 | /// AsyncVRSReader spawns a background thread to process the async jobs. 123 | class OssAsyncVRSReader : public OssVRSReader { 124 | public: 125 | explicit OssAsyncVRSReader(bool autoReadConfigurationRecord) 126 | : OssVRSReader(autoReadConfigurationRecord), readHandler_{*this} {} 127 | 128 | ~OssAsyncVRSReader() override; 129 | 130 | /// Read a stream's record, by record type & index. 131 | /// @param streamId: VRS stream id to read. 132 | /// @param recordType: record type to read, or "any". 133 | /// @param index: the index of the record to read. 134 | /// @return AwaitableRecord: by calling await, caller will receive the same data as calling 135 | /// readRecord 136 | AwaitableRecord asyncReadRecord(const string& streamId, const string& recordType, int index); 137 | 138 | /// Read a specifc record, by index. 139 | /// @param index: a record index. 140 | /// @return AwaitableRecord: by calling await, caller will receive the same data as calling 141 | /// readRecord 142 | AwaitableRecord asyncReadRecord(int index); 143 | 144 | private: 145 | AsyncReadHandler readHandler_; 146 | }; 147 | 148 | /// \brief The async MultiVRSReader class 149 | /// This class extends VRSReader and adds asynchronous APIs to read records. 150 | /// AsyncVRSReader spawns a background thread to process the async jobs. 151 | class OssAsyncMultiVRSReader : public OssMultiVRSReader { 152 | public: 153 | explicit OssAsyncMultiVRSReader(bool autoReadConfigurationRecord) 154 | : OssMultiVRSReader(autoReadConfigurationRecord), readHandler_{*this} {} 155 | 156 | ~OssAsyncMultiVRSReader() override; 157 | 158 | /// Read a stream's record, by record type & index. 159 | /// @param streamId: VRS stream id to read. 160 | /// @param recordType: record type to read, or "any". 161 | /// @param index: the index of the record to read. 162 | /// @return AwaitableRecord: by calling await, caller will receive the same data as calling 163 | /// readRecord 164 | AwaitableRecord asyncReadRecord(const string& streamId, const string& recordType, int index); 165 | 166 | /// Read a specifc record, by index. 167 | /// @param index: a record index. 168 | /// @return AwaitableRecord: by calling await, caller will receive the same data as calling 169 | /// readRecord 170 | AwaitableRecord asyncReadRecord(int index); 171 | 172 | private: 173 | AsyncReadHandler readHandler_; 174 | }; 175 | 176 | /// Binds methods and classes for AsyncVRSReader. 177 | void pybind_asyncvrsreaders(py::module& m); 178 | } // namespace pyvrs 179 | 180 | #if IS_VRS_OSS_CODE() 181 | using PyAsyncReader = pyvrs::OssAsyncVRSReader; 182 | using PyAsyncMultiReader = pyvrs::OssAsyncMultiVRSReader; 183 | #else 184 | #include "AsyncVRSReader_fb.h" 185 | #endif 186 | -------------------------------------------------------------------------------- /csrc/reader/FactoryHelper.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | #include "../utils/PyUtils.h" 25 | 26 | // This file only exists to isolate the factory code of VRSReader.cpp in a separate file. 27 | // The factory aspects of this machinery mean that we actually need to be able to resolve names 28 | // without namespace specifications, hence these "using namespace" declarations in this file. 29 | // If we remove those the stringifying macros won't generate the correct names. 30 | 31 | namespace { 32 | 33 | using namespace std; 34 | using namespace pyvrs; 35 | using namespace vrs; 36 | 37 | using PyObjector = function; 38 | using PyObjectorString = 39 | function; 40 | 41 | class PyObjectorRegistry : public map { 42 | public: 43 | void map(PyObject* dic, const DataPiece* piece) { 44 | auto iter = find(piece->getElementTypeName()); 45 | if (iter != end()) { 46 | iter->second(dic, piece); 47 | } 48 | } 49 | }; 50 | 51 | class PyObjectorStringRegistry : public map { 52 | public: 53 | void map(PyObject* dic, const DataPiece* piece, const string& encoding) { 54 | auto iter = find(piece->getElementTypeName()); 55 | if (iter != end()) { 56 | iter->second(dic, piece, encoding); 57 | } 58 | } 59 | }; 60 | 61 | template 62 | void DataPieceValuePyObjector(PyObject* dic, const DataPiece* piece) { 63 | if (piece->isAvailable()) { 64 | string label = piece->getLabel(); 65 | string type = typeName(piece); 66 | T v = reinterpret_cast*>(piece)->get(); 67 | pyDict_SetItemWithDecRef(dic, Py_BuildValue("(s,s)", label.c_str(), type.c_str()), pyObject(v)); 68 | } 69 | } 70 | 71 | static PyObjectorRegistry& getDataPieceValuePyObjectorRegistry() { 72 | static PyObjectorRegistry sInstance; 73 | return sInstance; 74 | } 75 | 76 | template 77 | struct DataPieceValuePyObjectorRegistrerer { 78 | explicit DataPieceValuePyObjectorRegistrerer() { 79 | getDataPieceValuePyObjectorRegistry()[getTypeName()] = DataPieceValuePyObjector; 80 | } 81 | }; 82 | 83 | template 84 | void DataPieceArrayPyObjector(PyObject* dic, const DataPiece* piece) { 85 | if (piece->isAvailable()) { 86 | string label = piece->getLabel(); 87 | string type = typeName(piece, "_array"); 88 | vector values; 89 | reinterpret_cast*>(piece)->get(values); 90 | PyObject* list = PyList_New(static_cast(values.size())); 91 | for (size_t i = 0; i < values.size(); i++) { 92 | PyList_SetItem(list, static_cast(i), pyObject(values[i])); 93 | } 94 | pyDict_SetItemWithDecRef(dic, Py_BuildValue("(s,s)", label.c_str(), type.c_str()), list); 95 | } 96 | } 97 | 98 | static PyObjectorRegistry& getDataPieceArrayPyObjectorRegistry() { 99 | static PyObjectorRegistry sInstance; 100 | return sInstance; 101 | } 102 | 103 | template 104 | struct DataPieceArrayPyObjectorRegistrerer { 105 | explicit DataPieceArrayPyObjectorRegistrerer() { 106 | getDataPieceArrayPyObjectorRegistry()[getTypeName()] = DataPieceArrayPyObjector; 107 | } 108 | }; 109 | 110 | template 111 | void DataPieceVectorPyObjector(PyObject* dic, const DataPiece* piece) { 112 | if (piece->isAvailable()) { 113 | string label = piece->getLabel(); 114 | string type = typeName(piece, "_vector"); 115 | vector values; 116 | reinterpret_cast*>(piece)->get(values); 117 | PyObject* list = PyList_New(static_cast(values.size())); 118 | for (size_t i = 0; i < values.size(); i++) { 119 | PyList_SetItem(list, static_cast(i), pyObject(values[i])); 120 | } 121 | pyDict_SetItemWithDecRef(dic, Py_BuildValue("(s,s)", label.c_str(), type.c_str()), list); 122 | } 123 | } 124 | 125 | static PyObjectorRegistry& getDataPieceVectorPyObjectorRegistry() { 126 | static PyObjectorRegistry sInstance; 127 | return sInstance; 128 | } 129 | 130 | template 131 | struct DataPieceVectorPyObjectorRegistrerer { 132 | explicit DataPieceVectorPyObjectorRegistrerer() { 133 | getDataPieceVectorPyObjectorRegistry()[getTypeName()] = DataPieceVectorPyObjector; 134 | } 135 | }; 136 | 137 | template 138 | void DataPieceStringMapPyObjector(PyObject* dic, const DataPiece* piece, const string& encoding) { 139 | if (piece->isAvailable()) { 140 | string label = piece->getLabel(); 141 | string type = typeName(piece, "_string_map"); 142 | map values; 143 | reinterpret_cast*>(piece)->get(values); 144 | PyObject* newDic = PyDict_New(); 145 | string errors; 146 | for (const auto& iter : values) { 147 | pyDict_SetItemWithDecRef( 148 | newDic, unicodeDecode(iter.first, encoding, errors), pyObject(iter.second)); 149 | } 150 | pyDict_SetItemWithDecRef(dic, Py_BuildValue("(s,s)", label.c_str(), type.c_str()), newDic); 151 | } 152 | } 153 | 154 | static PyObjectorStringRegistry& getDataPieceStringMapPyObjectorRegistry() { 155 | static PyObjectorStringRegistry sInstance; 156 | return sInstance; 157 | } 158 | 159 | template 160 | struct DataPieceStringMapPyObjectorRegistrerer { 161 | explicit DataPieceStringMapPyObjectorRegistrerer() { 162 | getDataPieceStringMapPyObjectorRegistry()[getTypeName()] = DataPieceStringMapPyObjector; 163 | } 164 | }; 165 | 166 | #define DEFINE_DATA_PIECE(DATA_PIECE_TYPE, TEMPLATE_TYPE) \ 167 | static DataPiece##DATA_PIECE_TYPE##PyObjectorRegistrerer \ 168 | sDataPiece##DATA_PIECE_TYPE##PyObjectorRegistrerer_##TEMPLATE_TYPE; 169 | 170 | #define DEFINE_DATA_PIECE_TYPE(TEMPLATE_TYPE) \ 171 | DEFINE_DATA_PIECE(Value, TEMPLATE_TYPE) \ 172 | DEFINE_DATA_PIECE(Array, TEMPLATE_TYPE) \ 173 | DEFINE_DATA_PIECE(Vector, TEMPLATE_TYPE) \ 174 | DEFINE_DATA_PIECE(StringMap, TEMPLATE_TYPE) 175 | 176 | // Define & generate the code for each POD type supported. 177 | #define POD_MACRO DEFINE_DATA_PIECE_TYPE 178 | #include 179 | 180 | DEFINE_DATA_PIECE(Vector, string) 181 | DEFINE_DATA_PIECE(StringMap, string) 182 | 183 | } // namespace 184 | 185 | namespace pyvrs { 186 | // To prevent the inclusion of this expensive header in more than one file, this function should 187 | // trigger a linker error if included more than once 188 | int singleIncludeBreakCheck() { 189 | return 0; 190 | } 191 | } // namespace pyvrs 192 | -------------------------------------------------------------------------------- /csrc/reader/FilteredFileReader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "FilteredFileReader.h" 18 | 19 | #include "../VrsBindings.h" 20 | #include "../utils/PyUtils.h" 21 | 22 | namespace py = pybind11; 23 | 24 | namespace pyvrs { 25 | 26 | using namespace vrs::utils; 27 | 28 | FilteredFileReader::FilteredFileReader(const std::string& filePath) { 29 | initVrsBindings(); 30 | filteredReader_.setSource(filePath); 31 | filteredReader_.openFile(); 32 | } 33 | 34 | void FilteredFileReader::after(double minTime, bool isRelativeMinTime) { 35 | filteredReader_.setMinTime(minTime, isRelativeMinTime); 36 | } 37 | 38 | void FilteredFileReader::before(double maxTime, bool isRelativeMaxTime) { 39 | filteredReader_.setMaxTime(maxTime, isRelativeMaxTime); 40 | } 41 | 42 | void FilteredFileReader::range( 43 | double minTime, 44 | double maxTime, 45 | bool isRelativeMinTime, 46 | bool isRelativeMaxTime) { 47 | filteredReader_.setMinTime(minTime, isRelativeMinTime); 48 | filteredReader_.setMaxTime(maxTime, isRelativeMaxTime); 49 | } 50 | 51 | vrs::utils::FilteredFileReader& FilteredFileReader::getFilteredReader() { 52 | filteredReader_.applyFilters(filters_); 53 | return filteredReader_; 54 | } 55 | 56 | void pybind_filtered_filereader(py::module& m) { 57 | auto filteredReader = py::class_>( 58 | m, "FilteredFileReader") 59 | .def(py::init()) 60 | .def("after", &pyvrs::FilteredFileReader::after) 61 | .def("before", &pyvrs::FilteredFileReader::before) 62 | .def("range", &pyvrs::FilteredFileReader::range); 63 | 64 | #if IS_VRS_FB_INTERNAL() 65 | pybind_fbfiltered_filereader(filteredReader); 66 | #endif 67 | } 68 | 69 | } // namespace pyvrs 70 | -------------------------------------------------------------------------------- /csrc/reader/FilteredFileReader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #if IS_VRS_FB_INTERNAL() 28 | #include "FilteredFileReader_fb.h" 29 | #endif 30 | 31 | namespace pyvrs { 32 | 33 | using namespace vrs; 34 | namespace py = pybind11; 35 | 36 | /// @brief The FilteredFileReader class 37 | /// This class enables selective reading of VRS files. 38 | #if IS_VRS_OSS_CODE() 39 | class FilteredFileReader { 40 | public: 41 | explicit FilteredFileReader(const std::string& filePath); 42 | 43 | void after(double minTime, bool isRelativeMinTime = false); 44 | void before(double maxTime, bool isRelativeMaxTime = false); 45 | void range( 46 | double minTime, 47 | double maxTime, 48 | bool isRelativeMinTime = false, 49 | bool isRelativeMaxTime = false); 50 | 51 | vrs::utils::FilteredFileReader& getFilteredReader(); 52 | 53 | private: 54 | vrs::utils::FilteredFileReader filteredReader_; 55 | vrs::utils::RecordFilterParams filters_; 56 | }; 57 | #endif 58 | 59 | /// Binds methods and classes for FilteredFileReader. 60 | void pybind_filtered_filereader(py::module& m); 61 | } // namespace pyvrs 62 | -------------------------------------------------------------------------------- /csrc/reader/Reader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "Reader.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "AsyncVRSReader.h" 23 | #include "FilteredFileReader.h" 24 | #include "MultiVRSReader.h" 25 | #include "VRSReader.h" 26 | 27 | namespace pyvrs { 28 | namespace py = pybind11; 29 | 30 | string extractAudioTrack(pyvrs::FilteredFileReader& filteredReader, const string& wavFilePath) { 31 | initVrsBindings(); 32 | return vrs::utils::extractAudioTrack(filteredReader.getFilteredReader(), wavFilePath); 33 | } 34 | 35 | void pybind_reader(py::module& m) { 36 | py::enum_(m, "ImageConversion") 37 | .value("OFF", pyvrs::ImageConversion::Off) 38 | .value("DECOMPRESS", pyvrs::ImageConversion::Decompress) 39 | .value("NORMALIZE", pyvrs::ImageConversion::Normalize) 40 | .value("NORMALIZE_GREY8", pyvrs::ImageConversion::NormalizeGrey8) 41 | .value("RAW_BUFFER", pyvrs::ImageConversion::RawBuffer) 42 | .value("RECORD_UNREAD_BYTES_BACKDOOR", pyvrs::ImageConversion::RecordUnreadBytesBackdoor) 43 | .export_values(); 44 | 45 | pybind_vrsreader(m); 46 | pybind_multivrsreader(m); 47 | pybind_asyncvrsreaders(m); 48 | 49 | #if IS_VRS_FB_INTERNAL() 50 | pybind_filtered_filereader(m); 51 | #endif 52 | 53 | m.def( 54 | "extract_audio_track", 55 | &extractAudioTrack, 56 | "Extract audio track from given FilteredFileReader"); 57 | } 58 | } // namespace pyvrs 59 | -------------------------------------------------------------------------------- /csrc/reader/Reader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | namespace pyvrs { 25 | namespace py = pybind11; 26 | 27 | /// Binds methods and classes for Reader. 28 | void pybind_reader(py::module& m); 29 | 30 | } // namespace pyvrs 31 | -------------------------------------------------------------------------------- /csrc/reader/VRSReaderBase.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | namespace pyvrs { 25 | 26 | namespace py = pybind11; 27 | 28 | /// @brief The VRSReaderBase class 29 | /// This class is the base class for VRSReader and MultiVRSReader. 30 | /// It provides the minimum common functionality between both classes needed to factorize 31 | /// pyvrs::OssAsyncVRSReader and pyvrs::OssAsyncMultiVRSReader 32 | class VRSReaderBase { 33 | public: 34 | virtual ~VRSReaderBase() = default; 35 | 36 | virtual py::object readRecord(int index) = 0; 37 | }; 38 | 39 | } // namespace pyvrs 40 | -------------------------------------------------------------------------------- /csrc/utils/PyBuffer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #define PY_SSIZE_T_CLEAN 19 | #include 20 | 21 | #include 22 | 23 | #include 24 | 25 | namespace pyvrs { 26 | namespace py = pybind11; 27 | using namespace vrs; 28 | 29 | /// \brief Image spec class. 30 | /// 31 | /// This class is vrs::ImageContentBlockSpec for Python bindings. 32 | /// The fields are exposed as a read only property but the methods required for dict interface is 33 | /// supported so that user can treat PyImageContentBlockSpec object as Python Dictionary to iterate 34 | /// on. 35 | class PyImageContentBlockSpec { 36 | public: 37 | PyImageContentBlockSpec() = default; 38 | explicit PyImageContentBlockSpec(const ImageContentBlockSpec& spec) : spec_{spec} {} 39 | 40 | uint32_t getWidth() const { 41 | return spec_.getWidth(); 42 | } 43 | 44 | uint32_t getHeight() const { 45 | return spec_.getHeight(); 46 | } 47 | 48 | uint32_t getStride() const { 49 | if (spec_.getImageFormat() == ImageFormat::RAW) { 50 | return spec_.getStride(); 51 | } 52 | return 0; 53 | } 54 | 55 | string getPixelFormatAsString() const { 56 | return spec_.getPixelFormatAsString(); 57 | } 58 | 59 | string getImageFormatAsString() const { 60 | return spec_.getImageFormatAsString(); 61 | } 62 | 63 | uint32_t getBytesPerPixel() const { 64 | if (spec_.getImageFormat() == ImageFormat::RAW) { 65 | return static_cast(spec_.getBytesPerPixel()); 66 | } 67 | return 0; 68 | } 69 | 70 | uint32_t getBlockSize() const { 71 | return static_cast(spec_.getBlockSize()); 72 | } 73 | 74 | size_t getRawImageSize() const { 75 | return spec_.getRawImageSize(); 76 | } 77 | 78 | string getCodecName() const { 79 | return spec_.getCodecName(); 80 | } 81 | 82 | uint8_t getCodecQuality() const { 83 | return spec_.getCodecQuality(); 84 | } 85 | 86 | uint32_t getKeyFrameTimestamp() const { 87 | return spec_.getKeyFrameTimestamp(); 88 | } 89 | 90 | uint32_t getKeyFrameIndex() const { 91 | return spec_.getKeyFrameIndex(); 92 | } 93 | 94 | uint8_t getChannelCountPerPixel() const { 95 | if (spec_.getImageFormat() == ImageFormat::RAW) { 96 | return spec_.getChannelCountPerPixel(); 97 | } 98 | return 0; 99 | } 100 | 101 | ImageFormat getImageFormat() const { 102 | return spec_.getImageFormat(); 103 | } 104 | 105 | ImageContentBlockSpec getImageContentBlockSpec() const { 106 | return spec_; 107 | } 108 | 109 | PixelFormat getPixelFormat() const { 110 | return spec_.getPixelFormat(); 111 | } 112 | 113 | string asString() const { 114 | return spec_.asString(); 115 | } 116 | 117 | ImageContentBlockSpec& getImageContentBlockSpec() { 118 | return spec_; 119 | } 120 | 121 | void initAttributesMap(); 122 | map attributesMap; 123 | 124 | private: 125 | ImageContentBlockSpec spec_; 126 | }; 127 | 128 | /// \brief Audio spec class. 129 | /// 130 | /// This class is vrs::AudioContentBlockSpec for Python bindings. 131 | /// The fields are exposed as a read only property but the methods required for dict interface is 132 | /// supported so that user can treat PyAudioContentBlockSpec object as Python Dictionary to iterate 133 | /// on. 134 | class PyAudioContentBlockSpec { 135 | public: 136 | explicit PyAudioContentBlockSpec(const AudioContentBlockSpec& spec) : spec_{spec} {} 137 | 138 | uint32_t getSampleCount() const { 139 | return spec_.getSampleCount(); 140 | } 141 | 142 | string getSampleFormatAsString() const { 143 | return spec_.getSampleFormatAsString(); 144 | } 145 | 146 | uint8_t getSampleFrameStride() const { 147 | return spec_.getSampleFrameStride(); 148 | } 149 | 150 | uint8_t getChannelCount() const { 151 | return spec_.getChannelCount(); 152 | } 153 | 154 | uint32_t getSampleRate() const { 155 | return spec_.getSampleRate(); 156 | } 157 | 158 | uint32_t getBlockSize() const { 159 | return static_cast(spec_.getBlockSize()); 160 | } 161 | 162 | AudioSampleFormat getSampleFormat() const { 163 | return spec_.getSampleFormat(); 164 | } 165 | 166 | string asString() const { 167 | return spec_.asString(); 168 | } 169 | 170 | void initAttributesMap(); 171 | map attributesMap; 172 | 173 | private: 174 | AudioContentBlockSpec spec_; 175 | }; 176 | 177 | /// \brief Class for content block in VRS Record. 178 | /// 179 | /// This class is vrs::ContentBlock for Python bindings. 180 | /// The fields are exposed as a read only property but the methods required for dict interface is 181 | /// supported so that user can treat ContentBlock object as Python Dictionary to iterate 182 | /// on. 183 | class PyContentBlock { 184 | public: 185 | explicit PyContentBlock(const ContentBlock& block) : block_{block} {} 186 | 187 | int getBufferSize() const { 188 | size_t blockSize = block_.getBlockSize(); 189 | return blockSize == ContentBlock::kSizeUnknown ? -1 : static_cast(blockSize); 190 | } 191 | 192 | string asString() const { 193 | return block_.asString(); 194 | } 195 | 196 | void initAttributesMap(); 197 | map attributesMap; 198 | 199 | private: 200 | ContentBlock block_; 201 | }; 202 | 203 | /// \brief Helper struct to hold an image data. 204 | /// Exposed to Python using protocol_buffer, optimized using the content spec. 205 | /// This class auto converts buffer to follow the pixel format on read. 206 | struct ImageBuffer { 207 | ImageBuffer() = default; 208 | ImageBuffer(const PyImageContentBlockSpec& imageSpec, const vector& imageBytes) 209 | : spec{imageSpec}, bytes{imageBytes} {} 210 | ImageBuffer(const PyImageContentBlockSpec& imageSpec, vector&& bytes) 211 | : spec{imageSpec}, bytes{std::move(bytes)} {} 212 | ImageBuffer(const PyImageContentBlockSpec& imageSpec, const py::buffer& b) : spec{imageSpec} { 213 | initBytesFromPyBuffer(b); 214 | } 215 | ImageBuffer(const PyImageContentBlockSpec& imageSpec, int64_t recordIndex, const py::buffer& b) 216 | : spec{imageSpec}, recordIndex{recordIndex} { 217 | initBytesFromPyBuffer(b); 218 | } 219 | ImageBuffer(const ImageContentBlockSpec& imageSpec, const vector& imageBytes) 220 | : spec{imageSpec}, bytes{imageBytes} {} 221 | ImageBuffer(const ImageContentBlockSpec& imageSpec, vector&& bytes) 222 | : spec{imageSpec}, bytes{std::move(bytes)} {} 223 | ImageBuffer(const ImageContentBlockSpec& imageSpec, const py::buffer& b) : spec{imageSpec} { 224 | initBytesFromPyBuffer(b); 225 | } 226 | ImageBuffer(const ImageContentBlockSpec& imageSpec, int64_t recordIndex, const py::buffer& b) 227 | : spec{imageSpec}, recordIndex{recordIndex} { 228 | initBytesFromPyBuffer(b); 229 | } 230 | void initBytesFromPyBuffer(const py::buffer& b); 231 | 232 | ImageBuffer jxlCompress(float quality) const; 233 | ImageBuffer jpgCompress(uint32_t quality) const; 234 | ImageBuffer decompress() const; 235 | 236 | PyImageContentBlockSpec spec; 237 | vector bytes; 238 | int64_t recordIndex{-1}; // only used by AsyncImageFilter 239 | }; 240 | 241 | /// \brief Helper class to hold a content block binary data. 242 | /// Exposed to Python using protocol_buffer, optimized using the content spec. 243 | struct ContentBlockBuffer { 244 | explicit ContentBlockBuffer(const ContentBlock& block) 245 | : spec{block}, structuredArray{false}, bytesAdjusted{false} {} 246 | ImageBuffer jxlCompress(float quality) const; 247 | ImageBuffer jpgCompress(uint32_t quality) const; 248 | ImageBuffer decompress() const; 249 | 250 | ContentBlock spec; 251 | vector bytes; 252 | bool structuredArray; // should the buffer be returned as a 1 dimension uint8_t array? 253 | bool bytesAdjusted; // was the buffer endian swapped and/or realigned? 254 | }; 255 | 256 | /// \brief Helper struct to hold a binary data. 257 | /// Exposed to Python using protocol_buffer, optimized using the content spec. 258 | /// This class reads data as is without converting buffer. 259 | struct BinaryBuffer { 260 | explicit BinaryBuffer( 261 | uint8_t* data_, 262 | size_t size_, 263 | size_t itemsize_, 264 | const string& format_, 265 | vector shape_ = {}) 266 | : data{data_}, size{size_}, itemsize{itemsize_}, format{format_}, shape{shape_} { 267 | // If shape is not specified, we treat this buffer as 1d array. 268 | if (shape.empty()) { 269 | shape.push_back(size); 270 | } 271 | } 272 | uint8_t* data; 273 | size_t size; 274 | size_t itemsize; 275 | string format; 276 | vector shape; 277 | }; 278 | 279 | /// Convert buffer to py::buffer_info following the spec. 280 | py::buffer_info convertContentBlockBuffer(ContentBlockBuffer& block); 281 | py::buffer_info convertImageBlockBuffer(ImageBuffer& block); 282 | 283 | /// Binds methods and classes for PyBuffer. 284 | void pybind_buffer(py::module& m); 285 | } // namespace pyvrs 286 | -------------------------------------------------------------------------------- /csrc/utils/PyExceptions.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "PyExceptions.h" 18 | 19 | namespace pyvrs { 20 | 21 | void pybind_exception(py::module& m) { 22 | // Register custom exceptions 23 | static py::exception timestampNotFoundError( 24 | m, "TimestampNotFoundError"); 25 | static py::exception streamNotFoundError(m, "StreamNotFoundError"); 26 | py::register_exception_translator([](std::exception_ptr p) { 27 | try { 28 | if (p) { 29 | std::rethrow_exception(p); 30 | } 31 | } catch (const pyvrs::TimestampNotFoundError& e) { 32 | timestampNotFoundError(e.what()); 33 | } catch (const pyvrs::StreamNotFoundError& e) { 34 | streamNotFoundError(e.what()); 35 | } 36 | }); 37 | } 38 | 39 | } // namespace pyvrs 40 | -------------------------------------------------------------------------------- /csrc/utils/PyExceptions.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #define PY_SSIZE_T_CLEAN 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | namespace pyvrs { 34 | namespace py = pybind11; 35 | 36 | /// Following classes are custom exceptions for VRS. 37 | /// \brief Custom exception class VRS when record doesn't exist for timestamp. 38 | class TimestampNotFoundError : public std::exception { 39 | public: 40 | explicit TimestampNotFoundError( 41 | double timestamp, 42 | double epsilon = 0, 43 | vrs::StreamId streamId = {}, 44 | vrs::Record::Type recordType = vrs::Record::Type::UNDEFINED) { 45 | std::stringstream ss; 46 | 47 | ss << "Record not found"; 48 | if (streamId.isValid()) { 49 | ss << fmt::format(" for stream {}", streamId.getFullName()); 50 | } 51 | if (recordType != vrs::Record::Type::UNDEFINED) { 52 | ss << fmt::format(" for record type {}", toString(recordType)); 53 | } 54 | if (epsilon != 0) { 55 | ss << fmt::format(" in range ({0}-{1})-({0}+{1})", timestamp, epsilon); 56 | } else { 57 | ss << fmt::format(" at timestamp {}", timestamp); 58 | } 59 | message_ = ss.str(); 60 | } 61 | 62 | const char* what() const noexcept override { 63 | return message_.c_str(); 64 | } 65 | 66 | private: 67 | std::string message_; 68 | }; 69 | 70 | /// \brief Custom exception class for VRS when the stream doesn't exist. 71 | class StreamNotFoundError : public std::exception { 72 | public: 73 | explicit StreamNotFoundError( 74 | vrs::RecordableTypeId recordableTypeId, 75 | const std::set& availableStreamIds) 76 | : StreamNotFoundError(vrs::toString(recordableTypeId), availableStreamIds) {} 77 | explicit StreamNotFoundError( 78 | const std::string& streamId, 79 | const std::set& availableStreamIds) { 80 | std::stringstream ss; 81 | ss << fmt::format("No matching stream for {}. Available streams are:\n", streamId); 82 | for (auto it : availableStreamIds) { 83 | ss << " " << it.getFullName() << "\n"; 84 | } 85 | message_ = ss.str(); 86 | } 87 | const char* what() const noexcept override { 88 | return message_.c_str(); 89 | } 90 | 91 | private: 92 | std::string message_; 93 | }; 94 | 95 | /// Binds methods and classes for PyExceptions. 96 | void pybind_exception(py::module& m); 97 | } // namespace pyvrs 98 | -------------------------------------------------------------------------------- /csrc/utils/PyFileSpec.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // This include *must* be before any STL include! See Python C API doc. 18 | #include 19 | #include 20 | 21 | #define PY_SSIZE_T_CLEAN 22 | #include 23 | 24 | #include "PyFileSpec.h" 25 | 26 | namespace pyvrs { 27 | namespace py = pybind11; 28 | #if IS_VRS_OSS_CODE() 29 | void pybind_filespec(py::module& m) { 30 | py::class_(m, "FileSpec") 31 | .def(py::init<>()) 32 | .def(py::init()) 33 | .def("get_easy_path", &PyFileSpec::getEasyPath) 34 | .def("get_filehandler_name", &PyFileSpec::getFileHandlerName) 35 | .def("get_chunks", &PyFileSpec::getChunks) 36 | .def("get_chunk_sizes", &PyFileSpec::getChunkSizes) 37 | .def("get_filename", &PyFileSpec::getFileName) 38 | .def("get_uri", &PyFileSpec::getUri) 39 | .def("__str__", &PyFileSpec::toJson); 40 | } 41 | #endif 42 | } // namespace pyvrs 43 | -------------------------------------------------------------------------------- /csrc/utils/PyFileSpec.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include "../VrsBindings.h" 33 | 34 | namespace pyvrs { 35 | namespace py = pybind11; 36 | 37 | class OssPyFileSpec { 38 | public: 39 | OssPyFileSpec() {} 40 | explicit OssPyFileSpec(const std::string& path) { 41 | initVrsBindings(); 42 | int status = vrs::RecordFileReader::vrsFilePathToFileSpec(path, spec_, true); 43 | if (status != 0) { 44 | throw py::value_error( 45 | fmt::format("Invalid path '{}': {}", path, vrs::errorCodeToMessageWithCode(status))); 46 | } 47 | } 48 | 49 | std::string getEasyPath() const { 50 | return spec_.getEasyPath(); 51 | } 52 | 53 | const std::string& getFileHandlerName() const { 54 | return spec_.fileHandlerName; 55 | } 56 | 57 | const std::vector& getChunks() const { 58 | return spec_.chunks; 59 | } 60 | 61 | const std::vector& getChunkSizes() const { 62 | return spec_.chunkSizes; 63 | } 64 | 65 | const std::string& getFileName() const { 66 | return spec_.fileName; 67 | } 68 | 69 | const std::string& getUri() const { 70 | return spec_.uri; 71 | } 72 | 73 | const vrs::FileSpec& getSpec() const { 74 | return spec_; 75 | } 76 | 77 | const std::string toJson() const { 78 | return spec_.toJson(); 79 | } 80 | 81 | protected: 82 | vrs::FileSpec spec_; 83 | }; 84 | 85 | void pybind_filespec(py::module& m); 86 | 87 | } // namespace pyvrs 88 | 89 | #if IS_VRS_OSS_CODE() 90 | using PyFileSpec = pyvrs::OssPyFileSpec; 91 | #else 92 | #include "PyFileSpec_fb.h" 93 | #endif 94 | 95 | template <> 96 | struct fmt::formatter { 97 | template 98 | constexpr auto parse(ParseContext& ctx) const -> decltype(ctx.begin()) { 99 | return ctx.begin(); 100 | } 101 | 102 | template 103 | auto format(const vrs::FileSpec& spec, FormatContext& ctx) const { 104 | return fmt::format_to(ctx.out(), "{}", spec.getEasyPath()); 105 | } 106 | }; 107 | -------------------------------------------------------------------------------- /csrc/utils/PyRecord.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | 29 | #include "PyBuffer.h" 30 | 31 | namespace pyvrs { 32 | 33 | namespace py = pybind11; 34 | using namespace vrs; 35 | using namespace std; 36 | 37 | /// \brief Cache for VRS Record's content. 38 | /// 39 | /// When a record is read, RecordCache is used to hold the record content. 40 | struct RecordCache { 41 | uint32_t recordFormatVersion; 42 | vector datalayoutBlocks; 43 | vector images; 44 | vector audioBlocks; 45 | vector customBlocks; 46 | vector unsupportedBlocks; 47 | 48 | void clear() { 49 | recordFormatVersion = 0; 50 | datalayoutBlocks.clear(); 51 | images.clear(); 52 | audioBlocks.clear(); 53 | customBlocks.clear(); 54 | unsupportedBlocks.clear(); 55 | } 56 | }; 57 | 58 | /// \brief The VRS Record class 59 | /// 60 | /// This class is a VRS record for Python bindings. 61 | /// The fields are exposed as a read only property but the methods required for dict interface is 62 | /// supported so that user can treat PyRecord object as Python Dictionary to iterate on. 63 | struct PyRecord { 64 | explicit PyRecord(const IndexRecord::RecordInfo& info, int32_t recordIndex_); 65 | explicit PyRecord(const IndexRecord::RecordInfo& info, int32_t recordIndex_, RecordCache& record); 66 | 67 | int32_t recordIndex; 68 | string recordType; 69 | double recordTimestamp; 70 | string streamId; 71 | 72 | uint32_t recordFormatVersion; 73 | vector datalayoutBlocks; 74 | vector imageBlocks; 75 | vector audioBlocks; 76 | vector customBlocks; 77 | vector unsupportedBlocks; 78 | 79 | vector audioSpecs; 80 | vector customBlockSpecs; 81 | vector imageSpecs; 82 | 83 | /// members and methods to set up dictionary interface. 84 | void initAttributesMap(); 85 | map attributesMap; 86 | }; 87 | 88 | /// Binds methods and classes for PyRecord. 89 | void pybind_record(py::module& m); 90 | } // namespace pyvrs 91 | -------------------------------------------------------------------------------- /csrc/utils/PyUtils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "PyUtils.h" 18 | 19 | #include 20 | 21 | #define DEFAULT_LOG_CHANNEL "PyUtils" 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | namespace { 28 | const char* kUtf8 = "utf-8"; 29 | } 30 | 31 | namespace pyvrs { 32 | 33 | string typeName(const DataPiece* piece, const char* suffix) { 34 | string name = piece->getElementTypeName(); 35 | // Remove trailing "_t", if any.. 36 | if (name.length() >= 2 && name.back() == 't' && name[name.length() - 2] == '_') { 37 | name.resize(name.length() - 2); 38 | } 39 | return suffix == nullptr ? name : name + suffix; 40 | } 41 | 42 | string lowercaseTypeName(Record::Type type) { 43 | string typeName = toString(type); 44 | std::transform(typeName.begin(), typeName.end(), typeName.begin(), ::tolower); 45 | return typeName; 46 | } 47 | 48 | string toupper(const string& s) { 49 | string t{s}; 50 | std::transform(t.begin(), t.end(), t.begin(), ::toupper); 51 | return t; 52 | } 53 | 54 | void pyDict_SetItemWithDecRef(PyObject* dict, PyObject* key, PyObject* value) { 55 | PyDict_SetItem(dict, key, value); 56 | Py_DECREF(key); 57 | Py_DECREF(value); 58 | } 59 | 60 | PyObject* unicodeDecode(const string& str, const string& encoding, const string& errors) { 61 | const char* s = str.c_str(); 62 | Py_ssize_t size = str.size(); 63 | 64 | if (encoding == "utf-8-safe") { 65 | // magic encoding name to try utf-8, but not fail if the encoding is not recognized. 66 | PyObject* decodedStr = PyUnicode_Decode(s, size, kUtf8, errors.c_str()); 67 | if (decodedStr != nullptr) { 68 | return decodedStr; 69 | } 70 | PyErr_Clear(); 71 | // if utf-8 fails, make a safe string, even if it's transformed... 72 | return pyObject(helpers::make_printable(str)); 73 | } 74 | PyObject* decodedStr = PyUnicode_Decode(s, size, encoding.c_str(), errors.c_str()); 75 | if (decodedStr != nullptr) { 76 | return decodedStr; 77 | } 78 | 79 | stringstream ss; 80 | ss << "Failed to decode \"" << s << "\" with encoding \"" << encoding << "\"."; 81 | // Try to decode in the few most common encodings so we can give a better hint for how to resolve 82 | // the issue. 83 | if (PyUnicode_DecodeASCII(s, size, nullptr)) { 84 | ss << " Try using \"ascii\" for encoding instead."; 85 | } else if (PyUnicode_DecodeLatin1(s, size, nullptr)) { 86 | ss << " Try using \"latin1\" for encoding instead."; 87 | } else { 88 | ss << " Encoding is neither \"ascii\" or \"latin1\""; 89 | } 90 | 91 | XR_LOGE("{}", ss.str()); 92 | 93 | PyErr_Clear(); 94 | PyUnicodeDecodeError_Create(encoding.c_str(), s, size, 0, size, ss.str().c_str()); 95 | throw pybind11::error_already_set(); 96 | } 97 | 98 | } // namespace pyvrs 99 | -------------------------------------------------------------------------------- /csrc/utils/PyUtils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #define PY_SSIZE_T_CLEAN 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | #include 28 | #include 29 | 30 | namespace pyvrs { 31 | 32 | using namespace std; 33 | using namespace vrs; 34 | namespace py = pybind11; 35 | 36 | string typeName(const DataPiece* piece, const char* suffix = nullptr); 37 | 38 | string lowercaseTypeName(Record::Type type); 39 | 40 | string toupper(const string& s); 41 | 42 | /// Convinent methods for pybind layer. 43 | /// PyDict_SetItem doesn't decrement the key and the value, this function decrease the reference 44 | /// count for key & value as well. 45 | void pyDict_SetItemWithDecRef(PyObject* dict, PyObject* key, PyObject* value); 46 | 47 | PyObject* unicodeDecode(const string& str, const string& encoding, const string& errors); 48 | 49 | /// pyWrap converts PyObject -> py::object 50 | inline py::object pyWrap(PyObject* object) { 51 | return py::reinterpret_steal(object); 52 | } 53 | 54 | /// pyObject converts C++ object into PyObject. 55 | template 56 | PyObject* pyObject(const T&); 57 | 58 | template <> 59 | inline PyObject* pyObject(const Bool& v) { 60 | PyObject* result = v.operator bool() ? Py_True : Py_False; 61 | Py_INCREF(result); 62 | return result; 63 | } 64 | 65 | template <> 66 | inline PyObject* pyObject(const char& v) { 67 | return PyBytes_FromStringAndSize(&v, 1); 68 | } 69 | 70 | template <> 71 | inline PyObject* pyObject(const uint8_t& v) { 72 | return PyLong_FromUnsignedLong(v); 73 | } 74 | 75 | template <> 76 | inline PyObject* pyObject(const int8_t& v) { 77 | return PyLong_FromLong(v); 78 | } 79 | 80 | template <> 81 | inline PyObject* pyObject(const uint16_t& v) { 82 | return PyLong_FromUnsignedLong(v); 83 | } 84 | 85 | template <> 86 | inline PyObject* pyObject(const int16_t& v) { 87 | return PyLong_FromLong(v); 88 | } 89 | 90 | template <> 91 | inline PyObject* pyObject(const uint32_t& v) { 92 | return PyLong_FromUnsignedLong(v); 93 | } 94 | 95 | template <> 96 | inline PyObject* pyObject(const int32_t& v) { 97 | return PyLong_FromLong(v); 98 | } 99 | 100 | template <> 101 | inline PyObject* pyObject(const uint64_t& v) { 102 | return PyLong_FromUnsignedLongLong(v); 103 | } 104 | 105 | template <> 106 | inline PyObject* pyObject(const int64_t& v) { 107 | return PyLong_FromLongLong(v); 108 | } 109 | 110 | template <> 111 | inline PyObject* pyObject(const double& d) { 112 | return PyFloat_FromDouble(d); 113 | } 114 | 115 | template <> 116 | inline PyObject* pyObject(const float& f) { 117 | return PyFloat_FromDouble(static_cast(f)); 118 | } 119 | 120 | template <> 121 | inline PyObject* pyObject(const Point2Df& p) { 122 | return Py_BuildValue("(d,d)", static_cast(p.x()), static_cast(p.y())); 123 | } 124 | 125 | template <> 126 | inline PyObject* pyObject(const Point2Dd& p) { 127 | return Py_BuildValue("(d,d)", p.x(), p.y()); 128 | } 129 | 130 | template <> 131 | inline PyObject* pyObject(const Point2Di& p) { 132 | return Py_BuildValue("(i,i)", p.x(), p.y()); 133 | } 134 | 135 | template <> 136 | inline PyObject* pyObject(const Point3Df& p) { 137 | return Py_BuildValue( 138 | "(d,d,d)", 139 | static_cast(p.x()), 140 | static_cast(p.y()), 141 | static_cast(p.z())); 142 | } 143 | 144 | template <> 145 | inline PyObject* pyObject(const Point3Dd& p) { 146 | return Py_BuildValue("(d,d,d)", p.x(), p.y(), p.z()); 147 | } 148 | 149 | template <> 150 | inline PyObject* pyObject(const Point3Di& p) { 151 | return Py_BuildValue("(i,i,i)", p.x(), p.y(), p.z()); 152 | } 153 | 154 | template <> 155 | inline PyObject* pyObject(const Point4Df& p) { 156 | return Py_BuildValue( 157 | "(d,d,d,d)", 158 | static_cast(p.x()), 159 | static_cast(p.y()), 160 | static_cast(p.z()), 161 | static_cast(p.w())); 162 | } 163 | 164 | template <> 165 | inline PyObject* pyObject(const Point4Dd& p) { 166 | return Py_BuildValue("(d,d,d,d)", p.x(), p.y(), p.z(), p.w()); 167 | } 168 | 169 | template <> 170 | inline PyObject* pyObject(const Point4Di& p) { 171 | return Py_BuildValue("(i,i,i,i)", p.x(), p.y(), p.z(), p.w()); 172 | } 173 | 174 | template <> 175 | inline PyObject* pyObject(const Matrix3Df& p) { 176 | return Py_BuildValue( 177 | "((d,d,d),(d,d,d),(d,d,d))", 178 | static_cast(p[0][0]), 179 | static_cast(p[0][1]), 180 | static_cast(p[0][2]), 181 | static_cast(p[1][0]), 182 | static_cast(p[1][1]), 183 | static_cast(p[1][2]), 184 | static_cast(p[2][0]), 185 | static_cast(p[2][1]), 186 | static_cast(p[2][2])); 187 | } 188 | 189 | template <> 190 | inline PyObject* pyObject(const Matrix2Dd& p) { 191 | return Py_BuildValue("((d,d),(d,d))", p[0][0], p[0][1], p[1][0], p[1][1]); 192 | } 193 | 194 | template <> 195 | inline PyObject* pyObject(const Matrix2Di& p) { 196 | return Py_BuildValue("((i,i),(i,i))", p[0][0], p[0][1], p[1][0], p[1][1]); 197 | } 198 | 199 | template <> 200 | inline PyObject* pyObject(const Matrix2Df& p) { 201 | return Py_BuildValue( 202 | "((d,d),(d,d))", 203 | static_cast(p[0][0]), 204 | static_cast(p[0][1]), 205 | static_cast(p[1][0]), 206 | static_cast(p[1][1])); 207 | } 208 | 209 | template <> 210 | inline PyObject* pyObject(const Matrix3Dd& p) { 211 | return Py_BuildValue( 212 | "((d,d,d),(d,d,d),(d,d,d))", 213 | p[0][0], 214 | p[0][1], 215 | p[0][2], 216 | p[1][0], 217 | p[1][1], 218 | p[1][2], 219 | p[2][0], 220 | p[2][1], 221 | p[2][2]); 222 | } 223 | 224 | template <> 225 | inline PyObject* pyObject(const Matrix3Di& p) { 226 | return Py_BuildValue( 227 | "((i,i,i),(i,i,i),(i,i,i))", 228 | p[0][0], 229 | p[0][1], 230 | p[0][2], 231 | p[1][0], 232 | p[1][1], 233 | p[1][2], 234 | p[2][0], 235 | p[2][1], 236 | p[2][2]); 237 | } 238 | 239 | template <> 240 | inline PyObject* pyObject(const Matrix4Df& p) { 241 | return Py_BuildValue( 242 | "((d,d,d,d),(d,d,d,d),(d,d,d,d),(d,d,d,d))", 243 | static_cast(p[0][0]), 244 | static_cast(p[0][1]), 245 | static_cast(p[0][2]), 246 | static_cast(p[0][3]), 247 | static_cast(p[1][0]), 248 | static_cast(p[1][1]), 249 | static_cast(p[1][2]), 250 | static_cast(p[1][3]), 251 | static_cast(p[2][0]), 252 | static_cast(p[2][1]), 253 | static_cast(p[2][2]), 254 | static_cast(p[2][3]), 255 | static_cast(p[3][0]), 256 | static_cast(p[3][1]), 257 | static_cast(p[3][2]), 258 | static_cast(p[3][3])); 259 | } 260 | 261 | template <> 262 | inline PyObject* pyObject(const Matrix4Dd& p) { 263 | return Py_BuildValue( 264 | "((d,d,d,d),(d,d,d,d),(d,d,d,d),(d,d,d,d))", 265 | p[0][0], 266 | p[0][1], 267 | p[0][2], 268 | p[0][3], 269 | p[1][0], 270 | p[1][1], 271 | p[1][2], 272 | p[1][3], 273 | p[2][0], 274 | p[2][1], 275 | p[2][2], 276 | p[2][3], 277 | p[3][0], 278 | p[3][1], 279 | p[3][2], 280 | p[3][3]); 281 | } 282 | 283 | template <> 284 | inline PyObject* pyObject(const Matrix4Di& p) { 285 | return Py_BuildValue( 286 | "((i,i,i,i),(i,i,i,i),(i,i,i,i),(i,i,i,i))", 287 | p[0][0], 288 | p[0][1], 289 | p[0][2], 290 | p[0][3], 291 | p[1][0], 292 | p[1][1], 293 | p[1][2], 294 | p[1][3], 295 | p[2][0], 296 | p[2][1], 297 | p[2][2], 298 | p[2][3], 299 | p[3][0], 300 | p[3][1], 301 | p[3][2], 302 | p[3][3]); 303 | } 304 | 305 | template <> 306 | inline PyObject* pyObject(const string& s) { 307 | return Py_BuildValue("s", s.c_str()); 308 | } 309 | 310 | inline PyObject* pyObject(const char* s) { 311 | return Py_BuildValue("s", s); 312 | } 313 | 314 | #define PYWRAP(val) pyWrap(pyObject(val)) 315 | #define DEF_GETITEM(m, class) \ 316 | m.def("__getitem__", [](class& dict, const std::string& key) { \ 317 | dict.initAttributesMap(); \ 318 | try { \ 319 | return dict.attributesMap.at(key); \ 320 | } catch (const std::out_of_range&) { \ 321 | throw py::key_error("key '" + key + "' does not exist."); \ 322 | } \ 323 | }); 324 | 325 | #define DEF_LEN(m, class) \ 326 | m.def( \ 327 | "__len__", \ 328 | [](class& dict) { \ 329 | dict.initAttributesMap(); \ 330 | return dict.attributesMap.size(); \ 331 | }, \ 332 | py::keep_alive<0, 1>()); 333 | #define DEF_ITER(m, class) \ 334 | m.def( \ 335 | "__iter__", \ 336 | [](class& dict) { \ 337 | dict.initAttributesMap(); \ 338 | return py::make_key_iterator(dict.attributesMap.begin(), dict.attributesMap.end()); \ 339 | }, \ 340 | py::keep_alive<0, 1>()); 341 | #define DEF_ITEM(m, class) \ 342 | m.def( \ 343 | "items", \ 344 | [](class& dict) { \ 345 | dict.initAttributesMap(); \ 346 | return py::make_iterator(dict.attributesMap.begin(), dict.attributesMap.end()); \ 347 | }, \ 348 | py::keep_alive<0, 1>()); 349 | #define DEF_DICT_FUNC(m, class) \ 350 | DEF_GETITEM(m, class) \ 351 | DEF_LEN(m, class) \ 352 | DEF_ITEM(m, class) 353 | 354 | } // namespace pyvrs 355 | -------------------------------------------------------------------------------- /csrc/utils/Utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "Utils.h" 18 | 19 | #include 20 | #include 21 | 22 | #include "../VrsBindings.h" 23 | #include "PyBuffer.h" 24 | #include "PyExceptions.h" 25 | #include "PyFileSpec.h" 26 | #include "PyRecord.h" 27 | 28 | namespace pyvrs { 29 | namespace py = pybind11; 30 | #if IS_VRS_OSS_CODE() 31 | static string recordsChecksum(const string& path, bool showProgress) { 32 | initVrsBindings(); 33 | return vrs::utils::recordsChecksum(path, showProgress); 34 | } 35 | 36 | static string verbatimChecksum(const string& path, bool showProgress) { 37 | initVrsBindings(); 38 | return vrs::utils::verbatimChecksum(path, showProgress); 39 | } 40 | 41 | void pybind_utils(py::module& m) { 42 | py::enum_(m, "CompressionPreset") 43 | .value("NONE", vrs::CompressionPreset::None) 44 | .value("LZ4_FAST", vrs::CompressionPreset::Lz4Fast) 45 | .value("LZ4_TIGHT", vrs::CompressionPreset::Lz4Tight) 46 | .value("ZSTD_FAST", vrs::CompressionPreset::ZstdFast) 47 | .value("ZSTD_LIGHT", vrs::CompressionPreset::ZstdLight) 48 | .value("ZSTD_MEDIUM", vrs::CompressionPreset::ZstdMedium) 49 | .value("ZSTD_TIGHT", vrs::CompressionPreset::ZstdTight) 50 | .value("ZSTD_MAX", vrs::CompressionPreset::ZstdMax) 51 | .value("DEFAULT", vrs::CompressionPreset::Default) 52 | .export_values(); 53 | m.def("records_checksum", &recordsChecksum, "Calculate a VRS file's logical checksum"); 54 | m.def("verbatim_checksum", &verbatimChecksum, "Calculate a file's checksum"); 55 | pybind_exception(m); 56 | pybind_record(m); 57 | pybind_buffer(m); 58 | pybind_filespec(m); 59 | } 60 | #endif 61 | } // namespace pyvrs 62 | -------------------------------------------------------------------------------- /csrc/utils/Utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace pyvrs { 22 | namespace py = pybind11; 23 | 24 | /// Binds methods and classes under utils/ directory. 25 | void pybind_utils(py::module& m); 26 | 27 | } // namespace pyvrs 28 | -------------------------------------------------------------------------------- /csrc/writer/PyDataPiece.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | namespace pyvrs { 28 | 29 | namespace py = pybind11; 30 | using namespace vrs; 31 | 32 | class DataPieceWrapper { 33 | public: 34 | virtual ~DataPieceWrapper() = default; 35 | }; 36 | 37 | // We should treat DataPieceString separatedly from other DataPiece types. 38 | class DataPieceStringWrapper : public DataPieceWrapper { 39 | public: 40 | void set(std::string& v) { 41 | dpv_->stage(v); 42 | } 43 | void setDataPiece(DataPieceString* dpv) { 44 | dpv_ = dpv; 45 | } 46 | 47 | private: 48 | DataPieceString* dpv_; 49 | }; 50 | 51 | #define DEFINE_DATA_PIECE_VALUE_WRAPPER(TEMPLATE_TYPE) \ 52 | class DataPieceValue##TEMPLATE_TYPE##Wrapper : public DataPieceWrapper { \ 53 | public: \ 54 | void set(TEMPLATE_TYPE& v) { \ 55 | dpv_->set(v); \ 56 | } \ 57 | void setDataPiece(DataPieceValue* dpv) { \ 58 | dpv_ = dpv; \ 59 | } \ 60 | \ 61 | private: \ 62 | DataPieceValue* dpv_; \ 63 | }; 64 | 65 | #define DEFINE_DATA_PIECE_VECTOR_WRAPPER(TEMPLATE_TYPE) \ 66 | class DataPieceVector##TEMPLATE_TYPE##Wrapper : public DataPieceWrapper { \ 67 | public: \ 68 | void set(std::vector& v) { \ 69 | dpv_->stage(v); \ 70 | } \ 71 | void setDataPiece(DataPieceVector* dpv) { \ 72 | dpv_ = dpv; \ 73 | } \ 74 | \ 75 | private: \ 76 | DataPieceVector* dpv_; \ 77 | }; 78 | 79 | #define DEFINE_DATA_PIECE_ARRAY_WRAPPER(TEMPLATE_TYPE) \ 80 | class DataPieceArray##TEMPLATE_TYPE##Wrapper : public DataPieceWrapper { \ 81 | public: \ 82 | void set(std::vector& v) { \ 83 | if (v.size() > dpv_->getArraySize()) { \ 84 | throw py::value_error("Given array does not fit in target field " + dpv_->getLabel()); \ 85 | } \ 86 | dpv_->set(v); \ 87 | } \ 88 | void setDataPiece(DataPieceArray* dpv) { \ 89 | dpv_ = dpv; \ 90 | } \ 91 | \ 92 | private: \ 93 | DataPieceArray* dpv_; \ 94 | }; 95 | 96 | #define DEFINE_DATA_PIECE_MAP_WRAPPER(TEMPLATE_TYPE) \ 97 | class DataPieceStringMap##TEMPLATE_TYPE##Wrapper : public DataPieceWrapper { \ 98 | public: \ 99 | void set(std::map& v) { \ 100 | dpv_->stage(v); \ 101 | } \ 102 | void setDataPiece(DataPieceStringMap* dpv) { \ 103 | dpv_ = dpv; \ 104 | } \ 105 | \ 106 | private: \ 107 | DataPieceStringMap* dpv_; \ 108 | }; 109 | 110 | #define DEFINE_ALL_DATA_PIECE_WRAPPER(TEMPLATE_TYPE) \ 111 | DEFINE_DATA_PIECE_VALUE_WRAPPER(TEMPLATE_TYPE) \ 112 | DEFINE_DATA_PIECE_VECTOR_WRAPPER(TEMPLATE_TYPE) \ 113 | DEFINE_DATA_PIECE_ARRAY_WRAPPER(TEMPLATE_TYPE) \ 114 | DEFINE_DATA_PIECE_MAP_WRAPPER(TEMPLATE_TYPE) 115 | 116 | // Define DataPieceWrapper classes 117 | // Define & generate the code for each POD type supported. 118 | #define POD_MACRO DEFINE_ALL_DATA_PIECE_WRAPPER 119 | #include 120 | 121 | DEFINE_DATA_PIECE_MAP_WRAPPER(string) 122 | DEFINE_DATA_PIECE_VECTOR_WRAPPER(string) 123 | 124 | } // namespace pyvrs 125 | -------------------------------------------------------------------------------- /csrc/writer/PyRecordable.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // This include *must* be before any STL include! See Python C API doc. 18 | #define PY_SSIZE_T_CLEAN 19 | #include // IWYU pragma: keepo 20 | 21 | // Includes needed for bindings (including marshalling STL containers) 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #define DEFAULT_LOG_CHANNEL "PyRecordable" 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | #include "PyRecordable.h" 34 | #include "VRSWriter.h" 35 | 36 | namespace py = pybind11; 37 | 38 | namespace pyvrs { 39 | 40 | bool PyRecordable::addRecordFormat(const PyRecordFormat* recordFormat) { 41 | return Recordable::addRecordFormat( 42 | recordFormat->getRecordType(), 43 | recordFormat->getFormatVersion(), 44 | recordFormat->getRecordFormat(), 45 | recordFormat->getDataLayouts()); 46 | } 47 | 48 | DataSource PyRecordFormat::getDataSource( 49 | const DataSourceChunk& src1, 50 | const DataSourceChunk& src2, 51 | const DataSourceChunk& src3) const { 52 | if (dataLayouts_.size() == 1) { 53 | return DataSource(*dataLayouts_[0], src1, src2, src3); 54 | } 55 | if (dataLayouts_.size() == 2) { 56 | return DataSource(*dataLayouts_[0], *dataLayouts_[1], src1, src2, src3); 57 | } 58 | return DataSource(src1, src2, src3); 59 | } 60 | 61 | const std::vector PyRecordFormat::getDataLayouts() const { 62 | std::vector dataLayouts = {}; 63 | for (auto& dataLayout : dataLayouts_) { 64 | dataLayouts.emplace_back(dataLayout.get()); 65 | } 66 | return dataLayouts; 67 | } 68 | 69 | RecordFormat PyRecordFormat::getRecordFormat() const { 70 | RecordFormat format; 71 | // default behavior of DataSource::copyTo is to copy all dataLayouts buffer first, 72 | // then copy other buffers. To make things simple, we will use the same order here. 73 | for (auto& dataLayout : dataLayouts_) { 74 | format = format + dataLayout->getContentBlock(); 75 | } 76 | for (auto& contentBlocks : additionalContentBlocks_) { 77 | for (ContentBlock contentBlock : contentBlocks) { 78 | format = format + contentBlock; 79 | } 80 | } 81 | return format; 82 | } 83 | 84 | #define ADD_DATA_PIECE_WRAPPER_SUPPORT(DATA_PIECE_TYPE, TEMPLATE_TYPE) \ 85 | if (piece->getTypeName() == "DataPiece" #DATA_PIECE_TYPE "<" #TEMPLATE_TYPE ">") { \ 86 | std::unique_ptr ptr = \ 87 | std::make_unique(); \ 88 | ptr->setDataPiece(reinterpret_cast*>(piece)); \ 89 | dataPieceMaps[i][piece->getLabel()] = std::move(ptr); \ 90 | } 91 | 92 | #define ADD_ALL_DATA_PIECE_WRAPPER_SUPPORT(TEMPLATE_TYPE) \ 93 | ADD_DATA_PIECE_WRAPPER_SUPPORT(Value, TEMPLATE_TYPE) \ 94 | ADD_DATA_PIECE_WRAPPER_SUPPORT(Array, TEMPLATE_TYPE) \ 95 | ADD_DATA_PIECE_WRAPPER_SUPPORT(Vector, TEMPLATE_TYPE) \ 96 | ADD_DATA_PIECE_WRAPPER_SUPPORT(StringMap, TEMPLATE_TYPE) 97 | 98 | std::vector>> 99 | PyRecordFormat::getMembers() { 100 | std::vector>> dataPieceMaps( 101 | dataLayouts_.size()); 102 | for (size_t i = 0; i < dataLayouts_.size(); i++) { 103 | dataLayouts_[i]->forEachDataPiece([&dataPieceMaps, &i](vrs::DataPiece* piece) { 104 | // Define & generate the code for each POD type supported. 105 | #define POD_MACRO ADD_ALL_DATA_PIECE_WRAPPER_SUPPORT 106 | #include 107 | ADD_DATA_PIECE_WRAPPER_SUPPORT(Vector, string) 108 | ADD_DATA_PIECE_WRAPPER_SUPPORT(StringMap, string) 109 | 110 | if (piece->getTypeName() == "DataPieceString") { 111 | std::unique_ptr ptr = 112 | std::make_unique(); 113 | ptr->setDataPiece(reinterpret_cast(piece)); 114 | dataPieceMaps[i][piece->getLabel()] = std::move(ptr); 115 | } 116 | }); 117 | } 118 | 119 | return dataPieceMaps; 120 | } 121 | 122 | void PyStream::init( 123 | RecordableTypeId typeId, 124 | const string& deviceFlavor, 125 | std::unique_ptr&& configurationRecordFormat, 126 | std::unique_ptr&& dataRecordFormat, 127 | std::unique_ptr&& stateRecordFormat) { 128 | recordable_ = std::make_unique(typeId, deviceFlavor); 129 | if (configurationRecordFormat != nullptr) { 130 | recordFormatMap_.insert( 131 | std::make_pair(Record::Type::CONFIGURATION, std::move(configurationRecordFormat))); 132 | } else { 133 | recordFormatMap_.insert(std::make_pair( 134 | Record::Type::CONFIGURATION, 135 | std::make_unique(Record::Type::CONFIGURATION))); 136 | } 137 | if (dataRecordFormat != nullptr) { 138 | recordFormatMap_.insert(std::make_pair(Record::Type::DATA, std::move(dataRecordFormat))); 139 | } else { 140 | recordFormatMap_.insert( 141 | std::make_pair(Record::Type::DATA, std::make_unique(Record::Type::DATA))); 142 | } 143 | if (stateRecordFormat != nullptr) { 144 | recordFormatMap_.insert(std::make_pair(Record::Type::STATE, std::move(stateRecordFormat))); 145 | } else { 146 | recordFormatMap_.insert( 147 | std::make_pair(Record::Type::STATE, std::make_unique(Record::Type::STATE))); 148 | } 149 | } 150 | 151 | PyStream::PyStream(PyStream&& other) { 152 | other.recordable_ = std::move(recordable_); 153 | for (auto& recordFormat : recordFormatMap_) { 154 | other.recordFormatMap_.insert( 155 | std::make_pair(recordFormat.first, std::move(recordFormat.second))); 156 | } 157 | } 158 | 159 | PyRecordFormat* PyStream::createRecordFormat(Record::Type recordType) { 160 | PyRecordFormat* recordFormat = nullptr; 161 | auto iter = recordFormatMap_.find(recordType); 162 | if (iter != recordFormatMap_.end()) { 163 | recordFormat = iter->second.get(); 164 | recordable_->addRecordFormat(recordFormat); 165 | return recordFormat; 166 | } 167 | return recordFormat; 168 | } 169 | 170 | const Record* PyStream::createRecord(double timestamp, const PyRecordFormat* recordFormat) { 171 | return recordable_->createRecord( 172 | timestamp, 173 | recordFormat->getRecordType(), 174 | recordFormat->getFormatVersion(), 175 | recordFormat->getDataSource()); 176 | } 177 | 178 | const Record* 179 | PyStream::createRecord(double timestamp, const PyRecordFormat* recordFormat, py::array buffer) { 180 | py::buffer_info info = buffer.request(); 181 | size_t size = info.itemsize; 182 | for (py::ssize_t i = 0; i < info.ndim; i++) { 183 | size *= info.shape[i]; 184 | } 185 | return recordable_->createRecord( 186 | timestamp, 187 | recordFormat->getRecordType(), 188 | recordFormat->getFormatVersion(), 189 | recordFormat->getDataSource(DataSourceChunk(info.ptr, size))); 190 | } 191 | 192 | const Record* PyStream::createRecord( 193 | double timestamp, 194 | const PyRecordFormat* recordFormat, 195 | py::array buffer, 196 | py::array buffer2) { 197 | py::buffer_info info = buffer.request(); 198 | py::buffer_info info2 = buffer2.request(); 199 | size_t size = info.itemsize; 200 | for (py::ssize_t i = 0; i < info.ndim; i++) { 201 | size *= info.shape[i]; 202 | } 203 | size_t size2 = info2.itemsize; 204 | for (py::ssize_t i = 0; i < info2.ndim; i++) { 205 | size2 *= info2.shape[i]; 206 | } 207 | return recordable_->createRecord( 208 | timestamp, 209 | recordFormat->getRecordType(), 210 | recordFormat->getFormatVersion(), 211 | recordFormat->getDataSource( 212 | DataSourceChunk(info.ptr, size), DataSourceChunk(info2.ptr, size2))); 213 | } 214 | 215 | const Record* PyStream::createRecord( 216 | double timestamp, 217 | const PyRecordFormat* recordFormat, 218 | py::array buffer, 219 | py::array buffer2, 220 | py::array buffer3) { 221 | py::buffer_info info = buffer.request(); 222 | py::buffer_info info2 = buffer2.request(); 223 | py::buffer_info info3 = buffer3.request(); 224 | size_t size = info.itemsize; 225 | for (py::ssize_t i = 0; i < info.ndim; i++) { 226 | size *= info.shape[i]; 227 | } 228 | size_t size2 = info2.itemsize; 229 | for (py::ssize_t i = 0; i < info2.ndim; i++) { 230 | size2 *= info2.shape[i]; 231 | } 232 | size_t size3 = info3.itemsize; 233 | for (py::ssize_t i = 0; i < info3.ndim; i++) { 234 | size3 *= info3.shape[i]; 235 | } 236 | return recordable_->createRecord( 237 | timestamp, 238 | recordFormat->getRecordType(), 239 | recordFormat->getFormatVersion(), 240 | recordFormat->getDataSource( 241 | DataSourceChunk(info.ptr, size), 242 | DataSourceChunk(info2.ptr, size2), 243 | DataSourceChunk(info3.ptr, size3))); 244 | } 245 | 246 | void PyStream::setCompression(CompressionPreset preset) { 247 | recordable_->setCompression(preset); 248 | } 249 | 250 | void PyStream::setTag(const std::string& tagName, const std::string& tagValue) { 251 | recordable_->setTag(tagName, tagValue); 252 | } 253 | 254 | std::string PyStream::getStreamID() { 255 | return recordable_->getStreamId().getNumericName(); 256 | } 257 | 258 | std::vector PyRecordFormat::getJsonDataLayouts() const { 259 | std::vector v; 260 | for (auto& dataLayout : dataLayouts_) { 261 | v.push_back(dataLayout->asJson()); 262 | } 263 | 264 | return v; 265 | } 266 | 267 | } // namespace pyvrs 268 | -------------------------------------------------------------------------------- /csrc/writer/PyRecordable.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | 30 | #include "VRSWriter.h" 31 | 32 | namespace pyvrs { 33 | 34 | class PyStream; 35 | 36 | namespace py = pybind11; 37 | using namespace vrs; 38 | 39 | class PyRecordFormat { 40 | public: 41 | ~PyRecordFormat() = default; 42 | PyRecordFormat() {} 43 | PyRecordFormat( 44 | const Record::Type& recordType, 45 | uint32_t formatVersion = 0, 46 | std::unique_ptr&& dataLayout = nullptr) 47 | : recordType_(recordType), formatVersion_(formatVersion) { 48 | if (dataLayout) { 49 | dataLayouts_.push_back(std::move(dataLayout)); 50 | } 51 | } 52 | 53 | PyRecordFormat( 54 | const Record::Type& recordType, 55 | uint32_t formatVersion, 56 | std::unique_ptr&& dataLayout, 57 | const std::vector& additionalContentBlocks) 58 | : recordType_(recordType), formatVersion_(formatVersion) { 59 | if (dataLayout) { 60 | dataLayouts_.push_back(std::move(dataLayout)); 61 | } 62 | additionalContentBlocks_.push_back(additionalContentBlocks); 63 | } 64 | 65 | PyRecordFormat( 66 | const Record::Type& recordType, 67 | uint32_t formatVersion, 68 | std::unique_ptr&& dataLayout1, 69 | std::unique_ptr&& dataLayout2, 70 | const std::vector& additionalContentBlocks1, 71 | const std::vector& additionalContentBlocks2) 72 | : recordType_(recordType), formatVersion_(formatVersion) { 73 | if (dataLayout1) { 74 | dataLayouts_.push_back(std::move(dataLayout1)); 75 | } 76 | if (dataLayout2) { 77 | dataLayouts_.push_back(std::move(dataLayout2)); 78 | } 79 | additionalContentBlocks_.push_back(additionalContentBlocks1); 80 | additionalContentBlocks_.push_back(additionalContentBlocks2); 81 | } 82 | 83 | PyRecordFormat(PyRecordFormat&& other) { 84 | dataLayouts_.swap(other.dataLayouts_); 85 | additionalContentBlocks_ = other.additionalContentBlocks_; 86 | recordType_ = other.recordType_; 87 | formatVersion_ = other.formatVersion_; 88 | } 89 | 90 | const std::vector getDataLayouts() const; 91 | 92 | Record::Type getRecordType() const { 93 | return recordType_; 94 | } 95 | 96 | uint32_t getFormatVersion() const { 97 | return formatVersion_; 98 | } 99 | 100 | DataSource getDataSource( 101 | const DataSourceChunk& src1 = {}, 102 | const DataSourceChunk& src2 = {}, 103 | const DataSourceChunk& src3 = {}) const; 104 | 105 | RecordFormat getRecordFormat() const; 106 | 107 | std::vector getJsonDataLayouts() const; 108 | 109 | std::vector>> getMembers(); 110 | 111 | private: 112 | std::vector> dataLayouts_; 113 | std::vector> additionalContentBlocks_; 114 | Record::Type recordType_; 115 | uint32_t formatVersion_; 116 | }; 117 | 118 | class PyRecordable : public Recordable { 119 | public: 120 | using Recordable::createRecord; 121 | PyRecordable(RecordableTypeId typeId, const string& deviceFlavor) 122 | : Recordable(typeId, deviceFlavor) {} 123 | 124 | bool addRecordFormat(const PyRecordFormat* recordFormat); 125 | 126 | const Record* createConfigurationRecord() override { 127 | // PYBIND11_OVERLOAD_PURE(const Record*, Recordable, createConfigurationRecord, ); 128 | return NULL; 129 | } 130 | 131 | const Record* createStateRecord() override { 132 | // PYBIND11_OVERLOAD_PURE(const Record*, Recordable, createStateRecord, ); 133 | return NULL; 134 | } 135 | }; 136 | 137 | class PyStream { 138 | public: 139 | ~PyStream(){}; 140 | 141 | PyStream( 142 | RecordableTypeId typeId, 143 | std::unique_ptr&& configurationRecordFormat = nullptr, 144 | std::unique_ptr&& dataRecordFormat = nullptr, 145 | std::unique_ptr&& stateRecordFormat = nullptr) { 146 | init( 147 | typeId, 148 | {}, 149 | std::move(configurationRecordFormat), 150 | std::move(dataRecordFormat), 151 | std::move(stateRecordFormat)); 152 | } 153 | PyStream( 154 | RecordableTypeId typeId, 155 | const string& deviceFlavor, 156 | std::unique_ptr&& configurationRecordFormat = nullptr, 157 | std::unique_ptr&& dataRecordFormat = nullptr, 158 | std::unique_ptr&& stateRecordFormat = nullptr) { 159 | init( 160 | typeId, 161 | deviceFlavor, 162 | std::move(configurationRecordFormat), 163 | std::move(dataRecordFormat), 164 | std::move(stateRecordFormat)); 165 | } 166 | 167 | PyStream(PyStream&& other); 168 | 169 | PyRecordable* getRecordable() { 170 | return recordable_.get(); 171 | } 172 | 173 | PyRecordFormat* createRecordFormat(Record::Type recordType); 174 | 175 | const Record* createRecord(double timestamp, const PyRecordFormat* recordFormat); 176 | 177 | const Record* 178 | createRecord(double timestamp, const PyRecordFormat* recordFormat, py::array buffer); 179 | 180 | const Record* createRecord( 181 | double timestamp, 182 | const PyRecordFormat* recordFormat, 183 | py::array buffer, 184 | py::array buffer2); 185 | 186 | const Record* createRecord( 187 | double timestamp, 188 | const PyRecordFormat* recordFormat, 189 | py::array buffer, 190 | py::array buffer2, 191 | py::array buffer3); 192 | 193 | void setCompression(CompressionPreset preset); 194 | 195 | void setTag(const std::string& tagName, const std::string& tagValue); 196 | 197 | std::string getStreamID(); 198 | 199 | private: 200 | void init( 201 | RecordableTypeId typeId, 202 | const string& deviceFlavor, 203 | std::unique_ptr&& configurationRecordFormat, 204 | std::unique_ptr&& dataRecordFormat, 205 | std::unique_ptr&& stateRecordFormat); 206 | 207 | std::unique_ptr recordable_; 208 | std::map> recordFormatMap_; 209 | }; 210 | 211 | } // namespace pyvrs 212 | -------------------------------------------------------------------------------- /csrc/writer/StreamFactory.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "StreamFactory.h" 18 | 19 | #include 20 | 21 | namespace pyvrs { 22 | 23 | StreamFactory& StreamFactory::getInstance() { 24 | static StreamFactory instance; 25 | return instance; 26 | } 27 | 28 | void StreamFactory::registerStreamCreationFunction( 29 | const std::string& name, 30 | std::function()> func) { 31 | streamCreationFunctionMap_[name] = func; 32 | } 33 | 34 | std::unique_ptr StreamFactory::createStream(const std::string& name) { 35 | auto func = streamCreationFunctionMap_.find(name); 36 | if (func != streamCreationFunctionMap_.end()) { 37 | return func->second(); 38 | } 39 | return std::unique_ptr(nullptr); 40 | } 41 | 42 | void StreamFactory::registerFlavoredStreamCreationFunction( 43 | const std::string& name, 44 | FlavoredStreamCreationFunc func) { 45 | flavoredStreamCreationFunctionMap_[name] = func; 46 | } 47 | 48 | std::unique_ptr StreamFactory::createFlavoredStream( 49 | const std::string& name, 50 | const std::string& flavor) { 51 | auto func = flavoredStreamCreationFunctionMap_.find(name); 52 | if (func != flavoredStreamCreationFunctionMap_.end()) { 53 | return func->second(flavor); 54 | } 55 | return std::unique_ptr(nullptr); 56 | } 57 | 58 | } // namespace pyvrs 59 | -------------------------------------------------------------------------------- /csrc/writer/StreamFactory.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "PyRecordable.h" 22 | 23 | namespace pyvrs { 24 | 25 | using StreamCreationFunc = std::function()>; 26 | using FlavoredStreamCreationFunc = std::function(const string& flavor)>; 27 | 28 | class StreamFactory { 29 | public: 30 | static StreamFactory& getInstance(); 31 | 32 | void registerStreamCreationFunction(const std::string& name, StreamCreationFunc func); 33 | std::unique_ptr createStream(const std::string& name); 34 | 35 | void registerFlavoredStreamCreationFunction( 36 | const std::string& name, 37 | FlavoredStreamCreationFunc func); 38 | std::unique_ptr createFlavoredStream( 39 | const std::string& name, 40 | const std::string& flavor); 41 | 42 | protected: 43 | StreamFactory() = default; 44 | virtual ~StreamFactory() = default; 45 | 46 | private: 47 | std::map streamCreationFunctionMap_ = {}; 48 | std::map flavoredStreamCreationFunctionMap_ = {}; 49 | }; 50 | 51 | } // namespace pyvrs 52 | -------------------------------------------------------------------------------- /csrc/writer/VRSWriter.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // This include *must* be before any STL include! See Python C API doc. 18 | #define PY_SSIZE_T_CLEAN 19 | #include // IWYU pragma: keepo 20 | 21 | #include "VRSWriter.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | // Includes needed for bindings (including marshalling STL containers) 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #define DEFAULT_LOG_CHANNEL "VRSWriter" 36 | #include 37 | 38 | #include "../VrsBindings.h" 39 | #include "../utils/PyUtils.h" 40 | #include "StreamFactory.h" 41 | 42 | // Open source DataLayout definitions 43 | #include "datalayouts/SampleDataLayout.h" 44 | 45 | namespace py = pybind11; 46 | 47 | using PyW = pyvrs::VRSWriter; 48 | 49 | namespace pyvrs { 50 | 51 | VRSWriter::VRSWriter() { 52 | init(); 53 | } 54 | 55 | VRSWriter::~VRSWriter() { 56 | close(); 57 | } 58 | 59 | void VRSWriter::init() { 60 | initVrsBindings(); 61 | writer_.setCompressionThreadPoolSize(std::thread::hardware_concurrency()); 62 | writer_.trackBackgroundThreadQueueByteSize(); 63 | 64 | /// Register open source stream writers (begin) 65 | StreamFactory::getInstance().registerStreamCreationFunction( 66 | "sample", []() { return createSampleStream(); }); 67 | StreamFactory::getInstance().registerStreamCreationFunction( 68 | "sample_with_flavor_1", []() { return createSampleStream("flavor_1"); }); 69 | StreamFactory::getInstance().registerStreamCreationFunction( 70 | "sample_with_flavor_2", []() { return createSampleStream("flavor_2"); }); 71 | StreamFactory::getInstance().registerFlavoredStreamCreationFunction( 72 | "flavored_sample", [](const string& flavor) { return createSampleStream(flavor); }); 73 | StreamFactory::getInstance().registerStreamCreationFunction( 74 | "sample_with_image", createSampleStreamWithImage); 75 | StreamFactory::getInstance().registerStreamCreationFunction( 76 | "sample_with_multiple_data_layout", createSampleStreamWithMultipleDataLayout); 77 | /// Register open source stream writers (end) 78 | 79 | #if IS_VRS_FB_INTERNAL() 80 | registerFbOnlyStreamWriters(); 81 | #endif 82 | } 83 | 84 | void VRSWriter::resetNewInstanceIds() { 85 | Recordable::resetNewInstanceIds(); 86 | } 87 | 88 | int VRSWriter::create(const std::string& filePath) { 89 | return writer_.createFileAsync(filePath); 90 | } 91 | 92 | PyStream* VRSWriter::createStream(const std::string& name) { 93 | streams_.emplace_back(StreamFactory::getInstance().createStream(name)); 94 | auto stream = streams_.back().get(); 95 | if (stream == nullptr) { 96 | streams_.pop_back(); 97 | throw py::value_error("Unsupported stream name " + name); 98 | } 99 | writer_.addRecordable(stream->getRecordable()); 100 | return stream; 101 | } 102 | 103 | PyStream* VRSWriter::createFlavoredStream(const std::string& name, const std::string& flavor) { 104 | streams_.emplace_back(StreamFactory::getInstance().createFlavoredStream(name, flavor)); 105 | auto stream = streams_.back().get(); 106 | if (stream == nullptr) { 107 | streams_.pop_back(); 108 | throw py::value_error("Unsupported stream name " + name); 109 | } 110 | writer_.addRecordable(stream->getRecordable()); 111 | return stream; 112 | } 113 | 114 | void VRSWriter::setTag(const std::string& tagName, const std::string& tagValue) { 115 | writer_.setTag(tagName, tagValue); 116 | } 117 | 118 | int VRSWriter::writeRecords(double maxTimestamp) { 119 | return writer_.writeRecordsAsync(maxTimestamp); 120 | } 121 | 122 | uint64_t VRSWriter::getBackgroundThreadQueueByteSize() { 123 | return writer_.getBackgroundThreadQueueByteSize(); 124 | } 125 | 126 | int VRSWriter::close() { 127 | return writer_.waitForFileClosed(); 128 | } 129 | 130 | } // namespace pyvrs 131 | -------------------------------------------------------------------------------- /csrc/writer/VRSWriter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #if IS_VRS_FB_INTERNAL() 28 | #include "VRSWriter_headers_fb.hpp" 29 | #endif 30 | 31 | #include 32 | #include 33 | #include 34 | 35 | #include "PyDataPiece.h" 36 | 37 | namespace pyvrs { 38 | 39 | namespace py = pybind11; 40 | using namespace vrs; 41 | 42 | class PyStream; 43 | 44 | /// @brief The VRSWriter class 45 | /// This class is a VRS file writer, optimized for Python bindings. 46 | class VRSWriter { 47 | public: 48 | VRSWriter(); 49 | ~VRSWriter(); 50 | 51 | void init(); 52 | 53 | /// Recordable instance ids are automatically assigned when Recordable objects are created. 54 | /// This guarantees that each Recordable gets a unique ID. 55 | /// WARNING! If your code relies on specific instance IDs, your design is weak, and you are 56 | /// setting up your project for a world of pain in the future. 57 | /// Use flavors and tag pairs to identify your streams instead. 58 | /// However, when many files are generated successively, it can lead to high instance 59 | /// id values, which can be confusing, and even problematic for unit tests. 60 | /// Use this API to reset the instance counters for each device type, so that the next devices 61 | /// will get an instance id of 1. 62 | /// ATTENTION! if you call this API at the wrong time, you can end up with multiple devices with 63 | /// the same id, and end up in a messy situation. Avoid this API if you can! 64 | void resetNewInstanceIds(); 65 | 66 | int create(const std::string& filePath); 67 | 68 | PyStream* createStream(const std::string& name); 69 | PyStream* createFlavoredStream(const std::string& name, const std::string& flavor); 70 | 71 | void setTag(const std::string& tagName, const std::string& tagValue); 72 | 73 | void addRecordable(Recordable* recordable); 74 | 75 | int writeRecords(double maxTimestamp); 76 | 77 | uint64_t getBackgroundThreadQueueByteSize(); 78 | 79 | int close(); 80 | 81 | #if IS_VRS_FB_INTERNAL() 82 | #include "VRSWriter_methods_fb.hpp" 83 | #endif 84 | 85 | private: 86 | RecordFileWriter writer_; 87 | std::vector> streams_; 88 | }; 89 | 90 | } // namespace pyvrs 91 | -------------------------------------------------------------------------------- /csrc/writer/Writer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // This include *must* be before any STL include! See Python C API doc. 18 | #include 19 | #include 20 | #define PY_SSIZE_T_CLEAN 21 | #include 22 | 23 | #include "Writer.h" 24 | 25 | #include 26 | #include 27 | 28 | // Includes needed for bindings (including marshalling STL containers) 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | 37 | #include "PyRecordable.h" 38 | #include "VRSWriter.h" 39 | 40 | namespace pyvrs { 41 | namespace py = pybind11; 42 | using namespace std; 43 | using namespace vrs; 44 | 45 | #define DEFINE_ALL_DATA_PIECE_PYBIND(TEMPLATE_TYPE) \ 46 | DEFINE_DATA_PIECE_PYBIND(Value, TEMPLATE_TYPE) \ 47 | DEFINE_DATA_PIECE_PYBIND(Array, TEMPLATE_TYPE) \ 48 | DEFINE_DATA_PIECE_PYBIND(Vector, TEMPLATE_TYPE) \ 49 | DEFINE_DATA_PIECE_PYBIND(StringMap, TEMPLATE_TYPE) 50 | 51 | #define DEFINE_DATA_PIECE_PYBIND(DATAPIECE_TYPE, TEMPLATE_TYPE) \ 52 | py::class_( \ 53 | m, "DataPiece" #DATAPIECE_TYPE #TEMPLATE_TYPE "Wrapper") \ 54 | .def(py::init<>()) \ 55 | .def("set", &pyvrs::DataPiece##DATAPIECE_TYPE##TEMPLATE_TYPE##Wrapper::set); 56 | 57 | #define DEFINE_MATRIX_DATA_PIECE_PYBIND(DATAPIECE_TYPE, MATRIX_TYPE, MATRIX_SIZE) \ 58 | py::class_(m, #DATAPIECE_TYPE) \ 59 | .def(py::init([](const std::vector>& mat) { \ 60 | vrs::DATAPIECE_TYPE matrix; \ 61 | if (mat.size() != MATRIX_SIZE) { \ 62 | throw py::value_error("Matrix size must be " #MATRIX_SIZE "x" #MATRIX_SIZE); \ 63 | } \ 64 | for (int i = 0; i < mat.size(); i++) { \ 65 | if (mat[i].size() != MATRIX_SIZE) { \ 66 | throw py::value_error("Matrix size must be " #MATRIX_SIZE "x" #MATRIX_SIZE); \ 67 | } \ 68 | for (int j = 0; j < mat[i].size(); j++) { \ 69 | matrix[i][j] = mat[i][j]; \ 70 | } \ 71 | } \ 72 | return matrix; \ 73 | })); 74 | 75 | void pybind_writer(py::module& m) { 76 | py::class_>( 77 | m, "RecordFormat") 78 | .def("getMembers", &pyvrs::PyRecordFormat::getMembers) 79 | .def("getJsonDataLayouts", &pyvrs::PyRecordFormat::getJsonDataLayouts); 80 | 81 | py::class_>(m, "Stream") 82 | .def("createRecordFormat", &pyvrs::PyStream::createRecordFormat) 83 | .def( 84 | "createRecord", 85 | py::overload_cast(&pyvrs::PyStream::createRecord)) 86 | .def( 87 | "createRecord", 88 | py::overload_cast( 89 | &pyvrs::PyStream::createRecord)) 90 | .def( 91 | "createRecord", 92 | py::overload_cast( 93 | &pyvrs::PyStream::createRecord)) 94 | .def( 95 | "createRecord", 96 | py::overload_cast( 97 | &pyvrs::PyStream::createRecord)) 98 | .def("setCompression", &pyvrs::PyStream::setCompression) 99 | .def("setTag", &pyvrs::PyStream::setTag) 100 | .def("getStreamID", &pyvrs::PyStream::getStreamID); 101 | 102 | py::class_(m, "Writer") 103 | .def(py::init<>()) 104 | .def("resetNewInstanceIds", &pyvrs::VRSWriter::resetNewInstanceIds) 105 | .def("create", py::overload_cast(&pyvrs::VRSWriter::create)) 106 | .def("createStream", &pyvrs::VRSWriter::createStream, py::return_value_policy::reference) 107 | .def( 108 | "createFlavoredStream", 109 | &pyvrs::VRSWriter::createFlavoredStream, 110 | py::return_value_policy::reference) 111 | .def("setTag", &pyvrs::VRSWriter::setTag) 112 | .def("writeRecords", &pyvrs::VRSWriter::writeRecords) 113 | .def("getBackgroundThreadQueueByteSize", &pyvrs::VRSWriter::getBackgroundThreadQueueByteSize) 114 | .def("close", &pyvrs::VRSWriter::close) 115 | #if IS_VRS_FB_INTERNAL() 116 | #include "Writer_fb.hpp" 117 | #endif 118 | ; 119 | 120 | py::class_(m, "DataPieceWrapper").def(py::init<>()); 121 | 122 | py::class_(m, "DataPieceStringWrapper") 123 | .def(py::init<>()) 124 | .def("set", &pyvrs::DataPieceStringWrapper::set); 125 | 126 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix2Dd, double, 2) 127 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix2Df, float, 2) 128 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix2Di, int, 2) 129 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix3Dd, double, 3) 130 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix3Df, float, 3) 131 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix3Di, int, 3) 132 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix4Dd, double, 4) 133 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix4Df, float, 4) 134 | DEFINE_MATRIX_DATA_PIECE_PYBIND(Matrix4Di, int, 4) 135 | 136 | // Define & generate the code for each POD type supported. 137 | #define POD_MACRO DEFINE_ALL_DATA_PIECE_PYBIND 138 | #include 139 | 140 | DEFINE_DATA_PIECE_PYBIND(Vector, string) 141 | DEFINE_DATA_PIECE_PYBIND(StringMap, string) 142 | } 143 | } // namespace pyvrs 144 | -------------------------------------------------------------------------------- /csrc/writer/Writer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace pyvrs { 22 | namespace py = pybind11; 23 | 24 | void pybind_writer(py::module& m); 25 | 26 | } // namespace pyvrs 27 | -------------------------------------------------------------------------------- /csrc/writer/datalayouts/SampleDataLayout.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "SampleDataLayout.h" 18 | 19 | #include 20 | 21 | #include "../PyRecordable.h" 22 | 23 | namespace pyvrs { 24 | 25 | /* 26 | * Terms: 27 | * - PyStream: Wrapper class of Recordable that represents stream. 28 | * This class can contain 3 RecordFormats, Configuration, Data and State RecordFormat. 29 | * ex. Camera stream, imu stream. 30 | * - PyRecordFormat: Wrapper class of RecordFormat. 31 | * You should add your own DataLayouts and ContentBlocks to define how each record 32 | * will look like when you write it to file. 33 | * 34 | * This sample code demonstrates how to set up PyRecordFormat and PyStream. 35 | */ 36 | 37 | /* 38 | * Use this template to create a stream with records containing only metadata (ex. imu stream). 39 | * - Configuration RecordFormat: 1 DataLayout 40 | * - Data RecordFormat: 1 DataLayout 41 | */ 42 | std::unique_ptr createSampleStream(const string& flavor) { 43 | // RecordFormat for configuration record. 44 | auto configurationRecordFormat = std::make_unique( 45 | Record::Type::CONFIGURATION, 1, std::make_unique()); 46 | 47 | // If your record format only need to hold a raw DataLayout, you can directly pass it. 48 | auto dataRecordFormat = 49 | std::make_unique(Record::Type::DATA, 1, std::make_unique()); 50 | 51 | return flavor.empty() ? std::make_unique( 52 | RecordableTypeId::UnitTest1, 53 | std::move(configurationRecordFormat), 54 | std::move(dataRecordFormat)) 55 | : std::make_unique( 56 | RecordableTypeId::UnitTest1, 57 | flavor, 58 | std::move(configurationRecordFormat), 59 | std::move(dataRecordFormat)); 60 | } 61 | 62 | /* 63 | * Use this template to create a stream with records containing metadata plus an image. 64 | * - Configuration RecordFormat: 1 DataLayout 65 | * - Data RecordFormat: 1 DataLayout + 1 image 66 | * You can change the ContentBlock from image to something else (audio, custom). 67 | */ 68 | std::unique_ptr createSampleStreamWithImage() { 69 | // RecordFormat for configuration record. 70 | auto configurationRecordFormat = std::make_unique( 71 | Record::Type::CONFIGURATION, 1, std::make_unique()); 72 | 73 | // To add image in record, you need to create an additional ContentBlocks 74 | std::vector dataContentBlocks = {ContentBlock(ImageFormat::RAW)}; 75 | 76 | // If you want to add additional content blocks, you have to create a wrapper 77 | auto dataRecordFormat = std::make_unique( 78 | Record::Type::DATA, 1, std::make_unique(), dataContentBlocks); 79 | 80 | return std::make_unique( 81 | RecordableTypeId::UnitTest1, 82 | std::move(configurationRecordFormat), 83 | std::move(dataRecordFormat)); 84 | } 85 | 86 | /* 87 | * Use this template to create a stream with two metadata blocks plus one image (Polaris records?) 88 | * - Configuration RecordFormat: 1 DataLayout 89 | * - Data RecordFormat: 2 DataLayouts + 1 image 90 | 91 | */ 92 | std::unique_ptr createSampleStreamWithMultipleDataLayout() { 93 | // RecordFormat for configuration record. 94 | auto configurationRecordFormat = std::make_unique( 95 | Record::Type::CONFIGURATION, 1, std::make_unique()); 96 | 97 | std::vector dataContentBlock = {ContentBlock(ImageFormat::RAW)}; 98 | std::vector dataContentBlock2 = {}; 99 | 100 | // Pass a vector of unique_ptr to the constructor of record format. 101 | auto dataRecordFormat = std::make_unique( 102 | Record::Type::DATA, 103 | 1, 104 | std::make_unique(), 105 | std::make_unique(), 106 | dataContentBlock, 107 | dataContentBlock2); 108 | 109 | return std::make_unique( 110 | RecordableTypeId::UnitTest1, 111 | std::move(configurationRecordFormat), 112 | std::move(dataRecordFormat)); 113 | } 114 | } // namespace pyvrs 115 | -------------------------------------------------------------------------------- /csrc/writer/datalayouts/SampleDataLayout.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace pyvrs { 24 | 25 | using namespace vrs; 26 | 27 | class MyConfiguration : public AutoDataLayout { 28 | public: 29 | DataPieceValue width{datalayout_conventions::kImageWidth}; 30 | DataPieceValue height{ 31 | datalayout_conventions::kImageHeight}; 32 | DataPieceValue pixelFormat{ 33 | datalayout_conventions::kImagePixelFormat}; 34 | AutoDataLayoutEnd endLayout; 35 | }; 36 | 37 | class MyMetadata : public AutoDataLayout { 38 | public: 39 | DataPieceValue roomTemperature{"room_temperature"}; 40 | DataPieceValue cameraId{"camera_id"}; 41 | DataPieceStringMap stringIntMap{"some_string_int_map"}; 42 | DataPieceArray arrayOfDouble{"doubles", 3}; 43 | DataPieceString aString{"some_string"}; 44 | 45 | AutoDataLayoutEnd endLayout; 46 | }; 47 | 48 | class PyStream; 49 | std::unique_ptr createSampleStream(const string& flavor = {}); 50 | std::unique_ptr createSampleStreamWithImage(); 51 | std::unique_ptr createSampleStreamWithMultipleDataLayout(); 52 | 53 | } // namespace pyvrs 54 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Minimal makefile for Sphinx documentation 16 | # 17 | 18 | # You can set these variables from the command line, and also 19 | # from the environment for the first two. 20 | SPHINXOPTS ?= 21 | SPHINXBUILD ?= sphinx-build 22 | SOURCEDIR = source 23 | BUILDDIR = build 24 | 25 | # Put it first so that "make" without argument is like "make help". 26 | help: 27 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 28 | 29 | .PHONY: help Makefile 30 | 31 | # Catch-all target: route all unknown targets to Sphinx using the new 32 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 33 | %: Makefile 34 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 35 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx_rtd_theme 3 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Configuration file for the Sphinx documentation builder. 16 | # 17 | # This file only contains a selection of the most common options. For a full 18 | # list see the documentation: 19 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 20 | 21 | # -- Path setup -------------------------------------------------------------- 22 | 23 | # If extensions (or modules to document with autodoc) are in another directory, 24 | # add these directories to sys.path here. If the directory is relative to the 25 | # documentation root, use os.path.abspath to make it absolute, like shown here. 26 | # 27 | 28 | import os 29 | import sys 30 | 31 | sys.path.insert(0, os.path.abspath("../..")) 32 | 33 | try: 34 | import sphinx_rtd_theme 35 | except ImportError: 36 | sphinx_rtd_theme = None 37 | 38 | # -- Project information ----------------------------------------------------- 39 | 40 | project = "pyvrs" 41 | copyright = "2022, Meta Reality Labs Research" 42 | author = "Meta Reality Labs Research" 43 | 44 | 45 | # -- General configuration --------------------------------------------------- 46 | 47 | # Add any Sphinx extension module names here, as strings. They can be 48 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 49 | # ones. 50 | extensions = [ 51 | "sphinx.ext.autodoc", 52 | "sphinx.ext.napoleon", 53 | "sphinx.ext.intersphinx", 54 | "sphinx.ext.todo", 55 | "sphinx.ext.coverage", 56 | "sphinx.ext.viewcode", 57 | "sphinx.ext.githubpages", 58 | ] 59 | 60 | napoleon_google_docstring = True 61 | 62 | # Add any paths that contain templates here, relative to this directory. 63 | templates_path = ["_templates"] 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | # This pattern also affects html_static_path and html_extra_path. 68 | exclude_patterns = [] 69 | 70 | 71 | # -- Options for HTML output ------------------------------------------------- 72 | 73 | # The theme to use for HTML and HTML Help pages. See the documentation for 74 | # a list of builtin themes. 75 | # 76 | if sphinx_rtd_theme: 77 | html_theme = "sphinx_rtd_theme" 78 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 79 | else: 80 | html_theme = "default" 81 | 82 | # Add any paths that contain custom static files (such as style sheets) here, 83 | # relative to this directory. They are copied after the builtin static files, 84 | # so a file named "default.css" will overwrite the builtin "default.css". 85 | html_static_path = ["_static"] 86 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. pyvrs documentation master file, created by 2 | sphinx-quickstart on Wed Jul 20 13:53:28 2022. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pyvrs's documentation! 7 | ================================= 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | modules/index 13 | -------------------------------------------------------------------------------- /docs/source/modules/base.rst: -------------------------------------------------------------------------------- 1 | pyvrs.base 2 | ================= 3 | 4 | .. automodule:: pyvrs.base 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules/filter.rst: -------------------------------------------------------------------------------- 1 | pyvrs.filter module 2 | =================== 3 | 4 | .. automodule:: pyvrs.filter 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules/index.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================== 3 | 4 | .. toctree:: 5 | 6 | base 7 | filter 8 | reader 9 | record 10 | slice 11 | utils 12 | -------------------------------------------------------------------------------- /docs/source/modules/reader.rst: -------------------------------------------------------------------------------- 1 | pyvrs.reader module 2 | =================== 3 | 4 | .. automodule:: pyvrs.reader 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules/record.rst: -------------------------------------------------------------------------------- 1 | pyvrs.record module 2 | =================== 3 | 4 | .. automodule:: pyvrs.record 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules/slice.rst: -------------------------------------------------------------------------------- 1 | pyvrs.slice module 2 | ================== 3 | 4 | .. automodule:: pyvrs.slice 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/modules/utils.rst: -------------------------------------------------------------------------------- 1 | pyvrs.utils module 2 | ================== 3 | 4 | .. automodule:: pyvrs.utils 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /pixi.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "pyvrs" 3 | version = "1.2.2" 4 | description = "pyvrs is a Python interface for C++ library VRS using pybind11." 5 | channels = ["conda-forge"] 6 | platforms = ["win-64"] 7 | 8 | [tasks] 9 | 10 | install_pyvrs = "python -m pip install -e ." 11 | clean = "rm -rf ./build ./wheels vrs.egg-info/" 12 | 13 | [build-dependencies] 14 | cmake = "3.28.3" 15 | cxx-compiler = "1.6.0.*" 16 | ninja = "1.11.1" 17 | 18 | [dependencies] 19 | gtest = "1.14.0" 20 | 21 | boost = "1.84.0" 22 | fmt = "10.1.1" 23 | libjpeg-turbo = "2.1.4.*" 24 | libpng = "1.6.39.*" 25 | lz4 = "4.3.2.*" 26 | portaudio = "19.6.0.*" 27 | xxhash = "0.8.2" 28 | zlib = "1.2.13.*" 29 | 30 | # Install Python dependencies 31 | python = "3.11.*" 32 | pip = "*" 33 | -------------------------------------------------------------------------------- /pyvrs/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from vrsbindings import ( 17 | AsyncMultiReader, 18 | AsyncReader, 19 | AudioSpec, 20 | AwaitableRecord, 21 | BinaryBuffer, 22 | CompressionPreset, 23 | ContentBlock, 24 | ContentBuffer, 25 | extract_audio_track, 26 | FileSpec, 27 | ImageBuffer, 28 | ImageConversion, 29 | ImageFormat, 30 | ImageSpec, 31 | MultiReader, 32 | PixelFormat, 33 | Reader, 34 | Record, 35 | recordable_type_id_name, 36 | RecordableId, 37 | RecordableTypeId, 38 | records_checksum, 39 | RecordType, 40 | StreamNotFoundError, 41 | TimestampNotFoundError, 42 | verbatim_checksum, 43 | VRSRecord, 44 | ) 45 | 46 | from .reader import AsyncVRSReader, SyncVRSReader 47 | 48 | __all__ = [ 49 | "AsyncVRSReader", 50 | "SyncVRSReader", 51 | "AsyncReader", 52 | "AsyncMultiReader", 53 | "AudioSpec", 54 | "AwaitableRecord", 55 | "BinaryBuffer", 56 | "CompressionPreset", 57 | "ContentBlock", 58 | "ContentBuffer", 59 | "extract_audio_track", 60 | "FileSpec", 61 | "ImageBuffer", 62 | "ImageConversion", 63 | "ImageFormat", 64 | "ImageSpec", 65 | "MultiReader", 66 | "PixelFormat", 67 | "Reader", 68 | "Record", 69 | "recordable_type_id_name", 70 | "RecordableId", 71 | "RecordableTypeId", 72 | "records_checksum", 73 | "RecordType", 74 | "StreamNotFoundError", 75 | "TimestampNotFoundError", 76 | "verbatim_checksum", 77 | "VRSRecord", 78 | ] 79 | -------------------------------------------------------------------------------- /pyvrs/datalayout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # import the native pybind11 VRS bindings (which pyvrs wraps) 17 | import json 18 | import pprint 19 | import re 20 | from typing import Any, Dict 21 | 22 | from . import RecordType 23 | 24 | 25 | class VRSDataLayout: 26 | def __init__( 27 | self, 28 | members: Dict[str, Any], 29 | data_layout: str, 30 | data_layout_type: RecordType, 31 | index: int, 32 | ) -> None: 33 | self.__dict__["members"] = members 34 | self.__dict__["data_layout"] = data_layout 35 | self.__dict__["data_layout_type"] = data_layout_type 36 | self.__dict__["index"] = index 37 | 38 | def __str__(self) -> str: 39 | if self.data_layout is None: 40 | return "(null)" 41 | data_layout = json.loads(self.data_layout)["data_layout"] 42 | list = [] 43 | for field in data_layout: 44 | d = {} 45 | for k, v in field.items(): 46 | if k not in ["offset", "size", "index"]: 47 | d[k] = type_conversion(v) if k == "type" else v 48 | 49 | if len(d) > 0: 50 | list.append(dict_to_str(d)) 51 | 52 | s = "\n".join([""] + list) 53 | return s 54 | 55 | def __getattr__(self, name: str): 56 | if name in self.__dict__["members"]: 57 | return self.__dict__["members"].get(name) 58 | raise AttributeError 59 | 60 | def __setattr__(self, name: str, value: Any) -> None: 61 | if name in self.__dict__["members"]: 62 | self.__dict__["members"].get(name).set(value) 63 | return 64 | raise AttributeError 65 | 66 | def __getitem__(self, name): 67 | return self.__getattr__(name) 68 | 69 | def __setitem__(self, name, value): 70 | return self.__setattr__(name, value) 71 | 72 | 73 | def dict_to_str(d: Dict[str, str]) -> str: 74 | list = [] 75 | if len(d) < 4: 76 | return " " + pprint.pformat(d) 77 | for k, v in d.items(): 78 | list.append(f" '{k}': '{v}',") 79 | s = "\n".join([" {"] + list + [" }"]) 80 | return s 81 | 82 | 83 | def type_conversion(value: str) -> str: 84 | if value == "DataPieceString": 85 | return "str" 86 | type_map = { 87 | "float": "float", 88 | "double": "float", 89 | "string": "str", 90 | "uint8_t": "int", 91 | "int8_t": "int", 92 | "uint16_t": "int", 93 | "int16_t": "int", 94 | "uint32_t": "int", 95 | "int32_t": "int", 96 | "uint64_t": "int", 97 | "int64_t": "int", 98 | "Point2Di": "(int, int)", 99 | "Point3Df": "(float, float, float)", 100 | "Point4Df": "(float, float, float, float)", 101 | } 102 | m = re.match(r"(?P\w+)<(?P\w+)>", value) 103 | datapiece_typename = m.group("datapiece_typename") 104 | typename = m.group("typename") 105 | 106 | if datapiece_typename == "DataPieceValue": 107 | return type_map[typename] 108 | if datapiece_typename in ["DataPieceVector", "DataPieceArray"]: 109 | return f"List[{type_map[typename]}]" 110 | if datapiece_typename == "DataPieceStringMap": 111 | return f"Dict[str, {type_map[typename]}]" 112 | return "None" 113 | -------------------------------------------------------------------------------- /pyvrs/record.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from functools import partial 17 | from typing import Callable, overload, Sequence, TypeVar, Union 18 | 19 | import numpy as np 20 | 21 | from . import VRSRecord as _VRSRecord 22 | 23 | from .utils import stringify_metadata_keys 24 | 25 | T = TypeVar("T") 26 | 27 | 28 | class VRSRecord: 29 | """Represents a single VRS Record.""" 30 | 31 | def __init__(self, record: _VRSRecord) -> None: 32 | self._record = record 33 | 34 | def __repr__(self) -> str: 35 | return self._record.__repr__() 36 | 37 | def __str__(self) -> str: 38 | return self._record.__str__() 39 | 40 | def __getitem__(self, key: str): 41 | return self._record[key] 42 | 43 | @property 44 | def format_version(self) -> int: 45 | """The format version of this VRS record.""" 46 | return self._record.format_version 47 | 48 | @property 49 | def n_audio_blocks(self) -> int: 50 | """The number of audio blocks present in this VRS record.""" 51 | return self._record.n_audio_blocks 52 | 53 | @property 54 | def n_custom_blocks(self) -> int: 55 | """The number of custom blocks present in this VRS record.""" 56 | return self._record.n_custom_blocks 57 | 58 | @property 59 | def n_image_blocks(self) -> int: 60 | """The number of image blocks present in this VRS record.""" 61 | return self._record.n_image_blocks 62 | 63 | @property 64 | def n_metadata_blocks(self) -> int: 65 | """The number of metadata blocks present in this VRS record.""" 66 | return self._record.n_metadata_blocks 67 | 68 | @property 69 | def n_blocks_in_total(self) -> int: 70 | """The total number of blocks on this record.""" 71 | return self._record.n_blocks_in_total 72 | 73 | @property 74 | def record_type(self) -> str: 75 | """The type of this VRS record, e.g. 'configuration' or 'data'.""" 76 | return self._record.record_type 77 | 78 | @property 79 | def record_index(self) -> int: 80 | """The absolute index of this VRS record in the VRS file.""" 81 | return self._record.record_index 82 | 83 | @property 84 | def stream_id(self) -> str: 85 | """A unique identifier for the type of device that recorded this record, along with an 86 | enumeration. E.g.: '1001-1'. 87 | """ 88 | return self._record.stream_id 89 | 90 | @property 91 | def timestamp(self) -> float: 92 | """The timestamp associated with this record.""" 93 | return self._record.timestamp 94 | 95 | @property 96 | def audio_blocks(self) -> "VRSBlocks": 97 | """The list of audio blocks associated with this record.""" 98 | return VRSBlocks( 99 | partial(lambda x, idx: np.asarray(x[idx]), self._record.audio_blocks), 100 | range(self.n_audio_blocks), 101 | ) 102 | 103 | @property 104 | def audio_specs(self) -> "VRSBlocks": 105 | """The list of audio block specs associated with this record.""" 106 | return VRSBlocks( 107 | partial(lambda x, idx: x[idx], self._record.audio_specs), 108 | range(self.n_audio_blocks), 109 | ) 110 | 111 | @property 112 | def custom_blocks(self) -> "VRSBlocks": 113 | """The list of custom blocks associated with this record.""" 114 | return VRSBlocks( 115 | partial(lambda x, idx: np.asarray(x[idx]), self._record.custom_blocks), 116 | range(self.n_custom_blocks), 117 | ) 118 | 119 | @property 120 | def custom_block_specs(self) -> "VRSBlocks": 121 | """The list of custom block specs associated with this record.""" 122 | return VRSBlocks( 123 | partial(lambda x, idx: x[idx], self._record.custom_block_specs), 124 | range(self.n_custom_blocks), 125 | ) 126 | 127 | @property 128 | def image_blocks(self) -> "VRSBlocks": 129 | """The list of image blocks associated with this record.""" 130 | return VRSBlocks( 131 | partial(lambda x, idx: np.asarray(x[idx]), self._record.image_blocks), 132 | range(self.n_image_blocks), 133 | ) 134 | 135 | @property 136 | def image_specs(self) -> "VRSBlocks": 137 | """The list of image block specs associated with this record.""" 138 | return VRSBlocks( 139 | partial(lambda x, idx: x[idx], self._record.image_specs), 140 | range(self.n_image_blocks), 141 | ) 142 | 143 | @property 144 | def metadata_blocks(self) -> "VRSBlocks": 145 | """The list of metadata blocks associated with this record.""" 146 | return VRSBlocks( 147 | partial( 148 | lambda x, idx: stringify_metadata_keys(x[idx]), 149 | self._record.metadata_blocks, 150 | ), 151 | range(self.n_metadata_blocks), 152 | ) 153 | 154 | 155 | class VRSBlocks(Sequence): 156 | """A representation of a list of Record blocks (image, metadata, audio or custom). 157 | Accessing elements returns the type directly. 158 | """ 159 | 160 | def __init__(self, get_func: Callable[[int], T], range_: range) -> None: 161 | self._range = range_ 162 | self._get_func = get_func 163 | 164 | @overload 165 | def __getitem__(self, i: int) -> T: ... 166 | 167 | @overload 168 | def __getitem__(self, i: slice) -> "VRSBlocks": ... 169 | 170 | def __getitem__(self, i: Union[int, slice]) -> Union[T, "VRSBlocks"]: 171 | if isinstance(i, int) or hasattr(i, "__index__"): 172 | return self._get_func(self._range[i]) 173 | else: 174 | # A slice or unknown type is passed. Let list handle it directly. 175 | return VRSBlocks(self._get_func, self._range[i]) 176 | 177 | def __len__(self) -> int: 178 | return len(self._range) 179 | -------------------------------------------------------------------------------- /pyvrs/slice.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from pathlib import Path 17 | from typing import List, overload, Sequence, TYPE_CHECKING, Union 18 | 19 | if TYPE_CHECKING: 20 | from .reader import AsyncVRSReader, VRSReader 21 | 22 | from .record import VRSRecord 23 | 24 | 25 | class VRSReaderSlice(Sequence): 26 | """A slice into a VRS file, i.e. an arbitrary collection of records. Can be further sliced and 27 | indexed, but looses some of the smarts present in the full VRSReader (e.g. the ability to 28 | filter) and some richer properties like temporal information. 29 | """ 30 | 31 | def __init__(self, path: Union[str, Path], r, indices: List[int]) -> None: 32 | self._path = Path(path) 33 | self._reader = r 34 | self._indices = indices 35 | 36 | @overload 37 | def __getitem__(self, i: int) -> VRSRecord: ... 38 | 39 | @overload 40 | def __getitem__(self, i: slice) -> "VRSReaderSlice": ... 41 | 42 | def __getitem__(self, i: Union[int, slice]) -> Union[VRSRecord, "VRSReaderSlice"]: 43 | return index_or_slice_records(self._path, self._reader, self._indices, i) 44 | 45 | def __len__(self) -> int: 46 | return len(self._indices) 47 | 48 | def __repr__(self) -> str: 49 | return f"VRSReaderSlice(path={str(self._path)!r}, len={len(self._indices)})" 50 | 51 | def __str__(self) -> str: 52 | return "\n".join( 53 | [str(self._path), f"Slice containing {len(self._indices)} records"] 54 | ) 55 | 56 | 57 | class AsyncVRSReaderSlice(Sequence): 58 | """A slice into a VRS file, i.e. an arbitrary collection of records. Can be further sliced and 59 | indexed, but looses some of the smarts present in the full VRSReader (e.g. the ability to 60 | filter) and some richer properties like temporal information. 61 | This should be created only via AsyncVRSReader.async_read_record call. 62 | """ 63 | 64 | def __init__(self, path: Union[str, Path], r, indices: List[int]) -> None: 65 | self._path = Path(path) 66 | self._reader = r 67 | self._indices = indices 68 | 69 | def __aiter__(self): 70 | self.index = 0 71 | return self 72 | 73 | async def __anext__(self): 74 | if self.index == len(self): 75 | raise StopAsyncIteration 76 | result = await self[self.index] 77 | self.index += 1 78 | return result 79 | 80 | @overload 81 | async def __getitem__(self, i: int) -> VRSRecord: ... 82 | 83 | @overload 84 | async def __getitem__(self, i: slice) -> "AsyncVRSReaderSlice": ... 85 | 86 | async def __getitem__( 87 | self, i: Union[int, slice] 88 | ) -> Union[VRSRecord, "AsyncVRSReaderSlice"]: 89 | return await async_index_or_slice_records( 90 | self._path, self._reader, self._indices, i 91 | ) 92 | 93 | def __len__(self) -> int: 94 | return len(self._indices) 95 | 96 | def __repr__(self) -> str: 97 | return ( 98 | f"AsyncVRSReaderSlice(path={str(self._path)!r}, len={len(self._indices)})" 99 | ) 100 | 101 | def __str__(self) -> str: 102 | return "\n".join( 103 | [str(self._path), f"Slice containing {len(self._indices)} records"] 104 | ) 105 | 106 | 107 | def index_or_slice_records( 108 | path: Path, 109 | reader: "VRSReader", 110 | vrs_indices: List[int], 111 | indices: Union[int, slice], 112 | ) -> Union[VRSRecord, VRSReaderSlice]: 113 | """Shared logic to index or slice into a VRSReader or VRSReaderSlice. Returns either a 114 | VRSReaderSlice (if i is a slice or iterable) or a VRSRecord (if i is an integer). 115 | 116 | Args: 117 | path: a file path 118 | reader: a reader instance which is based on the class extended from BaseVRSReader 119 | vrs_indices: a list of absolute indices for the VRS file that you are interested in. 120 | If you apply filter, this list should be the one after applying the filter. 121 | indices: A set of indices or an index for vrs_indices that you want to read. 122 | 123 | Returns: 124 | VRSReaderSlice if the given indices is a slice, otherwise read a record and returns VRSRecord object. 125 | """ 126 | if isinstance(indices, int) or hasattr(indices, "__index__"): 127 | record = reader.read_record(vrs_indices[indices]) 128 | return VRSRecord(record) 129 | else: 130 | # A slice or unknown type is passed. Let list handle it directly. 131 | return VRSReaderSlice(path, reader, vrs_indices[indices]) 132 | 133 | 134 | async def async_index_or_slice_records( 135 | path: Path, 136 | reader: "AsyncVRSReader", 137 | vrs_indices: List[int], 138 | indices: Union[int, slice], 139 | ) -> Union[VRSRecord, AsyncVRSReaderSlice]: 140 | """Shared logic to index or slice into a AsyncVRSReader or AsyncVRSReaderSlice. Returns either a 141 | AsyncVRSReaderSlice (if i is a slice or iterable) or a VRSRecord (if i is an integer). 142 | 143 | Args: 144 | path: a file path 145 | reader: a reader instance which is based on the class extended from AsyncVRSReader 146 | vrs_indices: a list of absolute indices for the VRS file that you are interested in. 147 | If you apply filter, this list should be the one after applying the filter. 148 | indices: A set of indices or an index for vrs_indices that you want to read. 149 | 150 | Returns: 151 | AsyncVRSReaderSlice if the given indices is a slice, otherwise read a record and returns VRSRecord object. 152 | """ 153 | if isinstance(indices, int) or hasattr(indices, "__index__"): 154 | record = await reader.async_read_record(vrs_indices[indices]) 155 | return VRSRecord(record) 156 | else: 157 | # A slice or unknown type is passed. Let list handle it directly. 158 | return AsyncVRSReaderSlice(path, reader, vrs_indices[indices]) 159 | -------------------------------------------------------------------------------- /pyvrs/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from fnmatch import fnmatch 17 | from typing import Any, Callable, Dict, Iterable, Mapping, Set, Tuple, TypeVar 18 | 19 | from . import recordable_type_id_name 20 | 21 | T = TypeVar("T") 22 | 23 | 24 | def get_recordable_type_id_name(stream_id: str) -> str: 25 | """Retrieve a human readable string what the recordable is. 26 | Args: 27 | stream_id: stream_id you are interested in. 28 | Returns: 29 | The name of recordable_type_id for the given stream id. 30 | """ 31 | return recordable_type_id_name(stream_id) 32 | 33 | 34 | def _filter_set_on_condition( 35 | s: Set[T], items: Iterable[T], cond_func: Callable[[T, T], bool] 36 | ) -> Set[T]: 37 | """Restrict down a set by a list of items and a comparator. For each element in the set, if for 38 | any item in the items the comparator returns True, the element is retained in the filtered set 39 | that is returned. Note that this does not mutate the original set, it returns a filtered copy. 40 | """ 41 | filtered = set() 42 | for a in s: 43 | for b in items: 44 | if cond_func(a, b): 45 | filtered.add(a) 46 | break 47 | return filtered 48 | 49 | 50 | def filter_by_stream_ids( 51 | available_stream_ids: Set[str], regex_stream_ids: Iterable[str] 52 | ) -> Set[str]: 53 | """Filter a set of stream_ids based on the provided regex stream_ids. 54 | Args: 55 | available_stream_ids: A set of stream_ids in a file. 56 | regex_stream_ids: regex of the stream ids that you are interested in. 57 | Returns: 58 | A set of stream_ids in available_stream_ids that matches the one of the regex in regex_stream_ids. 59 | """ 60 | return _filter_set_on_condition(available_stream_ids, regex_stream_ids, fnmatch) 61 | 62 | 63 | def filter_by_record_type( 64 | available_record_types: Set[str], requested_record_types: Iterable[str] 65 | ) -> Set[str]: 66 | """Filter a set record types based on the provided record types. Filter only succeeds on 67 | exact matches. 68 | Args: 69 | available_record_types: A set of record types in a file. 70 | requested_record_types: A set of record types you are interested in. 71 | Returns: 72 | A set of record types in available_record_types that is also in requested_record_types. 73 | """ 74 | return _filter_set_on_condition( 75 | available_record_types, requested_record_types, lambda a, b: a == b 76 | ) 77 | 78 | 79 | def string_of_set(s: Set) -> str: 80 | """Return a pretty string representation of a set (with elements ordered).""" 81 | return "{" + ", ".join(sorted(s)) + "}" 82 | 83 | 84 | def tags_to_justified_table_str(tags: Mapping[str, Any]) -> str: 85 | """Returns a nice table representation of tags for easy viewing.""" 86 | if len(tags) == 0: 87 | return "File contains no file tags." 88 | tag_width = max(len(t) for t in tags) 89 | table_content = ["{}| {}".format(k.rjust(tag_width), tags[k]) for k in sorted(tags)] 90 | table_width = max(len(content) for content in table_content) 91 | return "\n".join( 92 | [" FILE TAGS ".center(table_width, "-"), *table_content, "-" * table_width] 93 | ) 94 | 95 | 96 | def _unique_string_for_key(name: str, type_: str, dict_: Mapping[str, Any]) -> str: 97 | r"""Builds a unique key for a name type pair.""" 98 | i = 1 99 | while True: 100 | # default to name. If we hit an existing key, keep adding brackets, e.g. name<>. 101 | new_key = name + ("<" * i) + type_ + (">" * i) 102 | if new_key not in dict_: 103 | return new_key 104 | i += 1 105 | 106 | 107 | def stringify_metadata_keys( 108 | metadata_dict: Dict[Tuple[str, str], Any], 109 | ) -> Dict[str, Any]: 110 | r"""remove unambiguous types from metadata dicts. If the type is overloaded, converts to a 111 | key string representation.""" 112 | filtered: Dict[str, Any] = {} 113 | ambiguous_names: Set[str] = set() 114 | removed_types: Dict[str, str] = {} 115 | 116 | for (name, type_), value in metadata_dict.items(): 117 | if name in ambiguous_names: 118 | # this name is already overloaded - build a string from the name and type. 119 | filtered[_unique_string_for_key(name, type_, filtered)] = value 120 | elif name in filtered: 121 | # first time we find a collision. Get the old value and type.. 122 | old_value = filtered.pop(name) 123 | old_type = removed_types.pop(name) 124 | 125 | # and update the filtered dict with both the old entry and the new. 126 | filtered[_unique_string_for_key(name, old_type, filtered)] = old_value 127 | filtered[_unique_string_for_key(name, type_, filtered)] = value 128 | 129 | # blacklist so we don't hit this again. 130 | ambiguous_names.add(name) 131 | else: 132 | # this name seems unambiguous so far, just add it without type... 133 | filtered[name] = value 134 | # ...and keep this type around in case we need to disambiguate later. 135 | removed_types[name] = type_ 136 | 137 | return filtered 138 | -------------------------------------------------------------------------------- /release_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | from typing import Tuple 17 | 18 | 19 | def get_current_version() -> Tuple[str, str]: 20 | current_version = open("version.txt", "r").read().strip() 21 | current_tag = "v" + current_version 22 | return current_version, current_tag 23 | 24 | 25 | def get_next_version(release_type) -> Tuple[Tuple[int, int, int], str, str]: 26 | current_version = open("version.txt", "r").read().strip() 27 | version_list = [int(x) for x in current_version.strip("'").split(".")] 28 | major, minor, patch = version_list[0], version_list[1], version_list[2] 29 | if release_type == "patch": 30 | patch += 1 31 | elif release_type == "minor": 32 | minor += 1 33 | patch = 0 34 | elif release_type == "major": 35 | major += 1 36 | minor = patch = 0 37 | else: 38 | raise ValueError( 39 | "Incorrect release type specified. Acceptable types are major, minor and patch." 40 | ) 41 | 42 | new_version_tuple = (major, minor, patch) 43 | new_version_str = ".".join([str(x) for x in new_version_tuple]) 44 | new_tag_str = "v" + new_version_str 45 | return new_version_tuple, new_version_str, new_tag_str 46 | 47 | 48 | def update_version(new_version_tuple) -> None: 49 | """ 50 | given the current version, update the version to the 51 | next version depending on the type of release. 52 | """ 53 | 54 | with open("version.txt", "w") as writer: 55 | writer.write(".".join([str(x) for x in new_version_tuple])) 56 | 57 | 58 | def main(args): 59 | if args.release_type: 60 | if args.release_type in ["major", "minor", "patch"]: 61 | new_version_tuple, new_version, new_tag = get_next_version( 62 | args.release_type 63 | ) 64 | print(new_version, new_tag) 65 | else: 66 | raise ValueError("Incorrect release type specified") 67 | 68 | if args.update_version: 69 | update_version(new_version_tuple) 70 | 71 | if args.get_current_version: 72 | current_version, current_tag = get_current_version() 73 | print(current_version, current_tag) 74 | 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser(description="Versioning utils") 78 | parser.add_argument( 79 | "--release-type", 80 | type=str, 81 | required=False, 82 | help="type of release = major/minor/patch", 83 | ) 84 | parser.add_argument( 85 | "--update-version", 86 | action="store_true", 87 | required=False, 88 | help="updates the version in version.txt", 89 | ) 90 | parser.add_argument( 91 | "--get-current-version", 92 | action="store_true", 93 | required=False, 94 | help="get the current version in version.txt", 95 | ) 96 | 97 | args = parser.parse_args() 98 | main(args) 99 | -------------------------------------------------------------------------------- /scripts/build-macos-arm64-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Parse command line argument 17 | if [ "$#" -ne 1 ]; then 18 | echo "Usage: $0 [test|prod]" 19 | exit 1 20 | fi 21 | 22 | if [ "$1" == "test" ]; then 23 | TWINE_REPOSITORY="https://test.pypi.org/legacy/" 24 | else 25 | TWINE_REPOSITORY="https://upload.pypi.org/legacy/" 26 | fi 27 | 28 | 29 | # Use clang and not gcc 30 | export CC=/usr/bin/cc 31 | export CXX=/usr/bin/c++ 32 | 33 | # Build vrs using the following 34 | # cd /tmp; git clone https://github.com/facebookresearch/vrs.git -b v1.0.4 \ 35 | # && mkdir vrs_Build && cd vrs_Build \ 36 | # && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -DCMAKE_OSX_ARCHITECTURES="arm64" ../vrs/ \ 37 | # && sudo make -j2 install 38 | 39 | cd .. 40 | 41 | export ARCHFLAGS="-arch arm64" 42 | export CMAKE_ARGS="-DCMAKE_PREFIX_PATH=/Users/${USER}/homebrew" 43 | 44 | # Set environment variables for cibuildwheel 45 | export CIBW_PLATFORM="macos" 46 | export CIBW_OUTPUT_DIR="dist" 47 | export CIBW_ARCHS="arm64" 48 | export CIBW_BUILD_VERBOSITY="3" 49 | export CIBW_BUILD="cp39-*64 cp310-*64 cp311-*64 cp312-*64" 50 | export CIBW_BEFORE_BUILD_MACOS="arch -arm64 brew install boost cmake fmt glog jpeg-turbo libpng lz4 xxhash zstd opus" 51 | export CIBW_SKIP="*-manylinux_i686 *musllinux*" 52 | 53 | # Build wheels for all specified versions 54 | python -m pip install cibuildwheel==2.17.0 55 | python -m cibuildwheel --output-dir dist 56 | 57 | # # Upload to PyPI 58 | twine upload --repository-url "$TWINE_REPOSITORY" dist/* 59 | 60 | # # Clean up 61 | rm -rf dist 62 | -------------------------------------------------------------------------------- /scripts/install-macos-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | brew install cmake git ninja googletest glog fmt \ 17 | jpeg-turbo libpng \ 18 | lz4 zstd xxhash boost opus 19 | -------------------------------------------------------------------------------- /scripts/install-manylinux-deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Update the mirror list for Centos mirror deprecation 17 | sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* 18 | sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* 19 | 20 | # Install VRS dependencies 21 | yum install -y cmake git ninja-build gtest-devel \ 22 | lz4-devel libzstd-devel xxhash-devel libpng-devel 23 | 24 | cd /tmp && curl -kOL http://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz \ 25 | && tar -zxf opus-1.5.2.tar.gz \ 26 | && cd opus-1.5.2 \ 27 | && ./configure \ 28 | && make -j4 \ 29 | && make install \ 30 | && rm -rf /tmp/opus-1.5.2.tar.gz /tmp/opus-1.5.2; 31 | 32 | cd /tmp && git clone https://github.com/fmtlib/fmt.git -b 8.1.1 \ 33 | && cd fmt \ 34 | && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE -DFMT_TEST=OFF .;make -j4 install; rm -rf /tmp/fmt; 35 | 36 | cd /tmp && git clone https://github.com/libjpeg-turbo/libjpeg-turbo.git -b 2.1.4 \ 37 | && cd libjpeg-turbo \ 38 | && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_JPEG8=1 -DCMAKE_INSTALL_DEFAULT_PREFIX=/usr .;make -j4 install; rm -rf /tmp/libjpeg-turbo; 39 | 40 | cd /tmp && git clone --recursive https://github.com/boostorg/boost.git -b boost-1.81.0 \ 41 | && cd boost \ 42 | && ./bootstrap.sh \ 43 | && ./b2 install \ 44 | && rm -rf /tmp/boost; 45 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import os 17 | import re 18 | import shutil 19 | import subprocess 20 | import sys 21 | from pathlib import Path 22 | 23 | from setuptools import Extension, find_packages, setup 24 | from setuptools.command.build_ext import build_ext 25 | 26 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 27 | 28 | 29 | def _get_sha(): 30 | sha = "Unknown" 31 | try: 32 | sha = ( 33 | subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=str(ROOT_DIR)) 34 | .decode("ascii") 35 | .strip() 36 | ) 37 | except Exception: 38 | pass 39 | return sha 40 | 41 | 42 | def get_version(): 43 | path = os.path.join(ROOT_DIR, "version.txt") 44 | version = open(path, "r").read().strip() 45 | 46 | if os.getenv("PYVRS_TEST_BUILD"): 47 | sha = _get_sha() 48 | if sha != "Unknown": 49 | version += "+" + sha[:7] 50 | 51 | return version 52 | 53 | 54 | class CMakeExtension(Extension): 55 | def __init__(self, name, sourcedir=""): 56 | Extension.__init__(self, name, sources=[]) 57 | self.sourcedir = os.path.abspath(sourcedir) 58 | 59 | 60 | class CMakeBuild(build_ext): 61 | def run(self): 62 | try: 63 | subprocess.check_output(["cmake", "--version"]) 64 | except OSError: 65 | raise RuntimeError("CMake is not available.") 66 | super().run() 67 | 68 | def build_extension(self, ext): 69 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 70 | 71 | if not extdir.endswith(os.path.sep): 72 | extdir += os.path.sep 73 | 74 | if "DEBUG" in os.environ: 75 | cfg = "Debug" if os.environ["DEBUG"] == "1" else "Release" 76 | else: 77 | cfg = "Debug" if self.debug else "Release" 78 | 79 | cmake_args = [ 80 | f"-DCMAKE_BUILD_TYPE={cfg}", 81 | f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", 82 | f"-DPYTHON_EXECUTABLE={sys.executable}", 83 | "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", 84 | ] 85 | build_args = ["--target", os.path.basename(ext.name)] 86 | 87 | if "CMAKE_ARGS" in os.environ: 88 | cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] 89 | 90 | if sys.platform.startswith("win"): 91 | build_args += ["--config", "Release"] 92 | else: 93 | # Default to Ninja 94 | if "CMAKE_GENERATOR" not in os.environ: 95 | cmake_args += ["-GNinja"] 96 | 97 | if sys.platform.startswith("darwin"): 98 | # Cross-compile support for macOS - respect ARCHFLAGS if set 99 | archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) 100 | if archs: 101 | cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] 102 | 103 | if not os.path.exists(self.build_temp): 104 | os.makedirs(self.build_temp) 105 | subprocess.check_call( 106 | ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp 107 | ) 108 | subprocess.check_call( 109 | ["cmake", "--build", "."] + build_args, cwd=self.build_temp 110 | ) 111 | 112 | if sys.platform.startswith("win"): 113 | [ 114 | shutil.copy(os.path.join(f"{extdir}/Release/", f), extdir) 115 | for f in os.listdir(f"{extdir}/Release/") 116 | ] 117 | 118 | 119 | def main(): 120 | with open(os.path.join(ROOT_DIR, "README.md"), encoding="utf-8") as f: 121 | long_description = f.read() 122 | 123 | setup( 124 | name="vrs", 125 | version=get_version(), 126 | description="Python API for VRS", 127 | long_description=long_description, 128 | long_description_content_type="text/markdown", 129 | url="https://github.com/facebookresearch/pyvrs", 130 | author="Meta Reality Labs Research", 131 | license="Apache-2.0", 132 | install_requires=["numpy", "typing", "dataclasses"], 133 | python_requires=">=3.9", 134 | packages=find_packages(), 135 | zip_safe=False, 136 | ext_modules=[CMakeExtension("vrsbindings", sourcedir=ROOT_DIR)], 137 | cmdclass={ 138 | "build_ext": CMakeBuild, 139 | }, 140 | ) 141 | 142 | 143 | if __name__ == "__main__": 144 | main() 145 | -------------------------------------------------------------------------------- /test/pyvrs_async_reader_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import asyncio 17 | import functools 18 | import unittest 19 | from pathlib import Path 20 | from typing import Any 21 | 22 | import numpy as np 23 | import pkg_resources 24 | from pyvrs import AsyncVRSReader, SyncVRSReader 25 | 26 | 27 | def test_recording_path() -> Path: 28 | return Path(pkg_resources.resource_filename(__name__, "test_data/synthetic.vrs")) 29 | 30 | 31 | def async_test(func): 32 | if not asyncio.iscoroutinefunction(func): 33 | # If you are investigating why you have an `async def` and this fires, 34 | # make sure that you don't use `yield` in the test function, as `async def` + `yield` 35 | # is an async generator! 36 | raise TypeError("Only 'async def' is supported, please fix your call site") 37 | 38 | @functools.wraps(func) 39 | def wrapper(*args: Any, **kws: Any) -> None: 40 | asyncio.run(func(*args, **kws)) 41 | 42 | return wrapper 43 | 44 | 45 | class TestAsyncRead(unittest.TestCase): 46 | @unittest.skip 47 | @async_test 48 | async def test_async_read(self): 49 | reader = SyncVRSReader( 50 | test_recording_path(), 51 | auto_read_configuration_records=True, 52 | ) 53 | async_reader = AsyncVRSReader( 54 | test_recording_path(), 55 | auto_read_configuration_records=True, 56 | ) 57 | filtered_reader = reader.filtered_by_fields( 58 | stream_ids="100-1", 59 | record_types="data", 60 | ) 61 | async_filtered_reader = async_reader.filtered_by_fields( 62 | stream_ids="100-1", 63 | record_types="data", 64 | ) 65 | self.assertEqual(len(filtered_reader), 500) 66 | self.assertEqual(len(async_filtered_reader), 500) 67 | 68 | records = [record for record in filtered_reader[:500]] 69 | async_records = [record async for record in async_filtered_reader] 70 | async_records_widht_slice = [ 71 | record async for record in await async_filtered_reader[:500] 72 | ] 73 | 74 | self.assertEqual(len(records), len(async_records)) 75 | self.assertEqual(len(records), len(async_records_widht_slice)) 76 | for i in range(len(records)): 77 | b1 = records[i].image_blocks[0] 78 | b2 = async_records[i].image_blocks[0] 79 | b3 = async_records_widht_slice[i].image_blocks[0] 80 | self.assertTrue(np.array_equal(b1, b2)) 81 | self.assertTrue(np.array_equal(b1, b3)) 82 | 83 | 84 | if __name__ == "__main__": 85 | unittest.main() 86 | -------------------------------------------------------------------------------- /test/pyvrs_utils_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | from pyvrs.utils import stringify_metadata_keys 19 | 20 | 21 | class TestMetadataStringKeys(unittest.TestCase): 22 | def test_basic_stringify_types(self): 23 | self.assertEqual( 24 | stringify_metadata_keys( 25 | {("varnamea", "typenamea"): "aval", ("varnameb", "typenameb"): "bval"} 26 | ), 27 | {"varnamea": "aval", "varnameb": "bval"}, 28 | ) 29 | 30 | def test_stringify_types_with_collision(self): 31 | self.assertEqual( 32 | stringify_metadata_keys( 33 | { 34 | ("varname", "typenamea"): "aval", 35 | ("varname", "typenameb"): "bval", 36 | ("varnameother", "typenameb"): "cval", 37 | } 38 | ), 39 | { 40 | "varname": "aval", 41 | "varname": "bval", 42 | "varnameother": "cval", 43 | }, 44 | ) 45 | 46 | def test_stringify_types_with_ambiguity(self): 47 | self.assertEqual( 48 | stringify_metadata_keys( 49 | { 50 | ("acceleration", "int"): 2, 51 | ("acceleration", "int"): 3, 52 | ("acceleration", "float"): 4, 53 | ("other", "int"): 5, 54 | } 55 | ), 56 | { 57 | "acceleration": 2, 58 | "acceleration<>": 3, 59 | "acceleration": 4, 60 | "other": 5, 61 | }, 62 | ) 63 | 64 | 65 | if __name__ == "__main__": 66 | unittest.main() 67 | -------------------------------------------------------------------------------- /test/test_data/synthetic.vrs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebookresearch/pyvrs/1fab090a6516834bb2db83b4a302463568e6dd1b/test/test_data/synthetic.vrs -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 1.2.2 2 | --------------------------------------------------------------------------------