├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── databento ├── __init__.py ├── common │ ├── __init__.py │ ├── bentologging.py │ ├── constants.py │ ├── cram.py │ ├── dbnstore.py │ ├── enums.py │ ├── error.py │ ├── http.py │ ├── iterator.py │ ├── parsing.py │ ├── publishers.py │ ├── symbology.py │ ├── system.py │ ├── types.py │ └── validation.py ├── historical │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── batch.py │ │ ├── metadata.py │ │ ├── symbology.py │ │ └── timeseries.py │ └── client.py ├── live │ ├── __init__.py │ ├── client.py │ ├── gateway.py │ ├── protocol.py │ └── session.py ├── py.typed ├── reference │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── adjustment.py │ │ ├── corporate.py │ │ └── security.py │ └── client.py └── version.py ├── examples ├── historical_batch_download.py ├── historical_batch_download_async.py ├── historical_batch_list_files.py ├── historical_batch_list_jobs.py ├── historical_batch_submit_job.py ├── historical_metadata.py ├── historical_metadata_get_billable_size.py ├── historical_metadata_get_cost.py ├── historical_metadata_get_dataset_condition.py ├── historical_metadata_get_dataset_range.py ├── historical_metadata_get_record_count.py ├── historical_metadata_list_unit_price.py ├── historical_symbology_resolve.py ├── historical_timeseries_async.py ├── historical_timeseries_disk_io.py ├── historical_timeseries_from_file.py ├── historical_timeseries_replay.py ├── historical_timeseries_to_df.py ├── historical_timeseries_to_file.py ├── historical_timeseries_to_ndarray.py ├── live_smoke_test.py ├── reference_adjustment_factors.py ├── reference_corporate_actions.py ├── reference_security_master_get_last.py └── reference_security_master_get_range.py ├── notebooks └── quickstart.ipynb ├── pyproject.toml ├── scripts ├── build.sh ├── lint.sh └── test.sh └── tests ├── __init__.py ├── conftest.py ├── data ├── EQUS.MINI │ ├── test_data.bbo-1m.dbn.zst │ ├── test_data.bbo-1s.dbn.zst │ ├── test_data.definition.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── GLBX.MDP3 │ ├── test_data.bbo-1m.dbn.zst │ ├── test_data.bbo-1s.dbn.zst │ ├── test_data.definition.dbn.zst │ ├── test_data.mbo.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.mbp-10.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.statistics.dbn.zst │ ├── test_data.status.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── IFEU.IMPACT │ ├── test_data.bbo-1m.dbn.zst │ ├── test_data.bbo-1s.dbn.zst │ ├── test_data.definition.dbn.zst │ ├── test_data.mbo.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.mbp-10.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.statistics.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── LIVE │ └── test_data.live.dbn.zst ├── NDEX.IMPACT │ ├── test_data.bbo-1m.dbn.zst │ ├── test_data.bbo-1s.dbn.zst │ ├── test_data.definition.dbn.zst │ ├── test_data.mbo.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.mbp-10.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.statistics.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── OPRA.PILLAR │ ├── test_data.cbbo-1m.dbn.zst │ ├── test_data.cbbo-1s.dbn.zst │ ├── test_data.definition.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.statistics.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── REFERENCE │ ├── test_data.adjustment-factors.jsonl │ ├── test_data.corporate-actions-pit.jsonl │ ├── test_data.corporate-actions.jsonl │ └── test_data.security-master.jsonl ├── XNAS.ITCH │ ├── test_data.definition.dbn.zst │ ├── test_data.imbalance.dbn.zst │ ├── test_data.mbo.dbn.zst │ ├── test_data.mbp-1.dbn.zst │ ├── test_data.mbp-10.dbn.zst │ ├── test_data.ohlcv-1d.dbn.zst │ ├── test_data.ohlcv-1h.dbn.zst │ ├── test_data.ohlcv-1m.dbn.zst │ ├── test_data.ohlcv-1s.dbn.zst │ ├── test_data.tbbo.dbn.zst │ └── test_data.trades.dbn.zst ├── __init__.py └── generator.py ├── mockliveserver ├── __init__.py ├── __main__.py ├── controller.py ├── fixture.py ├── server.py └── source.py ├── test_bento_compression.py ├── test_bento_data_source.py ├── test_common_cram.py ├── test_common_enums.py ├── test_common_iterator.py ├── test_common_parsing.py ├── test_common_symbology.py ├── test_common_validation.py ├── test_historical_batch.py ├── test_historical_bento.py ├── test_historical_client.py ├── test_historical_data.py ├── test_historical_error.py ├── test_historical_metadata.py ├── test_historical_timeseries.py ├── test_historical_warnings.py ├── test_live_client.py ├── test_live_client_reconnect.py ├── test_live_gateway_messages.py ├── test_live_protocol.py ├── test_live_session.py ├── test_reference_adjustment.py ├── test_reference_corporate.py ├── test_reference_security.py └── test_release.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set standard UNIX/Linux/macOS line endings 2 | * text eol=lf 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Expected Behavior 2 | Add here. 3 | 4 | ### Actual Behavior 5 | Add here. 6 | 7 | ### Steps to Reproduce the Problem 8 | 9 | 1. 10 | 2. 11 | 3. 12 | 13 | ### Specifications 14 | 15 | - OS Platform: 16 | - Databento Version: 17 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Please include a summary of the change and which issue is fixed. 4 | Please also include relevant motivation and context. 5 | List any dependencies that are required for this change. 6 | 7 | Fixes # (issue) 8 | 9 | ### Type of change 10 | 11 | Please delete options that are not relevant. 12 | 13 | - [ ] Bug fix (non-breaking change which fixes an issue) 14 | - [ ] New feature (non-breaking change which adds functionality) 15 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 16 | - [ ] This change requires a documentation update 17 | 18 | ### How has this been tested? 19 | 20 | Please describe the tests that you ran to verify your changes. 21 | Provide instructions so we can reproduce. 22 | Please also list any relevant details for your test configuration. 23 | 24 | - [ ] Test A 25 | - [ ] Test B 26 | 27 | ## Checklist 28 | 29 | - [ ] My code builds locally with no new warnings (`scripts/build.sh`) 30 | - [ ] My code follows the style guidelines (`scripts/lint.sh`) 31 | - [ ] New and existing unit tests pass locally with my changes (`scripts/test.sh`) 32 | - [ ] I have made corresponding changes to the documentation 33 | - [ ] I have added tests that prove my fix is effective or that my feature works 34 | 35 | ### Declaration 36 | 37 | I confirm this contribution is made under an Apache 2.0 license and that I have the authority 38 | necessary to make this contribution on behalf of its copyright owner. 39 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_run: 5 | workflows: 6 | - test 7 | branches: [main] 8 | types: 9 | - completed 10 | 11 | jobs: 12 | release: 13 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: actions/setup-python@v5 18 | with: 19 | python-version: "3.10" 20 | - uses: snok/install-poetry@v1 21 | with: 22 | virtualenvs-create: true 23 | virtualenvs-in-project: true 24 | installer-parallel: true 25 | 26 | - name: Build 27 | run: | 28 | poetry build --no-interaction 29 | echo "RELEASE_NAME=$(poetry version)" >> $GITHUB_ENV 30 | echo "TAG_NAME=v$(poetry version -s)" >> $GITHUB_ENV 31 | echo "## Release notes" > NOTES.md 32 | sed -n '/^## /{n; :a; /^## /q; p; n; ba}' CHANGELOG.md >> NOTES.md 33 | 34 | - name: Release 35 | uses: softprops/action-gh-release@v1 36 | with: 37 | fail_on_unmatched_files: true 38 | append_body: true 39 | name: ${{ env.RELEASE_NAME }} 40 | tag_name: ${{ env.TAG_NAME }} 41 | body_path: ./NOTES.md 42 | files: ./dist/* 43 | 44 | - name: Publish 45 | env: 46 | POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }} 47 | run: poetry publish 48 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | 7 | jobs: 8 | test: 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | os: [ubuntu-latest, macos-latest, windows-latest] 13 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 14 | name: build - Python ${{ matrix.python-version }} (${{ matrix.os }}) 15 | runs-on: ${{ matrix.os }} 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: actions/setup-python@v5 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - uses: snok/install-poetry@v1 23 | with: 24 | virtualenvs-create: true 25 | virtualenvs-in-project: true 26 | installer-parallel: true 27 | 28 | - name: Build 29 | run: scripts/build.sh 30 | shell: bash 31 | 32 | - name: Test (release) 33 | timeout-minutes: 5 34 | if: ${{ github.ref == 'refs/heads/main' }} 35 | run: scripts/test.sh -vvv --release 36 | shell: bash 37 | 38 | - name: Test 39 | timeout-minutes: 5 40 | if: ${{ github.ref != 'refs/heads/main' }} 41 | run: scripts/test.sh -vvv 42 | shell: bash 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # Unit test / coverage reports 31 | htmlcov/ 32 | .tox/ 33 | .nox/ 34 | .coverage 35 | .coverage.* 36 | .cache 37 | nosetests.xml 38 | coverage.xml 39 | *.cover 40 | .hypothesis/ 41 | .pytest_cache/ 42 | 43 | # pyenv 44 | .python-version 45 | 46 | # Environments 47 | .env 48 | .venv 49 | env/ 50 | venv/ 51 | ENV/ 52 | env.bak/ 53 | venv.bak/ 54 | 55 | # Visual Studio Code 56 | .vscode/* 57 | !.vscode/settings.json 58 | !.vscode/tasks.json 59 | !.vscode/launch.json 60 | !.vscode/extensions.json 61 | 62 | # Allow Jupyter tutorials 63 | !notebooks/* 64 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | info@nautechsystems.io. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for taking the time to contribute to our project. 2 | We welcome feedback through discussions and issues on GitHub, as well as our [community Slack](https://databento.com/support). 3 | While we don't merge pull requests directly due to the open-source repository being a downstream 4 | mirror of our internal codebase, we can commit the changes upstream with the original author. 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # databento-python 2 | 3 | [![test](https://github.com/databento/databento-python/actions/workflows/test.yml/badge.svg?branch=dev)](https://github.com/databento/databento-python/actions/workflows/test.yml) 4 | ![python](https://img.shields.io/pypi/pyversions/databento.svg) 5 | [![pypi-version](https://img.shields.io/pypi/v/databento)](https://pypi.org/project/databento) 6 | [![license](https://img.shields.io/github/license/databento/databento-python?color=blue)](./LICENSE) 7 | [![code-style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 8 | [![Slack](https://img.shields.io/badge/join_Slack-community-darkblue.svg?logo=slack)](https://to.dbn.to/slack) 9 | 10 | The official Python client library for [Databento](https://databento.com). 11 | 12 | Key features include: 13 | - Fast, lightweight access to both live and historical data from [multiple markets](https://databento.com/docs/faqs/venues-and-publishers). 14 | - [Multiple schemas](https://databento.com/docs/schemas-and-data-formats/whats-a-schema?historical=python&live=python) such as MBO, MBP, top of book, OHLCV, last sale, and more. 15 | - [Fully normalized](https://databento.com/docs/standards-and-conventions/normalization?historical=python&live=python), i.e. identical message schemas for both live and historical data, across multiple asset classes. 16 | - Provides mappings between different symbology systems, including [smart symbology](https://databento.com/docs/api-reference-historical/basics/symbology?historical=python&live=python) for futures rollovers. 17 | - [Point-in-time]() instrument definitions, free of look-ahead bias and retroactive adjustments. 18 | - Reads and stores market data in an extremely efficient file format using [Databento Binary Encoding](https://databento.com/docs/standards-and-conventions/databento-binary-encoding?historical=python&live=python). 19 | - Event-driven [market replay](https://databento.com/docs/api-reference-historical/helpers/bento-replay?historical=python&live=python), including at high-frequency order book granularity. 20 | - Support for [batch download](https://databento.com/docs/faqs/streaming-vs-batch-download?historical=python&live=python) of flat files. 21 | - Support for [pandas](https://pandas.pydata.org/docs/), CSV, and JSON. 22 | 23 | ## Documentation 24 | The best place to begin is with our [Getting started](https://databento.com/docs/quickstart?historical=python&live=python) guide. 25 | 26 | You can find our full client API reference on the [Historical Reference](https://databento.com/docs/api-reference-historical?historical=python&live=python) and 27 | [Live Reference](https://databento.com/docs/reference-live?historical=python&live=python) sections of our documentation. See also the 28 | [Examples](https://databento.com/docs/examples?historical=python&live=python) section for various tutorials and code samples. 29 | 30 | ## Requirements 31 | The library is fully compatible with the latest distribution of Anaconda 3.9 and above. 32 | The minimum dependencies as found in the `pyproject.toml` are also listed below: 33 | - python = "^3.9" 34 | - aiohttp = "^3.8.3" 35 | - databento-dbn = "0.35.1" 36 | - numpy= ">=1.23.5" 37 | - pandas = ">=1.5.3" 38 | - pip-system-certs = ">=4.0" (Windows only) 39 | - pyarrow = ">=13.0.0" 40 | - requests = ">=2.25.1" 41 | - zstandard = ">=0.21.0" 42 | 43 | ## Installation 44 | To install the latest stable version of the package from PyPI: 45 | 46 | pip install -U databento 47 | 48 | ## Usage 49 | The library needs to be configured with an API key from your account. 50 | [Sign up](https://databento.com/signup) for free and you will automatically 51 | receive a set of API keys to start with. Each API key is a 32-character 52 | string starting with `db-`, that can be found on the API Keys page of your [Databento user portal](https://databento.com/platform/keys). 53 | 54 | A simple Databento application looks like this: 55 | 56 | ```python 57 | import databento as db 58 | 59 | client = db.Historical('YOUR_API_KEY') 60 | data = client.timeseries.get_range( 61 | dataset='GLBX.MDP3', 62 | symbols='ES.FUT', 63 | stype_in='parent', 64 | start='2022-06-10T14:30', 65 | end='2022-06-10T14:40', 66 | ) 67 | 68 | data.replay(callback=print) # market replay, with `print` as event handler 69 | ``` 70 | 71 | Replace `YOUR_API_KEY` with an actual API key, then run this program. 72 | 73 | This uses `.replay()` to access the entire block of data 74 | and dispatch each data event to an event handler. You can also use 75 | `.to_df()` or `.to_ndarray()` to cast the data into a Pandas `DataFrame` or numpy `ndarray`: 76 | 77 | ```python 78 | df = data.to_df() # to DataFrame 79 | array = data.to_ndarray() # to ndarray 80 | ``` 81 | 82 | Note that the API key was also passed as a parameter, which is 83 | [not recommended for production applications](https://databento.com/docs/portal/api-keys?historical=python&live=python). 84 | Instead, you can leave out this parameter to pass your API key via the `DATABENTO_API_KEY` environment variable: 85 | 86 | ```python 87 | import databento as db 88 | 89 | # Pass as parameter 90 | client = db.Historical('YOUR_API_KEY') 91 | 92 | # Or, pass as `DATABENTO_API_KEY` environment variable 93 | client = db.Historical() 94 | ``` 95 | 96 | ## License 97 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 98 | -------------------------------------------------------------------------------- /databento/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import warnings 3 | 4 | from databento_dbn import DBN_VERSION 5 | from databento_dbn import FIXED_PRICE_SCALE 6 | from databento_dbn import UNDEF_ORDER_SIZE 7 | from databento_dbn import UNDEF_PRICE 8 | from databento_dbn import UNDEF_STAT_QUANTITY 9 | from databento_dbn import UNDEF_TIMESTAMP 10 | from databento_dbn import Action 11 | from databento_dbn import BidAskPair 12 | from databento_dbn import CMBP1Msg 13 | from databento_dbn import Compression 14 | from databento_dbn import ConsolidatedBidAskPair 15 | from databento_dbn import Encoding 16 | from databento_dbn import ErrorMsg 17 | from databento_dbn import ImbalanceMsg 18 | from databento_dbn import InstrumentClass 19 | from databento_dbn import InstrumentDefMsg 20 | from databento_dbn import MatchAlgorithm 21 | from databento_dbn import MBOMsg 22 | from databento_dbn import MBP1Msg 23 | from databento_dbn import MBP10Msg 24 | from databento_dbn import Metadata 25 | from databento_dbn import OHLCVMsg 26 | from databento_dbn import RType 27 | from databento_dbn import Schema 28 | from databento_dbn import SecurityUpdateAction 29 | from databento_dbn import Side 30 | from databento_dbn import StatMsg 31 | from databento_dbn import StatType 32 | from databento_dbn import StatUpdateAction 33 | from databento_dbn import StatusAction 34 | from databento_dbn import StatusMsg 35 | from databento_dbn import StatusReason 36 | from databento_dbn import SType 37 | from databento_dbn import SymbolMappingMsg 38 | from databento_dbn import SystemMsg 39 | from databento_dbn import TradeMsg 40 | from databento_dbn import TradingEvent 41 | from databento_dbn import TriState 42 | from databento_dbn import UserDefinedInstrument 43 | from databento_dbn import VersionUpgradePolicy 44 | from databento_dbn.v2 import BBO1MMsg 45 | from databento_dbn.v2 import BBO1SMsg 46 | from databento_dbn.v2 import CBBO1MMsg 47 | from databento_dbn.v2 import CBBO1SMsg 48 | from databento_dbn.v2 import TBBOMsg 49 | from databento_dbn.v2 import TCBBOMsg 50 | 51 | from databento.common import API_VERSION 52 | from databento.common import bentologging 53 | from databento.common import symbology 54 | from databento.common.dbnstore import DBNStore 55 | from databento.common.enums import Delivery 56 | from databento.common.enums import FeedMode 57 | from databento.common.enums import HistoricalGateway 58 | from databento.common.enums import Packaging 59 | from databento.common.enums import ReconnectPolicy 60 | from databento.common.enums import RecordFlags 61 | from databento.common.enums import RollRule 62 | from databento.common.enums import SplitDuration 63 | from databento.common.enums import SymbologyResolution 64 | from databento.common.error import BentoClientError 65 | from databento.common.error import BentoError 66 | from databento.common.error import BentoHttpError 67 | from databento.common.error import BentoServerError 68 | from databento.common.publishers import Dataset 69 | from databento.common.publishers import Publisher 70 | from databento.common.publishers import Venue 71 | from databento.common.symbology import InstrumentMap 72 | from databento.common.types import DBNRecord 73 | from databento.historical.client import Historical 74 | from databento.live.client import Live 75 | from databento.reference.client import Reference 76 | from databento.version import __version__ # noqa 77 | 78 | 79 | __all__ = [ 80 | "API_VERSION", 81 | "DBN_VERSION", 82 | "FIXED_PRICE_SCALE", 83 | "UNDEF_ORDER_SIZE", 84 | "UNDEF_PRICE", 85 | "UNDEF_STAT_QUANTITY", 86 | "UNDEF_TIMESTAMP", 87 | "Action", 88 | "BBO1MMsg", 89 | "BBO1SMsg", 90 | "BentoClientError", 91 | "BentoError", 92 | "BentoHttpError", 93 | "BentoServerError", 94 | "BidAskPair", 95 | "CBBO1MMsg", 96 | "CBBO1SMsg", 97 | "CMBP1Msg", 98 | "Compression", 99 | "ConsolidatedBidAskPair", 100 | "DBNRecord", 101 | "DBNStore", 102 | "Dataset", 103 | "Delivery", 104 | "Encoding", 105 | "ErrorMsg", 106 | "FeedMode", 107 | "Historical", 108 | "HistoricalGateway", 109 | "ImbalanceMsg", 110 | "InstrumentClass", 111 | "InstrumentDefMsg", 112 | "InstrumentMap", 113 | "Live", 114 | "MBOMsg", 115 | "MBP1Msg", 116 | "MBP10Msg", 117 | "MatchAlgorithm", 118 | "Metadata", 119 | "OHLCVMsg", 120 | "Packaging", 121 | "Publisher", 122 | "RType", 123 | "ReconnectPolicy", 124 | "RecordFlags", 125 | "Reference", 126 | "RollRule", 127 | "SType", 128 | "Schema", 129 | "SecurityUpdateAction", 130 | "Side", 131 | "SplitDuration", 132 | "StatMsg", 133 | "StatType", 134 | "StatUpdateAction", 135 | "StatusAction", 136 | "StatusMsg", 137 | "StatusReason", 138 | "SymbolMappingMsg", 139 | "SymbologyResolution", 140 | "SystemMsg", 141 | "TBBOMsg", 142 | "TBBOMsg", 143 | "TCBBOMsg", 144 | "TradeMsg", 145 | "TradingEvent", 146 | "TriState", 147 | "UserDefinedInstrument", 148 | "Venue", 149 | "VersionUpgradePolicy", 150 | ] 151 | 152 | # Setup logging 153 | logging.getLogger(__name__).addHandler(logging.NullHandler()) 154 | 155 | # Setup deprecation warnings 156 | warnings.simplefilter("always", DeprecationWarning) 157 | 158 | # Convenience imports 159 | enable_logging = bentologging.enable_logging 160 | read_dbn = DBNStore.from_file 161 | map_symbols_csv = symbology.map_symbols_csv 162 | map_symbols_json = symbology.map_symbols_json 163 | -------------------------------------------------------------------------------- /databento/common/__init__.py: -------------------------------------------------------------------------------- 1 | API_VERSION = 0 2 | -------------------------------------------------------------------------------- /databento/common/bentologging.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | 5 | 6 | def enable_logging(level: int | str = logging.INFO) -> None: 7 | """ 8 | Enable logging for the Databento module. This function should be used for 9 | simple applications and examples. It is advisable to configure your own 10 | logging for serious applications. 11 | 12 | Parameters 13 | ---------- 14 | level : str or int, default 'INFO' 15 | The log level to configure. 16 | 17 | See Also 18 | -------- 19 | logging 20 | 21 | """ 22 | # Create a basic formatter 23 | formatter = logging.Formatter( 24 | fmt=logging.BASIC_FORMAT, 25 | ) 26 | 27 | # Construct a stream handler for stderr 28 | handler = logging.StreamHandler() 29 | handler.setFormatter(formatter) 30 | handler.setLevel(level=level) 31 | 32 | # Add the handler to the databento logger 33 | databento_logger = logging.getLogger("databento") 34 | databento_logger.setLevel(level=level) 35 | databento_logger.addHandler(handler) 36 | -------------------------------------------------------------------------------- /databento/common/constants.py: -------------------------------------------------------------------------------- 1 | from typing import Final 2 | 3 | import numpy as np 4 | from databento_dbn import BBOMsg 5 | from databento_dbn import CBBOMsg 6 | from databento_dbn import CMBP1Msg 7 | from databento_dbn import ImbalanceMsg 8 | from databento_dbn import InstrumentDefMsg 9 | from databento_dbn import MBOMsg 10 | from databento_dbn import MBP1Msg 11 | from databento_dbn import MBP10Msg 12 | from databento_dbn import OHLCVMsg 13 | from databento_dbn import Schema 14 | from databento_dbn import StatMsg 15 | from databento_dbn import StatusMsg 16 | from databento_dbn import TradeMsg 17 | from databento_dbn import v1 18 | from databento_dbn import v2 19 | 20 | from databento.common.types import DBNRecord 21 | 22 | 23 | ALL_SYMBOLS: Final = "ALL_SYMBOLS" 24 | 25 | 26 | DEFINITION_TYPE_MAX_MAP: Final = { 27 | x[0]: np.iinfo(x[1]).max for x in InstrumentDefMsg._dtypes if not isinstance(x[1], str) 28 | } 29 | 30 | HTTP_STREAMING_READ_SIZE: Final = 2**12 31 | 32 | SCHEMA_STRUCT_MAP: Final[dict[Schema, type[DBNRecord]]] = { 33 | Schema.DEFINITION: InstrumentDefMsg, 34 | Schema.IMBALANCE: ImbalanceMsg, 35 | Schema.MBO: MBOMsg, 36 | Schema.MBP_1: MBP1Msg, 37 | Schema.MBP_10: MBP10Msg, 38 | Schema.OHLCV_1S: OHLCVMsg, 39 | Schema.OHLCV_1M: OHLCVMsg, 40 | Schema.OHLCV_1H: OHLCVMsg, 41 | Schema.OHLCV_1D: OHLCVMsg, 42 | Schema.OHLCV_EOD: OHLCVMsg, 43 | Schema.STATISTICS: StatMsg, 44 | Schema.STATUS: StatusMsg, 45 | Schema.TBBO: MBP1Msg, 46 | Schema.TRADES: TradeMsg, 47 | Schema.CMBP_1: CMBP1Msg, 48 | Schema.CBBO_1S: CBBOMsg, 49 | Schema.CBBO_1M: CBBOMsg, 50 | Schema.TCBBO: CBBOMsg, 51 | Schema.BBO_1S: BBOMsg, 52 | Schema.BBO_1M: BBOMsg, 53 | } 54 | 55 | SCHEMA_STRUCT_MAP_V2: Final[dict[Schema, type[DBNRecord]]] = { 56 | Schema.DEFINITION: v2.InstrumentDefMsg, 57 | Schema.IMBALANCE: v2.ImbalanceMsg, 58 | Schema.MBO: v2.MBOMsg, 59 | Schema.MBP_1: v2.MBP1Msg, 60 | Schema.MBP_10: v2.MBP10Msg, 61 | Schema.OHLCV_1S: v2.OHLCVMsg, 62 | Schema.OHLCV_1M: v2.OHLCVMsg, 63 | Schema.OHLCV_1H: v2.OHLCVMsg, 64 | Schema.OHLCV_1D: v2.OHLCVMsg, 65 | Schema.STATISTICS: v2.StatMsg, 66 | Schema.STATUS: v2.StatusMsg, 67 | Schema.TBBO: v2.MBP1Msg, 68 | Schema.TRADES: v2.TradeMsg, 69 | Schema.CMBP_1: v2.CMBP1Msg, 70 | Schema.CBBO_1S: v2.CBBOMsg, 71 | Schema.CBBO_1M: v2.CBBOMsg, 72 | Schema.TCBBO: v2.CBBOMsg, 73 | Schema.BBO_1S: v2.BBOMsg, 74 | Schema.BBO_1M: v2.BBOMsg, 75 | } 76 | 77 | SCHEMA_STRUCT_MAP_V1: Final[dict[Schema, type[DBNRecord]]] = { 78 | Schema.DEFINITION: v1.InstrumentDefMsg, 79 | Schema.IMBALANCE: v1.ImbalanceMsg, 80 | Schema.MBO: v1.MBOMsg, 81 | Schema.MBP_1: v1.MBP1Msg, 82 | Schema.MBP_10: v1.MBP10Msg, 83 | Schema.OHLCV_1S: v1.OHLCVMsg, 84 | Schema.OHLCV_1M: v1.OHLCVMsg, 85 | Schema.OHLCV_1H: v1.OHLCVMsg, 86 | Schema.OHLCV_1D: v1.OHLCVMsg, 87 | Schema.STATISTICS: v1.StatMsg, 88 | Schema.STATUS: v1.StatusMsg, 89 | Schema.TBBO: v1.MBP1Msg, 90 | Schema.TRADES: v1.TradeMsg, 91 | Schema.CMBP_1: v1.CMBP1Msg, 92 | Schema.CBBO_1S: v1.CBBOMsg, 93 | Schema.CBBO_1M: v1.CBBOMsg, 94 | Schema.TCBBO: v1.CBBOMsg, 95 | Schema.BBO_1S: v1.BBOMsg, 96 | Schema.BBO_1M: v1.BBOMsg, 97 | } 98 | 99 | 100 | CORPORATE_ACTIONS_DATETIME_COLUMNS: Final[list[str]] = [ 101 | "ts_record", 102 | "ts_created", 103 | ] 104 | 105 | CORPORATE_ACTIONS_DATE_COLUMNS: Final[list[str]] = [ 106 | "event_date", 107 | "event_created_date", 108 | "effective_date", 109 | "ex_date", 110 | "record_date", 111 | "listing_date", 112 | "delisting_date", 113 | "payment_date", 114 | "duebills_redemption_date", 115 | "from_date", 116 | "to_date", 117 | "registration_date", 118 | "start_date", 119 | "end_date", 120 | "open_date", 121 | "close_date", 122 | "start_subscription_date", 123 | "end_subscription_date", 124 | "option_election_date", 125 | "withdrawal_right_from_date", 126 | "withdrawal_rights_to_date", 127 | "notification_date", 128 | "financial_year_end_date", 129 | "exp_completion_date", 130 | ] 131 | 132 | ADJUSTMENT_FACTORS_DATETIME_COLUMNS: Final[list[str]] = [ 133 | "ts_created", 134 | ] 135 | 136 | ADJUSTMENT_FACTORS_DATE_COLUMNS: Final[list[str]] = [ 137 | "ex_date", 138 | ] 139 | 140 | SECURITY_MASTER_DATETIME_COLUMNS: Final[list[str]] = [ 141 | "ts_record", 142 | "ts_effective", 143 | "ts_created", 144 | ] 145 | 146 | SECURITY_MASTER_DATE_COLUMNS: Final[list[str]] = [ 147 | "listing_created_date", 148 | "listing_date", 149 | "delisting_date", 150 | "shares_outstanding_date", 151 | ] 152 | -------------------------------------------------------------------------------- /databento/common/cram.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions for handling challenge-response authentication. 3 | """ 4 | 5 | import argparse 6 | import hashlib 7 | import os 8 | import sys 9 | from typing import Final 10 | 11 | 12 | BUCKET_ID_LENGTH: Final = 5 13 | 14 | 15 | def get_challenge_response(challenge: str, key: str) -> str: 16 | """ 17 | Return the response for a given challenge-response authentication mechanism 18 | (CRAM) code provided by a Databento service. 19 | 20 | A valid API key is hashed with the challenge string. 21 | 22 | Parameters 23 | ---------- 24 | challenge : str 25 | The CRAM challenge string. 26 | key : str 27 | The user API key for authentication. 28 | 29 | Returns 30 | ------- 31 | str 32 | 33 | """ 34 | bucket_id = key[-BUCKET_ID_LENGTH:] 35 | sha = hashlib.sha256(f"{challenge}|{key}".encode()).hexdigest() 36 | return f"{sha}-{bucket_id}" 37 | 38 | 39 | if __name__ == "__main__": 40 | parser = argparse.ArgumentParser( 41 | description="Script for computing a CRAM response.", 42 | ) 43 | parser.add_argument( 44 | "challenge", 45 | help="The CRAM challenge string", 46 | ) 47 | parser.add_argument( 48 | "key", 49 | nargs="?", 50 | default=os.environ.get("DATABENTO_API_KEY"), 51 | help="An API key; defaults to the value of DATABENTO_API_KEY if set", 52 | ) 53 | arguments = parser.parse_args(sys.argv[1:]) 54 | 55 | if arguments.key is None: 56 | parser.print_usage() 57 | exit(1) 58 | 59 | print( 60 | get_challenge_response( 61 | challenge=arguments.challenge, 62 | key=arguments.key, 63 | ), 64 | ) 65 | -------------------------------------------------------------------------------- /databento/common/enums.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from enum import Enum 4 | from enum import Flag 5 | from enum import IntFlag 6 | from enum import unique 7 | from typing import Callable 8 | from typing import TypeVar 9 | 10 | 11 | M = TypeVar("M", bound=Enum) 12 | 13 | 14 | def coercible(enum_type: type[M]) -> type[M]: 15 | """ 16 | Decorate coercible enumerations. 17 | 18 | Decorating an Enum class with this function will intercept calls to 19 | __new__ and perform a type coercion for the passed value. The type conversion 20 | function is chosen based on the subclass of the Enum type. 21 | 22 | Currently supported subclasses types: 23 | int 24 | values are passed to int() 25 | str 26 | values are passed to str(), the result is also lowercased 27 | 28 | Parameters 29 | ---------- 30 | enum_type : EnumMeta 31 | The deocrated Enum type. 32 | 33 | Returns 34 | ------- 35 | EnumMeta 36 | 37 | Raises 38 | ------ 39 | ValueError 40 | If an invalid value of the Enum is given. 41 | 42 | Notes 43 | ----- 44 | This decorator makes some assumptions about your Enum class. 45 | 1. Your attribute names are all UPPERCASE 46 | 2. Your attribute values are all lowercase 47 | 48 | """ 49 | _new: Callable[[type[M], object], M] = enum_type.__new__ 50 | 51 | def _cast_str(value: object) -> str: 52 | return str(value).lower() 53 | 54 | coerce_fn: Callable[[object], str | int] 55 | if issubclass(enum_type, int): 56 | coerce_fn = int 57 | elif issubclass(enum_type, str): 58 | coerce_fn = _cast_str 59 | else: 60 | raise TypeError(f"{enum_type} does not a subclass a coercible type.") 61 | 62 | def coerced_new(enum: type[M], value: object) -> M: 63 | if value is None: 64 | raise ValueError( 65 | f"value `{value}` is not coercible to {enum_type.__name__}.", 66 | ) 67 | try: 68 | return _new(enum, coerce_fn(value)) 69 | except ValueError: 70 | name_to_try = str(value).replace(".", "_").replace("-", "_").upper() 71 | named = enum._member_map_.get(name_to_try) 72 | if named is not None: 73 | return named 74 | enum_values = list(value for value in enum._value2member_map_) 75 | 76 | raise ValueError( 77 | f"The `{value}` was not a valid value of {enum_type.__name__}" 78 | f", was '{value}'. Use any of {enum_values}.", 79 | ) from None 80 | 81 | setattr(enum_type, "__new__", coerced_new) 82 | 83 | return enum_type 84 | 85 | 86 | class StringyMixin: 87 | """ 88 | Mixin class for overloading __str__ on Enum types. This will use the 89 | Enumerations subclass, if any, to modify the behavior of str(). 90 | 91 | For subclasses of enum.Flag a comma separated string of names is 92 | returned. For integer enumerations, the lowercase member name is 93 | returned. For string enumerations, the value is returned. 94 | 95 | """ 96 | 97 | def __str__(self) -> str: 98 | if isinstance(self, Flag): 99 | return ", ".join(f.name.lower() for f in self.__class__ if f in self) 100 | if isinstance(self, int): 101 | return getattr(self, "name").lower() 102 | return getattr(self, "value") 103 | 104 | 105 | @unique 106 | @coercible 107 | class HistoricalGateway(StringyMixin, str, Enum): 108 | """ 109 | Represents a historical data center gateway location. 110 | """ 111 | 112 | BO1 = "https://hist.databento.com" 113 | 114 | 115 | @unique 116 | @coercible 117 | class FeedMode(StringyMixin, str, Enum): 118 | """ 119 | Represents a data feed mode. 120 | """ 121 | 122 | HISTORICAL = "historical" 123 | HISTORICAL_STREAMING = "historical-streaming" 124 | LIVE = "live" 125 | 126 | 127 | @unique 128 | @coercible 129 | class SplitDuration(StringyMixin, str, Enum): 130 | """ 131 | Represents the duration before splitting for each batched data file. 132 | """ 133 | 134 | DAY = "day" 135 | WEEK = "week" 136 | MONTH = "month" 137 | NONE = "none" 138 | 139 | 140 | @unique 141 | @coercible 142 | class Packaging(StringyMixin, str, Enum): 143 | """ 144 | Represents the packaging method for batched data files. 145 | """ 146 | 147 | NONE = "none" 148 | ZIP = "zip" 149 | TAR = "tar" 150 | 151 | 152 | @unique 153 | @coercible 154 | class Delivery(StringyMixin, str, Enum): 155 | """ 156 | Represents the delivery mechanism for batched data. 157 | """ 158 | 159 | DOWNLOAD = "download" 160 | S3 = "s3" 161 | DISK = "disk" 162 | 163 | 164 | @unique 165 | @coercible 166 | class RollRule(StringyMixin, str, Enum): 167 | """ 168 | Represents a smart symbology roll rule. 169 | """ 170 | 171 | VOLUME = "volume" 172 | OPEN_INTEREST = "open_interst" 173 | CALENDAR = "calendar" 174 | 175 | 176 | @unique 177 | @coercible 178 | class SymbologyResolution(StringyMixin, str, Enum): 179 | """ 180 | Status code of symbology resolution. 181 | 182 | - OK: All symbol mappings resolved. 183 | - PARTIAL: One or more symbols did not resolve on at least one date. 184 | - NOT_FOUND: One or more symbols where not found on any date in range. 185 | 186 | """ 187 | 188 | OK = "ok" 189 | PARTIAL = "partial" 190 | NOT_FOUND = "not_found" 191 | 192 | 193 | @unique 194 | @coercible 195 | # Ignore type to work around mypy bug https://github.com/python/mypy/issues/9319 196 | class RecordFlags(StringyMixin, IntFlag): # type: ignore 197 | """ 198 | Represents record flags. 199 | 200 | F_LAST 201 | Marks the last record in a single event for a given `instrument_id`. 202 | F_TOB 203 | Indicates a top-of-book message, not an individual order. 204 | F_SNAPSHOT 205 | Message sourced from a replay, such as a snapshot server. 206 | F_MBP 207 | Aggregated price level message, not an individual order. 208 | F_BAD_TS_RECV 209 | The `ts_recv` value is inaccurate (clock issues or reordering). 210 | F_MAYBE_BAD_BOOK 211 | Indicates an unrecoverable gap was detected in the channel. 212 | 213 | Other bits are reserved and have no current meaning. 214 | 215 | """ 216 | 217 | F_LAST = 128 218 | F_TOB = 64 219 | F_SNAPSHOT = 32 220 | F_MBP = 16 221 | F_BAD_TS_RECV = 8 222 | F_MAYBE_BAD_BOOK = 4 223 | 224 | 225 | @unique 226 | @coercible 227 | class ReconnectPolicy(StringyMixin, str, Enum): 228 | """ 229 | Live session reconnection policy. 230 | """ 231 | 232 | NONE = "none" 233 | RECONNECT = "reconnect" 234 | 235 | 236 | @unique 237 | @coercible 238 | class PriceType(StringyMixin, str, Enum): 239 | """ 240 | Price type for DataFrame price fields. 241 | """ 242 | 243 | FIXED = "fixed" 244 | FLOAT = "float" 245 | DECIMAL = "decimal" 246 | -------------------------------------------------------------------------------- /databento/common/error.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import Mapping 4 | from typing import Any 5 | 6 | 7 | class BentoError(Exception): 8 | """ 9 | Represents a Databento specific error. 10 | """ 11 | 12 | 13 | class BentoHttpError(BentoError): 14 | """ 15 | Represents a Databento specific HTTP error. 16 | """ 17 | 18 | def __init__( 19 | self, 20 | http_status: int, 21 | http_body: bytes | str | None = None, 22 | json_body: dict[str, Any] | None = None, 23 | message: str | None = None, 24 | headers: Mapping[str, Any] | None = None, 25 | ) -> None: 26 | super().__init__(message) 27 | 28 | if http_body and isinstance(http_body, bytes): 29 | try: 30 | http_body = http_body.decode("utf-8") 31 | except UnicodeDecodeError: 32 | http_body = ( 33 | "" 34 | ) 35 | 36 | self.http_status = http_status 37 | self.http_body = http_body 38 | self.json_body = json_body 39 | 40 | try: 41 | json_message = self.json_body["detail"]["message"] 42 | json_case = self.json_body["detail"]["case"] 43 | json_docs = self.json_body["detail"]["docs"] 44 | except (TypeError, KeyError): 45 | self.message = message 46 | else: 47 | composed_message = [ 48 | json_case, 49 | json_message, 50 | ] 51 | if json_docs: 52 | composed_message.append(f"documentation: {json_docs}") 53 | 54 | self.message = "\n".join(composed_message) 55 | 56 | self.headers = headers or {} 57 | self.request_id = self.headers.get("request-id", None) 58 | 59 | def __str__(self) -> str: 60 | msg = self.message or "" 61 | msg = f"{self.http_status} {msg}" 62 | if self.request_id is not None: 63 | return f"Request {self.request_id}: {msg}" 64 | else: 65 | return msg 66 | 67 | def __repr__(self) -> str: 68 | return ( 69 | f"{type(self).__name__}(" 70 | f"request_id={self.request_id}, " 71 | f"http_status={self.http_status}, " 72 | f"message={self.message})" 73 | ) 74 | 75 | 76 | class BentoServerError(BentoHttpError): 77 | """ 78 | Represents a Databento specific server side 500 series HTTP error. 79 | """ 80 | 81 | def __init__( 82 | self, 83 | http_status: int, 84 | http_body: bytes | str | None = None, 85 | json_body: dict[str, Any] | None = None, 86 | message: str | None = None, 87 | headers: Mapping[str, Any] | None = None, 88 | ) -> None: 89 | super().__init__( 90 | http_status=http_status, 91 | http_body=http_body, 92 | json_body=json_body, 93 | message=message, 94 | headers=headers, 95 | ) 96 | 97 | 98 | class BentoClientError(BentoHttpError): 99 | """ 100 | Represents a Databento specific client side 400 series HTTP error. 101 | """ 102 | 103 | def __init__( 104 | self, 105 | http_status: int, 106 | http_body: bytes | str | None = None, 107 | json_body: dict[str, Any] | None = None, 108 | message: str | None = None, 109 | headers: Mapping[str, Any] | None = None, 110 | ) -> None: 111 | super().__init__( 112 | http_status=http_status, 113 | http_body=http_body, 114 | json_body=json_body, 115 | message=message, 116 | headers=headers, 117 | ) 118 | 119 | 120 | class BentoWarning(Warning): 121 | """ 122 | Represents a Databento specific warning. 123 | """ 124 | 125 | 126 | class BentoDeprecationWarning(BentoWarning): 127 | """ 128 | Represents a Databento deprecation warning. 129 | """ 130 | -------------------------------------------------------------------------------- /databento/common/iterator.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from collections.abc import Iterable 3 | from typing import TypeVar 4 | 5 | 6 | _C = TypeVar("_C") 7 | 8 | 9 | def chunk(iterable: Iterable[_C], size: int) -> Iterable[tuple[_C, ...]]: 10 | """ 11 | Break an iterable into chunks with a length of at most `size`. 12 | 13 | Parameters 14 | ---------- 15 | iterable: Iterable[_C] 16 | The iterable to break up. 17 | size : int 18 | The maximum size of each chunk. 19 | 20 | Returns 21 | ------- 22 | Iterable[_C] 23 | 24 | Raises 25 | ------ 26 | ValueError 27 | If `size` is less than 1. 28 | 29 | """ 30 | if size < 1: 31 | raise ValueError("size must be at least 1") 32 | 33 | it = iter(iterable) 34 | return iter(lambda: tuple(itertools.islice(it, size)), ()) 35 | -------------------------------------------------------------------------------- /databento/common/system.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import re 3 | from typing import Final 4 | 5 | from databento.version import __version__ 6 | 7 | 8 | TOKEN_PATTERN: Final = re.compile(r"[^a-zA-Z0-9\.]") 9 | 10 | PLATFORM_NAME: Final = TOKEN_PATTERN.sub("-", platform.system()) 11 | PLATFORM_VERSION: Final = TOKEN_PATTERN.sub("-", platform.release()) 12 | PYTHON_VERSION: Final = TOKEN_PATTERN.sub("-", platform.python_version()) 13 | 14 | USER_AGENT: Final = ( 15 | f"Databento/{__version__} Python/{PYTHON_VERSION} {PLATFORM_NAME}/{PLATFORM_VERSION}" 16 | ) 17 | -------------------------------------------------------------------------------- /databento/common/types.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | from typing import Callable 3 | from typing import Generic 4 | from typing import TypedDict 5 | from typing import TypeVar 6 | from typing import Union 7 | 8 | import databento_dbn 9 | import pandas as pd 10 | 11 | 12 | DBNRecord = Union[ 13 | databento_dbn.BBOMsg, 14 | databento_dbn.CBBOMsg, 15 | databento_dbn.CMBP1Msg, 16 | databento_dbn.MBOMsg, 17 | databento_dbn.MBP1Msg, 18 | databento_dbn.MBP10Msg, 19 | databento_dbn.TradeMsg, 20 | databento_dbn.OHLCVMsg, 21 | databento_dbn.ImbalanceMsg, 22 | databento_dbn.InstrumentDefMsg, 23 | databento_dbn.InstrumentDefMsgV1, 24 | databento_dbn.InstrumentDefMsgV2, 25 | databento_dbn.StatMsg, 26 | databento_dbn.StatMsgV1, 27 | databento_dbn.StatusMsg, 28 | databento_dbn.SymbolMappingMsg, 29 | databento_dbn.SymbolMappingMsgV1, 30 | databento_dbn.SystemMsg, 31 | databento_dbn.SystemMsgV1, 32 | databento_dbn.ErrorMsg, 33 | databento_dbn.ErrorMsgV1, 34 | ] 35 | 36 | RecordCallback = Callable[[DBNRecord], None] 37 | ExceptionCallback = Callable[[Exception], None] 38 | ReconnectCallback = Callable[[pd.Timestamp, pd.Timestamp], None] 39 | 40 | _T = TypeVar("_T") 41 | 42 | 43 | class Default(Generic[_T]): 44 | """ 45 | A container for a default value. This is to be used when a callable wants 46 | to detect if a default parameter value is being used. 47 | 48 | Example 49 | ------- 50 | def foo(param=Default[int](10)): 51 | if isinstance(param, Default): 52 | print(f"param={param.value} (default)") 53 | else: 54 | print(f"param={param.value}") 55 | 56 | """ 57 | 58 | def __init__(self, value: _T): 59 | self._value = value 60 | 61 | @property 62 | def value(self) -> _T: 63 | """ 64 | The default value. 65 | 66 | Returns 67 | ------- 68 | _T 69 | 70 | """ 71 | return self._value 72 | 73 | 74 | class MappingIntervalDict(TypedDict): 75 | """ 76 | Represents a symbol mapping over a start and end date range interval. 77 | 78 | Parameters 79 | ---------- 80 | start_date : dt.date 81 | The start of the mapping period. 82 | end_date : dt.date 83 | The end of the mapping period. 84 | symbol : str 85 | The symbol value. 86 | 87 | """ 88 | 89 | start_date: dt.date 90 | end_date: dt.date 91 | symbol: str 92 | -------------------------------------------------------------------------------- /databento/historical/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/historical/__init__.py -------------------------------------------------------------------------------- /databento/historical/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/historical/api/__init__.py -------------------------------------------------------------------------------- /databento/historical/api/symbology.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import Iterable 4 | from datetime import date 5 | from typing import Any 6 | 7 | from databento_dbn import SType 8 | from requests import Response 9 | 10 | from databento.common import API_VERSION 11 | from databento.common.http import BentoHttpAPI 12 | from databento.common.parsing import datetime_to_date_string 13 | from databento.common.parsing import optional_date_to_string 14 | from databento.common.parsing import optional_symbols_list_to_list 15 | from databento.common.publishers import Dataset 16 | from databento.common.validation import validate_enum 17 | from databento.common.validation import validate_semantic_string 18 | 19 | 20 | class SymbologyHttpAPI(BentoHttpAPI): 21 | """ 22 | Provides request methods for the symbology HTTP API endpoints. 23 | """ 24 | 25 | def __init__(self, key: str, gateway: str) -> None: 26 | super().__init__(key=key, gateway=gateway) 27 | self._base_url = gateway + f"/v{API_VERSION}/symbology" 28 | 29 | def resolve( 30 | self, 31 | dataset: Dataset | str, 32 | symbols: Iterable[str | int] | str | int, 33 | stype_in: SType | str, 34 | stype_out: SType | str, 35 | start_date: date | str, 36 | end_date: date | str | None = None, 37 | ) -> dict[str, Any]: 38 | """ 39 | Request symbology mappings resolution from Databento. 40 | 41 | Makes a `POST /symbology.resolve` HTTP request. 42 | 43 | Parameters 44 | ---------- 45 | dataset : Dataset or str 46 | The dataset code (string identifier) for the request. 47 | symbols : Iterable[str | int] or str or int, optional 48 | The symbols to resolve. Takes up to 2,000 symbols per request. 49 | stype_in : SType or str, default 'raw_symbol' 50 | The input symbology type to resolve from. 51 | stype_out : SType or str, default 'instrument_id' 52 | The output symbology type to resolve to. 53 | start_date : date or str 54 | The start date (UTC) of the request time range (inclusive). 55 | end_date : date or str, optional 56 | The end date (UTC) of the request time range (exclusive). 57 | 58 | Returns 59 | ------- 60 | dict[str, Any] 61 | A result including a map of input symbol to output symbol across a 62 | date range. 63 | 64 | """ 65 | stype_in_valid = validate_enum(stype_in, SType, "stype_in") 66 | symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid) 67 | data: dict[str, object | None] = { 68 | "dataset": validate_semantic_string(dataset, "dataset"), 69 | "symbols": ",".join(symbols_list), 70 | "stype_in": str(stype_in_valid), 71 | "stype_out": str(validate_enum(stype_out, SType, "stype_out")), 72 | "start_date": datetime_to_date_string(start_date), 73 | "end_date": optional_date_to_string(end_date), 74 | } 75 | 76 | response: Response = self._post( 77 | url=self._base_url + ".resolve", 78 | data=data, 79 | basic_auth=True, 80 | ) 81 | 82 | return response.json() 83 | -------------------------------------------------------------------------------- /databento/historical/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | 6 | from databento.common.enums import HistoricalGateway 7 | from databento.common.validation import validate_gateway 8 | from databento.historical.api.batch import BatchHttpAPI 9 | from databento.historical.api.metadata import MetadataHttpAPI 10 | from databento.historical.api.symbology import SymbologyHttpAPI 11 | from databento.historical.api.timeseries import TimeseriesHttpAPI 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class Historical: 18 | """ 19 | Provides a client connection class for requesting historical data and 20 | metadata from the Databento API servers. 21 | 22 | Parameters 23 | ---------- 24 | key : str, optional 25 | The user API key for authentication. 26 | If `None` then the `DATABENTO_API_KEY` environment variable is used. 27 | gateway : HistoricalGateway or str, default HistoricalGateway.BO1 28 | The API server gateway. 29 | If `None` then the default gateway is used. 30 | 31 | Examples 32 | -------- 33 | > import databento as db 34 | > client = db.Historical('YOUR_API_KEY') 35 | 36 | """ 37 | 38 | def __init__( 39 | self, 40 | key: str | None = None, 41 | gateway: HistoricalGateway | str = HistoricalGateway.BO1, 42 | ): 43 | if key is None: 44 | key = os.environ.get("DATABENTO_API_KEY") 45 | if key is None or not isinstance(key, str) or key.isspace(): 46 | raise ValueError(f"invalid API key, was {key}") 47 | 48 | try: 49 | gateway = HistoricalGateway(gateway) 50 | except ValueError: 51 | gateway = validate_gateway(str(gateway)) 52 | 53 | self._key = key 54 | self._gateway = gateway 55 | 56 | self.batch = BatchHttpAPI(key=key, gateway=gateway) 57 | self.metadata = MetadataHttpAPI(key=key, gateway=gateway) 58 | self.symbology = SymbologyHttpAPI(key=key, gateway=gateway) 59 | self.timeseries = TimeseriesHttpAPI(key=key, gateway=gateway) 60 | 61 | # Not logging security sensitive `key` 62 | logger.info("Initialized %s(gateway=%s)", type(self).__name__, self.gateway) 63 | 64 | @property 65 | def key(self) -> str: 66 | """ 67 | Return the user API key for the client. 68 | 69 | Returns 70 | ------- 71 | str 72 | 73 | """ 74 | return self._key 75 | 76 | @property 77 | def gateway(self) -> str: 78 | """ 79 | Return the API server gateway for the client. 80 | 81 | Returns 82 | ------- 83 | str 84 | 85 | """ 86 | return self._gateway 87 | -------------------------------------------------------------------------------- /databento/live/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/live/__init__.py -------------------------------------------------------------------------------- /databento/live/gateway.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import dataclasses 4 | import logging 5 | from io import BytesIO 6 | from operator import attrgetter 7 | from typing import SupportsBytes 8 | from typing import TypeVar 9 | 10 | from databento_dbn import Encoding 11 | from databento_dbn import Schema 12 | from databento_dbn import SType 13 | 14 | from databento.common.publishers import Dataset 15 | from databento.common.system import USER_AGENT 16 | 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | T = TypeVar("T", bound="GatewayControl") 21 | 22 | 23 | @dataclasses.dataclass 24 | class GatewayControl(SupportsBytes): 25 | """ 26 | Base class for gateway control messages. 27 | """ 28 | 29 | @classmethod 30 | def parse(cls: type[T], line: str | bytes) -> T: 31 | """ 32 | Parse a message of type `T` from a string. 33 | 34 | Parameters 35 | ---------- 36 | line : str | bytes 37 | The data to parse into a GatewayControl message. 38 | 39 | Returns 40 | ------- 41 | T 42 | 43 | Raises 44 | ------ 45 | ValueError 46 | If the line fails to parse. 47 | 48 | """ 49 | if isinstance(line, bytes): 50 | line = line.decode("utf-8") 51 | 52 | if not line.endswith("\n"): 53 | raise ValueError(f"'{line!r}' does not end with a newline") 54 | 55 | split_tokens = [t.partition("=") for t in line.strip().split("|")] 56 | data_dict = {k: v for k, _, v in split_tokens} 57 | 58 | try: 59 | return cls(**data_dict) 60 | except TypeError: 61 | raise ValueError( 62 | f"'{line!r}'is not a parseable {cls.__name__}", 63 | ) from None 64 | 65 | def __str__(self) -> str: 66 | fields = tuple(map(attrgetter("name"), dataclasses.fields(self))) 67 | values = tuple(getattr(self, f) for f in fields) 68 | tokens = "|".join(f"{k}={v}" for k, v in zip(fields, values) if v is not None) 69 | return f"{tokens}\n" 70 | 71 | def __bytes__(self) -> bytes: 72 | return str(self).encode("utf-8") 73 | 74 | 75 | @dataclasses.dataclass 76 | class Greeting(GatewayControl): 77 | """ 78 | A greeting message is sent by the gateway upon connection. 79 | """ 80 | 81 | lsg_version: str 82 | 83 | 84 | @dataclasses.dataclass 85 | class ChallengeRequest(GatewayControl): 86 | """ 87 | A challenge request is sent by the gateway upon connection. 88 | """ 89 | 90 | cram: str 91 | 92 | 93 | @dataclasses.dataclass 94 | class AuthenticationResponse(GatewayControl): 95 | """ 96 | An authentication response is sent by the gateway after a valid 97 | authentication request is sent to the gateway. 98 | """ 99 | 100 | success: str 101 | error: str | None = None 102 | session_id: str | None = None 103 | 104 | 105 | @dataclasses.dataclass 106 | class AuthenticationRequest(GatewayControl): 107 | """ 108 | An authentication request is sent to the gateway after a challenge response 109 | is received. 110 | 111 | This is required to authenticate a user. 112 | 113 | """ 114 | 115 | auth: str 116 | dataset: Dataset | str 117 | encoding: Encoding = Encoding.DBN 118 | details: str | None = None 119 | ts_out: str = "0" 120 | heartbeat_interval_s: int | None = None 121 | client: str = USER_AGENT 122 | 123 | 124 | @dataclasses.dataclass 125 | class SubscriptionRequest(GatewayControl): 126 | """ 127 | A subscription request is sent to the gateway upon request from the client. 128 | """ 129 | 130 | schema: Schema | str 131 | stype_in: SType 132 | symbols: str 133 | start: int | None = None 134 | snapshot: int = 0 135 | id: int | None = None 136 | is_last: int = 1 137 | 138 | 139 | @dataclasses.dataclass 140 | class SessionStart(GatewayControl): 141 | """ 142 | A session start message is sent to the gateway upon request from the 143 | client. 144 | """ 145 | 146 | start_session: str = "0" 147 | 148 | 149 | def parse_gateway_message(line: str) -> GatewayControl: 150 | """ 151 | Parse a gateway message from a string. 152 | 153 | Returns 154 | ------- 155 | GatewayControl 156 | 157 | Raises 158 | ------ 159 | ValueError 160 | If `line` is not a parseable GatewayControl message. 161 | 162 | """ 163 | for message_cls in GatewayControl.__subclasses__(): 164 | try: 165 | return message_cls.parse(line) 166 | except ValueError: 167 | continue 168 | raise ValueError(f"'{line.strip()}' is not a parseable gateway message") 169 | 170 | 171 | class GatewayDecoder: 172 | """ 173 | Decoder for gateway control messages. 174 | """ 175 | 176 | def __init__(self) -> None: 177 | self.__buffer = BytesIO() 178 | 179 | @property 180 | def buffer(self) -> BytesIO: 181 | """ 182 | The internal buffer for decoding messages. 183 | 184 | Returns 185 | ------- 186 | BytesIO 187 | 188 | """ 189 | return self.__buffer 190 | 191 | def write(self, data: bytes) -> None: 192 | """ 193 | Write data to the decoder's buffer. This will make the data available 194 | for decoding. 195 | 196 | Parameters 197 | ---------- 198 | data : bytes 199 | The data to write. 200 | 201 | """ 202 | self.__buffer.seek(0, 2) # seek to end 203 | self.__buffer.write(data) 204 | 205 | def decode(self) -> list[GatewayControl]: 206 | """ 207 | Decode messages from the decoder's buffer. This will consume decoded 208 | data from the buffer. 209 | 210 | Returns 211 | ------- 212 | list[GatewayControl] 213 | 214 | """ 215 | self.__buffer.seek(0) # rewind 216 | buffer_lines = self.__buffer.getvalue().splitlines(keepends=True) 217 | 218 | cursor = 0 219 | messages = [] 220 | for line in buffer_lines: 221 | if not line.endswith(b"\n"): 222 | break 223 | try: 224 | message = parse_gateway_message(line.decode("utf-8")) 225 | except ValueError: 226 | logger.exception("could not parse gateway message: %s", line) 227 | raise 228 | else: 229 | cursor += len(line) 230 | messages.append(message) 231 | 232 | self.__buffer = BytesIO(self.__buffer.getvalue()[cursor:]) 233 | return messages 234 | -------------------------------------------------------------------------------- /databento/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/py.typed -------------------------------------------------------------------------------- /databento/reference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/reference/__init__.py -------------------------------------------------------------------------------- /databento/reference/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/databento/reference/api/__init__.py -------------------------------------------------------------------------------- /databento/reference/api/adjustment.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import Iterable 4 | from datetime import date 5 | from datetime import datetime 6 | 7 | import pandas as pd 8 | from databento_dbn import Compression 9 | from databento_dbn import SType 10 | 11 | from databento.common import API_VERSION 12 | from databento.common.constants import ADJUSTMENT_FACTORS_DATE_COLUMNS 13 | from databento.common.constants import ADJUSTMENT_FACTORS_DATETIME_COLUMNS 14 | from databento.common.http import BentoHttpAPI 15 | from databento.common.parsing import convert_date_columns 16 | from databento.common.parsing import convert_datetime_columns 17 | from databento.common.parsing import convert_jsonl_to_df 18 | from databento.common.parsing import datetime_to_string 19 | from databento.common.parsing import optional_datetime_to_string 20 | from databento.common.parsing import optional_string_to_list 21 | from databento.common.parsing import optional_symbols_list_to_list 22 | 23 | 24 | class AdjustmentFactorsHttpAPI(BentoHttpAPI): 25 | """ 26 | Provides request methods for the adjustment factors HTTP API endpoints. 27 | """ 28 | 29 | def __init__(self, key: str, gateway: str) -> None: 30 | super().__init__(key=key, gateway=gateway) 31 | self._base_url = gateway + f"/v{API_VERSION}/adjustment_factors" 32 | 33 | def get_range( 34 | self, 35 | start: pd.Timestamp | datetime | date | str | int, 36 | end: pd.Timestamp | datetime | date | str | int | None = None, 37 | symbols: Iterable[str] | str | None = None, 38 | stype_in: SType | str = "raw_symbol", 39 | countries: Iterable[str] | str | None = None, 40 | security_types: Iterable[str] | str | None = None, 41 | ) -> pd.DataFrame: 42 | """ 43 | Request a new adjustment factors time series from Databento. 44 | 45 | Makes a `POST /adjustment_factors.get_range` HTTP request. 46 | 47 | The `ex_date` column will be used to filter the time range and order the records. 48 | It will also be set as the index of the resulting data frame. 49 | 50 | Parameters 51 | ---------- 52 | start : pd.Timestamp, datetime, date, str, or int 53 | The start datetime of the request time range (inclusive) based on `ex_date`. 54 | Assumes UTC as timezone unless passed a tz-aware object. 55 | If an integer is passed, then this represents nanoseconds since the UNIX epoch. 56 | end : pd.Timestamp, datetime, date, str, or int, optional 57 | The end datetime of the request time range (exclusive) based on `ex_date`. 58 | Assumes UTC as timezone unless passed a tz-aware object. 59 | If an integer is passed, then this represents nanoseconds since the UNIX epoch. 60 | Defaults to the forward filled value of `start` based on the resolution provided. 61 | symbols : Iterable[str] or str, optional 62 | The symbols to filter for. Takes up to 2,000 symbols per request. 63 | If more than 1 symbol is specified, the data is merged and sorted by time. 64 | If 'ALL_SYMBOLS' or `None` then will select **all** symbols. 65 | stype_in : SType or str, default 'raw_symbol' 66 | The input symbology type to resolve from. 67 | Use any of 'raw_symbol', 'nasdaq_symbol', 'isin', 'us_code'. 68 | countries : Iterable[str] or str, optional 69 | The listing countries to filter for. 70 | Takes any number of two letter ISO 3166-1 alpha-2 country codes per request. 71 | If not specified then will select **all** listing countries by default. 72 | See [CNTRY](https://databento.com/docs/standards-and-conventions/reference-data-enums#cntry) enum. 73 | security_types : Iterable[str] or str, optional 74 | The security types to filter for. 75 | Takes any number of security types per request. 76 | If not specified then will select **all** security types by default. 77 | See [SECTYPE](https://databento.com/docs/standards-and-conventions/reference-data-enums#sectype) enum. 78 | 79 | Returns 80 | ------- 81 | pandas.DataFrame 82 | The data converted into a data frame. 83 | 84 | """ 85 | symbols_list = optional_symbols_list_to_list(symbols, SType.RAW_SYMBOL) 86 | countries = optional_string_to_list(countries) 87 | security_types = optional_string_to_list(security_types) 88 | 89 | data: dict[str, object | None] = { 90 | "start": datetime_to_string(start), 91 | "end": optional_datetime_to_string(end), 92 | "symbols": ",".join(symbols_list), 93 | "stype_in": stype_in, 94 | "countries": ",".join(countries) if countries else None, 95 | "security_types": ",".join(security_types) if security_types else None, 96 | "compression": str(Compression.ZSTD), # Always request zstd 97 | } 98 | 99 | response = self._post( 100 | url=self._base_url + ".get_range", 101 | data=data, 102 | basic_auth=True, 103 | ) 104 | 105 | df = convert_jsonl_to_df(response.content, compressed=True) 106 | if df.empty: 107 | return df 108 | 109 | convert_datetime_columns(df, ADJUSTMENT_FACTORS_DATETIME_COLUMNS) 110 | convert_date_columns(df, ADJUSTMENT_FACTORS_DATE_COLUMNS) 111 | 112 | df.set_index("ex_date", inplace=True) 113 | df.sort_index(inplace=True) 114 | 115 | return df 116 | -------------------------------------------------------------------------------- /databento/reference/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | 6 | from databento.common.enums import HistoricalGateway 7 | from databento.common.validation import validate_gateway 8 | from databento.reference.api.adjustment import AdjustmentFactorsHttpAPI 9 | from databento.reference.api.corporate import CorporateActionsHttpAPI 10 | from databento.reference.api.security import SecurityMasterHttpAPI 11 | 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class Reference: 17 | """ 18 | Provides a client connection class for requesting reference data from the 19 | Databento API servers. 20 | 21 | Parameters 22 | ---------- 23 | key : str, optional 24 | The user API key for authentication. 25 | If `None` then the `DATABENTO_API_KEY` environment variable is used. 26 | gateway : HistoricalGateway or str, default HistoricalGateway.BO1 27 | The API server gateway. 28 | If `None` then the default gateway is used. 29 | 30 | Examples 31 | -------- 32 | > import databento as db 33 | > client = db.Reference('YOUR_API_KEY') 34 | 35 | """ 36 | 37 | def __init__( 38 | self, 39 | key: str | None = None, 40 | gateway: HistoricalGateway | str = HistoricalGateway.BO1, 41 | ): 42 | if key is None: 43 | key = os.environ.get("DATABENTO_API_KEY") 44 | if key is None or not isinstance(key, str) or key.isspace(): 45 | raise ValueError(f"invalid API key, was {key}") 46 | 47 | try: 48 | gateway = HistoricalGateway(gateway) 49 | except ValueError: 50 | gateway = validate_gateway(str(gateway)) 51 | 52 | self._key = key 53 | self._gateway = gateway 54 | 55 | self.adjustment_factors = AdjustmentFactorsHttpAPI(key=key, gateway=gateway) 56 | self.corporate_actions = CorporateActionsHttpAPI(key=key, gateway=gateway) 57 | self.security_master = SecurityMasterHttpAPI(key=key, gateway=gateway) 58 | 59 | # Not logging security sensitive `key` 60 | logger.info("Initialized %s(gateway=%s)", type(self).__name__, self.gateway) 61 | 62 | @property 63 | def key(self) -> str: 64 | """ 65 | Return the user API key for the client. 66 | 67 | Returns 68 | ------- 69 | str 70 | 71 | """ 72 | return self._key 73 | 74 | @property 75 | def gateway(self) -> str: 76 | """ 77 | Return the API server gateway for the client. 78 | 79 | Returns 80 | ------- 81 | str 82 | 83 | """ 84 | return self._gateway 85 | -------------------------------------------------------------------------------- /databento/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.56.0" 2 | -------------------------------------------------------------------------------- /examples/historical_batch_download.py: -------------------------------------------------------------------------------- 1 | from databento import Historical 2 | 3 | 4 | if __name__ == "__main__": 5 | key = "YOUR_API_KEY" 6 | client = Historical(key=key) 7 | 8 | # Will download all job files to a `my_data/YOUR_JOB_ID/` directory 9 | downloaded_files = client.batch.download( 10 | output_dir="my_data", 11 | job_id="YOUR_JOB_ID", # <-- Discover this from `.list_jobs(...)` 12 | ) 13 | 14 | for file in downloaded_files: 15 | print(f"Downloaded {file.name}") 16 | -------------------------------------------------------------------------------- /examples/historical_batch_download_async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from databento import Historical 4 | 5 | 6 | async def example_download_batch_job_async() -> None: 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | # Will download all job files to a `my_data/YOUR_JOB_ID/` directory 11 | downloaded_files = await client.batch.download_async( 12 | output_dir="my_data", 13 | job_id="YOUR_JOB_ID", # <-- Discover this from `.list_jobs(...)` 14 | ) 15 | 16 | for file in downloaded_files: 17 | print(f"Downloaded {file.name}") 18 | 19 | 20 | if __name__ == "__main__": 21 | asyncio.run(example_download_batch_job_async()) 22 | -------------------------------------------------------------------------------- /examples/historical_batch_list_files.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | response = client.batch.list_files( 11 | job_id="YOUR_JOB_ID", # <-- Discover this from `.list_jobs(...)` 12 | ) 13 | 14 | pprint(response) 15 | -------------------------------------------------------------------------------- /examples/historical_batch_list_jobs.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | response = client.batch.list_jobs( 11 | states="received,queued,processing,done", # Included states 12 | since=None, # <-- Filter for jobs 'since' the given time 13 | ) 14 | 15 | pprint(response) 16 | -------------------------------------------------------------------------------- /examples/historical_batch_submit_job.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | response = client.batch.submit_job( 11 | dataset="GLBX.MDP3", 12 | symbols=["ESM2"], 13 | schema="mbo", 14 | start="2022-06-10T12:00", 15 | end="2022-06-10T14:00", 16 | limit=1000, # <-- Limiting batch request to 1000 records only 17 | encoding="csv", 18 | compression="zstd", 19 | delivery="download", 20 | ) 21 | 22 | pprint(response) 23 | -------------------------------------------------------------------------------- /examples/historical_metadata.py: -------------------------------------------------------------------------------- 1 | from databento import Historical 2 | 3 | 4 | if __name__ == "__main__": 5 | key = "YOUR_API_KEY" 6 | client = Historical(key=key) 7 | 8 | print(client.metadata.list_publishers()) 9 | print(client.metadata.list_datasets()) 10 | print(client.metadata.list_schemas(dataset="GLBX.MDP3")) 11 | print(client.metadata.list_fields(schema="trades", encoding="dbn")) 12 | -------------------------------------------------------------------------------- /examples/historical_metadata_get_billable_size.py: -------------------------------------------------------------------------------- 1 | from databento import Historical 2 | 3 | 4 | if __name__ == "__main__": 5 | key = "YOUR_API_KEY" 6 | client = Historical(key=key) 7 | 8 | size: int = client.metadata.get_billable_size( 9 | dataset="GLBX.MDP3", 10 | symbols=["ESM2"], 11 | schema="mbo", 12 | start="2022-06-10T12:00", 13 | end="2022-06-10T14:00", 14 | ) 15 | 16 | print(size) 17 | -------------------------------------------------------------------------------- /examples/historical_metadata_get_cost.py: -------------------------------------------------------------------------------- 1 | from databento import Historical 2 | 3 | 4 | if __name__ == "__main__": 5 | key = "YOUR_API_KEY" 6 | client = Historical(key=key) 7 | 8 | cost: float = client.metadata.get_cost( 9 | dataset="GLBX.MDP3", 10 | symbols="ESM2", 11 | schema="mbo", 12 | start="2022-06-10", 13 | end="2022-06-15", 14 | ) 15 | 16 | print(cost) 17 | -------------------------------------------------------------------------------- /examples/historical_metadata_get_dataset_condition.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | condition = client.metadata.get_dataset_condition(dataset="XNAS.ITCH") 11 | 12 | pprint(condition) 13 | -------------------------------------------------------------------------------- /examples/historical_metadata_get_dataset_range.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | available_range = client.metadata.get_dataset_range(dataset="GLBX.MDP3") 11 | 12 | pprint(available_range) 13 | -------------------------------------------------------------------------------- /examples/historical_metadata_get_record_count.py: -------------------------------------------------------------------------------- 1 | from databento import Historical 2 | 3 | 4 | if __name__ == "__main__": 5 | key = "YOUR_API_KEY" 6 | client = Historical(key=key) 7 | 8 | count: int = client.metadata.get_record_count( 9 | dataset="GLBX.MDP3", 10 | symbols=["ESM2"], 11 | schema="mbo", 12 | start="2022-06-10T12:00", 13 | end="2022-06-10T14:00", 14 | ) 15 | 16 | print(count) 17 | -------------------------------------------------------------------------------- /examples/historical_metadata_list_unit_price.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | 5 | 6 | if __name__ == "__main__": 7 | key = "YOUR_API_KEY" 8 | client = Historical(key=key) 9 | 10 | unit_prices = client.metadata.list_unit_prices(dataset="GLBX.MDP3") 11 | 12 | pprint(unit_prices) 13 | -------------------------------------------------------------------------------- /examples/historical_symbology_resolve.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import Historical 4 | from databento import SType 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Historical(key=key) 10 | 11 | response = client.symbology.resolve( 12 | dataset="GLBX.MDP3", 13 | symbols=["ESM2"], 14 | stype_in=SType.RAW_SYMBOL, 15 | stype_out=SType.INSTRUMENT_ID, 16 | start_date="2022-06-01", 17 | end_date="2022-06-30", 18 | ) 19 | 20 | pprint(response) 21 | -------------------------------------------------------------------------------- /examples/historical_timeseries_async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pprint import pprint 3 | 4 | from databento import DBNStore 5 | from databento import Historical 6 | 7 | 8 | async def example_get_range_async() -> None: 9 | key = "YOUR_API_KEY" 10 | client = Historical(key=key) 11 | 12 | data: DBNStore = await client.timeseries.get_range_async( 13 | dataset="GLBX.MDP3", 14 | symbols=["ESM2"], 15 | schema="mbo", 16 | start="2022-06-10T12:00", 17 | end="2022-06-10T14:00", 18 | limit=1000, # <-- Limiting response to 1000 records only 19 | ) 20 | pprint(data.to_df()) 21 | 22 | 23 | if __name__ == "__main__": 24 | asyncio.run(example_get_range_async()) 25 | -------------------------------------------------------------------------------- /examples/historical_timeseries_disk_io.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import DBNStore 4 | from databento import Historical 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Historical(key=key) 10 | 11 | path = "my_data.dbn" 12 | 13 | # Execute request through client 14 | data: DBNStore = client.timeseries.get_range( 15 | dataset="GLBX.MDP3", 16 | symbols=["ESM2"], 17 | schema="mbo", 18 | start="2022-06-10T12:00", 19 | end="2022-06-10T14:00", 20 | limit=1000, # <-- Limiting response to 1000 records only 21 | path=path, 22 | ) 23 | 24 | pprint(data.to_ndarray()) 25 | -------------------------------------------------------------------------------- /examples/historical_timeseries_from_file.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from databento import DBNStore 4 | 5 | 6 | if __name__ == "__main__": 7 | ts_start = datetime.datetime.utcnow() 8 | 9 | # Can load from file path (if exists) 10 | data = DBNStore.from_file(path="my_data.dbn") 11 | 12 | print(data.to_df()) 13 | print(datetime.datetime.utcnow() - ts_start) 14 | -------------------------------------------------------------------------------- /examples/historical_timeseries_replay.py: -------------------------------------------------------------------------------- 1 | from databento import DBNStore 2 | from databento import Historical 3 | 4 | 5 | if __name__ == "__main__": 6 | key = "YOUR_API_KEY" 7 | client = Historical(key=key) 8 | 9 | data: DBNStore = client.timeseries.get_range( 10 | dataset="GLBX.MDP3", 11 | symbols=["ESM2"], 12 | schema="trades", 13 | start="2022-06-10T12:00", 14 | end="2022-06-10T14:00", 15 | limit=1000, # <-- Limiting response to 1000 records only 16 | ) 17 | 18 | data.replay(callback=print) 19 | -------------------------------------------------------------------------------- /examples/historical_timeseries_to_df.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import DBNStore 4 | from databento import Historical 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Historical(key=key) 10 | 11 | data: DBNStore = client.timeseries.get_range( 12 | dataset="GLBX.MDP3", 13 | symbols=["ESM2"], 14 | schema="trades", 15 | start="2022-06-10T12:00", 16 | end="2022-06-10T14:00", 17 | limit=1000, # <-- Limiting response to 1000 records only 18 | ) 19 | 20 | # Convert to pandas dataframe 21 | pprint(data.to_df()) 22 | -------------------------------------------------------------------------------- /examples/historical_timeseries_to_file.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import DBNStore 4 | from databento import Historical 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Historical(key=key) 10 | 11 | data: DBNStore = client.timeseries.get_range( 12 | dataset="GLBX.MDP3", 13 | symbols=["ESM2"], 14 | schema="trades", 15 | start="2022-06-10T12:00", 16 | end="2022-06-10T14:00", 17 | limit=1000, # <-- Limiting response to 1000 records only 18 | ) # -> DBNStore 19 | 20 | path = "my_data.dbn" 21 | data.to_file(path=path) 22 | 23 | data = DBNStore.from_file(path=path) 24 | 25 | # Data now loaded into memory 26 | pprint(data.to_df()) 27 | -------------------------------------------------------------------------------- /examples/historical_timeseries_to_ndarray.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | from databento import DBNStore 4 | from databento import Historical 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Historical(key=key) 10 | 11 | data: DBNStore = client.timeseries.get_range( 12 | dataset="GLBX.MDP3", 13 | symbols=["ESM2"], 14 | schema="mbp-1", 15 | start="2022-06-10T12:00", 16 | end="2022-06-10T14:00", 17 | limit=1000, # <-- Limiting response to 1000 records only 18 | ) 19 | 20 | # Convert to numpy ndarray 21 | pprint(data.to_ndarray()) 22 | -------------------------------------------------------------------------------- /examples/live_smoke_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse 4 | import os 5 | import typing 6 | 7 | from databento import Dataset 8 | from databento import Live 9 | from databento import RecordFlags 10 | from databento import Schema 11 | from databento import SType 12 | from databento_dbn import ErrorMsg 13 | from databento_dbn import MBOMsg 14 | from databento_dbn import RType 15 | from databento_dbn import SymbolMappingMsg 16 | 17 | 18 | def parse_args() -> argparse.Namespace: 19 | parser = argparse.ArgumentParser(prog="Python client") 20 | parser.add_argument("--gateway", type=str, help="Gateway to connect") 21 | parser.add_argument("--port", type=int, default=13000, help="Gatewat port to connect") 22 | parser.add_argument( 23 | "--api-key-env-var", 24 | type=str, 25 | help="Gateway to connect as Gateway::port", 26 | default="DATABENTO_API_KEY", 27 | ) 28 | parser.add_argument("--dataset", type=Dataset, help="Dataset") 29 | parser.add_argument("--schema", type=Schema, help="Schema") 30 | parser.add_argument("--stype", type=SType, help="SType") 31 | parser.add_argument("--symbols", type=str, help="Symbols") 32 | parser.add_argument("--start", type=str, default=None, help="Start time (rfc-339)") 33 | parser.add_argument( 34 | "--use-snapshot", 35 | action="store_true", 36 | help="Whether or not to request snapshot subscription", 37 | ) 38 | 39 | return parser.parse_args() 40 | 41 | 42 | def run_client(args: argparse.Namespace) -> None: 43 | client = Live(key=get_api_key(args.api_key_env_var), gateway=args.gateway, port=args.port) 44 | 45 | client.subscribe( 46 | dataset=args.dataset, 47 | schema=args.schema, 48 | stype_in=args.stype, 49 | symbols=args.symbols, 50 | start=args.start, 51 | ) 52 | 53 | print("Starting client...") 54 | 55 | for record in client: 56 | if is_expected_record(args, record): 57 | print(f"Received expected record {record}") 58 | break 59 | elif isinstance(record, ErrorMsg): 60 | raise ValueError(f"Received error {record.err}") 61 | else: 62 | print(f"{record}") 63 | 64 | print("Finished client") 65 | 66 | 67 | def run_client_with_snapshot(args: argparse.Namespace) -> None: 68 | client = Live(key=get_api_key(args.api_key_env_var), gateway=args.gateway, port=args.port) 69 | 70 | client.subscribe( 71 | dataset=args.dataset, 72 | schema=args.schema, 73 | stype_in=args.stype, 74 | symbols=args.symbols, 75 | snapshot=True, 76 | ) 77 | 78 | received_snapshot_record = False 79 | 80 | print("Starting client...") 81 | 82 | for record in client: 83 | if isinstance(record, SymbolMappingMsg): 84 | continue 85 | elif isinstance(record, MBOMsg): 86 | if record.flags & RecordFlags.F_SNAPSHOT: 87 | received_snapshot_record = True 88 | else: 89 | print(f"Received expected record {record}") 90 | break 91 | elif isinstance(record, ErrorMsg): 92 | raise ValueError(f"Received error {record.err}") 93 | else: 94 | raise ValueError(f"Received unexpected record {record}") 95 | 96 | print("Finished client") 97 | 98 | assert received_snapshot_record 99 | 100 | 101 | def is_expected_record(args: argparse.Namespace, record: typing.Any) -> bool: 102 | try: 103 | start = int(args.start) 104 | except Exception: 105 | start = None 106 | 107 | # For start != 0 we stop at SymbolMappingMsg so that the tests can be run outside trading hours 108 | should_expect_symbol_mapping = args.stype != SType.INSTRUMENT_ID and ( 109 | start is None or start != 0 110 | ) 111 | if should_expect_symbol_mapping: 112 | return isinstance(record, SymbolMappingMsg) 113 | else: 114 | return record.rtype == RType.from_schema(args.schema) 115 | 116 | 117 | def get_api_key(api_key_name: str) -> str: 118 | api_key = os.getenv(api_key_name) 119 | if not api_key: 120 | raise ValueError(f"Invalid api_key {api_key_name}") 121 | 122 | return api_key 123 | 124 | 125 | def main() -> None: 126 | args = parse_args() 127 | 128 | if args.use_snapshot: 129 | run_client_with_snapshot(args) 130 | else: 131 | run_client(args) 132 | 133 | 134 | if __name__ == "__main__": 135 | main() 136 | -------------------------------------------------------------------------------- /examples/reference_adjustment_factors.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | import pandas as pd 4 | from databento import Reference 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Reference(key=key) 10 | 11 | response: pd.DataFrame = client.adjustment_factors.get_range( 12 | symbols="TSLA", 13 | stype_in="raw_symbol", 14 | start="2020", 15 | ) 16 | 17 | pprint(response.head()) 18 | -------------------------------------------------------------------------------- /examples/reference_corporate_actions.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | import pandas as pd 4 | from databento import Reference 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Reference(key=key) 10 | 11 | response: pd.DataFrame = client.corporate_actions.get_range( 12 | symbols="AAPL,MSFT,TSLA", 13 | stype_in="raw_symbol", 14 | start="2023", 15 | end="2024-04", 16 | events="DIV,LIQ", 17 | countries="US", 18 | ) 19 | 20 | pprint(response.head()) 21 | -------------------------------------------------------------------------------- /examples/reference_security_master_get_last.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | import pandas as pd 4 | from databento import Reference 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Reference(key=key) 10 | 11 | response: pd.DataFrame = client.security_master.get_last( 12 | symbols="AAPL,AMZN,MSFT,TQQQ", 13 | countries="US", 14 | ) 15 | 16 | pprint(response.head()) 17 | -------------------------------------------------------------------------------- /examples/reference_security_master_get_range.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | 3 | import pandas as pd 4 | from databento import Reference 5 | 6 | 7 | if __name__ == "__main__": 8 | key = "YOUR_API_KEY" 9 | client = Reference(key=key) 10 | 11 | response: pd.DataFrame = client.security_master.get_range( 12 | symbols="AAPL,AMZN,MSFT,TQQQ", 13 | start="2024-01", 14 | countries="US", 15 | ) 16 | 17 | pprint(response.head()) 18 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "databento" 3 | version = "0.56.0" 4 | description = "Official Python client library for Databento" 5 | authors = [ 6 | "Databento ", 7 | ] 8 | license = "Apache License 2.0" 9 | packages = [ 10 | {include = "databento"}, 11 | {include = "databento/py.typed"}, 12 | ] 13 | classifiers = [ 14 | "Development Status :: 4 - Beta", 15 | "Operating System :: OS Independent", 16 | "Topic :: Software Development :: Libraries", 17 | "Topic :: Software Development :: Libraries :: Python Modules", 18 | "Topic :: Office/Business :: Financial", 19 | "Topic :: Office/Business :: Financial :: Investment", 20 | ] 21 | readme = "README.md" 22 | documentation = "https://databento.com/docs" 23 | homepage = "https://databento.com" 24 | repository = "https://github.com/databento/databento-python" 25 | 26 | [tool.poetry.urls] 27 | "Bug Tracker" = "https://github.com/databento/databento-python/issues" 28 | 29 | [tool.poetry.dependencies] 30 | python = "^3.9" 31 | aiohttp = [ 32 | {version = "^3.8.3", python = "<3.12"}, 33 | {version = "^3.9.0", python = "^3.12"} 34 | ] 35 | databento-dbn = "0.35.1" 36 | numpy = [ 37 | {version = ">=1.23.5", python = "<3.12"}, 38 | {version = ">=1.26.0", python = "^3.12"} 39 | ] 40 | pandas = ">=1.5.3" 41 | pip-system-certs = {version=">=4.0", markers="platform_system == 'Windows'"} 42 | pyarrow = ">=13.0.0" 43 | requests = ">=2.25.1" 44 | zstandard = ">=0.21.0" 45 | 46 | [tool.poetry.group.dev.dependencies] 47 | black = "^23.9.1" 48 | mypy = "1.5.1" 49 | pytest = "^7.4.2" 50 | pytest-asyncio = "==0.21.1" 51 | ruff = "^0.0.291" 52 | types-requests = "^2.30.0.0" 53 | tomli = "^2.0.1" 54 | teamcity-messages = "^1.32" 55 | types-pytz = "^2024.1.0.20240203" 56 | types-aiofiles = "^23.2.0.20240403" 57 | 58 | [build-system] 59 | requires = ["poetry-core"] 60 | build-backend = "poetry.core.masonry.api" 61 | 62 | [tool.black] 63 | line_length = 100 64 | 65 | [tool.mypy] 66 | python_version = 3.9 67 | disallow_untyped_defs = true 68 | disallow_any_generics = true 69 | disallow_subclassing_any = true 70 | ignore_missing_imports = true 71 | namespace_packages = true 72 | no_strict_optional = true 73 | warn_no_return = true 74 | warn_unused_configs = true 75 | warn_unused_ignores = true 76 | plugins = ["numpy.typing.mypy_plugin"] 77 | 78 | [tool.pytest.ini_options] 79 | testpaths = ["tests"] 80 | asyncio_mode = "auto" 81 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | poetry install --with=dev --no-interaction --no-root 3 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Running $(poetry run mypy --version)..." 3 | poetry run mypy . 4 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | poetry run pytest tests . $@ 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | import sys 4 | 5 | 6 | sys.path.append(os.path.join(os.path.dirname(__file__), ".")) 7 | 8 | TESTS_ROOT = pathlib.Path(__file__).absolute().parent 9 | PROJECT_ROOT = pathlib.Path(__file__).absolute().parent.parent 10 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pytest fixtures. 3 | """ 4 | 5 | import asyncio 6 | import logging 7 | import pathlib 8 | import random 9 | import string 10 | from collections.abc import AsyncGenerator 11 | from collections.abc import Generator 12 | from collections.abc import Iterable 13 | from typing import Callable 14 | 15 | import databento.live.session 16 | import pytest 17 | from databento import historical 18 | from databento import live 19 | from databento import reference 20 | from databento.common.publishers import Dataset 21 | from databento_dbn import Schema 22 | 23 | from tests import TESTS_ROOT 24 | from tests.mockliveserver.fixture import MockLiveServerInterface 25 | from tests.mockliveserver.fixture import fixture_mock_live_server # noqa 26 | 27 | 28 | def pytest_addoption(parser: pytest.Parser) -> None: 29 | """ 30 | Customize pytest cli options. This should not be invoked directly. 31 | 32 | Parameters 33 | ---------- 34 | parser : pytest.Parser 35 | The pytest argument parser. 36 | 37 | See Also 38 | -------- 39 | pytest.addoption 40 | 41 | """ 42 | # Add a --release flag 43 | parser.addoption( 44 | "--release", 45 | action="store_true", 46 | help="indicates release tests should be run", 47 | ) 48 | 49 | 50 | def pytest_configure(config: pytest.Config) -> None: 51 | """ 52 | Configure pytest. This should not be invoked directly. 53 | 54 | Parameters 55 | ---------- 56 | config : pytest.Config 57 | The pytest configuration. 58 | 59 | """ 60 | # Add custom mark for `release` 61 | config.addinivalue_line( 62 | "markers", 63 | "release: mark tests as release tests (run with --release)", 64 | ) 65 | 66 | 67 | def pytest_collection_modifyitems( 68 | config: pytest.Config, 69 | items: Iterable[pytest.Item], 70 | ) -> None: 71 | """ 72 | Customize test items. This should not be invoked directly. 73 | 74 | Parameters 75 | ---------- 76 | config : pytest.Config 77 | The pytest configuration. 78 | items : Iterable[pytest.Item] 79 | The pytest test item. 80 | 81 | """ 82 | skip_release = pytest.mark.skip( 83 | reason="skipping release test (invoke pytest with --release to execute)", 84 | ) 85 | 86 | for item in items: 87 | # Skip release tests if `--release` was not specified 88 | if "release" in item.keywords and not config.getoption("--release"): 89 | item.add_marker(skip_release) 90 | 91 | 92 | @pytest.fixture(autouse=True) 93 | def fixture_log_capture( 94 | caplog: pytest.LogCaptureFixture, 95 | ) -> Generator[None, None, None]: 96 | with caplog.at_level(logging.DEBUG): 97 | yield 98 | 99 | 100 | @pytest.fixture(name="event_loop", scope="module") 101 | def fixture_event_loop() -> Generator[asyncio.AbstractEventLoop, None, None]: 102 | policy = asyncio.get_event_loop_policy() 103 | loop = policy.new_event_loop() 104 | yield loop 105 | loop.close() 106 | 107 | 108 | @pytest.fixture(name="live_test_data_path") 109 | def fixture_live_test_data_path() -> pathlib.Path: 110 | """ 111 | Fixture to retrieve the live stub data path. 112 | 113 | Returns 114 | ------- 115 | pathlib.Path 116 | 117 | See Also 118 | -------- 119 | live_test_data 120 | 121 | """ 122 | return TESTS_ROOT / "data" / "LIVE" / "test_data.live.dbn.zst" 123 | 124 | 125 | @pytest.fixture(name="test_data_path") 126 | def fixture_test_data_path() -> Callable[[Dataset, Schema], pathlib.Path]: 127 | """ 128 | Fixture to retrieve stub data paths. 129 | 130 | Parameters 131 | ---------- 132 | dataset: Dataset, 133 | The dataset of the stub data to request. 134 | schema : Schema 135 | The schema of the stub data path to request. 136 | 137 | Returns 138 | ------- 139 | Callable[[Dataset, Schema], pathlib.Path] 140 | 141 | See Also 142 | -------- 143 | test_data 144 | 145 | """ 146 | 147 | def func(dataset: Dataset, schema: Schema) -> pathlib.Path: 148 | path = TESTS_ROOT / "data" / dataset / f"test_data.{schema}.dbn.zst" 149 | if not path.exists(): 150 | pytest.skip(f"no test data for {dataset} {schema}") 151 | return path 152 | 153 | return func 154 | 155 | 156 | @pytest.fixture(name="live_test_data") 157 | def fixture_live_test_data( 158 | live_test_data_path: pathlib.Path, 159 | ) -> bytes: 160 | """ 161 | Fixture to retrieve live stub test data. 162 | 163 | Returns 164 | ------- 165 | bytes 166 | 167 | See Also 168 | -------- 169 | live_test_data_path 170 | 171 | """ 172 | return live_test_data_path.read_bytes() 173 | 174 | 175 | @pytest.fixture(name="test_data") 176 | def fixture_test_data( 177 | test_data_path: Callable[[Dataset, Schema], pathlib.Path], 178 | ) -> Callable[[Dataset, Schema], bytes]: 179 | """ 180 | Fixture to retrieve stub test data. 181 | 182 | Parameters 183 | ---------- 184 | test_data_path : Callable 185 | The test_data_path fixture. 186 | 187 | Returns 188 | ------- 189 | Callable[[Dataset, Schema], bytes] 190 | 191 | See Also 192 | -------- 193 | test_data_path 194 | 195 | """ 196 | 197 | def func(dataset: Dataset, schema: Schema) -> bytes: 198 | return test_data_path(dataset, schema).read_bytes() 199 | 200 | return func 201 | 202 | 203 | @pytest.fixture(name="test_api_key") 204 | def fixture_test_api_key() -> str: 205 | """ 206 | Generate a random API key for testing. API keys are 32 characters in 207 | length, the first three of which are "db-". 208 | 209 | Returns 210 | ------- 211 | str 212 | 213 | """ 214 | chars = string.ascii_letters + string.digits 215 | random_str = "".join(random.choice(chars) for _ in range(29)) # noqa: S311 216 | return f"db-{random_str}" 217 | 218 | 219 | @pytest.fixture(name="test_live_api_key") 220 | async def fixture_test_live_api_key( 221 | test_api_key: str, 222 | mock_live_server: MockLiveServerInterface, 223 | ) -> AsyncGenerator[str, None]: 224 | async with mock_live_server.api_key_context(test_api_key): 225 | yield test_api_key 226 | 227 | 228 | @pytest.fixture(name="historical_client") 229 | def fixture_historical_client( 230 | test_api_key: str, 231 | ) -> Generator[historical.client.Historical, None, None]: 232 | """ 233 | Fixture for a Historical client. 234 | 235 | Yields 236 | ------ 237 | Historical 238 | 239 | """ 240 | test_client = historical.client.Historical( 241 | key=test_api_key, 242 | gateway="localhost", 243 | ) 244 | yield test_client 245 | 246 | 247 | @pytest.fixture(name="reference_client") 248 | def fixture_reference_client( 249 | test_api_key: str, 250 | ) -> Generator[reference.client.Reference, None, None]: 251 | """ 252 | Fixture for a Reference client. 253 | 254 | Yields 255 | ------ 256 | Reference 257 | 258 | """ 259 | test_client = reference.client.Reference( 260 | key=test_api_key, 261 | gateway="localhost", 262 | ) 263 | yield test_client 264 | 265 | 266 | @pytest.fixture(name="live_client") 267 | async def fixture_live_client( 268 | test_live_api_key: str, 269 | mock_live_server: MockLiveServerInterface, 270 | monkeypatch: pytest.MonkeyPatch, 271 | ) -> AsyncGenerator[live.client.Live, None]: 272 | """ 273 | Fixture for a Live client to connect to the MockLiveServer. 274 | 275 | Yields 276 | ------ 277 | Live 278 | 279 | """ 280 | monkeypatch.setattr( 281 | databento.live.session, 282 | "AUTH_TIMEOUT_SECONDS", 283 | 0.5, 284 | ) 285 | monkeypatch.setattr( 286 | databento.live.session, 287 | "CONNECT_TIMEOUT_SECONDS", 288 | 0.5, 289 | ) 290 | 291 | test_client = live.client.Live( 292 | key=test_live_api_key, 293 | gateway=mock_live_server.host, 294 | port=mock_live_server.port, 295 | ) 296 | 297 | with mock_live_server.test_context(): 298 | yield test_client 299 | 300 | if test_client.is_connected(): 301 | test_client.stop() 302 | await test_client.wait_for_close() 303 | -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/EQUS.MINI/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/EQUS.MINI/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.mbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.statistics.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.status.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.status.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/GLBX.MDP3/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/GLBX.MDP3/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/IFEU.IMPACT/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/LIVE/test_data.live.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/LIVE/test_data.live.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/NDEX.IMPACT/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/OPRA.PILLAR/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/REFERENCE/test_data.adjustment-factors.jsonl: -------------------------------------------------------------------------------- 1 | {"security_id":39827,"event_id":197496,"event_type":"DIV","issuer_name":"Microsoft Corporation","security_type":"EQS","primary_exchange":"USNASD","operating_mic":"XNAS","nasdaq_symbol":"MSFT","local_code":"MSFT","local_code_resulting":null,"isin":"US5949181045","isin_resulting":null,"us_code":"594918104","status":"A","ex_date":"2007-02-13","factor":0.996544574982723,"close":28.94,"currency":"USD","sentiment":1.00362819626814,"reason":17,"detail":"QTR Dividend (cash) of USD0.100000000/EQS [DIVPAYRATE=0.100000000] [DIVCURRENCY=USD] [DIVPERIOD=QTR] [OPTION=1] [EXCHGCD=USNASD]","ts_created":"2024-06-10T07:21:25.845228Z"} 2 | {"security_id":39827,"event_id":197496,"event_type":"DIV","issuer_name":"Microsoft Corporation","security_type":"EQS","primary_exchange":"USNASD","operating_mic":"XNAS","nasdaq_symbol":"MSFT","local_code":"MSFT","local_code_resulting":null,"isin":"US5949181045","isin_resulting":null,"us_code":"594918104","status":"A","ex_date":"2007-02-13","factor":0.996544574982723,"close":28.94,"currency":"USD","sentiment":1.00362819626814,"reason":17,"detail":"QTR Dividend (cash) of USD0.100000000/EQS [DIVPAYRATE=0.100000000] [DIVCURRENCY=USD] [DIVPERIOD=QTR] [OPTION=1] [EXCHGCD=USNASD]","ts_created":"2024-06-10T07:21:25.845228Z"} 3 | -------------------------------------------------------------------------------- /tests/data/REFERENCE/test_data.corporate-actions-pit.jsonl: -------------------------------------------------------------------------------- 1 | {"ts_record":"2023-10-10T04:37:14Z","event_unique_id":"U-40179043276-1888618","event_id":"E-9043276-RSPLT","listing_id":"L-1888618","listing_group_id":"LG-888618","security_id":"S-799713","issuer_id":"I-119138","event_action":"U","event":"RSPLT","event_subtype":"CONSD","event_date_label":"ex_date","event_date":"2023-10-10","event_created_date":"2023-02-09","effective_date":null,"ex_date":"2023-10-10","record_date":null,"record_date_id":"D-9043276","related_event":null,"related_event_id":null,"global_status":"A","listing_status":"T","listing_source":"M","listing_date":"2008-03-18","delisting_date":null,"issuer_name":"Bowmo Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USOTC","exchange":"USOTC","operating_mic":"OTCM","symbol":"BOMO","nasdaq_symbol":"BOMO","local_code":"BOMO","isin":"US2288912064","us_code":"228891206","bbg_comp_id":"BBG000VCV3H9","bbg_comp_ticker":"BOMO US","figi":"BBG000VCV726","figi_ticker":"BOMO UV","listing_country":"US","register_country":"US","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Pink Current Information","segment_mic":"PINC","mand_volu_flag":"M","rd_priority":1,"lot_size":100,"par_value":0.01,"par_value_currency":"USD","payment_date":null,"duebills_redemption_date":null,"from_date":null,"to_date":null,"registration_date":null,"start_date":null,"end_date":null,"open_date":null,"close_date":null,"start_subscription_date":null,"end_subscription_date":null,"option_election_date":null,"withdrawal_rights_from_date":null,"withdrawal_rights_to_date":null,"notification_date":null,"financial_year_end_date":null,"exp_completion_date":null,"payment_type":"S","option_id":"1","serial_id":"1","default_option_flag":true,"rate_currency":"USD","ratio_old":1000.0,"ratio_new":1.0,"fraction":null,"outturn_style":"NEWO","outturn_security_type":"EQS","outturn_security_id":"S-799713","outturn_isin":"US2288913054","outturn_us_code":"228891305","outturn_local_code":"BOMOD","outturn_bbg_comp_id":"BBG000VCV3H9","outturn_bbg_comp_ticker":"BOMO US","outturn_figi":"BBG000VCV726","outturn_figi_ticker":"BOMO UV","min_offer_qty":null,"max_offer_qty":null,"min_qualify_qty":null,"max_qualify_qty":null,"min_accept_qty":null,"max_accept_qty":null,"tender_strike_price":null,"tender_price_step":null,"option_expiry_time":null,"option_expiry_tz":null,"withdrawal_rights_flag":null,"withdrawal_rights_expiry_time":null,"withdrawal_rights_expiry_tz":null,"expiry_time":null,"expiry_tz":null,"date_info":{},"rate_info":{"par_value_old":0.01,"par_value_new":0.01},"event_info":{},"ts_created":"2024-07-22T01:43:15.099184Z"} 2 | {"ts_record":"2023-11-01T00:00:00Z","event_unique_id":"U-40179043276-1888618","event_id":"E-9043276-RSPLT","listing_id":"L-1888618","listing_group_id":"LG-888618","security_id":"S-799713","issuer_id":"I-119138","event_action":"U","event":"RSPLT","event_subtype":"CONSD","event_date_label":"ex_date","event_date":"2023-10-10","event_created_date":"2023-02-09","effective_date":null,"ex_date":"2023-10-10","record_date":null,"record_date_id":"D-9043276","related_event":null,"related_event_id":null,"global_status":"A","listing_status":"T","listing_source":"M","listing_date":"2008-03-18","delisting_date":null,"issuer_name":"Bowmo Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USOTC","exchange":"USOTC","operating_mic":"OTCM","symbol":"BOMO","nasdaq_symbol":"BOMO","local_code":"BOMO","isin":"US2288912064","us_code":"228891206","bbg_comp_id":"BBG000VCV3H9","bbg_comp_ticker":"BOMO US","figi":"BBG000VCV726","figi_ticker":"BOMO UV","listing_country":"US","register_country":"US","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Pink Current Information","segment_mic":"PINC","mand_volu_flag":"M","rd_priority":1,"lot_size":100,"par_value":0.01,"par_value_currency":"USD","payment_date":null,"duebills_redemption_date":null,"from_date":null,"to_date":null,"registration_date":null,"start_date":null,"end_date":null,"open_date":null,"close_date":null,"start_subscription_date":null,"end_subscription_date":null,"option_election_date":null,"withdrawal_rights_from_date":null,"withdrawal_rights_to_date":null,"notification_date":null,"financial_year_end_date":null,"exp_completion_date":null,"payment_type":"S","option_id":"1","serial_id":"1","default_option_flag":true,"rate_currency":"USD","ratio_old":1000.0,"ratio_new":1.0,"fraction":null,"outturn_style":"NEWO","outturn_security_type":"EQS","outturn_security_id":"S-799713","outturn_isin":"US2288913054","outturn_us_code":"228891305","outturn_local_code":"BOMOD","outturn_bbg_comp_id":"BBG000VCV3H9","outturn_bbg_comp_ticker":"BOMO US","outturn_figi":"BBG000VCV726","outturn_figi_ticker":"BOMO UV","min_offer_qty":null,"max_offer_qty":null,"min_qualify_qty":null,"max_qualify_qty":null,"min_accept_qty":null,"max_accept_qty":null,"tender_strike_price":null,"tender_price_step":null,"option_expiry_time":null,"option_expiry_tz":null,"withdrawal_rights_flag":null,"withdrawal_rights_expiry_time":null,"withdrawal_rights_expiry_tz":null,"expiry_time":null,"expiry_tz":null,"date_info":{},"rate_info":{"par_value_old":0.01,"par_value_new":0.01},"event_info":{},"ts_created":"2024-07-23T00:00:00.000000Z"} 3 | -------------------------------------------------------------------------------- /tests/data/REFERENCE/test_data.corporate-actions.jsonl: -------------------------------------------------------------------------------- 1 | {"ts_record":"2023-10-10T04:37:14Z","event_unique_id":"U-40179043276-1888618","event_id":"E-9043276-RSPLT","listing_id":"L-1888618","listing_group_id":"LG-888618","security_id":"S-799713","issuer_id":"I-119138","event_action":"U","event":"RSPLT","event_subtype":"CONSD","event_date_label":"ex_date","event_date":"2023-10-10","event_created_date":"2023-02-09","effective_date":null,"ex_date":"2023-10-10","record_date":null,"record_date_id":"D-9043276","related_event":null,"related_event_id":null,"global_status":"A","listing_status":"T","listing_source":"M","listing_date":"2008-03-18","delisting_date":null,"issuer_name":"Bowmo Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USOTC","exchange":"USOTC","operating_mic":"OTCM","symbol":"BOMO","nasdaq_symbol":"BOMO","local_code":"BOMO","isin":"US2288912064","us_code":"228891206","bbg_comp_id":"BBG000VCV3H9","bbg_comp_ticker":"BOMO US","figi":"BBG000VCV726","figi_ticker":"BOMO UV","listing_country":"US","register_country":"US","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Pink Current Information","segment_mic":"PINC","mand_volu_flag":"M","rd_priority":1,"lot_size":100,"par_value":0.01,"par_value_currency":"USD","payment_date":null,"duebills_redemption_date":null,"from_date":null,"to_date":null,"registration_date":null,"start_date":null,"end_date":null,"open_date":null,"close_date":null,"start_subscription_date":null,"end_subscription_date":null,"option_election_date":null,"withdrawal_rights_from_date":null,"withdrawal_rights_to_date":null,"notification_date":null,"financial_year_end_date":null,"exp_completion_date":null,"payment_type":"S","option_id":"1","serial_id":"1","default_option_flag":true,"rate_currency":"USD","ratio_old":1000.0,"ratio_new":1.0,"fraction":null,"outturn_style":"NEWO","outturn_security_type":"EQS","outturn_security_id":"S-799713","outturn_isin":"US2288913054","outturn_us_code":"228891305","outturn_local_code":"BOMOD","outturn_bbg_comp_id":"BBG000VCV3H9","outturn_bbg_comp_ticker":"BOMO US","outturn_figi":"BBG000VCV726","outturn_figi_ticker":"BOMO UV","min_offer_qty":null,"max_offer_qty":null,"min_qualify_qty":null,"max_qualify_qty":null,"min_accept_qty":null,"max_accept_qty":null,"tender_strike_price":null,"tender_price_step":null,"option_expiry_time":null,"option_expiry_tz":null,"withdrawal_rights_flag":null,"withdrawal_rights_expiry_time":null,"withdrawal_rights_expiry_tz":null,"expiry_time":null,"expiry_tz":null,"date_info":{},"rate_info":{"par_value_old":0.01,"par_value_new":0.01},"event_info":{},"ts_created":"2024-07-22T01:43:15.099184Z"} 2 | {"ts_record":"2023-10-10T04:37:14Z","event_unique_id":"U-40179751345-16556634","event_id":"E-9751345-RSPLT","listing_id":"L-16556634","listing_group_id":"LG-6556634","security_id":"S-4633970","issuer_id":"I-175515","event_action":"U","event":"RSPLT","event_subtype":"CONSD","event_date_label":"ex_date","event_date":"2023-10-10","event_created_date":"1929-09-30","effective_date":null,"ex_date":"2023-10-10","record_date":null,"record_date_id":"D-9751345","related_event":null,"related_event_id":null,"global_status":"A","listing_status":"L","listing_source":"M","listing_date":"2015-10-29","delisting_date":null,"issuer_name":"Borqs Technologies Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USNASD","exchange":"USNASD","operating_mic":"XNAS","symbol":"BRQS","nasdaq_symbol":"BRQS","local_code":"BRQS","isin":"VGG1466B1452","us_code":"G1466B145","bbg_comp_id":"BBG00B9RG1J6","bbg_comp_ticker":"BRQS US","figi":"BBG00B9RG1W1","figi_ticker":"BRQS UR","listing_country":"US","register_country":"VG","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Capital Market","segment_mic":"XNCM","mand_volu_flag":"M","rd_priority":1,"lot_size":100,"par_value":null,"par_value_currency":"USD","payment_date":null,"duebills_redemption_date":null,"from_date":null,"to_date":null,"registration_date":null,"start_date":null,"end_date":null,"open_date":null,"close_date":null,"start_subscription_date":null,"end_subscription_date":null,"option_election_date":null,"withdrawal_rights_from_date":null,"withdrawal_rights_to_date":null,"notification_date":null,"financial_year_end_date":null,"exp_completion_date":null,"payment_type":"S","option_id":"1","serial_id":"1","default_option_flag":true,"rate_currency":"USD","ratio_old":12.0,"ratio_new":1.0,"fraction":"U","outturn_style":"NEWO","outturn_security_type":"EQS","outturn_security_id":"S-4633970","outturn_isin":"VGG1466B1452","outturn_us_code":"G1466B145","outturn_local_code":"BRQS","outturn_bbg_comp_id":"BBG00B9RG1J6","outturn_bbg_comp_ticker":"BRQS US","outturn_figi":"BBG00B9RG1W1","outturn_figi_ticker":"BRQS UR","min_offer_qty":null,"max_offer_qty":null,"min_qualify_qty":null,"max_qualify_qty":null,"min_accept_qty":null,"max_accept_qty":null,"tender_strike_price":null,"tender_price_step":null,"option_expiry_time":null,"option_expiry_tz":null,"withdrawal_rights_flag":null,"withdrawal_rights_expiry_time":null,"withdrawal_rights_expiry_tz":null,"expiry_time":null,"expiry_tz":null,"date_info":{},"rate_info":{"par_value_old":null,"par_value_new":null},"event_info":{},"ts_created":"2024-07-22T01:43:15.099184Z"} 3 | -------------------------------------------------------------------------------- /tests/data/REFERENCE/test_data.security-master.jsonl: -------------------------------------------------------------------------------- 1 | {"ts_record":"2024-08-07T23:00:00Z","ts_effective":"2024-08-07T00:00:00Z","listing_id":"L-135825","listing_group_id":"LG-7964616","security_id":"S-33449","issuer_id":"I-30017","listing_status":"L","listing_source":"M","listing_created_date":"2001-05-05","listing_date":"1980-12-12","delisting_date":null,"issuer_name":"Apple Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USNASD","exchange":"USNASD","operating_mic":"XNAS","symbol":"AAPL_KZ","nasdaq_symbol":"AAPL_KZ","local_code":"AAPL","isin":"US0378331005","us_code":"037833100","bbg_comp_id":"BBG000B9XRY4","bbg_comp_ticker":"AAPL US","figi":"BBG000B9Y5X2","figi_ticker":"AAPL UW","fisn":null,"lei":"HWUPKR0MPOU8FGXBT394","sic":"3571","cik":"0000320193","gics":null,"naics":"334111","cic":"US31","cfi":"ESVUFR","incorporation_country":"US","listing_country":"US","register_country":"US","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Global Select","segment_mic":"XNGS","structure":null,"lot_size":100,"par_value":0.00001,"par_value_currency":"USD","voting":"V","vote_per_sec":1.0,"shares_outstanding":15204137000,"shares_outstanding_date":"2024-07-19","ts_created":"2024-08-21T03:49:45.938634Z"} 2 | {"ts_record":"2024-08-07T23:00:00Z","ts_effective":"2024-08-15T00:00:00Z","listing_id":"L-135825","listing_group_id":"LG-7964616","security_id":"S-33449","issuer_id":"I-30017","listing_status":"L","listing_source":"M","listing_created_date":"2001-05-05","listing_date":"1980-12-12","delisting_date":null,"issuer_name":"Apple Inc","security_type":"EQS","security_description":"Ordinary Shares","primary_exchange":"USNASD","exchange":"USNASD","operating_mic":"XNAS","symbol":"AAPL_KZ","nasdaq_symbol":"AAPL_KZ","local_code":"AAPL","isin":"US0378331005","us_code":"037833100","bbg_comp_id":"BBG000B9XRY4","bbg_comp_ticker":"AAPL US","figi":"BBG000B9Y5X2","figi_ticker":"AAPL UW","fisn":null,"lei":"HWUPKR0MPOU8FGXBT394","sic":"3571","cik":"0000320193","gics":null,"naics":"334111","cic":"US31","cfi":"ESVUFR","incorporation_country":"US","listing_country":"US","register_country":"US","trading_currency":"USD","multi_currency":false,"segment_mic_name":"Global Select","segment_mic":"XNGS","structure":null,"lot_size":100,"par_value":0.00001,"par_value_currency":"USD","voting":"V","vote_per_sec":1.0,"shares_outstanding":15204137000,"shares_outstanding_date":"2024-07-19","ts_created":"2024-08-21T03:49:45.952760Z"} 3 | -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.definition.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.definition.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.mbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.mbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst -------------------------------------------------------------------------------- /tests/data/XNAS.ITCH/test_data.trades.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/XNAS.ITCH/test_data.trades.dbn.zst -------------------------------------------------------------------------------- /tests/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/data/__init__.py -------------------------------------------------------------------------------- /tests/data/generator.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility to generate stub data for testing purposes. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import argparse 8 | import asyncio 9 | import pathlib 10 | import sys 11 | import warnings 12 | from typing import Final 13 | 14 | import databento as db 15 | from databento.common.publishers import Dataset 16 | from databento_dbn import Schema 17 | 18 | from tests import TESTS_ROOT 19 | 20 | 21 | warnings.simplefilter("ignore") 22 | 23 | STUB_DATA_PATH: Final = TESTS_ROOT / "data" 24 | 25 | LIMIT: Final = 4 26 | 27 | MANIFEST: Final = { 28 | Dataset.GLBX_MDP3: [ 29 | (Schema.MBO, ["ESH1"], "2020-12-28"), 30 | (Schema.MBP_1, ["ESH1"], "2020-12-28"), 31 | (Schema.MBP_10, ["ESH1"], "2020-12-28"), 32 | (Schema.BBO_1S, ["ESH1"], "2020-12-28"), 33 | (Schema.BBO_1M, ["ESH1"], "2020-12-28"), 34 | (Schema.TBBO, ["ESH1"], "2020-12-28"), 35 | (Schema.TRADES, ["ESH1"], "2020-12-28"), 36 | (Schema.OHLCV_1S, ["ESH1"], "2020-12-28"), 37 | (Schema.OHLCV_1M, ["ESH1"], "2020-12-28"), 38 | (Schema.OHLCV_1H, ["ESH1"], "2020-12-28"), 39 | (Schema.OHLCV_1D, ["ESH1"], "2020-12-28"), 40 | (Schema.DEFINITION, ["ESH1"], "2020-12-28"), 41 | (Schema.STATISTICS, ["ESH1"], "2020-12-28"), 42 | (Schema.STATUS, ["ESH1"], "2020-12-28"), 43 | ], 44 | Dataset.XNAS_ITCH: [ 45 | (Schema.MBO, ["NVDA"], "2020-12-28"), 46 | (Schema.MBP_1, ["NVDA"], "2020-12-28"), 47 | (Schema.MBP_10, ["NVDA"], "2020-12-28"), 48 | (Schema.TBBO, ["NVDA"], "2020-12-28"), 49 | (Schema.TRADES, ["NVDA"], "2020-12-28"), 50 | (Schema.OHLCV_1S, ["NVDA"], "2020-12-28"), 51 | (Schema.OHLCV_1M, ["NVDA"], "2020-12-28"), 52 | (Schema.OHLCV_1H, ["NVDA"], "2020-12-28"), 53 | (Schema.OHLCV_1D, ["NVDA"], "2020-12-28"), 54 | (Schema.DEFINITION, ["NVDA"], "2020-12-28"), 55 | (Schema.IMBALANCE, ["NVDA"], "2020-12-28"), 56 | ], 57 | Dataset.OPRA_PILLAR: [ 58 | (Schema.MBP_1, ["AAPL 250221C00250000"], "2025-02-20"), 59 | (Schema.TBBO, ["AAPL 250221C00250000"], "2025-02-20"), 60 | (Schema.TRADES, ["AAPL 250221C00250000"], "2025-02-20"), 61 | (Schema.CBBO_1S, ["AAPL 250221C00250000"], "2025-02-20"), 62 | (Schema.CBBO_1M, ["AAPL 250221C00250000"], "2025-02-20"), 63 | (Schema.OHLCV_1S, ["AAPL 250221C00250000"], "2025-02-20"), 64 | (Schema.OHLCV_1M, ["AAPL 250221C00250000"], "2025-02-20"), 65 | (Schema.OHLCV_1H, ["AAPL 250221C00250000"], "2025-02-20"), 66 | (Schema.OHLCV_1D, ["AAPL 250221C00250000"], "2025-02-20"), 67 | (Schema.DEFINITION, ["AAPL 250221C00250000"], "2025-02-20"), 68 | (Schema.STATISTICS, ["AAPL 250221C00250000"], "2025-02-20"), 69 | ], 70 | Dataset.EQUS_MINI: [ 71 | (Schema.MBP_1, ["QQQ"], "2023-03-28"), 72 | (Schema.TBBO, ["QQQ"], "2023-03-28"), 73 | (Schema.TRADES, ["QQQ"], "2023-03-28"), 74 | (Schema.BBO_1S, ["QQQ"], "2023-03-28"), 75 | (Schema.BBO_1M, ["QQQ"], "2023-03-28"), 76 | (Schema.OHLCV_1S, ["QQQ"], "2023-03-28"), 77 | (Schema.OHLCV_1M, ["QQQ"], "2023-03-28"), 78 | (Schema.OHLCV_1H, ["QQQ"], "2023-03-28"), 79 | (Schema.OHLCV_1D, ["QQQ"], "2023-03-28"), 80 | (Schema.DEFINITION, ["QQQ"], "2023-03-28"), 81 | ], 82 | Dataset.IFEU_IMPACT: [ 83 | (Schema.MBO, ["BRN FMJ0024!"], "2024-01-18"), 84 | (Schema.MBP_1, ["BRN FMJ0024!"], "2024-01-18"), 85 | (Schema.MBP_10, ["BRN FMJ0024!"], "2024-01-18"), 86 | (Schema.BBO_1S, ["BRN FMJ0024!"], "2024-01-18"), 87 | (Schema.BBO_1M, ["BRN FMJ0024!"], "2024-01-18"), 88 | (Schema.TBBO, ["BRN FMJ0024!"], "2024-01-18"), 89 | (Schema.TRADES, ["BRN FMJ0024!"], "2024-01-18"), 90 | (Schema.OHLCV_1S, ["BRN FMJ0024!"], "2024-01-18"), 91 | (Schema.OHLCV_1M, ["BRN FMJ0024!"], "2024-01-18"), 92 | (Schema.OHLCV_1H, ["BRN FMJ0024!"], "2024-01-18"), 93 | (Schema.OHLCV_1D, ["BRN FMJ0024!"], "2024-01-18"), 94 | (Schema.DEFINITION, ["BRN FMJ0024!"], "2024-01-18"), 95 | (Schema.STATISTICS, ["BRN FMJ0024!"], "2024-01-18"), 96 | ], 97 | Dataset.NDEX_IMPACT: [ 98 | (Schema.MBO, ["TFM FMH0024!"], "2024-01-18"), 99 | (Schema.MBP_1, ["TFM FMH0024!"], "2024-01-18"), 100 | (Schema.MBP_10, ["TFM FMH0024!"], "2024-01-18"), 101 | (Schema.BBO_1S, ["TFM FMH0024!"], "2024-01-18"), 102 | (Schema.BBO_1M, ["TFM FMH0024!"], "2024-01-18"), 103 | (Schema.TBBO, ["TFM FMH0024!"], "2024-01-18"), 104 | (Schema.TRADES, ["TFM FMH0024!"], "2024-01-18"), 105 | (Schema.OHLCV_1S, ["TFM FMH0024!"], "2024-01-18"), 106 | (Schema.OHLCV_1M, ["TFM FMH0024!"], "2024-01-18"), 107 | (Schema.OHLCV_1H, ["TFM FMH0024!"], "2024-01-18"), 108 | (Schema.OHLCV_1D, ["TFM FMH0024!"], "2024-01-18"), 109 | (Schema.DEFINITION, ["TFM FMH0024!"], "2024-01-18"), 110 | (Schema.STATISTICS, ["TFM FMH0024!"], "2024-01-18"), 111 | ], 112 | } 113 | 114 | 115 | async def generate_stub_data( 116 | regenerate: bool = False, 117 | dataset_select: Dataset | None = None, 118 | ) -> None: 119 | client = db.Historical() 120 | 121 | tasks = [] 122 | for dataset, specifications in MANIFEST.items(): 123 | if dataset_select is not None and dataset != dataset_select: 124 | print(f"skipping dataset {dataset}") 125 | continue 126 | 127 | dataset_dir = STUB_DATA_PATH / dataset 128 | dataset_dir.mkdir(parents=True, exist_ok=True) 129 | for spec in specifications: 130 | schema, symbols, start = spec 131 | path = dataset_dir / f"test_data.{schema}.dbn.zst" 132 | if path.exists(): 133 | if not regenerate: 134 | continue 135 | path.unlink() 136 | 137 | tasks.append( 138 | asyncio.create_task( 139 | client.timeseries.get_range_async( 140 | dataset=dataset, 141 | schema=schema, 142 | symbols=symbols, 143 | start=start, 144 | limit=LIMIT, 145 | path=path, 146 | ), 147 | ), 148 | ) 149 | 150 | print(f"generating {len(tasks)} stub files", end="...", flush=True) 151 | stores = await asyncio.gather(*tasks) 152 | print("done") 153 | 154 | # Check for empty stubs 155 | for store in stores: 156 | num_records = len(list(store)) 157 | if num_records == 0: 158 | print( 159 | f"WARNING: {store.dataset} stub file {store._data_source.name} has no records!", 160 | ) 161 | 162 | 163 | if __name__ == "__main__": 164 | parser = argparse.ArgumentParser( 165 | description=f"Generate DBN stub data to: ./{STUB_DATA_PATH.relative_to(pathlib.Path.cwd())}", 166 | ) 167 | parser.add_argument( 168 | "-d", 169 | "--dataset", 170 | choices=tuple(str(x.value) for x in MANIFEST), 171 | default=None, 172 | ) 173 | parser.add_argument( 174 | "--regenerate", 175 | action="store_true", 176 | dest="regenerate", 177 | help="remove old stub files to regenerate existing data", 178 | ) 179 | 180 | parsed = parser.parse_args(sys.argv[1:]) 181 | asyncio.run( 182 | generate_stub_data( 183 | regenerate=parsed.regenerate, 184 | dataset_select=parsed.dataset, 185 | ), 186 | ) 187 | -------------------------------------------------------------------------------- /tests/mockliveserver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/databento-python/10c75b77d3852f560e54e0f2835102d16cd01c3f/tests/mockliveserver/__init__.py -------------------------------------------------------------------------------- /tests/mockliveserver/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import logging 4 | import os 5 | import sys 6 | from collections import defaultdict 7 | from socket import AF_INET 8 | 9 | from databento.common.publishers import Dataset 10 | from databento_dbn import Schema 11 | 12 | from tests.mockliveserver.controller import Controller 13 | from tests.mockliveserver.source import ReplayProtocol 14 | 15 | from .server import MockLiveServerProtocol 16 | from .server import SessionMode 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument( 24 | "host", 25 | help="the hostname to bind; defaults to `localhost`", 26 | nargs="?", 27 | default="localhost", 28 | ) 29 | parser.add_argument( 30 | "-p", 31 | "--port", 32 | help="the port to bind; defaults to an open port", 33 | type=int, 34 | default=0, 35 | ) 36 | parser.add_argument( 37 | "-e", 38 | "--echo", 39 | help="a file to write echos of gateway control messages to", 40 | default=os.devnull, 41 | ) 42 | parser.add_argument( 43 | "-v", 44 | "--verbose", 45 | action="store_true", 46 | help="enabled debug logging", 47 | ) 48 | 49 | 50 | async def main() -> None: 51 | params = parser.parse_args(sys.argv[1:]) 52 | 53 | # Setup console logging handler 54 | log_level = logging.DEBUG if params.verbose else logging.INFO 55 | logging.basicConfig( 56 | level=log_level, 57 | format="%(asctime)s %(levelname)s %(message)s", 58 | stream=sys.stderr, 59 | ) 60 | 61 | logger.info("mockliveserver starting") 62 | loop = asyncio.get_running_loop() 63 | 64 | api_key_table: dict[str, set[str]] = defaultdict(set) 65 | file_replay_table: dict[tuple[Dataset, Schema], ReplayProtocol] = {} 66 | sessions: list[MockLiveServerProtocol] = [] 67 | echo_stream = open(params.echo, "wb", buffering=0) 68 | 69 | def protocol_factory() -> MockLiveServerProtocol: 70 | protocol = MockLiveServerProtocol( 71 | loop=loop, 72 | mode=SessionMode.FILE_REPLAY, 73 | api_key_table=api_key_table, 74 | file_replay_table=file_replay_table, 75 | echo_stream=echo_stream, 76 | ) 77 | sessions.append(protocol) 78 | return protocol 79 | 80 | # Create server for incoming connections 81 | server = await loop.create_server( 82 | protocol_factory=protocol_factory, 83 | family=AF_INET, # force ipv4 84 | host=params.host, 85 | port=params.port, 86 | start_serving=True, 87 | ) 88 | ip, port, *_ = server._sockets[-1].getsockname() # type: ignore [attr-defined] 89 | 90 | # Create command interface for stdin 91 | _ = Controller( 92 | server=server, 93 | api_key_table=api_key_table, 94 | file_replay_table=file_replay_table, 95 | sessions=sessions, 96 | loop=loop, 97 | ) 98 | 99 | # Log Arguments 100 | logger.info("host: %s (%s)", params.host, ip) 101 | logger.info("port: %d", port) 102 | logger.info("echo: %s", params.echo) 103 | logger.info("verbose: %s", params.verbose) 104 | logger.info("mockliveserver now serving") 105 | 106 | try: 107 | await server.serve_forever() 108 | except asyncio.CancelledError: 109 | logger.info("terminating mock live server") 110 | 111 | echo_stream.close() 112 | 113 | 114 | asyncio.run(main()) 115 | -------------------------------------------------------------------------------- /tests/mockliveserver/controller.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | import asyncio 5 | import logging 6 | import sys 7 | from collections.abc import Mapping 8 | from collections.abc import MutableMapping 9 | from pathlib import Path 10 | 11 | from databento.common.cram import BUCKET_ID_LENGTH 12 | from databento.common.publishers import Dataset 13 | from databento_dbn import Schema 14 | 15 | from tests.mockliveserver.server import MockLiveServerProtocol 16 | from tests.mockliveserver.source import FileReplay 17 | from tests.mockliveserver.source import ReplayProtocol 18 | 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class Controller: 24 | command_parser = argparse.ArgumentParser(prog="mockliveserver") 25 | subparsers = command_parser.add_subparsers(dest="command") 26 | 27 | close_command = subparsers.add_parser("close", help="close the mock live server") 28 | 29 | active_count = subparsers.add_parser( 30 | "active_count", 31 | help="log the number of active connections", 32 | ) 33 | 34 | add_key = subparsers.add_parser("add_key", help="add an API key to the mock live server") 35 | add_key.add_argument("key", type=str) 36 | 37 | del_key = subparsers.add_parser("del_key", help="delete an API key from the mock live server") 38 | del_key.add_argument("key", type=str) 39 | 40 | add_dbn = subparsers.add_parser("add_dbn", help="add a dbn file for replay") 41 | add_dbn.add_argument("dataset", type=str) 42 | add_dbn.add_argument("schema", type=str) 43 | add_dbn.add_argument("dbn_file", type=str) 44 | 45 | disconnect = subparsers.add_parser("disconnect", help="disconnect a live session") 46 | disconnect.add_argument("session_id", type=str) 47 | 48 | def __init__( 49 | self, 50 | server: asyncio.base_events.Server, 51 | api_key_table: Mapping[str, set[str]], 52 | file_replay_table: MutableMapping[tuple[Dataset, Schema], ReplayProtocol], 53 | sessions: list[MockLiveServerProtocol], 54 | loop: asyncio.AbstractEventLoop, 55 | ) -> None: 56 | self._server = server 57 | self._api_key_table = api_key_table 58 | self._file_replay_table = file_replay_table 59 | self._sessions = sessions 60 | self._loop = loop 61 | 62 | self._read_task = loop.create_task(self._read_commands()) 63 | 64 | async def _read_commands(self) -> None: 65 | while self._server.is_serving(): 66 | line = await self._loop.run_in_executor(None, sys.stdin.readline) 67 | self.data_received(line.strip()) 68 | 69 | def data_received(self, command_str: str) -> None: 70 | params = self.command_parser.parse_args(command_str.split()) 71 | command_func = getattr(self, f"_command_{params.command}", None) 72 | if command_func is None: 73 | raise ValueError(f"{params.command} does not have a command handler") 74 | else: 75 | logger.info("received command: %s", command_str) 76 | command_params = dict(params._get_kwargs()) 77 | command_params.pop("command") 78 | try: 79 | command_func(**command_params) 80 | except Exception: 81 | logger.exception("error processing command: %s", params.command) 82 | print(f"nack: {command_str}", flush=True) 83 | else: 84 | print(f"ack: {command_str}", flush=True) 85 | 86 | def _command_close(self, *_: str) -> None: 87 | """ 88 | Close the server. 89 | """ 90 | self._read_task.cancel() 91 | self._loop.call_soon(self._server.close) 92 | 93 | def _command_active_count(self, *_: str) -> None: 94 | """ 95 | Log the number of active connections. 96 | """ 97 | # _active_count was removed in Python 3.13 98 | if hasattr(self._server, "_active_count"): 99 | count = self._server._active_count 100 | else: 101 | count = len(self._server._clients) # type: ignore [attr-defined] 102 | 103 | logger.info("active connections: %d", count) 104 | 105 | def _command_add_key(self, key: str) -> None: 106 | """ 107 | Add an API key to the server. 108 | """ 109 | if len(key) < BUCKET_ID_LENGTH: 110 | logger.error("api key must be at least %d characters long", BUCKET_ID_LENGTH) 111 | return 112 | 113 | bucket_id = key[-BUCKET_ID_LENGTH:] 114 | self._api_key_table[bucket_id].add(key) 115 | logger.info("added api key '%s'", key) 116 | 117 | def _command_del_key(self, key: str) -> None: 118 | """ 119 | Remove API key from the server. 120 | """ 121 | if len(key) < BUCKET_ID_LENGTH: 122 | logger.error("api key must be at least %d characters long", BUCKET_ID_LENGTH) 123 | return 124 | 125 | bucket_id = key[-BUCKET_ID_LENGTH:] 126 | self._api_key_table[bucket_id].remove(key) 127 | logger.info("deleted api key '%s'", key) 128 | 129 | def _command_add_dbn(self, dataset: str, schema: str, dbn_file: str) -> None: 130 | """ 131 | Add a DBN file for streaming. 132 | """ 133 | try: 134 | dataset_valid = Dataset(dataset) 135 | schema_valid = Schema(schema) 136 | except ValueError as exc: 137 | logger.error("invalid parameter value: %s", exc) 138 | return 139 | 140 | dbn_path = Path(dbn_file) 141 | if not dbn_path.exists() or not dbn_path.is_file(): 142 | logger.error("invalid file path: %s", dbn_path) 143 | return 144 | 145 | self._file_replay_table[(dataset_valid, schema_valid)] = FileReplay(dbn_path) 146 | 147 | def _command_disconnect(self, session_id: str | None) -> None: 148 | for session in self._sessions: 149 | if session_id is None or session.session_id == session_id: 150 | logger.info("disconnecting session %s", session.session_id) 151 | session.transport.abort() 152 | return 153 | raise ValueError(f"no session with id {session_id}") 154 | -------------------------------------------------------------------------------- /tests/mockliveserver/fixture.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import contextlib 3 | import os 4 | import pathlib 5 | import sys 6 | from asyncio.subprocess import Process 7 | from collections.abc import AsyncGenerator 8 | from collections.abc import Generator 9 | from typing import Callable 10 | from typing import TypeVar 11 | 12 | import pytest 13 | import pytest_asyncio 14 | from databento.common.publishers import Dataset 15 | from databento.live.gateway import GatewayControl 16 | from databento_dbn import Schema 17 | 18 | from tests import TESTS_ROOT 19 | 20 | 21 | class MockLiveServerInterface: 22 | """ 23 | Process wrapper for communicating with the mock live server. 24 | """ 25 | 26 | _GC = TypeVar("_GC", bound=GatewayControl) 27 | 28 | def __init__( 29 | self, 30 | process: Process, 31 | host: str, 32 | port: int, 33 | echo_file: pathlib.Path, 34 | ): 35 | self._process = process 36 | self._host = host 37 | self._port = port 38 | self._echo_fd = open(echo_file) 39 | 40 | @property 41 | def host(self) -> str: 42 | """ 43 | The mock live server host. 44 | 45 | Returns 46 | ------- 47 | str 48 | 49 | """ 50 | return self._host 51 | 52 | @property 53 | def port(self) -> int: 54 | """ 55 | The mock live server port. 56 | 57 | Returns 58 | ------- 59 | int 60 | 61 | """ 62 | return self._port 63 | 64 | @property 65 | def stdout(self) -> asyncio.StreamReader: 66 | if self._process.stdout is not None: 67 | return self._process.stdout 68 | raise RuntimeError("no stream reader for stdout") 69 | 70 | async def _send_command( 71 | self, 72 | command: str, 73 | timeout: float = 1.0, 74 | ) -> None: 75 | if self._process.stdin is None: 76 | raise RuntimeError("cannot write command to mock live server") 77 | self._process.stdin.write( 78 | f"{command.strip()}\n".encode(), 79 | ) 80 | 81 | try: 82 | line = await asyncio.wait_for(self.stdout.readline(), timeout) 83 | except asyncio.TimeoutError: 84 | raise RuntimeError("timeout waiting for command acknowledgement") 85 | 86 | line_str = line.decode("utf-8") 87 | 88 | if line_str.startswith(f"ack: {command}"): 89 | return 90 | elif line_str.startswith(f"nack: {command}"): 91 | raise RuntimeError(f"received nack for command: {command}") 92 | 93 | raise RuntimeError(f"invalid response from server: {line_str!r}") 94 | 95 | async def active_count(self) -> None: 96 | """ 97 | Send the "active_count" command. 98 | """ 99 | await self._send_command("active_count") 100 | 101 | async def add_key(self, api_key: str) -> None: 102 | """ 103 | Send the "add_key" command. 104 | 105 | Parameters 106 | ---------- 107 | api_key : str 108 | The API key to add. 109 | 110 | """ 111 | await self._send_command(f"add_key {api_key}") 112 | 113 | async def add_dbn(self, dataset: Dataset, schema: Schema, path: pathlib.Path) -> None: 114 | """ 115 | Send the "add_dbn" command. 116 | 117 | Parameters 118 | ---------- 119 | dataset : Dataset 120 | The DBN dataset. 121 | schema : Schema 122 | The DBN schema. 123 | path : pathlib.Path 124 | The path to the DBN file. 125 | 126 | """ 127 | await self._send_command(f"add_dbn {dataset} {schema} {path.resolve()}") 128 | 129 | async def del_key(self, api_key: str) -> None: 130 | """ 131 | Send the "del_key" command. 132 | 133 | Parameters 134 | ---------- 135 | api_key : str 136 | The API key to delete. 137 | 138 | """ 139 | await self._send_command(f"del_key {api_key}") 140 | 141 | async def disconnect(self, session_id: str) -> None: 142 | """ 143 | Send the "disconnect" command. 144 | 145 | Parameters 146 | ---------- 147 | session_id : str 148 | The live session ID to disconnect. 149 | 150 | """ 151 | await self._send_command(f"disconnect {session_id}") 152 | 153 | async def close(self) -> None: 154 | """ 155 | Send the "close" command. 156 | """ 157 | await self._send_command("close") 158 | 159 | def kill(self) -> None: 160 | """ 161 | Kill the mock live server. 162 | """ 163 | self._process.kill() 164 | 165 | @contextlib.contextmanager 166 | def test_context(self) -> Generator[None, None, None]: 167 | self._echo_fd.seek(0, os.SEEK_END) 168 | yield 169 | 170 | @contextlib.asynccontextmanager 171 | async def api_key_context(self, api_key: str) -> AsyncGenerator[str, None]: 172 | await self.add_key(api_key) 173 | yield api_key 174 | await self.del_key(api_key) 175 | 176 | async def wait_for_start(self) -> None: 177 | await self.active_count() 178 | 179 | async def wait_for_message_of_type( 180 | self, 181 | message_type: type[_GC], 182 | timeout: float = 1.0, 183 | ) -> _GC: 184 | """ 185 | Wait for a message of a given type. 186 | 187 | Parameters 188 | ---------- 189 | message_type : type[_GC] 190 | The type of GatewayControl message to wait for. 191 | timeout: float, default 1.0 192 | The maximum number of seconds to wait. 193 | 194 | Returns 195 | ------- 196 | _GC 197 | 198 | """ 199 | loop = asyncio.get_running_loop() 200 | deadline = loop.time() + timeout 201 | while self._process.returncode is None: 202 | line = await asyncio.wait_for( 203 | loop.run_in_executor(None, self._echo_fd.readline), 204 | timeout=max( 205 | 0, 206 | deadline - loop.time(), 207 | ), 208 | ) 209 | try: 210 | return message_type.parse(line) 211 | except ValueError: 212 | continue 213 | raise RuntimeError("Mock server is closed.") 214 | 215 | 216 | @pytest_asyncio.fixture(name="mock_live_server", scope="module") 217 | async def fixture_mock_live_server( 218 | unused_tcp_port_factory: Callable[[], int], 219 | tmp_path_factory: pytest.TempPathFactory, 220 | ) -> AsyncGenerator[MockLiveServerInterface, None]: 221 | port = unused_tcp_port_factory() 222 | echo_file = tmp_path_factory.mktemp("mockliveserver") / "echo.txt" 223 | echo_file.touch() 224 | 225 | process = await asyncio.subprocess.create_subprocess_exec( 226 | "python3", 227 | "-m", 228 | "tests.mockliveserver", 229 | "127.0.0.1", 230 | "--port", 231 | str(port), 232 | "--echo", 233 | echo_file.resolve(), 234 | "--verbose", 235 | executable=sys.executable, 236 | stdin=asyncio.subprocess.PIPE, 237 | stdout=asyncio.subprocess.PIPE, 238 | stderr=sys.stderr, 239 | ) 240 | 241 | interface = MockLiveServerInterface( 242 | process=process, 243 | host="127.0.0.1", 244 | port=port, 245 | echo_file=echo_file, 246 | ) 247 | 248 | await interface.wait_for_start() 249 | 250 | for dataset in Dataset: 251 | for schema in Schema.variants(): 252 | path = TESTS_ROOT / "data" / dataset / f"test_data.{schema}.dbn.zst" 253 | if path.exists(): 254 | await interface.add_dbn(dataset, schema, path) 255 | 256 | yield interface 257 | 258 | process.terminate() 259 | await process.wait() 260 | -------------------------------------------------------------------------------- /tests/mockliveserver/source.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Generator 2 | from collections.abc import Iterator 3 | from pathlib import Path 4 | from typing import IO 5 | from typing import Final 6 | from typing import Protocol 7 | 8 | import zstandard 9 | from databento.common.dbnstore import is_zstandard 10 | from databento_dbn import Compression 11 | 12 | 13 | FILE_READ_SIZE: Final = 2**10 14 | 15 | 16 | class ReplayProtocol(Protocol): 17 | @property 18 | def name(self) -> str: ... 19 | 20 | def __iter__(self) -> Iterator[bytes]: ... 21 | 22 | 23 | class FileReplay(ReplayProtocol): 24 | def __init__(self, dbn_file: Path): 25 | self._dbn_file = dbn_file 26 | self._compression = Compression.NONE 27 | 28 | with self._dbn_file.open("rb") as dbn: 29 | if is_zstandard(dbn): 30 | self._compression = Compression.ZSTD 31 | 32 | @property 33 | def name(self) -> str: 34 | return self._dbn_file.name 35 | 36 | def __iter__(self) -> Generator[bytes, None, None]: 37 | with self._dbn_file.open("rb") as dbn: 38 | if self._compression == Compression.ZSTD: 39 | reader: IO[bytes] = zstandard.ZstdDecompressor().stream_reader(dbn) 40 | else: 41 | reader = dbn 42 | while next_bytes := reader.read(FILE_READ_SIZE): 43 | yield next_bytes 44 | -------------------------------------------------------------------------------- /tests/test_bento_compression.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for DBNStore compression. 3 | """ 4 | 5 | from io import BytesIO 6 | 7 | import pytest 8 | from databento.common.dbnstore import is_dbn 9 | from databento.common.dbnstore import is_zstandard 10 | 11 | 12 | @pytest.mark.parametrize( 13 | "data,expected", 14 | [ 15 | pytest.param(b"DBN", True), 16 | pytest.param(b"123", False, id="mismatch"), 17 | pytest.param(b"", False, id="empty"), 18 | ], 19 | ) 20 | def test_is_dbn(data: bytes, expected: bool) -> None: 21 | """ 22 | Test that buffers that start with DBN are identified as DBN files. 23 | """ 24 | # Arrange, Act 25 | reader = BytesIO(data) 26 | 27 | # Assert 28 | assert is_dbn(reader) == expected 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "data,expected", 33 | [ 34 | pytest.param( 35 | 0x28_B5_2F_FD_04_58_6C_08_00_02_CE_33_38_30_8F_D3_18_88.to_bytes(18, "big"), 36 | True, 37 | id="standard_frame", 38 | ), 39 | pytest.param( 40 | 0x50_2A_4D_18_94_00_00_00_44_42_5A_01_47_4C_42_58_2E_4D.to_bytes(18, "big"), 41 | True, 42 | id="skippable_frame", 43 | ), 44 | pytest.param( 45 | 0x44_42_4E_01_C6_00_00_00_47_4C_42_58_2E_4D_44_50_33_00.to_bytes( 46 | 18, 47 | "little", 48 | ), 49 | False, 50 | id="mismatch", 51 | ), 52 | pytest.param(b"", False, id="empty"), 53 | ], 54 | ) 55 | def test_is_zstandard(data: bytes, expected: bool) -> None: 56 | """ 57 | Test that buffers that contain ZSTD data are correctly identified. 58 | """ 59 | # Arrange, Act 60 | reader = BytesIO(data) 61 | 62 | # Assert 63 | assert is_zstandard(reader) == expected 64 | -------------------------------------------------------------------------------- /tests/test_bento_data_source.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from typing import Callable 3 | 4 | import pytest 5 | from databento.common.dbnstore import FileDataSource 6 | from databento.common.dbnstore import MemoryDataSource 7 | from databento.common.publishers import Dataset 8 | from databento_dbn import Schema 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "dataset", 13 | [ 14 | Dataset.GLBX_MDP3, 15 | Dataset.XNAS_ITCH, 16 | Dataset.OPRA_PILLAR, 17 | Dataset.EQUS_MINI, 18 | Dataset.IFEU_IMPACT, 19 | Dataset.NDEX_IMPACT, 20 | ], 21 | ) 22 | @pytest.mark.parametrize("schema", [pytest.param(x) for x in Schema.variants()]) 23 | def test_memory_data_source( 24 | test_data: Callable[[Dataset, Schema], bytes], 25 | dataset: Dataset, 26 | schema: Schema, 27 | ) -> None: 28 | """ 29 | Test create of MemoryDataSource. 30 | """ 31 | # Arrange, Act 32 | data = test_data(dataset, schema) 33 | data_source = MemoryDataSource(data) 34 | 35 | # Assert 36 | assert len(data) == data_source.nbytes 37 | assert repr(data) == data_source.name 38 | 39 | 40 | @pytest.mark.parametrize( 41 | "dataset", 42 | [ 43 | Dataset.GLBX_MDP3, 44 | Dataset.XNAS_ITCH, 45 | Dataset.OPRA_PILLAR, 46 | Dataset.EQUS_MINI, 47 | Dataset.IFEU_IMPACT, 48 | Dataset.NDEX_IMPACT, 49 | ], 50 | ) 51 | @pytest.mark.parametrize("schema", [pytest.param(x) for x in Schema.variants()]) 52 | def test_file_data_source( 53 | test_data_path: Callable[[Dataset, Schema], pathlib.Path], 54 | dataset: Dataset, 55 | schema: Schema, 56 | ) -> None: 57 | """ 58 | Test create of FileDataSource. 59 | """ 60 | # Arrange, Act 61 | path = test_data_path(dataset, schema) 62 | data_source = FileDataSource(path) 63 | 64 | # Assert 65 | assert path.stat().st_size == data_source.nbytes 66 | assert path.name == data_source.name 67 | -------------------------------------------------------------------------------- /tests/test_common_cram.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for CRAM. 3 | """ 4 | 5 | import pytest 6 | from databento.common import cram 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "challenge,key,expected", 11 | [ 12 | pytest.param( 13 | "abcd1234", 14 | "db-unittestapikey1234567890FFFFF", 15 | "be87ce3d564b64481d4ad1902e2b41b26e3eef62b9de37d87eb4a1d4a5199b6f-FFFFF", 16 | ), 17 | ], 18 | ) 19 | def test_get_challenge_response( 20 | challenge: str, 21 | key: str, 22 | expected: str, 23 | ) -> None: 24 | """ 25 | A challenge response is of the form {hash}-{bucket_id}. 26 | 27 | - hash is a sha256 of the user's API key and CRAM challenge. 28 | - bucket_id is the last 5 characters of the user's API key. 29 | The hash calculated using the sha256 algorithm. 30 | The digest is the following string {key}|{challenge} where: 31 | - key is the user's API key 32 | - challenge is the CRAM challenge, this is salt for the hash. 33 | 34 | """ 35 | # Arrange, Act 36 | response = cram.get_challenge_response( 37 | challenge=challenge, 38 | key=key, 39 | ) 40 | 41 | # Assert 42 | assert response == expected 43 | -------------------------------------------------------------------------------- /tests/test_common_enums.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for databento.common.enums. 3 | """ 4 | 5 | from enum import Enum 6 | from enum import Flag 7 | from itertools import combinations 8 | from typing import Final 9 | 10 | import pytest 11 | from databento.common.enums import Delivery 12 | from databento.common.enums import FeedMode 13 | from databento.common.enums import HistoricalGateway 14 | from databento.common.enums import Packaging 15 | from databento.common.enums import RecordFlags 16 | from databento.common.enums import RollRule 17 | from databento.common.enums import SplitDuration 18 | from databento.common.enums import StringyMixin 19 | from databento.common.enums import SymbologyResolution 20 | from databento.common.publishers import Dataset 21 | from databento_dbn import Compression 22 | from databento_dbn import DBNError 23 | from databento_dbn import Encoding 24 | from databento_dbn import Schema 25 | from databento_dbn import SType 26 | 27 | 28 | NATIVE_ENUMS: Final = ( 29 | Dataset, 30 | FeedMode, 31 | HistoricalGateway, 32 | Packaging, 33 | Delivery, 34 | RecordFlags, 35 | RollRule, 36 | SplitDuration, 37 | SymbologyResolution, 38 | ) 39 | 40 | DBN_ENUMS: Final = ( 41 | Compression, 42 | Encoding, 43 | Schema, 44 | SType, 45 | ) 46 | 47 | 48 | @pytest.mark.parametrize( 49 | "enum_type", 50 | (pytest.param(enum) for enum in NATIVE_ENUMS if issubclass(enum, int)), 51 | ) 52 | def test_int_enum_string_coercion(enum_type: type[Enum]) -> None: 53 | """ 54 | Test the int coercion for integer enumerations. 55 | 56 | See: databento.common.enums.coercible 57 | 58 | """ 59 | # Arrange, Act, Assert 60 | for variant in enum_type: 61 | assert variant == enum_type(str(variant.value)) 62 | with pytest.raises(ValueError): 63 | enum_type("NaN") # sanity 64 | 65 | 66 | @pytest.mark.parametrize( 67 | "enum_type", 68 | (pytest.param(enum) for enum in NATIVE_ENUMS if issubclass(enum, str)), 69 | ) 70 | def test_str_enum_case_coercion(enum_type: type[Enum]) -> None: 71 | """ 72 | Test the lowercase name coercion for string enumerations. 73 | 74 | See: databento.common.enums.coercible 75 | 76 | """ 77 | # Arrange, Act, Assert 78 | for enum in enum_type: 79 | assert enum == enum_type(enum.value.lower()) 80 | assert enum == enum_type(enum.value.upper()) 81 | with pytest.raises(ValueError): 82 | enum_type("foo") # sanity 83 | 84 | 85 | @pytest.mark.parametrize( 86 | "enum_type", 87 | NATIVE_ENUMS, 88 | ) 89 | def test_enum_name_coercion(enum_type: type[Enum]) -> None: 90 | """ 91 | Test that enums can be coerced from the member names. 92 | 93 | This includes case and dash conversion to underscores. 94 | See: databento.common.enums.coercible 95 | 96 | """ 97 | # Arrange, Act 98 | if enum_type in (Compression, Encoding, Schema, SType): 99 | enum_it = iter(enum_type.variants()) # type: ignore [attr-defined] 100 | else: 101 | enum_it = iter(enum_type) 102 | 103 | # Assert 104 | for enum in enum_it: 105 | assert enum == enum_type(enum.name) 106 | assert enum == enum_type(enum.name.replace("_", "-")) 107 | assert enum == enum_type(enum.name.lower()) 108 | assert enum == enum_type(enum.name.upper()) 109 | with pytest.raises(ValueError): 110 | enum_type("bar") # sanity 111 | 112 | 113 | @pytest.mark.parametrize( 114 | "enum_type", 115 | DBN_ENUMS, 116 | ) 117 | def test_dbn_enum_name_coercion(enum_type: type[Enum]) -> None: 118 | """ 119 | Test that DBN enums can be coerced from the member names. 120 | 121 | This includes case and dash conversion to underscores. 122 | 123 | """ 124 | # Arrange, Act 125 | if enum_type in (Compression, Encoding, Schema, SType): 126 | enum_it = iter(enum_type.variants()) # type: ignore [attr-defined] 127 | else: 128 | enum_it = iter(enum_type) 129 | 130 | # Assert 131 | for enum in enum_it: 132 | assert enum == enum_type(enum.name) 133 | assert enum == enum_type(enum.name.replace("_", "-")) 134 | assert enum == enum_type(enum.name.lower()) 135 | assert enum == enum_type(enum.name.upper()) 136 | with pytest.raises(DBNError): 137 | enum_type("bar") # sanity 138 | 139 | 140 | @pytest.mark.parametrize( 141 | "enum_type", 142 | (pytest.param(enum) for enum in NATIVE_ENUMS), 143 | ) 144 | def test_enum_none_not_coercible(enum_type: type[Enum]) -> None: 145 | """ 146 | Test that None type is not coercible and raises a TypeError. 147 | 148 | See: databento.common.enum.coercible 149 | 150 | """ 151 | # Arrange, Act 152 | if enum_type == Compression: 153 | enum_type(None) 154 | else: 155 | # Assert 156 | with pytest.raises(ValueError): 157 | enum_type(None) 158 | 159 | 160 | @pytest.mark.parametrize( 161 | "enum_type", 162 | (pytest.param(enum) for enum in DBN_ENUMS), 163 | ) 164 | def test_dbn_enum_none_not_coercible(enum_type: type[Enum]) -> None: 165 | """ 166 | Test that None type is not coercible and raises a TypeError. 167 | """ 168 | # Arrange, Act 169 | if enum_type == Compression: 170 | enum_type(None) 171 | else: 172 | # Assert 173 | with pytest.raises(DBNError): 174 | enum_type(None) 175 | 176 | 177 | @pytest.mark.parametrize( 178 | "enum_type", 179 | (pytest.param(enum) for enum in NATIVE_ENUMS if issubclass(enum, int)), 180 | ) 181 | def test_int_enum_stringy_mixin(enum_type: type[Enum]) -> None: 182 | """ 183 | Test the StringyMixin for integer enumerations. 184 | 185 | See: databento.common.enum.StringyMixin 186 | 187 | """ 188 | # Arrange, Act 189 | if not issubclass(enum_type, StringyMixin): 190 | pytest.skip(f"{type(enum_type)} is not a subclass of StringyMixin") 191 | 192 | # Assert 193 | for enum in enum_type: 194 | assert str(enum) == enum.name.lower() 195 | 196 | 197 | @pytest.mark.parametrize( 198 | "enum_type", 199 | (pytest.param(enum) for enum in NATIVE_ENUMS if issubclass(enum, str)), 200 | ) 201 | def test_str_enum_stringy_mixin(enum_type: type[Enum]) -> None: 202 | """ 203 | Test the StringyMixin for string enumerations. 204 | 205 | See: databento.common.enum.StringyMixin 206 | 207 | """ 208 | # Arrange, Act 209 | if not issubclass(enum_type, StringyMixin): 210 | pytest.skip(f"{type(enum_type)} is not a subclass of StringyMixin") 211 | 212 | # Assert 213 | for enum in enum_type: 214 | assert str(enum) == enum.value 215 | 216 | 217 | @pytest.mark.parametrize( 218 | "enum_type", 219 | (pytest.param(enum) for enum in NATIVE_ENUMS if issubclass(enum, Flag)), 220 | ) 221 | def test_int_flags_stringy_mixin(enum_type: type[Flag]) -> None: 222 | """ 223 | Test that combinations of int flags are displayed properly. 224 | """ 225 | # Arrange, Act 226 | for value in map(sum, combinations(enum_type, 2)): # type: ignore [arg-type] 227 | record_flags = enum_type(value) 228 | 229 | # Assert 230 | assert str(record_flags) == ", ".join( 231 | f.name.lower() for f in enum_type if f in record_flags 232 | ) 233 | -------------------------------------------------------------------------------- /tests/test_common_iterator.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | 3 | import pytest 4 | from databento.common import iterator 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "things, size, expected", 9 | [ 10 | ( 11 | "abcdefg", 12 | 2, 13 | [ 14 | ("a", "b"), 15 | ("c", "d"), 16 | ("e", "f"), 17 | ("g",), 18 | ], 19 | ), 20 | ], 21 | ) 22 | def test_chunk( 23 | things: Iterable[object], 24 | size: int, 25 | expected: Iterable[tuple[object]], 26 | ) -> None: 27 | """ 28 | Test that an iterable is chunked property. 29 | """ 30 | # Arrange, Act, Assert 31 | chunks = [chunk for chunk in iterator.chunk(things, size)] 32 | assert chunks == expected 33 | -------------------------------------------------------------------------------- /tests/test_common_validation.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from enum import Enum 4 | from pathlib import Path 5 | from typing import Any 6 | 7 | import pytest 8 | from databento.common.validation import validate_enum 9 | from databento.common.validation import validate_file_write_path 10 | from databento.common.validation import validate_gateway 11 | from databento.common.validation import validate_maybe_enum 12 | from databento.common.validation import validate_path 13 | from databento.common.validation import validate_semantic_string 14 | from databento.common.validation import validate_smart_symbol 15 | from databento_dbn import Encoding 16 | 17 | 18 | @pytest.mark.parametrize( 19 | "value", 20 | [ 21 | [None, 0], 22 | ], 23 | ) 24 | def test_validate_path_given_wrong_types_raises_type_error( 25 | value: Any, 26 | ) -> None: 27 | # Arrange, Act, Assert 28 | with pytest.raises(TypeError): 29 | validate_path(value, "param") 30 | 31 | 32 | def test_validate_file_write_path( 33 | tmp_path: Path, 34 | ) -> None: 35 | # Arrange, Act, Assert 36 | test_file = tmp_path / "test.file" 37 | validate_file_write_path(test_file, "param") 38 | 39 | 40 | def test_validate_file_write_path_is_dir( 41 | tmp_path: Path, 42 | ) -> None: 43 | # Arrange, Act, Assert 44 | with pytest.raises(IsADirectoryError): 45 | validate_file_write_path(tmp_path, "param") 46 | 47 | 48 | def test_validate_file_write_path_exists( 49 | tmp_path: Path, 50 | ) -> None: 51 | # Arrange, Act, Assert 52 | test_file = tmp_path / "test.file" 53 | test_file.touch() 54 | with pytest.raises(FileExistsError): 55 | validate_file_write_path(test_file, "param") 56 | 57 | 58 | @pytest.mark.parametrize( 59 | "value, enum", 60 | [ 61 | [None, Encoding], 62 | ], 63 | ) 64 | def test_validate_enum_given_wrong_types_raises_type_error( 65 | value: Any, 66 | enum: type[Enum], 67 | ) -> None: 68 | # Arrange, Act, Assert 69 | with pytest.raises(ValueError): 70 | validate_enum(value, enum, "param") 71 | 72 | 73 | def test_validate_enum_given_invalid_value_raises_value_error() -> None: 74 | # Arrange, Act, Assert 75 | with pytest.raises(ValueError): 76 | validate_enum("invalid", Encoding, "encoding") 77 | 78 | 79 | @pytest.mark.parametrize( 80 | "value, enum, expected", 81 | [ 82 | ["dbn", Encoding, "dbn"], 83 | ["DBN", Encoding, "dbn"], 84 | [Encoding.DBN, Encoding, "dbn"], 85 | ], 86 | ) 87 | def test_validate_enum_given_valid_value_returns_expected_output( 88 | value: str | Enum, 89 | enum: type[Enum], 90 | expected: str | Enum, 91 | ) -> None: 92 | # Arrange, Act, Assert 93 | assert validate_enum(value, enum, "param") == expected 94 | 95 | 96 | def test_validate_maybe_enum_give_none_returns_none() -> None: 97 | # Arrange, Act, Assert 98 | assert validate_maybe_enum(None, Encoding, "encoding") is None 99 | 100 | 101 | @pytest.mark.parametrize( 102 | "url, expected", 103 | [ 104 | pytest.param("databento.com", "https://databento.com"), 105 | pytest.param("hist.databento.com", "https://hist.databento.com"), 106 | pytest.param("http://databento.com", "https://databento.com"), 107 | pytest.param("http://hist.databento.com", "https://hist.databento.com"), 108 | pytest.param("//", ValueError), 109 | pytest.param("", ValueError), 110 | ], 111 | ) 112 | def test_validate_gateway( 113 | url: str, 114 | expected: str | type[Exception], 115 | ) -> None: 116 | """ 117 | Tests several correct and malformed URLs. 118 | """ 119 | # Arrange, Act, Assert 120 | if isinstance(expected, str): 121 | assert validate_gateway(url) == expected 122 | else: 123 | with pytest.raises(expected): 124 | validate_gateway(url) 125 | 126 | 127 | @pytest.mark.parametrize( 128 | "symbol, expected", 129 | [ 130 | pytest.param("ES", "ES"), 131 | pytest.param("es", "ES"), 132 | pytest.param("ES.FUT", "ES.FUT"), 133 | pytest.param("es.opt", "ES.OPT"), 134 | pytest.param("ES.C.0", "ES.c.0"), 135 | pytest.param("es.c.5", "ES.c.5"), 136 | pytest.param(".v.2", ValueError), 137 | pytest.param("es..9", ValueError), 138 | pytest.param("es.n.", ValueError), 139 | pytest.param("es.c.5.0", ValueError), 140 | pytest.param("", ValueError), 141 | ], 142 | ) 143 | def test_validate_smart_symbol( 144 | symbol: str, 145 | expected: str | type[Exception], 146 | ) -> None: 147 | """ 148 | Test several correct smart symbols and invalid syntax. 149 | """ 150 | # Arrange, Act, Assert 151 | if isinstance(expected, str): 152 | assert validate_smart_symbol(symbol) == expected 153 | else: 154 | with pytest.raises(expected): 155 | validate_smart_symbol(symbol) 156 | 157 | 158 | @pytest.mark.parametrize( 159 | "value,expected", 160 | [ 161 | pytest.param("nick", "nick"), 162 | pytest.param("", ValueError, id="empty"), 163 | pytest.param(" ", ValueError, id="whitespace"), 164 | pytest.param("foo\x00", ValueError, id="unprintable"), 165 | ], 166 | ) 167 | def test_validate_semantic_string( 168 | value: str, 169 | expected: str | type[Exception], 170 | ) -> None: 171 | """ 172 | Test that validate_semantic_string rejects string which are: 173 | - empty 174 | - whitespace 175 | - contain unprintable characters 176 | """ 177 | # Arrange, Act, Assert 178 | if isinstance(expected, str): 179 | assert validate_semantic_string(value, "unittest") == expected 180 | else: 181 | with pytest.raises(expected): 182 | assert validate_semantic_string(value, "") 183 | -------------------------------------------------------------------------------- /tests/test_historical_client.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import pathlib 4 | from typing import Callable 5 | from unittest.mock import MagicMock 6 | 7 | import databento as db 8 | import pytest 9 | import requests 10 | from databento import DBNStore 11 | from databento import Historical 12 | from databento.common.enums import HistoricalGateway 13 | from databento.common.publishers import Dataset 14 | from databento_dbn import Schema 15 | 16 | 17 | def test_key_returns_expected() -> None: 18 | # Arrange 19 | key = "DUMMY_API_KEY" 20 | 21 | # Act 22 | client = db.Historical(key=key) 23 | 24 | # Assert 25 | assert client.key == "DUMMY_API_KEY" 26 | 27 | 28 | def test_default_host_returns_expected() -> None: 29 | # Arrange, Act 30 | historical_client = db.Historical(key="DUMMY_API_KEY") 31 | 32 | # Assert 33 | assert historical_client.gateway == "https://hist.databento.com" 34 | 35 | 36 | @pytest.mark.parametrize( 37 | "gateway, expected", 38 | [ 39 | [HistoricalGateway.BO1, "https://hist.databento.com"], 40 | ["bo1", "https://hist.databento.com"], 41 | ], 42 | ) 43 | def test_gateway_nearest_and_bo1_map_to_hist_databento( 44 | gateway: HistoricalGateway | str, 45 | expected: str, 46 | ) -> None: 47 | # Arrange, Act 48 | client = db.Historical(key="DUMMY_API_KEY", gateway=gateway) 49 | 50 | # Assert 51 | assert client.gateway == expected 52 | 53 | 54 | def test_custom_gateway_returns_expected() -> None: 55 | # Arrange 56 | ny4_gateway = "ny4.databento.com" 57 | 58 | # Act 59 | client = db.Historical(key="DUMMY_API_KEY", gateway=ny4_gateway) 60 | 61 | # Assert 62 | assert client.gateway == "https://ny4.databento.com" 63 | 64 | 65 | @pytest.mark.parametrize( 66 | "gateway", 67 | [ 68 | "//", 69 | "", 70 | ], 71 | ) 72 | def test_custom_gateway_error( 73 | gateway: str, 74 | ) -> None: 75 | """ 76 | Test that setting a custom gateway to an invalid url raises an exception. 77 | """ 78 | # Arrange, Act, Assert 79 | with pytest.raises(ValueError): 80 | db.Historical(key="DUMMY_API_KEY", gateway=gateway) 81 | 82 | 83 | @pytest.mark.parametrize( 84 | "gateway, expected", 85 | [ 86 | ["hist.databento.com", "https://hist.databento.com"], 87 | ["http://hist.databento.com", "https://hist.databento.com"], 88 | ], 89 | ) 90 | def test_custom_gateway_force_https( 91 | gateway: str, 92 | expected: str, 93 | ) -> None: 94 | """ 95 | Test that custom gateways are forced to the https scheme. 96 | """ 97 | # Arrange, Act 98 | client = db.Historical(key="DUMMY_API_KEY", gateway=gateway) 99 | 100 | # Assert 101 | assert client.gateway == expected 102 | 103 | 104 | def test_re_request_symbology_makes_expected_request( 105 | test_data_path: Callable[[Dataset, Schema], pathlib.Path], 106 | monkeypatch: pytest.MonkeyPatch, 107 | historical_client: Historical, 108 | ) -> None: 109 | # Arrange 110 | monkeypatch.setattr(requests, "post", mocked_post := MagicMock()) 111 | 112 | bento = DBNStore.from_file(path=test_data_path(Dataset.GLBX_MDP3, Schema.MBO)) 113 | 114 | # Act 115 | bento.request_symbology(historical_client) 116 | 117 | # Assert 118 | call = mocked_post.call_args.kwargs 119 | assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/symbology.resolve" 120 | assert call["data"] == { 121 | "dataset": "GLBX.MDP3", 122 | "symbols": "ESH1", 123 | "stype_in": "raw_symbol", 124 | "stype_out": "instrument_id", 125 | "start_date": "2020-12-28", 126 | "end_date": "2020-12-29", 127 | } 128 | assert sorted(call["headers"].keys()) == ["accept", "user-agent"] 129 | assert call["headers"]["accept"] == "application/json" 130 | assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) 131 | assert call["timeout"] == (100, 100) 132 | assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) 133 | 134 | 135 | def test_request_full_definitions_expected_request( 136 | test_data: Callable[[Dataset, Schema], bytes], 137 | test_data_path: Callable[[Dataset, Schema], pathlib.Path], 138 | monkeypatch: pytest.MonkeyPatch, 139 | historical_client: Historical, 140 | ) -> None: 141 | # Arrange 142 | monkeypatch.setattr(requests, "post", mocked_post := MagicMock()) 143 | 144 | # Create an MBO bento 145 | bento = DBNStore.from_file(path=test_data_path(Dataset.GLBX_MDP3, Schema.MBO)) 146 | 147 | # Mock from_bytes with the definition stub 148 | stream_bytes = test_data(Dataset.GLBX_MDP3, Schema.DEFINITION) 149 | monkeypatch.setattr( 150 | DBNStore, 151 | "from_bytes", 152 | MagicMock(return_value=DBNStore.from_bytes(stream_bytes)), 153 | ) 154 | 155 | # Act 156 | definition_bento = bento.request_full_definitions(historical_client) 157 | 158 | # Assert 159 | call = mocked_post.call_args.kwargs 160 | assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/timeseries.get_range" 161 | assert call["data"] == { 162 | "dataset": "GLBX.MDP3", 163 | "start": "2020-12-28T00:00:00+00:00", 164 | "end": "2020-12-29T00:00:00+00:00", 165 | "symbols": "ESH1", 166 | "schema": "definition", 167 | "stype_in": "raw_symbol", 168 | "stype_out": "instrument_id", 169 | "encoding": "dbn", 170 | "compression": "zstd", 171 | } 172 | assert sorted(call["headers"].keys()) == ["accept", "user-agent"] 173 | assert call["headers"]["accept"] == "application/json" 174 | assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) 175 | assert call["timeout"] == (100, 100) 176 | assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) 177 | assert len(stream_bytes) == definition_bento.nbytes 178 | -------------------------------------------------------------------------------- /tests/test_historical_data.py: -------------------------------------------------------------------------------- 1 | import databento 2 | import pytest 3 | from databento.common.constants import SCHEMA_STRUCT_MAP 4 | 5 | 6 | def test_mbo_fields() -> None: 7 | """ 8 | Test that columns match the MBO struct. 9 | """ 10 | # Arrange 11 | struct = SCHEMA_STRUCT_MAP[databento.Schema.MBO] 12 | 13 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 14 | fields.remove("hd") 15 | fields.remove("record_size") 16 | fields.remove("size_hint") 17 | 18 | # Act 19 | difference = fields.symmetric_difference(struct._ordered_fields) 20 | 21 | # Assert 22 | assert not difference 23 | 24 | 25 | @pytest.mark.parametrize( 26 | "schema,level_count", 27 | [ 28 | (databento.Schema.TBBO, 1), 29 | (databento.Schema.MBP_1, 1), 30 | (databento.Schema.MBP_10, 10), 31 | ], 32 | ) 33 | def test_mbp_fields( 34 | schema: databento.Schema, 35 | level_count: int, 36 | ) -> None: 37 | """ 38 | Test that columns match the MBP structs. 39 | """ 40 | # Arrange 41 | struct = SCHEMA_STRUCT_MAP[schema] 42 | 43 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 44 | fields.remove("hd") 45 | fields.remove("record_size") 46 | fields.remove("size_hint") 47 | 48 | # Act 49 | difference = fields.symmetric_difference(struct._ordered_fields) 50 | 51 | # Assert 52 | assert "levels" in difference 53 | 54 | # bid/ask size, price, ct for each level, plus the levels field 55 | assert len(difference) == 6 * level_count + 1 56 | 57 | 58 | @pytest.mark.parametrize( 59 | "schema", 60 | [ 61 | databento.Schema.OHLCV_1S, 62 | databento.Schema.OHLCV_1M, 63 | databento.Schema.OHLCV_1H, 64 | databento.Schema.OHLCV_1D, 65 | ], 66 | ) 67 | def test_ohlcv_fields( 68 | schema: databento.Schema, 69 | ) -> None: 70 | """ 71 | Test that columns match the OHLCV structs. 72 | """ 73 | # Arrange 74 | struct = SCHEMA_STRUCT_MAP[schema] 75 | 76 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 77 | fields.remove("hd") 78 | fields.remove("record_size") 79 | fields.remove("size_hint") 80 | 81 | # Act 82 | difference = fields.symmetric_difference(struct._ordered_fields) 83 | 84 | # Assert 85 | assert not difference 86 | 87 | 88 | def test_trades_struct() -> None: 89 | """ 90 | Test that columns match the Trades struct. 91 | """ 92 | # Arrange 93 | struct = SCHEMA_STRUCT_MAP[databento.Schema.TRADES] 94 | 95 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 96 | fields.remove("hd") 97 | fields.remove("record_size") 98 | fields.remove("size_hint") 99 | 100 | # Act 101 | difference = fields.symmetric_difference(struct._ordered_fields) 102 | 103 | # Assert 104 | assert not difference 105 | 106 | 107 | def test_definition_struct() -> None: 108 | """ 109 | Test that columns match the Definition struct. 110 | """ 111 | # Arrange 112 | struct = SCHEMA_STRUCT_MAP[databento.Schema.DEFINITION] 113 | 114 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 115 | fields.remove("hd") 116 | fields.remove("record_size") 117 | fields.remove("size_hint") 118 | 119 | # Act 120 | difference = fields.symmetric_difference(struct._ordered_fields) 121 | 122 | # Assert 123 | assert not difference 124 | 125 | 126 | def test_imbalance_struct() -> None: 127 | """ 128 | Test that columns match the Imbalance struct. 129 | """ 130 | # Arrange 131 | struct = SCHEMA_STRUCT_MAP[databento.Schema.IMBALANCE] 132 | 133 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 134 | fields.remove("hd") 135 | fields.remove("record_size") 136 | fields.remove("size_hint") 137 | 138 | # Act 139 | difference = fields.symmetric_difference(struct._ordered_fields) 140 | 141 | # Assert 142 | assert not difference 143 | 144 | 145 | def test_statistics_struct() -> None: 146 | """ 147 | Test that columns match the Statistics struct. 148 | """ 149 | # Arrange 150 | struct = SCHEMA_STRUCT_MAP[databento.Schema.STATISTICS] 151 | 152 | fields = set(f for f in dir(struct) if not f.startswith(("_", "pretty_"))) 153 | fields.remove("hd") 154 | fields.remove("record_size") 155 | fields.remove("size_hint") 156 | 157 | # Act 158 | difference = fields.symmetric_difference(struct._ordered_fields) 159 | 160 | # Assert 161 | assert not difference 162 | -------------------------------------------------------------------------------- /tests/test_historical_error.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import AsyncMock 2 | from unittest.mock import MagicMock 3 | 4 | import aiohttp 5 | import pytest 6 | import requests 7 | from databento.common.error import BentoClientError 8 | from databento.common.error import BentoServerError 9 | from databento.common.http import check_http_error 10 | from databento.common.http import check_http_error_async 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "status_code, expected_exception, message", 15 | [ 16 | pytest.param(404, BentoClientError, r"", id="404"), 17 | pytest.param(408, BentoClientError, r"timed out.$", id="408"), 18 | pytest.param(500, BentoServerError, r"", id="500"), 19 | pytest.param(504, BentoServerError, r"timed out.$", id="504"), 20 | ], 21 | ) 22 | def test_check_http_status( 23 | status_code: int, 24 | expected_exception: type[Exception], 25 | message: str, 26 | ) -> None: 27 | """ 28 | Test that responses with the given status code raise the expected 29 | exception. 30 | """ 31 | # Arrange 32 | response = requests.Response() 33 | response.status_code = status_code 34 | 35 | # Act, Assert 36 | with pytest.raises(expected_exception) as exc: 37 | check_http_error(response) 38 | 39 | exc.match(message) 40 | 41 | 42 | @pytest.mark.asyncio 43 | @pytest.mark.parametrize( 44 | "status_code, expected_exception, message", 45 | [ 46 | pytest.param(404, BentoClientError, r"", id="404"), 47 | pytest.param(408, BentoClientError, r"timed out.$", id="408"), 48 | pytest.param(500, BentoServerError, r"", id="500"), 49 | pytest.param(504, BentoServerError, r"timed out.$", id="504"), 50 | ], 51 | ) 52 | async def test_check_http_status_async( 53 | status_code: int, 54 | expected_exception: type[Exception], 55 | message: str, 56 | ) -> None: 57 | """ 58 | Test that responses with the given status code raise the expected 59 | exception. 60 | """ 61 | # Arrange 62 | response = MagicMock( 63 | spec=aiohttp.ClientResponse, 64 | status=status_code, 65 | json=AsyncMock(return_value={}), 66 | ) 67 | 68 | # Act, Assert 69 | with pytest.raises(expected_exception) as exc: 70 | await check_http_error_async(response) 71 | 72 | exc.match(message) 73 | 74 | 75 | def test_client_error_str_and_repr() -> None: 76 | # Arrange, Act 77 | error = BentoClientError( 78 | http_status=400, 79 | http_body=None, 80 | message="Bad Request", 81 | ) 82 | 83 | # Assert 84 | assert str(error) == "400 Bad Request" 85 | assert repr(error) == "BentoClientError(request_id=None, http_status=400, message=Bad Request)" 86 | 87 | 88 | def test_server_error_str_and_repr() -> None: 89 | # Arrange, Act 90 | error = BentoServerError( 91 | http_status=500, 92 | http_body=None, 93 | message="Internal Server Error", 94 | ) 95 | 96 | # Assert 97 | assert str(error) == "500 Internal Server Error" 98 | assert ( 99 | repr(error) 100 | == "BentoServerError(request_id=None, http_status=500, message=Internal Server Error)" 101 | ) 102 | -------------------------------------------------------------------------------- /tests/test_historical_timeseries.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Callable 3 | from unittest.mock import MagicMock 4 | 5 | import databento as db 6 | import pytest 7 | import requests 8 | from databento import DBNStore 9 | from databento.common.error import BentoServerError 10 | from databento.common.publishers import Dataset 11 | from databento.historical.client import Historical 12 | from databento_dbn import Schema 13 | 14 | 15 | def test_get_range_given_invalid_schema_raises_error( 16 | historical_client: Historical, 17 | ) -> None: 18 | # Arrange, Act, Assert 19 | with pytest.raises(ValueError): 20 | historical_client.timeseries.get_range( 21 | dataset="GLBX.MDP3", 22 | symbols="ESH1", 23 | schema="ticks", # <--- invalid 24 | start="2020-12-28", 25 | end="2020-12-28T23:00", 26 | ) 27 | 28 | 29 | def test_get_range_given_invalid_stype_in_raises_error( 30 | historical_client: Historical, 31 | ) -> None: 32 | # Arrange, Act, Assert 33 | with pytest.raises(ValueError): 34 | historical_client.timeseries.get_range( 35 | dataset="GLBX.MDP3", 36 | symbols="ESH1", 37 | schema="mbo", 38 | start="2020-12-28", 39 | end="2020-12-28T23:00", 40 | stype_in="zzz", # <--- invalid 41 | ) 42 | 43 | 44 | def test_get_range_given_invalid_stype_out_raises_error( 45 | historical_client: Historical, 46 | ) -> None: 47 | # Arrange, Act, Assert 48 | with pytest.raises(ValueError): 49 | historical_client.timeseries.get_range( 50 | dataset="GLBX.MDP3", 51 | symbols="ESH1", 52 | schema="mbo", 53 | start="2020-12-28", 54 | end="2020-12-28T23:00", 55 | stype_out="zzz", # <--- invalid 56 | ) 57 | 58 | 59 | def test_get_range_error_no_file_write( 60 | monkeypatch: pytest.MonkeyPatch, 61 | tmp_path: Path, 62 | historical_client: Historical, 63 | ) -> None: 64 | # Arrange 65 | mocked_response = MagicMock() 66 | mocked_response.__enter__.return_value = MagicMock( 67 | status_code=500, 68 | json=MagicMock( 69 | return_value={}, 70 | ), 71 | ) 72 | monkeypatch.setattr(requests, "post", MagicMock(return_value=mocked_response)) 73 | 74 | output_file = tmp_path / "output.dbn" 75 | 76 | # Act 77 | with pytest.raises(BentoServerError): 78 | historical_client.timeseries.get_range( 79 | dataset="GLBX.MDP3", 80 | symbols="ES.c.0", 81 | stype_in="continuous", 82 | schema="trades", 83 | start="2020-12-28T12:00", 84 | end="2020-12-29", 85 | path=output_file, 86 | ) 87 | 88 | # Assert 89 | assert not output_file.exists() 90 | 91 | 92 | def test_get_range_sends_expected_request( 93 | test_data: Callable[[Dataset, Schema], bytes], 94 | monkeypatch: pytest.MonkeyPatch, 95 | historical_client: Historical, 96 | ) -> None: 97 | # Arrange 98 | monkeypatch.setattr(requests, "post", mocked_post := MagicMock()) 99 | stream_bytes = test_data(Dataset.GLBX_MDP3, Schema.TRADES) 100 | 101 | monkeypatch.setattr( 102 | DBNStore, 103 | "from_bytes", 104 | MagicMock(return_value=DBNStore.from_bytes(stream_bytes)), 105 | ) 106 | 107 | # Act 108 | historical_client.timeseries.get_range( 109 | dataset="GLBX.MDP3", 110 | symbols="ES.c.0", 111 | stype_in="continuous", 112 | schema="trades", 113 | start="2020-12-28T12:00", 114 | end="2020-12-29", 115 | ) 116 | 117 | # Assert 118 | call = mocked_post.call_args.kwargs 119 | assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/timeseries.get_range" 120 | assert sorted(call["headers"].keys()) == ["accept", "user-agent"] 121 | assert call["headers"]["accept"] == "application/json" 122 | assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) 123 | assert call["data"] == { 124 | "dataset": "GLBX.MDP3", 125 | "start": "2020-12-28T12:00", 126 | "end": "2020-12-29", 127 | "symbols": "ES.c.0", 128 | "schema": "trades", 129 | "stype_in": "continuous", 130 | "stype_out": "instrument_id", 131 | "encoding": "dbn", 132 | "compression": "zstd", 133 | } 134 | assert call["timeout"] == (100, 100) 135 | assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) 136 | 137 | 138 | def test_get_range_with_limit_sends_expected_request( 139 | test_data: Callable[[Dataset, Schema], bytes], 140 | monkeypatch: pytest.MonkeyPatch, 141 | historical_client: Historical, 142 | ) -> None: 143 | # Arrange 144 | monkeypatch.setattr(requests, "post", mocked_post := MagicMock()) 145 | 146 | # Mock from_bytes with the definition stub 147 | stream_bytes = test_data(Dataset.GLBX_MDP3, Schema.TRADES) 148 | monkeypatch.setattr( 149 | DBNStore, 150 | "from_bytes", 151 | MagicMock(return_value=DBNStore.from_bytes(stream_bytes)), 152 | ) 153 | 154 | # Act 155 | historical_client.timeseries.get_range( 156 | dataset="GLBX.MDP3", 157 | symbols="ESH1", 158 | schema="trades", 159 | start="2020-12-28T12:00", 160 | end="2020-12-29", 161 | limit=1000000, 162 | ) 163 | 164 | # Assert 165 | call = mocked_post.call_args.kwargs 166 | assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/timeseries.get_range" 167 | assert sorted(call["headers"].keys()) == ["accept", "user-agent"] 168 | assert call["headers"]["accept"] == "application/json" 169 | assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) 170 | assert call["data"] == { 171 | "dataset": "GLBX.MDP3", 172 | "start": "2020-12-28T12:00", 173 | "end": "2020-12-29", 174 | "limit": "1000000", 175 | "symbols": "ESH1", 176 | "schema": "trades", 177 | "stype_in": "raw_symbol", 178 | "stype_out": "instrument_id", 179 | "encoding": "dbn", 180 | "compression": "zstd", 181 | } 182 | assert call["timeout"] == (100, 100) 183 | assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) 184 | -------------------------------------------------------------------------------- /tests/test_historical_warnings.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | from databento.common.http import check_backend_warnings 5 | from requests import Response 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "header_field", 10 | [ 11 | "X-Warning", 12 | "x-warning", 13 | ], 14 | ) 15 | @pytest.mark.parametrize( 16 | "category, message, expected_category", 17 | [ 18 | pytest.param("Warning", "this is a test", "BentoWarning"), 19 | pytest.param( 20 | "DeprecationWarning", 21 | "you're too old!", 22 | "BentoDeprecationWarning", 23 | ), 24 | pytest.param("Warning", "edge: case", "BentoWarning"), 25 | pytest.param("UnknownWarning", "", "BentoWarning"), 26 | ], 27 | ) 28 | def test_backend_warning( 29 | header_field: str, 30 | category: str, 31 | message: str, 32 | expected_category: str, 33 | ) -> None: 34 | """ 35 | Test that a backend warning in a response header is correctly parsed as a 36 | type of BentoWarning. 37 | """ 38 | # Arrange 39 | response = Response() 40 | expected = f'["{category}: {message}"]' 41 | response.headers[header_field] = expected 42 | 43 | # Act 44 | with pytest.warns() as warnings: 45 | check_backend_warnings(response) 46 | 47 | # Assert 48 | assert len(warnings) == 1 49 | assert warnings.list[0].category.__name__ == expected_category 50 | assert str(warnings.list[0].message) == message 51 | 52 | 53 | @pytest.mark.parametrize( 54 | "header_field", 55 | [ 56 | "X-Warning", 57 | "x-warning", 58 | ], 59 | ) 60 | def test_multiple_backend_warning( 61 | header_field: str, 62 | ) -> None: 63 | """ 64 | Test that multiple backend warnings in a response header are supported. 65 | """ 66 | # Arrange 67 | response = Response() 68 | backend_warnings = [ 69 | "Warning: this is a test", 70 | "DeprecationWarning: you're too old!", 71 | ] 72 | response.headers[header_field] = json.dumps(backend_warnings) 73 | 74 | # Act 75 | with pytest.warns() as warnings: 76 | check_backend_warnings(response) 77 | 78 | # Assert 79 | assert len(warnings) == len(backend_warnings) 80 | assert warnings.list[0].category.__name__ == "BentoWarning" 81 | assert str(warnings.list[0].message) == "this is a test" 82 | assert warnings.list[1].category.__name__ == "BentoDeprecationWarning" 83 | assert str(warnings.list[1].message) == "you're too old!" 84 | -------------------------------------------------------------------------------- /tests/test_live_client_reconnect.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | from unittest.mock import MagicMock 5 | 6 | import pandas as pd 7 | import pytest 8 | from databento import Dataset 9 | from databento import Schema 10 | from databento import SType 11 | from databento.common.enums import ReconnectPolicy 12 | from databento.common.types import DBNRecord 13 | from databento.live import client 14 | from databento.live.gateway import AuthenticationRequest 15 | from databento.live.gateway import SessionStart 16 | from databento.live.gateway import SubscriptionRequest 17 | 18 | from tests.mockliveserver.fixture import MockLiveServerInterface 19 | 20 | 21 | async def test_reconnect_policy_none( 22 | test_live_api_key: str, 23 | mock_live_server: MockLiveServerInterface, 24 | ) -> None: 25 | """ 26 | Test that a reconnect policy of "none" does not reconnect the client. 27 | """ 28 | # Arrange 29 | live_client = client.Live( 30 | key=test_live_api_key, 31 | gateway=mock_live_server.host, 32 | port=mock_live_server.port, 33 | reconnect_policy=ReconnectPolicy.NONE, 34 | ) 35 | 36 | live_client.subscribe( 37 | dataset=Dataset.GLBX_MDP3, 38 | schema=Schema.MBO, 39 | stype_in=SType.RAW_SYMBOL, 40 | symbols="TEST", 41 | ) 42 | live_client._session._protocol.disconnected.set_exception(ConnectionResetError) 43 | 44 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 45 | 46 | # Act 47 | await mock_live_server.disconnect( 48 | session_id=live_client._session.session_id, 49 | ) 50 | 51 | # Assert 52 | with pytest.raises(asyncio.TimeoutError): 53 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 54 | 55 | 56 | async def test_reconnect_before_start( 57 | test_live_api_key: str, 58 | mock_live_server: MockLiveServerInterface, 59 | reconnect_policy: ReconnectPolicy = ReconnectPolicy.RECONNECT, 60 | ) -> None: 61 | """ 62 | Test that a reconnect policy of "reconnect_do_not_replay" reconnects a 63 | client but does not send the session start command if the session was not 64 | streaming previously. 65 | """ 66 | # Arrange 67 | live_client = client.Live( 68 | key=test_live_api_key, 69 | gateway=mock_live_server.host, 70 | port=mock_live_server.port, 71 | reconnect_policy=reconnect_policy, 72 | ) 73 | 74 | live_client.subscribe( 75 | dataset=Dataset.GLBX_MDP3, 76 | schema=Schema.MBO, 77 | stype_in=SType.RAW_SYMBOL, 78 | symbols="TEST", 79 | ) 80 | 81 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 82 | 83 | records: list[DBNRecord] = [] 84 | live_client.add_callback(records.append) 85 | 86 | # Act 87 | live_client._session._protocol.disconnected.set_exception(ConnectionResetError) 88 | await mock_live_server.disconnect( 89 | session_id=live_client._session.session_id, 90 | ) 91 | 92 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 93 | 94 | live_client.stop() 95 | 96 | # Assert 97 | with pytest.raises(asyncio.TimeoutError): 98 | await mock_live_server.wait_for_message_of_type(SessionStart) 99 | 100 | 101 | @pytest.mark.parametrize( 102 | "schema", 103 | (pytest.param(schema, id=str(schema)) for schema in Schema.variants()), 104 | ) 105 | @pytest.mark.parametrize( 106 | "stype_in", 107 | (pytest.param(stype_in, id=str(stype_in)) for stype_in in SType.variants()), 108 | ) 109 | @pytest.mark.parametrize( 110 | "symbols", 111 | [ 112 | ("TEST0",), 113 | ("TEST0", "TEST1"), 114 | ("TEST0", "TEST1", "TEST2"), 115 | ], 116 | ) 117 | @pytest.mark.parametrize( 118 | "start,snapshot", 119 | [ 120 | (0, False), 121 | (None, True), 122 | ], 123 | ) 124 | async def test_reconnect_subscriptions( 125 | test_live_api_key: str, 126 | mock_live_server: MockLiveServerInterface, 127 | schema: Schema, 128 | stype_in: SType, 129 | symbols: tuple[str, ...], 130 | start: int | None, 131 | snapshot: bool, 132 | reconnect_policy: ReconnectPolicy = ReconnectPolicy.RECONNECT, 133 | ) -> None: 134 | """ 135 | Test that a reconnect policy of "reconnect_do_not_replay" re-sends the 136 | subscription requests with a start of `None`. 137 | """ 138 | # Arrange 139 | live_client = client.Live( 140 | key=test_live_api_key, 141 | gateway=mock_live_server.host, 142 | port=mock_live_server.port, 143 | reconnect_policy=reconnect_policy, 144 | ) 145 | 146 | for symbol in symbols: 147 | live_client.subscribe( 148 | dataset=Dataset.GLBX_MDP3, 149 | schema=schema, 150 | stype_in=stype_in, 151 | symbols=symbol, 152 | start=start, 153 | snapshot=snapshot, 154 | ) 155 | 156 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 157 | 158 | # Act 159 | live_client._session._protocol.disconnected.set_exception(ConnectionResetError) 160 | await mock_live_server.disconnect( 161 | session_id=live_client._session.session_id, 162 | ) 163 | 164 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 165 | 166 | reconnect_subscriptions: list[SubscriptionRequest] = [] 167 | for _ in range(len(symbols)): 168 | request = await mock_live_server.wait_for_message_of_type(SubscriptionRequest) 169 | reconnect_subscriptions.append(request) 170 | 171 | live_client.stop() 172 | 173 | # Assert 174 | for i, symbol in enumerate(symbols): 175 | sub = reconnect_subscriptions[i] 176 | assert sub.schema == schema 177 | assert sub.stype_in == stype_in 178 | assert sub.symbols == symbol 179 | assert sub.start is None 180 | assert sub.snapshot == str(int(snapshot)) 181 | 182 | 183 | async def test_reconnect_callback( 184 | test_live_api_key: str, 185 | mock_live_server: MockLiveServerInterface, 186 | reconnect_policy: ReconnectPolicy = ReconnectPolicy.RECONNECT, 187 | ) -> None: 188 | """ 189 | Test that a reconnect policy of "reconnect_do_not_replay" will cause a user 190 | supplied reconnection callback to be executed when a reconnection occurs. 191 | """ 192 | # Arrange 193 | live_client = client.Live( 194 | key=test_live_api_key, 195 | gateway=mock_live_server.host, 196 | port=mock_live_server.port, 197 | reconnect_policy=reconnect_policy, 198 | ) 199 | 200 | live_client.subscribe( 201 | dataset=Dataset.GLBX_MDP3, 202 | schema=Schema.MBO, 203 | stype_in=SType.RAW_SYMBOL, 204 | symbols="TEST", 205 | ) 206 | 207 | reconnect_callback = MagicMock() 208 | live_client.add_reconnect_callback(reconnect_callback) 209 | 210 | await mock_live_server.wait_for_message_of_type(AuthenticationRequest) 211 | 212 | # Act 213 | live_client.start() 214 | live_client._session._protocol.disconnected.set_exception(ConnectionResetError) 215 | 216 | await mock_live_server.wait_for_message_of_type(SessionStart) 217 | 218 | await live_client.wait_for_close() 219 | 220 | # Assert 221 | reconnect_callback.assert_called() 222 | args, _ = reconnect_callback.call_args 223 | gap_start, gap_end = args 224 | assert isinstance(gap_start, pd.Timestamp) 225 | assert isinstance(gap_end, pd.Timestamp) 226 | -------------------------------------------------------------------------------- /tests/test_live_protocol.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from unittest.mock import MagicMock 3 | 4 | import pytest 5 | from databento.common.publishers import Dataset 6 | from databento.live.protocol import DatabentoLiveProtocol 7 | from databento_dbn import Schema 8 | from databento_dbn import SType 9 | 10 | from tests.mockliveserver.fixture import MockLiveServerInterface 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "dataset", 15 | [ 16 | Dataset.GLBX_MDP3, 17 | Dataset.XNAS_ITCH, 18 | Dataset.OPRA_PILLAR, 19 | Dataset.EQUS_MINI, 20 | Dataset.IFEU_IMPACT, 21 | Dataset.NDEX_IMPACT, 22 | ], 23 | ) 24 | async def test_protocol_connection( 25 | mock_live_server: MockLiveServerInterface, 26 | test_live_api_key: str, 27 | dataset: Dataset, 28 | ) -> None: 29 | """ 30 | Test the low-level DatabentoLiveProtocol can be used to establish a 31 | connection to the live subscription gateway. 32 | """ 33 | # Arrange 34 | transport, protocol = await asyncio.get_event_loop().create_connection( 35 | protocol_factory=lambda: DatabentoLiveProtocol( 36 | api_key=test_live_api_key, 37 | dataset=dataset, 38 | ), 39 | host=mock_live_server.host, 40 | port=mock_live_server.port, 41 | ) 42 | 43 | # Act, Assert 44 | await asyncio.wait_for(protocol.authenticated, timeout=1) 45 | transport.close() 46 | await asyncio.wait_for(protocol.disconnected, timeout=1) 47 | 48 | 49 | @pytest.mark.parametrize( 50 | "dataset", 51 | [ 52 | Dataset.GLBX_MDP3, 53 | Dataset.XNAS_ITCH, 54 | Dataset.OPRA_PILLAR, 55 | Dataset.EQUS_MINI, 56 | Dataset.IFEU_IMPACT, 57 | Dataset.NDEX_IMPACT, 58 | ], 59 | ) 60 | async def test_protocol_connection_streaming( 61 | monkeypatch: pytest.MonkeyPatch, 62 | mock_live_server: MockLiveServerInterface, 63 | test_live_api_key: str, 64 | dataset: Dataset, 65 | ) -> None: 66 | """ 67 | Test the low-level DatabentoLiveProtocol can be used to stream DBN records 68 | from the live subscription gateway. 69 | """ 70 | # Arrange 71 | monkeypatch.setattr( 72 | DatabentoLiveProtocol, 73 | "received_metadata", 74 | metadata_mock := MagicMock(), 75 | ) 76 | monkeypatch.setattr( 77 | DatabentoLiveProtocol, 78 | "received_record", 79 | record_mock := MagicMock(), 80 | ) 81 | 82 | _, protocol = await asyncio.get_event_loop().create_connection( 83 | protocol_factory=lambda: DatabentoLiveProtocol( 84 | api_key=test_live_api_key, 85 | dataset=dataset, 86 | ), 87 | host=mock_live_server.host, 88 | port=mock_live_server.port, 89 | ) 90 | 91 | await asyncio.wait_for(protocol.authenticated, timeout=1) 92 | 93 | # Act 94 | protocol.subscribe( 95 | schema=Schema.TRADES, 96 | symbols="TEST", 97 | stype_in=SType.RAW_SYMBOL, 98 | ) 99 | 100 | protocol.start() 101 | await asyncio.wait_for(protocol.disconnected, timeout=1) 102 | 103 | # Assert 104 | assert metadata_mock.call_count == 1 105 | assert record_mock.call_count == 4 106 | -------------------------------------------------------------------------------- /tests/test_live_session.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from databento.common.error import BentoError 3 | from databento.live.session import DBNQueue 4 | 5 | 6 | def test_dbn_queue_put( 7 | timeout: float = 0.01, 8 | ) -> None: 9 | """ 10 | Test that DBNQueue.put raises a BentoError if disabled. 11 | 12 | The `timeout` is required, otherwise we will block forever. 13 | 14 | """ 15 | # Arrange 16 | queue = DBNQueue() 17 | 18 | # Act, Assert 19 | with pytest.raises(BentoError): 20 | queue.put(None, timeout=timeout) 21 | 22 | queue.enable() 23 | queue.put(None, timeout=timeout) 24 | 25 | queue.disable() 26 | with pytest.raises(BentoError): 27 | queue.put(None, timeout=timeout) 28 | 29 | 30 | def test_dbn_queue_put_nowait() -> None: 31 | """ 32 | Test that DBNQueue.put_nowait raises a BentoError if disabled. 33 | """ 34 | # Arrange 35 | queue = DBNQueue() 36 | 37 | # Act, Assert 38 | with pytest.raises(BentoError): 39 | queue.put_nowait(None) 40 | 41 | queue.enable() 42 | queue.put_nowait(None) 43 | 44 | queue.disable() 45 | with pytest.raises(BentoError): 46 | queue.put_nowait(None) 47 | -------------------------------------------------------------------------------- /tests/test_reference_adjustment.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from collections.abc import Iterable 4 | from pathlib import Path 5 | from unittest.mock import MagicMock 6 | 7 | import databento as db 8 | import pytest 9 | import requests 10 | import zstandard 11 | from databento.reference.client import Reference 12 | 13 | from tests import TESTS_ROOT 14 | 15 | 16 | @pytest.mark.parametrize( 17 | ( 18 | "countries", 19 | "security_types", 20 | "expected_countries", 21 | "expected_security_types", 22 | ), 23 | [ 24 | [ 25 | None, 26 | None, 27 | None, 28 | None, 29 | ], 30 | [ 31 | [], 32 | [], 33 | None, 34 | None, 35 | ], 36 | [ 37 | "US", 38 | "EQS", 39 | "US", 40 | "EQS", 41 | ], 42 | [ 43 | "US,CA", 44 | "EQS,ETF", 45 | "US,CA", 46 | "EQS,ETF", 47 | ], 48 | [ 49 | ["US", "CA"], 50 | ["EQS", "ETF"], 51 | "US,CA", 52 | "EQS,ETF", 53 | ], 54 | ], 55 | ) 56 | def test_adjustment_factors_get_range_sends_expected_request( 57 | monkeypatch: pytest.MonkeyPatch, 58 | reference_client: Reference, 59 | countries: Iterable[str] | str | None, 60 | security_types: Iterable[str] | str | None, 61 | expected_countries: str, 62 | expected_security_types: str, 63 | ) -> None: 64 | # Arrange 65 | mock_response = MagicMock() 66 | mock_response.content = zstandard.compress(b'{"ex_date":"1970-01-01"}\n') 67 | mock_response.__enter__.return_value = mock_response 68 | mock_response.__exit__ = MagicMock() 69 | monkeypatch.setattr(requests, "post", mock_post := MagicMock(return_value=mock_response)) 70 | 71 | # Act 72 | reference_client.adjustment_factors.get_range( 73 | symbols="AAPL", 74 | stype_in="raw_symbol", 75 | start="2024-01", 76 | end="2024-04", 77 | countries=countries, 78 | security_types=security_types, 79 | ) 80 | 81 | # Assert 82 | call = mock_post.call_args.kwargs 83 | assert ( 84 | call["url"] == f"{reference_client.gateway}/v{db.API_VERSION}/adjustment_factors.get_range" 85 | ) 86 | assert sorted(call["headers"].keys()) == ["accept", "user-agent"] 87 | assert call["headers"]["accept"] == "application/json" 88 | assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/")) 89 | assert call["data"] == { 90 | "start": "2024-01", 91 | "end": "2024-04", 92 | "symbols": "AAPL", 93 | "stype_in": "raw_symbol", 94 | "countries": expected_countries, 95 | "security_types": expected_security_types, 96 | "compression": "zstd", 97 | } 98 | assert call["timeout"] == (100, 100) 99 | assert isinstance(call["auth"], requests.auth.HTTPBasicAuth) 100 | 101 | 102 | def test_adjustment_factors_get_range_when_empty_response( 103 | monkeypatch: pytest.MonkeyPatch, 104 | reference_client: Reference, 105 | ) -> None: 106 | # Arrange 107 | mock_response = MagicMock() 108 | mock_response.content = zstandard.compress(b"") 109 | mock_response.__enter__.return_value = mock_response 110 | mock_response.__exit__ = MagicMock() 111 | monkeypatch.setattr(requests, "post", MagicMock(return_value=mock_response)) 112 | 113 | # Act 114 | df_raw = reference_client.adjustment_factors.get_range( 115 | symbols="AAPL", 116 | stype_in="raw_symbol", 117 | start="2024-01", 118 | end="2024-04", 119 | ) 120 | 121 | # Assert 122 | assert df_raw.empty 123 | 124 | 125 | def test_adjustment_factors_get_range_response( 126 | monkeypatch: pytest.MonkeyPatch, 127 | reference_client: Reference, 128 | ) -> None: 129 | # Arrange 130 | mock_response = MagicMock() 131 | data_path = Path(TESTS_ROOT) / "data" / "REFERENCE" / "test_data.adjustment-factors.jsonl" 132 | mock_response.content = zstandard.compress(data_path.read_bytes()) 133 | mock_response.__enter__.return_value = mock_response 134 | mock_response.__exit__ = MagicMock() 135 | monkeypatch.setattr(requests, "post", MagicMock(return_value=mock_response)) 136 | 137 | # Act 138 | df_raw = reference_client.adjustment_factors.get_range( 139 | symbols="AAPL", 140 | stype_in="raw_symbol", 141 | start="2024-01", 142 | end="2024-04", 143 | ) 144 | 145 | # Assert 146 | assert len(df_raw) == 2 147 | assert df_raw.index.name == "ex_date" 148 | -------------------------------------------------------------------------------- /tests/test_release.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests specific to releasing a version of databento-python. 3 | """ 4 | 5 | import operator 6 | import re 7 | from datetime import date 8 | 9 | import databento 10 | import pytest 11 | import tomli 12 | 13 | from tests import PROJECT_ROOT 14 | 15 | 16 | CHANGELOG_RELEASE_TITLE = re.compile(r"## (?P\d+\.\d+\.\d+) - (?P.+)\n") 17 | 18 | 19 | @pytest.fixture(name="changelog") 20 | def fixture_changelog() -> str: 21 | """ 22 | Fixture for the text of CHANGELOG.md. 23 | 24 | Returns 25 | ------- 26 | str 27 | 28 | """ 29 | # Arrange, Act, Assert 30 | with open(PROJECT_ROOT / "CHANGELOG.md") as changelog: 31 | return changelog.read() 32 | 33 | 34 | @pytest.fixture(name="pyproject_version") 35 | def fixture_pyproject_version() -> str: 36 | """ 37 | Fixture for the version text of project.toml. 38 | 39 | Returns 40 | ------- 41 | str 42 | 43 | """ 44 | # Arrange, Act, Assert 45 | with open(PROJECT_ROOT / "pyproject.toml", "rb") as pyproject: 46 | data = tomli.load(pyproject) 47 | return data["tool"]["poetry"]["version"] 48 | 49 | 50 | @pytest.mark.release 51 | def test_release_changelog(changelog: str, pyproject_version: str) -> None: 52 | """ 53 | Test that CHANGELOG.md and pyproject.toml contain correct version 54 | information. 55 | 56 | This test verifies that: 57 | - The version in `version.py` matches the latest release note 58 | - The version in `version.py` matches the version in `pyproject.toml` 59 | - The versions are unique. 60 | - The versions are ascending. 61 | - The release dates are chronological. 62 | 63 | """ 64 | # Arrange, Act 65 | releases = CHANGELOG_RELEASE_TITLE.findall(changelog) 66 | 67 | try: 68 | versions = list(map(operator.itemgetter(0), releases)) 69 | version_tuples = [tuple(map(int, v.split("."))) for v in versions] 70 | except Exception: 71 | # This could happen if we have an irregular version string. 72 | raise AssertionError("Failed to parse version from CHANGELOG.md") 73 | 74 | try: 75 | date_strings = list(map(operator.itemgetter(1), releases)) 76 | dates = list(map(date.fromisoformat, date_strings)) 77 | except Exception: 78 | # This could happen if we have TBD as the release date. 79 | raise AssertionError("Failed to parse release date from CHANGELOG.md") 80 | 81 | # Assert 82 | # Ensure latest version matches `__version__` 83 | assert databento.__version__ == versions[0] 84 | 85 | # Ensure latest version matches pyproject.toml 86 | assert databento.__version__ == pyproject_version 87 | 88 | # Ensure versions are unique 89 | assert len(versions) == len(set(versions)) 90 | 91 | # Ensure version is ascending 92 | assert version_tuples == sorted(version_tuples, reverse=True) 93 | 94 | # Ensure release dates are chronological 95 | assert dates == sorted(dates, reverse=True) 96 | --------------------------------------------------------------------------------