├── python ├── python │ └── databento_dbn │ │ ├── py.typed │ │ ├── v3.py │ │ ├── v2.py │ │ ├── v1.py │ │ └── __init__.py ├── build.rs ├── Cargo.toml ├── pyproject.toml ├── README.md └── src │ ├── enums.rs │ └── encode.rs ├── scripts ├── build.sh ├── test.sh ├── format.sh ├── config.sh ├── get_version.sh ├── regenerate_test_data.sh ├── lint.sh └── bump_version.sh ├── tests └── data │ ├── test_data.mbo.dbn │ ├── test_data.mbo.dbz │ ├── test_data.tbbo.dbn │ ├── test_data.tbbo.dbz │ ├── test_data.bbo-1m.dbn │ ├── test_data.bbo-1s.dbn │ ├── test_data.cbbo-1s.dbn │ ├── test_data.cmbp-1.dbn │ ├── test_data.mbo.v3.dbn │ ├── test_data.mbp-1.dbn │ ├── test_data.mbp-1.dbz │ ├── test_data.mbp-10.dbn │ ├── test_data.mbp-10.dbz │ ├── test_data.status.dbn │ ├── test_data.trades.dbn │ ├── test_data.trades.dbz │ ├── test_data.imbalance.dbn │ ├── test_data.ohlcv-1d.dbn │ ├── test_data.ohlcv-1d.dbz │ ├── test_data.ohlcv-1h.dbn │ ├── test_data.ohlcv-1h.dbz │ ├── test_data.ohlcv-1m.dbn │ ├── test_data.ohlcv-1m.dbz │ ├── test_data.ohlcv-1s.dbn │ ├── test_data.ohlcv-1s.dbz │ ├── test_data.definition.dbn │ ├── test_data.definition.dbz │ ├── test_data.mbo.v1.dbn.zst │ ├── test_data.mbo.v2.dbn.zst │ ├── test_data.mbo.v3.dbn.zst │ ├── test_data.mbp-1.v1.dbn.zst │ ├── test_data.mbp-1.v2.dbn.zst │ ├── test_data.mbp-1.v3.dbn.zst │ ├── test_data.statistics.dbn │ ├── test_data.tbbo.v1.dbn.zst │ ├── test_data.tbbo.v2.dbn.zst │ ├── test_data.tbbo.v3.dbn.zst │ ├── test_data.bbo-1m.v2.dbn.zst │ ├── test_data.bbo-1m.v3.dbn.zst │ ├── test_data.bbo-1s.v2.dbn.zst │ ├── test_data.bbo-1s.v3.dbn.zst │ ├── test_data.cbbo-1s.v2.dbn.zst │ ├── test_data.cbbo-1s.v3.dbn.zst │ ├── test_data.cmbp-1.v2.dbn.zst │ ├── test_data.cmbp-1.v3.dbn.zst │ ├── test_data.mbp-10.v1.dbn.zst │ ├── test_data.mbp-10.v2.dbn.zst │ ├── test_data.mbp-10.v3.dbn.zst │ ├── test_data.status.v2.dbn.zst │ ├── test_data.status.v3.dbn.zst │ ├── test_data.trades.v1.dbn.zst │ ├── test_data.trades.v2.dbn.zst │ ├── test_data.trades.v3.dbn.zst │ ├── test_data.definition.v1.dbn.zst │ ├── test_data.definition.v2.dbn.zst │ ├── test_data.definition.v3.dbn.zst │ ├── test_data.imbalance.v1.dbn.zst │ ├── test_data.imbalance.v2.dbn.zst │ ├── test_data.imbalance.v3.dbn.zst │ ├── test_data.ohlcv-1d.v1.dbn.zst │ ├── test_data.ohlcv-1d.v2.dbn.zst │ ├── test_data.ohlcv-1d.v3.dbn.zst │ ├── test_data.ohlcv-1h.v1.dbn.zst │ ├── test_data.ohlcv-1h.v2.dbn.zst │ ├── test_data.ohlcv-1h.v3.dbn.zst │ ├── test_data.ohlcv-1m.v1.dbn.zst │ ├── test_data.ohlcv-1m.v2.dbn.zst │ ├── test_data.ohlcv-1m.v3.dbn.zst │ ├── test_data.ohlcv-1s.v1.dbn.zst │ ├── test_data.ohlcv-1s.v2.dbn.zst │ ├── test_data.ohlcv-1s.v3.dbn.zst │ ├── test_data.statistics.v1.dbn.zst │ ├── test_data.statistics.v2.dbn.zst │ ├── test_data.statistics.v3.dbn.zst │ ├── test_data.definition.dbn.frag.zst │ ├── test_data.definition.v1.dbn.frag │ ├── test_data.definition.v2.dbn.frag │ ├── test_data.definition.v3.dbn.frag │ ├── test_data.definition.v1.dbn.frag.zst │ ├── test_data.definition.v3.dbn.frag.zst │ └── multi-frame.definition.v1.dbn.frag.zst ├── c ├── src │ ├── lib.rs │ ├── cfile.rs │ ├── compat.rs │ ├── decode.rs │ └── metadata.rs ├── README.md ├── Cargo.toml ├── cbindgen.toml └── build.rs ├── .gitignore ├── rust ├── dbn-macros │ ├── tests │ │ └── ui │ │ │ ├── missing_rtype.rs │ │ │ ├── csv_serialize_invalid_dbn_attr.rs │ │ │ ├── json_serialize_invalid_dbn_attr.rs │ │ │ ├── csv_serialize_conflicting_dbn_attr.rs │ │ │ ├── json_serialize_conflicting_dbn_attr.rs │ │ │ ├── csv_serialize_duplicate_encode_orders.stderr │ │ │ ├── json_serialize_duplicate_encode_orders.stderr │ │ │ ├── csv_serialize_duplicate_encode_orders.rs │ │ │ ├── json_serialize_duplicate_encode_orders.rs │ │ │ ├── missing_rtype.stderr │ │ │ ├── json_serialize_invalid_dbn_attr.stderr │ │ │ ├── json_serialize_conflicting_dbn_attr.stderr │ │ │ ├── csv_serialize_invalid_dbn_attr.stderr │ │ │ └── csv_serialize_conflicting_dbn_attr.stderr │ ├── src │ │ ├── utils.rs │ │ ├── debug.rs │ │ ├── py_field_desc.rs │ │ ├── has_rtype.rs │ │ ├── lib.rs │ │ └── serialize.rs │ └── Cargo.toml ├── dbn │ ├── src │ │ ├── encode │ │ │ ├── csv.rs │ │ │ ├── json.rs │ │ │ └── dbn.rs │ │ ├── json_writer.rs │ │ ├── decode │ │ │ ├── dbn.rs │ │ │ ├── zstd.rs │ │ │ └── stream.rs │ │ ├── record │ │ │ ├── with_ts_out_methods.rs │ │ │ ├── traits.rs │ │ │ ├── record_methods_tests.rs │ │ │ ├── layout_tests.rs │ │ │ └── conv.rs │ │ ├── test_utils.rs │ │ ├── v2 │ │ │ └── impl_default.rs │ │ ├── error.rs │ │ ├── compat │ │ │ └── traits.rs │ │ ├── v3.rs │ │ ├── python │ │ │ ├── conversions.rs │ │ │ └── metadata.rs │ │ ├── v1 │ │ │ └── impl_default.rs │ │ ├── v1.rs │ │ ├── lib.rs │ │ ├── enums │ │ │ └── methods.rs │ │ ├── flags.rs │ │ └── v3 │ │ │ └── methods.rs │ ├── README.md │ └── Cargo.toml └── dbn-cli │ ├── Cargo.toml │ ├── src │ ├── filter.rs │ ├── main.rs │ └── encode.rs │ └── README.md ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ ├── config.yml │ └── bug_report.md ├── pull_request_template.md └── workflows │ └── build.yaml ├── CONTRIBUTING.md ├── Cargo.toml ├── README.md └── CODE_OF_CONDUCT.md /python/python/databento_dbn/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | cargo --version 3 | cargo build --all-features 4 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | cargo --version 3 | cargo test --all-features 4 | -------------------------------------------------------------------------------- /tests/data/test_data.mbo.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.dbn -------------------------------------------------------------------------------- /tests/data/test_data.mbo.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.dbz -------------------------------------------------------------------------------- /tests/data/test_data.tbbo.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.tbbo.dbn -------------------------------------------------------------------------------- /tests/data/test_data.tbbo.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.tbbo.dbz -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1m.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1m.dbn -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1s.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1s.dbn -------------------------------------------------------------------------------- /tests/data/test_data.cbbo-1s.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cbbo-1s.dbn -------------------------------------------------------------------------------- /tests/data/test_data.cmbp-1.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cmbp-1.dbn -------------------------------------------------------------------------------- /tests/data/test_data.mbo.v3.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.v3.dbn -------------------------------------------------------------------------------- /tests/data/test_data.mbp-1.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-1.dbn -------------------------------------------------------------------------------- /tests/data/test_data.mbp-1.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-1.dbz -------------------------------------------------------------------------------- /tests/data/test_data.mbp-10.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-10.dbn -------------------------------------------------------------------------------- /tests/data/test_data.mbp-10.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-10.dbz -------------------------------------------------------------------------------- /tests/data/test_data.status.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.status.dbn -------------------------------------------------------------------------------- /tests/data/test_data.trades.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.trades.dbn -------------------------------------------------------------------------------- /tests/data/test_data.trades.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.trades.dbz -------------------------------------------------------------------------------- /tests/data/test_data.imbalance.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.imbalance.dbn -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1d.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1d.dbn -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1d.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1d.dbz -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1h.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1h.dbn -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1h.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1h.dbz -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1m.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1m.dbn -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1m.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1m.dbz -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1s.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1s.dbn -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1s.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1s.dbz -------------------------------------------------------------------------------- /c/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod cfile; 2 | pub mod compat; 3 | pub mod decode; 4 | pub mod metadata; 5 | pub mod text_serialization; 6 | -------------------------------------------------------------------------------- /scripts/format.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | cargo --version 3 | cargo fmt --check # fails if anything is misformatted 4 | -------------------------------------------------------------------------------- /tests/data/test_data.definition.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.dbn -------------------------------------------------------------------------------- /tests/data/test_data.definition.dbz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.dbz -------------------------------------------------------------------------------- /tests/data/test_data.mbo.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbo.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbo.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbo.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-1.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-1.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-1.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-1.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-1.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-1.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.statistics.dbn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.statistics.dbn -------------------------------------------------------------------------------- /tests/data/test_data.tbbo.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.tbbo.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.tbbo.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.tbbo.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.tbbo.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.tbbo.v3.dbn.zst -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | .idea/ 3 | .profile/ 4 | .vscode/ 5 | .helix/ 6 | 7 | target/ 8 | **/*.rs.bk 9 | *.sw[po] 10 | -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1m.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1m.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1m.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1m.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1s.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1s.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.bbo-1s.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.bbo-1s.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.cbbo-1s.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cbbo-1s.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.cbbo-1s.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cbbo-1s.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.cmbp-1.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cmbp-1.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.cmbp-1.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.cmbp-1.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-10.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-10.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-10.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-10.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.mbp-10.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.mbp-10.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.status.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.status.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.status.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.status.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.trades.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.trades.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.trades.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.trades.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.trades.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.trades.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.imbalance.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.imbalance.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.imbalance.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.imbalance.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.imbalance.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.imbalance.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1d.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1d.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1d.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1d.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1d.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1d.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1h.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1h.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1h.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1h.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1h.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1h.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1m.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1m.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1m.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1m.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1m.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1m.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1s.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1s.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1s.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1s.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.ohlcv-1s.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.ohlcv-1s.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.statistics.v1.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.statistics.v1.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.statistics.v2.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.statistics.v2.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.statistics.v3.dbn.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.statistics.v3.dbn.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.dbn.frag.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.dbn.frag.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.v1.dbn.frag: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v1.dbn.frag -------------------------------------------------------------------------------- /tests/data/test_data.definition.v2.dbn.frag: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v2.dbn.frag -------------------------------------------------------------------------------- /tests/data/test_data.definition.v3.dbn.frag: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v3.dbn.frag -------------------------------------------------------------------------------- /tests/data/test_data.definition.v1.dbn.frag.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v1.dbn.frag.zst -------------------------------------------------------------------------------- /tests/data/test_data.definition.v3.dbn.frag.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/test_data.definition.v3.dbn.frag.zst -------------------------------------------------------------------------------- /tests/data/multi-frame.definition.v1.dbn.frag.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databento/dbn/HEAD/tests/data/multi-frame.definition.v1.dbn.frag.zst -------------------------------------------------------------------------------- /scripts/config.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | SCRIPTS_DIR="$(cd "$(dirname "$0")" || exit; pwd -P)" 4 | PROJECT_ROOT_DIR="$(dirname "${SCRIPTS_DIR}")" 5 | -------------------------------------------------------------------------------- /scripts/get_version.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | source "$(dirname "$0")/config.sh" 4 | grep -E '^version =' "${PROJECT_ROOT_DIR}/Cargo.toml" | cut -d'"' -f 2 5 | -------------------------------------------------------------------------------- /python/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | // Sets the correct linker arguments when building with `cargo` 3 | pyo3_build_config::add_extension_module_link_args(); 4 | } 5 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/missing_rtype.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::dbn_record; 2 | 3 | #[repr(C)] 4 | #[dbn_record] 5 | struct Record { 6 | pub a: u8, 7 | } 8 | 9 | fn main() {} 10 | -------------------------------------------------------------------------------- /rust/dbn/src/encode/csv.rs: -------------------------------------------------------------------------------- 1 | //! Encoding of DBN records into comma-separated values (CSV). 2 | 3 | pub(crate) mod serialize; 4 | mod sync; 5 | 6 | pub use sync::{Encoder, EncoderBuilder}; 7 | -------------------------------------------------------------------------------- /scripts/regenerate_test_data.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | source "$(dirname "$0")/config.sh" 4 | cd "${PROJECT_ROOT_DIR}/tests/data" 5 | find . -name '*.dbn*' -exec cargo run -- {} --output {} --force \; 6 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::CsvSerialize; 2 | 3 | #[derive(CsvSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(unknown)] 7 | pub a: u8, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::JsonSerialize; 2 | 3 | #[derive(JsonSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(unknown)] 7 | pub a: u8, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /c/README.md: -------------------------------------------------------------------------------- 1 | # dbn-c 2 | 3 | Work-in-progress C FFI bindings for the DBN crate, using [cbindgen](https://github.com/eqrion/cbindgen). 4 | 5 | ## License 6 | 7 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 8 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::CsvSerialize; 2 | 3 | #[derive(CsvSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(fixed_price, unix_nanos)] 7 | pub a: u8, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::JsonSerialize; 2 | 3 | #[derive(JsonSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(fixed_price, unix_nanos)] 7 | pub a: u8, 8 | } 9 | 10 | fn main() {} 11 | -------------------------------------------------------------------------------- /rust/dbn/src/json_writer.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | // Re-export for version and casing consistency 4 | pub use json_writer::{ 5 | JSONObjectWriter as JsonObjectWriter, JSONWriter as JsonWriter, 6 | PrettyJSONWriter as PrettyJsonWriter, NULL, 7 | }; 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Request a new feature to be added here 4 | labels: 5 | - enhancement 6 | --- 7 | 8 | # Feature Request 9 | 10 | Please provide a detailed description of your proposal, with some examples. 11 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_duplicate_encode_orders.stderr: -------------------------------------------------------------------------------- 1 | error: Specified duplicate encode order `1` for field 2 | --> tests/ui/csv_serialize_duplicate_encode_orders.rs:9:5 3 | | 4 | 9 | / #[dbn(encode_order(1))] 5 | 10 | | pub c: u8, 6 | | |_____________^ 7 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_duplicate_encode_orders.stderr: -------------------------------------------------------------------------------- 1 | error: Specified duplicate encode order `1` for field 2 | --> tests/ui/json_serialize_duplicate_encode_orders.rs:9:5 3 | | 4 | 9 | / #[dbn(encode_order(1))] 5 | 10 | | pub c: u8, 6 | | |_____________^ 7 | -------------------------------------------------------------------------------- /rust/dbn/src/encode/json.rs: -------------------------------------------------------------------------------- 1 | //! Encoding of DBN records into [JSON lines](https://jsonlines.org). 2 | 3 | pub(crate) mod serialize; 4 | mod sync; 5 | pub use sync::{Encoder, EncoderBuilder}; 6 | #[cfg(feature = "async")] 7 | mod r#async; 8 | #[cfg(feature = "async")] 9 | pub use r#async::Encoder as AsyncEncoder; 10 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_duplicate_encode_orders.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::CsvSerialize; 2 | 3 | #[derive(CsvSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(encode_order(1))] 7 | pub a: u8, 8 | pub b: u8, 9 | #[dbn(encode_order(1))] 10 | pub c: u8, 11 | } 12 | 13 | fn main() {} 14 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_duplicate_encode_orders.rs: -------------------------------------------------------------------------------- 1 | use dbn_macros::JsonSerialize; 2 | 3 | #[derive(JsonSerialize)] 4 | #[repr(C)] 5 | struct Record { 6 | #[dbn(encode_order(1))] 7 | pub a: u8, 8 | pub b: u8, 9 | #[dbn(encode_order(1))] 10 | pub c: u8, 11 | } 12 | 13 | fn main() {} 14 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/missing_rtype.stderr: -------------------------------------------------------------------------------- 1 | error: Need to specify at least one rtype to match against 2 | --> tests/ui/missing_rtype.rs:4:1 3 | | 4 | 4 | #[dbn_record] 5 | | ^^^^^^^^^^^^^ 6 | | 7 | = note: this error originates in the attribute macro `dbn_record` (in Nightly builds, run with -Z macro-backtrace for more info) 8 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | cargo --version 5 | cargo clippy --all-features -- --deny warnings 6 | # `cargo doc` does not have a `--deny warnings` flag like clippy, workaround from: 7 | # https://github.com/rust-lang/cargo/issues/8424#issuecomment-1070988443 8 | RUSTDOCFLAGS='--deny warnings' cargo doc --all-features --no-deps 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: General Questions 4 | url: https://github.com/databento/dbn/discussions 5 | about: Please ask questions like "How do I achieve x?" here. 6 | - name: DatabentoHQ 7 | url: https://twitter.com/DatabentoHQ 8 | about: Follow us on twitter for news and updates! 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for taking the time to contribute to our project. 2 | We welcome feedback through discussions and issues on GitHub, as well as our [community Slack](https://databento.com/support). 3 | While we don't merge pull requests directly due to the open-source repository being a downstream 4 | mirror of our internal codebase, we can commit the changes upstream with the original author. 5 | -------------------------------------------------------------------------------- /rust/dbn/src/encode/dbn.rs: -------------------------------------------------------------------------------- 1 | //! Encoding DBN records into DBN, Zstandard-compressed or not. 2 | mod sync; 3 | pub use sync::{Encoder, MetadataEncoder, RecordEncoder}; 4 | 5 | #[cfg(feature = "async")] 6 | mod r#async; 7 | #[cfg(feature = "async")] 8 | pub use r#async::{ 9 | Encoder as AsyncEncoder, MetadataEncoder as AsyncMetadataEncoder, 10 | RecordEncoder as AsyncRecordEncoder, 11 | }; 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug here 4 | labels: 5 | - bug 6 | --- 7 | 8 | # Bug Report 9 | 10 | ### Expected Behavior 11 | Add here... 12 | 13 | ### Actual Behavior 14 | Add here... 15 | 16 | ### Steps to Reproduce the Problem 17 | 18 | 1. 19 | 2. 20 | 3. 21 | 22 | ### Specifications 23 | 24 | - OS platform: 25 | - Rust version: 26 | - `dbn` version: 27 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/utils.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::{Ident, Span, TokenStream}; 2 | use proc_macro_crate::FoundCrate; 3 | use quote::quote; 4 | 5 | pub fn crate_name() -> TokenStream { 6 | match proc_macro_crate::crate_name("dbn").expect("dbn crate in Cargo.toml") { 7 | FoundCrate::Itself => quote!(crate), 8 | FoundCrate::Name(name) => { 9 | let ident = Ident::new(&name, Span::call_site()); 10 | quote!( ::#ident ) 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /rust/dbn-macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dbn-macros" 3 | description = "Proc macros for dbn crate" 4 | authors.workspace = true 5 | version.workspace = true 6 | edition.workspace = true 7 | license.workspace = true 8 | repository.workspace = true 9 | 10 | [lib] 11 | proc-macro = true 12 | 13 | [dependencies] 14 | proc-macro-crate = "3.4.0" 15 | proc-macro2 = "1.0.103" 16 | quote = "1.0.42" 17 | syn = { version = "2.0", features = ["full"] } 18 | 19 | [dev-dependencies] 20 | csv = { workspace = true } 21 | dbn = { path = "../dbn" } 22 | trybuild = "1.0.114" 23 | -------------------------------------------------------------------------------- /c/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dbn-c" 3 | description = "C bindings for working with Databento Binary Encoding (DBN)" 4 | # This crate should not be published 5 | publish = false 6 | authors.workspace = true 7 | version.workspace = true 8 | edition.workspace = true 9 | license.workspace = true 10 | repository.workspace = true 11 | 12 | [lib] 13 | name = "dbn_c" 14 | crate-type = ["staticlib"] 15 | 16 | [dependencies] 17 | anyhow = { workspace = true } 18 | dbn = { path = "../rust/dbn", features = [] } 19 | libc = "0.2.176" 20 | 21 | [build-dependencies] 22 | cbindgen = { version = "0.29.2", default-features = false } 23 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "c", 4 | "python", 5 | "rust/dbn-cli", 6 | "rust/dbn-macros", 7 | "rust/dbn" 8 | ] 9 | resolver = "2" 10 | 11 | [workspace.package] 12 | authors = ["Databento "] 13 | edition = "2021" 14 | version = "0.45.0" 15 | documentation = "https://databento.com/docs" 16 | repository = "https://github.com/databento/dbn" 17 | license = "Apache-2.0" 18 | 19 | [workspace.dependencies] 20 | anyhow = "1.0.100" 21 | csv = "1.4" 22 | pyo3 = "0.26.0" 23 | pyo3-build-config = "0.26.0" 24 | rstest = "0.26.1" 25 | serde = { version = "1.0", features = ["derive"] } 26 | time = "0.3.44" 27 | zstd = "0.13" 28 | -------------------------------------------------------------------------------- /python/python/databento_dbn/v3.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: F401 2 | from ._lib import BBOMsg 3 | from ._lib import CBBOMsg 4 | from ._lib import CMBP1Msg 5 | from ._lib import ErrorMsg 6 | from ._lib import ImbalanceMsg 7 | from ._lib import InstrumentDefMsg 8 | from ._lib import MBOMsg 9 | from ._lib import MBP1Msg 10 | from ._lib import MBP10Msg 11 | from ._lib import OHLCVMsg 12 | from ._lib import StatMsg 13 | from ._lib import StatusMsg 14 | from ._lib import SymbolMappingMsg 15 | from ._lib import SystemMsg 16 | from ._lib import TradeMsg 17 | 18 | 19 | # Aliases 20 | TBBOMsg = MBP1Msg 21 | BBO1SMsg = BBOMsg 22 | BBO1MMsg = BBOMsg 23 | TCBBOMsg = CMBP1Msg 24 | CBBO1SMsg = CBBOMsg 25 | CBBO1MMsg = CBBOMsg 26 | -------------------------------------------------------------------------------- /c/cbindgen.toml: -------------------------------------------------------------------------------- 1 | language = "C" 2 | pragma_once = true 3 | autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */" 4 | line_length = 100 5 | tab_width = 4 6 | sys_includes = ["stdio.h"] 7 | # Affects enum typedefs 8 | cpp_compat = true 9 | 10 | [export] 11 | prefix = "Dbn" 12 | renaming_overrides_prefixing = true 13 | 14 | [export.rename] 15 | "FILE" = "FILE" 16 | # Workaround for cbindgen not understanding constants defined in terms of other constants 17 | "SYMBOL_CSTR_LEN_V2" = "DbnSYMBOL_CSTR_LEN" 18 | "ASSET_CSTR_LEN" = "DbnASSET_CSTR_LEN_V3" 19 | 20 | [enum] 21 | prefix_with_name = true 22 | 23 | [parse] 24 | parse_deps = true 25 | include = ["dbn"] 26 | extra_bindings = ["dbn"] 27 | -------------------------------------------------------------------------------- /python/python/databento_dbn/v2.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: F401 2 | from ._lib import BBOMsg 3 | from ._lib import CBBOMsg 4 | from ._lib import CMBP1Msg 5 | from ._lib import ErrorMsg 6 | from ._lib import ImbalanceMsg 7 | from ._lib import InstrumentDefMsgV2 as InstrumentDefMsg 8 | from ._lib import MBOMsg 9 | from ._lib import MBP1Msg 10 | from ._lib import MBP10Msg 11 | from ._lib import OHLCVMsg 12 | from ._lib import StatMsgV1 as StatMsg 13 | from ._lib import StatusMsg 14 | from ._lib import SymbolMappingMsg 15 | from ._lib import SystemMsg 16 | from ._lib import TradeMsg 17 | 18 | 19 | # Aliases 20 | TBBOMsg = MBP1Msg 21 | BBO1SMsg = BBOMsg 22 | BBO1MMsg = BBOMsg 23 | TCBBOMsg = CMBP1Msg 24 | CBBO1SMsg = CBBOMsg 25 | CBBO1MMsg = CBBOMsg 26 | -------------------------------------------------------------------------------- /python/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "databento-dbn" 3 | description = "Python library written in Rust for working with Databento Binary Encoding (DBN)" 4 | # This crate should only be published as a Python package 5 | publish = false 6 | authors.workspace = true 7 | version.workspace = true 8 | edition.workspace = true 9 | license.workspace = true 10 | repository.workspace = true 11 | 12 | [lib] 13 | name = "databento_dbn" # Python modules can't contain dashes 14 | 15 | [dependencies] 16 | dbn = { path = "../rust/dbn", features = ["python"] } 17 | pyo3.workspace = true 18 | time.workspace = true 19 | 20 | [build-dependencies] 21 | pyo3-build-config.workspace = true 22 | 23 | [dev-dependencies] 24 | rstest.workspace = true 25 | zstd.workspace = true 26 | -------------------------------------------------------------------------------- /python/python/databento_dbn/v1.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: F401 2 | from ._lib import BBOMsg 3 | from ._lib import CBBOMsg 4 | from ._lib import CMBP1Msg 5 | from ._lib import ErrorMsgV1 as ErrorMsg 6 | from ._lib import ImbalanceMsg 7 | from ._lib import InstrumentDefMsgV1 as InstrumentDefMsg 8 | from ._lib import MBOMsg 9 | from ._lib import MBP1Msg 10 | from ._lib import MBP10Msg 11 | from ._lib import OHLCVMsg 12 | from ._lib import StatMsgV1 as StatMsg 13 | from ._lib import StatusMsg 14 | from ._lib import SymbolMappingMsgV1 as SymbolMappingMsg 15 | from ._lib import SystemMsgV1 as SystemMsg 16 | from ._lib import TradeMsg 17 | 18 | 19 | # Aliases 20 | TBBOMsg = MBP1Msg 21 | BBO1SMsg = BBOMsg 22 | BBO1MMsg = BBOMsg 23 | TCBBOMsg = CMBP1Msg 24 | CBBO1SMsg = CBBOMsg 25 | CBBO1MMsg = CBBOMsg 26 | -------------------------------------------------------------------------------- /rust/dbn/src/decode/dbn.rs: -------------------------------------------------------------------------------- 1 | //! Decoding of DBN files. 2 | pub(super) const DBN_PREFIX: &[u8] = b"DBN"; 3 | pub(super) const DBN_PREFIX_LEN: usize = DBN_PREFIX.len(); 4 | 5 | /// Returns `true` if `bytes` starts with valid uncompressed DBN. 6 | pub fn starts_with_prefix(bytes: &[u8]) -> bool { 7 | bytes.len() > DBN_PREFIX_LEN && &bytes[..DBN_PREFIX_LEN] == DBN_PREFIX 8 | } 9 | 10 | mod sync; 11 | pub(crate) use sync::decode_iso8601; 12 | pub use sync::{Decoder, MetadataDecoder, RecordDecoder}; 13 | pub mod fsm; 14 | 15 | #[cfg(feature = "async")] 16 | mod r#async; 17 | #[cfg(feature = "async")] 18 | pub use r#async::{ 19 | decode_metadata_with_fsm as async_decode_metadata_with_fsm, 20 | decode_record_ref_with_fsm as async_decode_record_ref_with_fsm, Decoder as AsyncDecoder, 21 | MetadataDecoder as AsyncMetadataDecoder, RecordDecoder as AsyncRecordDecoder, 22 | }; 23 | -------------------------------------------------------------------------------- /rust/dbn-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dbn-cli" 3 | description = "Command-line utility for converting Databento Binary Encoding (DBN) files to text-based formats" 4 | default-run = "dbn" 5 | keywords = ["market-data", "json", "csv", "conversion", "encoding"] 6 | # see https://crates.io/category_slugs 7 | categories = ["command-line-utilities", "encoding"] 8 | authors.workspace = true 9 | version.workspace = true 10 | edition.workspace = true 11 | license.workspace = true 12 | repository.workspace = true 13 | 14 | [[bin]] 15 | name = "dbn" 16 | path = "src/main.rs" 17 | 18 | [dependencies] 19 | dbn = { path = "../dbn", version = "=0.45.0", default-features = false } 20 | 21 | anyhow = { workspace = true } 22 | clap = { version = "4.5", features = ["derive", "wrap_help"] } 23 | serde = { workspace = true, features = ["derive"] } 24 | zstd = { workspace = true } 25 | 26 | [dev-dependencies] 27 | assert_cmd = "2.1" 28 | predicates = "3.1" 29 | rstest = { workspace = true } 30 | tempfile = "3.23" 31 | -------------------------------------------------------------------------------- /scripts/bump_version.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | # 3 | # Updates the version to ${1}. 4 | # 5 | 6 | source "$(dirname "$0")/config.sh" 7 | 8 | if [ -z "$1" ]; then 9 | echo "Usage: $0 " >&2 10 | echo "Example: $0 0.1.2" >&2 11 | exit 1 12 | fi 13 | 14 | OLD_VERSION="$("${SCRIPTS_DIR}/get_version.sh")" 15 | NEW_VERSION="$1" 16 | 17 | # Replace package versions 18 | find \ 19 | "${PROJECT_ROOT_DIR}" \ 20 | -type f \ 21 | -name "*.toml" \ 22 | -exec sed -Ei "s/^version\s*=\s*\"${OLD_VERSION}\"/version = \"${NEW_VERSION}\"/" {} \; 23 | # Replace dependency versions 24 | find \ 25 | "${PROJECT_ROOT_DIR}" \ 26 | -type f \ 27 | -name "*.toml" \ 28 | -exec sed -Ei "s/version\s*=\s*\"=${OLD_VERSION}\"/version = \"=${NEW_VERSION}\"/" {} \; 29 | # Replace Python TOML version 30 | find \ 31 | "${PROJECT_ROOT_DIR}" \ 32 | -type f \ 33 | -name "pyproject.toml" \ 34 | -exec sed -Ei "s/version\s*=\s*\"${OLD_VERSION}\"/version = \"=${NEW_VERSION}\"/" {} \; 35 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.stderr: -------------------------------------------------------------------------------- 1 | error: unrecognized dbn attr argument unknown 2 | --> tests/ui/json_serialize_invalid_dbn_attr.rs:6:11 3 | | 4 | 6 | #[dbn(unknown)] 5 | | ^^^^^^^ 6 | 7 | error[E0433]: failed to resolve: unresolved import 8 | --> tests/ui/json_serialize_invalid_dbn_attr.rs:3:10 9 | | 10 | 3 | #[derive(JsonSerialize)] 11 | | ^^^^^^^^^^^^^ unresolved import 12 | | 13 | = note: this error originates in the derive macro `JsonSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) 14 | 15 | error[E0603]: module `serialize` is private 16 | --> tests/ui/json_serialize_invalid_dbn_attr.rs:3:10 17 | | 18 | 3 | #[derive(JsonSerialize)] 19 | | ^^^^^^^^^^^^^ 20 | | | 21 | | private module 22 | | trait `WriteField` is not publicly re-exported 23 | | 24 | note: the module `serialize` is defined here 25 | --> $WORKSPACE/rust/dbn/src/encode/json.rs 26 | | 27 | | pub(crate) mod serialize; 28 | | ^^^^^^^^^^^^^^^^^^^^^^^^ 29 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.stderr: -------------------------------------------------------------------------------- 1 | error: Passed incompatible serialization arguments to dbn attr 2 | --> tests/ui/json_serialize_conflicting_dbn_attr.rs:6:5 3 | | 4 | 6 | #[dbn(fixed_price, unix_nanos)] 5 | | ^ 6 | 7 | error[E0433]: failed to resolve: unresolved import 8 | --> tests/ui/json_serialize_conflicting_dbn_attr.rs:3:10 9 | | 10 | 3 | #[derive(JsonSerialize)] 11 | | ^^^^^^^^^^^^^ unresolved import 12 | | 13 | = note: this error originates in the derive macro `JsonSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) 14 | 15 | error[E0603]: module `serialize` is private 16 | --> tests/ui/json_serialize_conflicting_dbn_attr.rs:3:10 17 | | 18 | 3 | #[derive(JsonSerialize)] 19 | | ^^^^^^^^^^^^^ 20 | | | 21 | | private module 22 | | trait `WriteField` is not publicly re-exported 23 | | 24 | note: the module `serialize` is defined here 25 | --> $WORKSPACE/rust/dbn/src/encode/json.rs 26 | | 27 | | pub(crate) mod serialize; 28 | | ^^^^^^^^^^^^^^^^^^^^^^^^ 29 | -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "databento-dbn" 3 | version = "0.45.0" 4 | description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" 5 | readme = "README.md" 6 | requires-python = ">=3.10" 7 | license = "Apache-2.0" 8 | authors = [{ name = "Databento", email = "support@databento.com" }] 9 | classifiers = [ 10 | "Programming Language :: Rust", 11 | "Programming Language :: Python :: Implementation :: CPython", 12 | ] 13 | dependencies = [] 14 | 15 | [project.urls] 16 | Homepage = "https://databento.com" 17 | Documentation = "https://databento.com/docs" 18 | Repository = "https://github.com/databento/dbn" 19 | "Bug Tracker" = "https://github.com/databento/dbn/issues" 20 | 21 | [tool.poetry] 22 | requires-poetry = ">=2.0" 23 | 24 | [tool.poetry.dependencies] 25 | python = ">=3.10,<3.15" 26 | 27 | [tool.poetry.group.dev.dependencies] 28 | maturin = ">=1.0" 29 | 30 | [build-system] 31 | requires = ["maturin>=1.0"] 32 | build-backend = "maturin" 33 | 34 | [tool.maturin] 35 | features = ["pyo3/extension-module"] 36 | python-source = "python" 37 | module-name = "databento_dbn._lib" 38 | -------------------------------------------------------------------------------- /c/build.rs: -------------------------------------------------------------------------------- 1 | //! Writes generated C header for DBN functions and symbols to 2 | //! ${target_directory}/include/dbn/dbn.h 3 | 4 | extern crate cbindgen; 5 | 6 | use std::{env, ffi::OsStr, fs, path::PathBuf}; 7 | 8 | fn find_target_dir() -> PathBuf { 9 | if let Some(target_dir) = env::var_os("CARGO_TARGET_DIR") { 10 | return PathBuf::from(target_dir); 11 | } 12 | let mut dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); 13 | loop { 14 | if dir.file_name() == Some(OsStr::new("target")) 15 | // Want to find the top directory containing a CACHEDIR.TAG file 16 | || (dir.join("CACHEDIR.TAG").exists() 17 | && !dir 18 | .parent().is_none_or(|p| p.join("CACHEDIR.TAG").exists())) 19 | { 20 | return dir; 21 | } 22 | assert!(dir.pop(), "Unable to determine target dir"); 23 | } 24 | } 25 | 26 | fn main() { 27 | let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); 28 | let target_dir = find_target_dir(); 29 | let include_dir = target_dir.join("include").join("dbn"); 30 | fs::create_dir_all(&include_dir).unwrap(); 31 | let out_path = include_dir.join("dbn.h"); 32 | 33 | cbindgen::generate(crate_dir) 34 | .expect("Unable to generate bindings") 35 | .write_to_file(out_path); 36 | } 37 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.stderr: -------------------------------------------------------------------------------- 1 | error: unrecognized dbn attr argument unknown 2 | --> tests/ui/csv_serialize_invalid_dbn_attr.rs:6:11 3 | | 4 | 6 | #[dbn(unknown)] 5 | | ^^^^^^^ 6 | 7 | error[E0603]: module `serialize` is private 8 | --> tests/ui/csv_serialize_invalid_dbn_attr.rs:3:10 9 | | 10 | 3 | #[derive(CsvSerialize)] 11 | | ^^^^^^^^^^^^ 12 | | | 13 | | private module 14 | | trait `WriteField` is not publicly re-exported 15 | | 16 | note: the module `serialize` is defined here 17 | --> $WORKSPACE/rust/dbn/src/encode/csv.rs 18 | | 19 | | pub(crate) mod serialize; 20 | | ^^^^^^^^^^^^^^^^^^^^^^^^ 21 | 22 | error[E0599]: no function or associated item named `write_header` found for type `u8` in the current scope 23 | --> tests/ui/csv_serialize_invalid_dbn_attr.rs:3:10 24 | | 25 | 3 | #[derive(CsvSerialize)] 26 | | ^^^^^^^^^^^^ function or associated item not found in `u8` 27 | | 28 | = help: items from traits can only be used if the trait is in scope 29 | = note: this error originates in the derive macro `CsvSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) 30 | help: trait `WriteField` which provides `write_header` is implemented but not in scope; perhaps you want to import it 31 | | 32 | 1 + use dbn::encode::csv::serialize::WriteField; 33 | | 34 | -------------------------------------------------------------------------------- /python/python/databento_dbn/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | from collections.abc import Sequence 3 | from typing import Protocol 4 | from typing import TypedDict 5 | 6 | # Import native module 7 | from ._lib import * # noqa: F403 8 | 9 | 10 | class MappingInterval(Protocol): 11 | """ 12 | Represents a symbol mapping over a start and end date range interval. 13 | 14 | Parameters 15 | ---------- 16 | start_date : dt.date 17 | The start of the mapping period. 18 | end_date : dt.date 19 | The end of the mapping period. 20 | symbol : str 21 | The symbol value. 22 | 23 | """ 24 | 25 | start_date: dt.date 26 | end_date: dt.date 27 | symbol: str 28 | 29 | 30 | class MappingIntervalDict(TypedDict): 31 | """ 32 | Represents a symbol mapping over a start and end date range interval. 33 | 34 | Parameters 35 | ---------- 36 | start_date : dt.date 37 | The start of the mapping period. 38 | end_date : dt.date 39 | The end of the mapping period. 40 | symbol : str 41 | The symbol value. 42 | 43 | """ 44 | 45 | start_date: dt.date 46 | end_date: dt.date 47 | symbol: str 48 | 49 | 50 | class SymbolMapping(Protocol): 51 | """ 52 | Represents the mappings for one native symbol. 53 | """ 54 | 55 | raw_symbol: str 56 | intervals: Sequence[MappingInterval] 57 | -------------------------------------------------------------------------------- /rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.stderr: -------------------------------------------------------------------------------- 1 | error: Passed incompatible serialization arguments to dbn attr 2 | --> tests/ui/csv_serialize_conflicting_dbn_attr.rs:6:5 3 | | 4 | 6 | #[dbn(fixed_price, unix_nanos)] 5 | | ^ 6 | 7 | error[E0603]: module `serialize` is private 8 | --> tests/ui/csv_serialize_conflicting_dbn_attr.rs:3:10 9 | | 10 | 3 | #[derive(CsvSerialize)] 11 | | ^^^^^^^^^^^^ 12 | | | 13 | | private module 14 | | trait `WriteField` is not publicly re-exported 15 | | 16 | note: the module `serialize` is defined here 17 | --> $WORKSPACE/rust/dbn/src/encode/csv.rs 18 | | 19 | | pub(crate) mod serialize; 20 | | ^^^^^^^^^^^^^^^^^^^^^^^^ 21 | 22 | error[E0599]: no function or associated item named `write_header` found for type `u8` in the current scope 23 | --> tests/ui/csv_serialize_conflicting_dbn_attr.rs:3:10 24 | | 25 | 3 | #[derive(CsvSerialize)] 26 | | ^^^^^^^^^^^^ function or associated item not found in `u8` 27 | | 28 | = help: items from traits can only be used if the trait is in scope 29 | = note: this error originates in the derive macro `CsvSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) 30 | help: trait `WriteField` which provides `write_header` is implemented but not in scope; perhaps you want to import it 31 | | 32 | 1 + use dbn::encode::csv::serialize::WriteField; 33 | | 34 | -------------------------------------------------------------------------------- /c/src/cfile.rs: -------------------------------------------------------------------------------- 1 | use std::ptr::NonNull; 2 | 3 | /// A non-owning wrapper around a `*mut libc::FILE`. 4 | pub struct CFileRef { 5 | ptr: NonNull, 6 | bytes_written: usize, 7 | } 8 | 9 | impl CFileRef { 10 | pub fn new(ptr: *mut libc::FILE) -> Option { 11 | NonNull::new(ptr).map(|ptr| Self { 12 | ptr, 13 | bytes_written: 0, 14 | }) 15 | } 16 | 17 | pub fn bytes_written(&self) -> usize { 18 | self.bytes_written 19 | } 20 | 21 | pub fn as_ptr(&mut self) -> *mut libc::FILE { 22 | self.ptr.as_ptr() 23 | } 24 | } 25 | 26 | impl std::io::Write for CFileRef { 27 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 28 | let ret = unsafe { 29 | libc::fwrite( 30 | buf.as_ptr() as *const libc::c_void, 31 | 1, 32 | buf.len(), 33 | self.as_ptr(), 34 | ) 35 | }; 36 | if ret > 0 { 37 | self.bytes_written += ret; 38 | Ok(ret) 39 | } else { 40 | Err(std::io::Error::last_os_error()) 41 | } 42 | } 43 | 44 | fn flush(&mut self) -> std::io::Result<()> { 45 | let ret = unsafe { libc::fflush(self.as_ptr()) }; 46 | if ret == 0 { 47 | Ok(()) 48 | } else { 49 | Err(std::io::Error::last_os_error()) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Pull request 2 | 3 | Please include a summary of the changes. 4 | Please also include relevant motivation and context. 5 | List any dependencies that are required for this change. 6 | 7 | Fixes # (issue) 8 | 9 | ## Type of change 10 | 11 | Please delete options that are not relevant. 12 | 13 | - [ ] Bug fix (non-breaking change which fixes an issue) 14 | - [ ] New feature (non-breaking change which adds functionality) 15 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 16 | - [ ] This change requires a documentation update 17 | 18 | ## How has this change been tested? 19 | 20 | Please describe the tests that you ran to verify your changes. 21 | Provide instructions so we can reproduce. 22 | Please also list any relevant details for your test configuration. 23 | 24 | - [ ] Test A 25 | - [ ] Test B 26 | 27 | ## Checklist 28 | 29 | - [ ] My code builds locally with no new warnings (`scripts/build.sh`) 30 | - [ ] My code follows the style guidelines (`scripts/lint.sh` and `scripts/format.sh`) 31 | - [ ] New and existing unit tests pass locally with my changes (`scripts/test.sh`) 32 | - [ ] I have made corresponding changes to the documentation 33 | - [ ] I have added tests that prove my fix is effective or that my feature works 34 | 35 | ## Declaration 36 | 37 | I confirm this contribution is made under an Apache 2.0 license and that I have the authority 38 | necessary to make this contribution on behalf of its copyright owner. 39 | -------------------------------------------------------------------------------- /rust/dbn/README.md: -------------------------------------------------------------------------------- 1 | # dbn 2 | 3 | [![build](https://github.com/databento/dbn/actions/workflows/build.yaml/badge.svg)](https://github.com/databento/dbn/actions/workflows/build.yaml) 4 | [![Documentation](https://img.shields.io/docsrs/dbn)](https://docs.rs/dbn/latest/dbn/) 5 | ![license](https://img.shields.io/github/license/databento/dbn?color=blue) 6 | [![Current Crates.io Version](https://img.shields.io/crates/v/dbn.svg)](https://crates.io/crates/dbn) 7 | 8 | The official crate for working with Databento Binary Encoding (DBN). 9 | For more information about DBN, read our [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). 10 | 11 | Check out the [databento crate](https://crates.io/crates/databento) for the official Databento Rust client. 12 | 13 | ## Installation 14 | 15 | To add the crate to an existing project, run the following command: 16 | ```sh 17 | cargo add dbn 18 | ``` 19 | 20 | ## Usage 21 | 22 | To read a DBN file with MBO data and print each row: 23 | ```rust 24 | use dbn::{ 25 | decode::dbn::Decoder, 26 | record::MboMsg, 27 | }; 28 | use streaming_iterator::StreamingIterator; 29 | 30 | let mut dbn_stream = Decoder::from_zstd_file("20201228.dbn.zst")?.decode_stream::()?; 31 | while let Some(mbo_msg) = dbn_stream.next() { 32 | println!("{mbo_msg:?}"); 33 | } 34 | ``` 35 | 36 | ## Documentation 37 | 38 | See [the docs](https://docs.rs/dbn) for more detailed usage. 39 | 40 | ## License 41 | 42 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 43 | -------------------------------------------------------------------------------- /rust/dbn/src/record/with_ts_out_methods.rs: -------------------------------------------------------------------------------- 1 | use std::mem; 2 | 3 | use crate::{record::as_u8_slice, HasRType, Record, RecordHeader, RecordMut, WithTsOut}; 4 | 5 | impl Record for WithTsOut { 6 | fn header(&self) -> &RecordHeader { 7 | self.rec.header() 8 | } 9 | 10 | fn raw_index_ts(&self) -> u64 { 11 | self.rec.raw_index_ts() 12 | } 13 | } 14 | 15 | impl RecordMut for WithTsOut { 16 | fn header_mut(&mut self) -> &mut RecordHeader { 17 | self.rec.header_mut() 18 | } 19 | } 20 | 21 | impl HasRType for WithTsOut { 22 | fn has_rtype(rtype: u8) -> bool { 23 | T::has_rtype(rtype) 24 | } 25 | } 26 | 27 | impl AsRef<[u8]> for WithTsOut 28 | where 29 | T: HasRType, 30 | { 31 | fn as_ref(&self) -> &[u8] { 32 | unsafe { as_u8_slice(self) } 33 | } 34 | } 35 | 36 | impl WithTsOut { 37 | /// Creates a new record with `ts_out`. Updates the `length` property in 38 | /// [`RecordHeader`] to ensure the additional field is accounted for. 39 | pub fn new(rec: T, ts_out: u64) -> Self { 40 | let mut res = Self { rec, ts_out }; 41 | res.header_mut().length = (mem::size_of_val(&res) / RecordHeader::LENGTH_MULTIPLIER) as u8; 42 | res 43 | } 44 | 45 | /// Parses the raw live gateway send timestamp into a datetime. 46 | pub fn ts_out(&self) -> time::OffsetDateTime { 47 | // u64::MAX is within maximum allowable range 48 | time::OffsetDateTime::from_unix_timestamp_nanos(self.ts_out as i128).unwrap() 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /c/src/compat.rs: -------------------------------------------------------------------------------- 1 | use dbn::{ 2 | compat::{ 3 | ErrorMsgV1, InstrumentDefMsgV1, InstrumentDefMsgV2, StatMsgV1, SymbolMappingMsgV1, 4 | SystemMsgV1, 5 | }, 6 | ErrorMsg, InstrumentDefMsg, StatMsg, SymbolMappingMsg, SystemMsg, 7 | }; 8 | 9 | /// Converts an V1 ErrorMsg to V2. 10 | #[no_mangle] 11 | pub extern "C" fn from_error_v1_to_v2(def_v1: &ErrorMsgV1) -> ErrorMsg { 12 | ErrorMsg::from(def_v1) 13 | } 14 | 15 | /// Converts an V1 InstrumentDefMsg to V2. 16 | #[no_mangle] 17 | pub extern "C" fn from_instrument_def_v1_to_v2(def_v1: &InstrumentDefMsgV1) -> InstrumentDefMsgV2 { 18 | InstrumentDefMsgV2::from(def_v1) 19 | } 20 | 21 | /// Converts a V1 InstrumentDefMsg to V3. 22 | #[no_mangle] 23 | pub extern "C" fn from_instrument_def_v1_to_v3(def_v1: &InstrumentDefMsgV1) -> InstrumentDefMsg { 24 | InstrumentDefMsg::from(def_v1) 25 | } 26 | 27 | /// Converts a V2 InstrumentDefMsg to V3. 28 | #[no_mangle] 29 | pub extern "C" fn from_instrument_def_v2_to_v3(def_v2: &InstrumentDefMsgV2) -> InstrumentDefMsg { 30 | InstrumentDefMsg::from(def_v2) 31 | } 32 | 33 | /// Converts a V1 StatMsg to V3. 34 | #[no_mangle] 35 | pub extern "C" fn from_stat_v1_to_v3(stat_v1: &StatMsgV1) -> StatMsg { 36 | StatMsg::from(stat_v1) 37 | } 38 | 39 | /// Converts an V1 SymbolMappingMsg to V2. 40 | #[no_mangle] 41 | pub extern "C" fn from_symbol_mapping_v1_to_v2(def_v1: &SymbolMappingMsgV1) -> SymbolMappingMsg { 42 | SymbolMappingMsg::from(def_v1) 43 | } 44 | 45 | /// Converts an V1 SystemMsg to V2. 46 | #[no_mangle] 47 | pub extern "C" fn from_system_v1_to_v2(def_v1: &SystemMsgV1) -> SystemMsg { 48 | SystemMsg::from(def_v1) 49 | } 50 | -------------------------------------------------------------------------------- /rust/dbn/src/test_utils.rs: -------------------------------------------------------------------------------- 1 | use fallible_streaming_iterator::FallibleStreamingIterator; 2 | 3 | use crate::{ 4 | decode::{private::LastRecord, DecodeRecordRef}, 5 | Error, HasRType, RecordRef, 6 | }; 7 | 8 | /// A testing shim to get a streaming iterator from a [`Vec`]. 9 | pub struct VecStream { 10 | vec: Vec, 11 | idx: isize, 12 | } 13 | 14 | impl VecStream { 15 | pub fn new(vec: Vec) -> Self { 16 | // initialize at -1 because `advance()` is always called before 17 | // `get()`. 18 | Self { vec, idx: -1 } 19 | } 20 | } 21 | 22 | impl FallibleStreamingIterator for VecStream { 23 | type Item = T; 24 | type Error = Error; 25 | 26 | fn advance(&mut self) -> Result<(), Error> { 27 | self.idx += 1; 28 | Ok(()) 29 | } 30 | 31 | fn get(&self) -> Option<&Self::Item> { 32 | self.vec.get(self.idx as usize) 33 | } 34 | } 35 | 36 | impl DecodeRecordRef for VecStream 37 | where 38 | T: HasRType, 39 | { 40 | fn decode_record_ref(&mut self) -> crate::Result>> { 41 | self.idx += 1; 42 | let Some(rec) = self.vec.get(self.idx as usize) else { 43 | return Ok(None); 44 | }; 45 | Ok(Some(RecordRef::from(rec))) 46 | } 47 | } 48 | 49 | impl LastRecord for VecStream 50 | where 51 | T: HasRType + AsRef<[u8]>, 52 | { 53 | fn last_record(&self) -> Option> { 54 | if self.vec.is_empty() { 55 | None 56 | } else { 57 | Some(RecordRef::from(self.vec.get(self.idx as usize).unwrap())) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # databento-dbn 2 | 3 | [![build](https://github.com/databento/dbn/actions/workflows/build.yaml/badge.svg)](https://github.com/databento/dbn/actions/workflows/build.yaml) 4 | ![python](https://img.shields.io/badge/python-3.10+-blue.svg) 5 | ![license](https://img.shields.io/github/license/databento/dbn?color=blue) 6 | [![pypi-version](https://img.shields.io/pypi/v/databento_dbn)](https://pypi.org/project/databento-dbn) 7 | 8 | Python bindings for the `dbn` Rust library, used by the [Databento Python client library](https://github.com/databento/databento-python). 9 | For more information about the encoding, read our [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). 10 | 11 | Using this library is for advanced users and is not fully documented or supported. 12 | 13 | ## Installation 14 | 15 | To install the latest stable version from PyPI: 16 | ```sh 17 | pip install -U databento-dbn 18 | ``` 19 | 20 | ## Usage and documentation 21 | 22 | See the [documentation](https://databento.com/docs/quickstart?historical=python&live=python) for the Python client library. 23 | 24 | ## Building 25 | 26 | `databento-dbn` is written in Rust, so you'll need to have [Rust installed](https://www.rust-lang.org/) 27 | as well as [Maturin](https://github.com/PyO3/maturin). 28 | 29 | To build, run the following commands: 30 | ```sh 31 | git clone https://github.com/databento/dbn 32 | cd dbn 33 | maturin build 34 | ``` 35 | 36 | To build the Python package and install it for the active Python interpreter in your `PATH`, run: 37 | ```sh 38 | maturin develop 39 | ``` 40 | This will install a package named `databento-dbn` in your current Python environment. 41 | 42 | ## License 43 | 44 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 45 | -------------------------------------------------------------------------------- /rust/dbn/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dbn" 3 | description = "Library for working with Databento Binary Encoding (DBN)" 4 | keywords = ["finance", "market-data", "conversion", "encoding", "trading"] 5 | # see https://crates.io/category_slugs 6 | categories = ["encoding"] 7 | authors.workspace = true 8 | version.workspace = true 9 | edition.workspace = true 10 | license.workspace = true 11 | repository.workspace = true 12 | 13 | [package.metadata.docs.rs] 14 | # Document all features on docs.rs 15 | all-features = true 16 | # To build locally: `RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features --open` 17 | rustdoc-args = ["--cfg", "docsrs"] 18 | 19 | [features] 20 | default = [] 21 | async = ["dep:async-compression", "dep:tokio"] 22 | python = ["dep:pyo3", "dep:strum"] 23 | serde = ["dep:serde", "time/parsing", "time/serde"] 24 | # Enables deriving the `Copy` trait for records. 25 | trivial_copy = [] 26 | 27 | [dependencies] 28 | dbn-macros = { version = "=0.45.0", path = "../dbn-macros" } 29 | 30 | async-compression = { version = "0.4.33", features = ["tokio", "zstd"], optional = true } 31 | csv = { workspace = true } 32 | fallible-streaming-iterator = { version = "0.1.9", features = ["std"] } 33 | # Fast integer to string conversion 34 | itoa = "1.0" 35 | num_enum = "0.7" 36 | pyo3 = { workspace = true, optional = true } 37 | json-writer = "0.4" 38 | oval = "2.0" 39 | serde = { workspace = true, features = ["derive"], optional = true } 40 | # extra enum traits for Python 41 | strum = { version = "0.27", features = ["derive"], optional = true } 42 | thiserror = "2.0" 43 | time = { workspace = true, features = ["formatting", "macros"] } 44 | tokio = { version = ">=1.41", features = ["fs", "io-util"], optional = true } 45 | zstd = { workspace = true } 46 | 47 | [dev-dependencies] 48 | rstest = { workspace = true } 49 | strum = { version = "0.27", features = ["derive"] } 50 | tokio = { version = "1", features = ["fs", "io-util", "macros", "rt-multi-thread"] } 51 | # Checking alignment and padding 52 | type-layout = "0.2.0" 53 | -------------------------------------------------------------------------------- /rust/dbn/src/decode/zstd.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Range; 2 | 3 | use super::FromLittleEndianSlice; 4 | 5 | /// Range of magic numbers for a Zstandard skippable frame. 6 | pub(crate) const ZSTD_SKIPPABLE_MAGIC_RANGE: Range = 0x184D2A50..0x184D2A60; 7 | /// Magic number for the beginning of a Zstandard frame. 8 | const ZSTD_MAGIC_NUMBER: u32 = 0xFD2FB528; 9 | 10 | pub fn starts_with_prefix(bytes: &[u8]) -> bool { 11 | if bytes.len() < 4 { 12 | return false; 13 | } 14 | let magic = u32::from_le_slice(&bytes[..4]); 15 | ZSTD_MAGIC_NUMBER == magic 16 | } 17 | 18 | /// Helper to create an async Zstandard decoder with multiple member support. 19 | #[cfg(feature = "async")] 20 | pub fn zstd_decoder(reader: R) -> async_compression::tokio::bufread::ZstdDecoder 21 | where 22 | R: tokio::io::AsyncBufReadExt + Unpin, 23 | { 24 | let mut zstd_decoder = async_compression::tokio::bufread::ZstdDecoder::new(reader); 25 | // explicitly enable decoding multiple frames 26 | zstd_decoder.multiple_members(true); 27 | zstd_decoder 28 | } 29 | 30 | #[cfg(test)] 31 | mod tests { 32 | use std::{fs::File, io::Read}; 33 | 34 | use rstest::rstest; 35 | 36 | use super::*; 37 | use crate::{decode::tests::TEST_DATA_PATH, Schema}; 38 | 39 | #[rstest] 40 | #[case::mbo(Schema::Mbo)] 41 | #[case::mbp1(Schema::Mbp1)] 42 | #[case::mbp10(Schema::Mbp10)] 43 | #[case::definition(Schema::Definition)] 44 | fn test_starts_with_prefix_valid(#[case] schema: Schema) { 45 | let mut file = 46 | File::open(format!("{TEST_DATA_PATH}/test_data.{schema}.v3.dbn.zst")).unwrap(); 47 | let mut buf = [0u8; 4]; 48 | file.read_exact(&mut buf).unwrap(); 49 | assert!(starts_with_prefix(buf.as_slice())); 50 | } 51 | 52 | #[rstest] 53 | fn test_starts_with_prefix_other() { 54 | let mut file = File::open(format!("{TEST_DATA_PATH}/test_data.mbo.v3.dbn")).unwrap(); 55 | let mut buf = [0u8; 4]; 56 | file.read_exact(&mut buf).unwrap(); 57 | assert!(!starts_with_prefix(buf.as_slice())); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbn 2 | 3 | [![build](https://github.com/databento/dbn/actions/workflows/build.yaml/badge.svg)](https://github.com/databento/dbn/actions/workflows/build.yaml) 4 | [![Documentation](https://img.shields.io/docsrs/dbn)](https://docs.rs/dbn/latest/dbn/) 5 | [![license](https://img.shields.io/github/license/databento/dbn?color=blue)](./LICENSE) 6 | [![Current Crates.io Version](https://img.shields.io/crates/v/dbn.svg)](https://crates.io/crates/dbn) 7 | [![pypi-version](https://img.shields.io/pypi/v/databento_dbn)](https://pypi.org/project/databento-dbn) 8 | [![Slack](https://img.shields.io/badge/join_Slack-community-darkblue.svg?logo=slack)](https://to.dbn.to/slack) 9 | 10 | **D**atabento **B**inary E**n**coding (DBN) is an extremely fast message encoding and storage format for normalized market data. 11 | The DBN specification includes a simple, self-describing metadata header and a fixed set of struct definitions, which enforce a standardized way to normalize market data. 12 | 13 | All official Databento client libraries use DBN under the hood, both as a data interchange format and for in-memory representation of data. 14 | DBN is also the default encoding for all Databento APIs, including live data streaming, historical data streaming, and batch flat files. 15 | 16 | This repository contains both libraries and a CLI tool for working with DBN files and streams. 17 | Python bindings for `dbn` are provided in the `databento_dbn` package. 18 | 19 | For more details, read our [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). 20 | 21 | ## Features 22 | 23 | - Performant binary encoding and decoding 24 | - Highly compressible with Zstandard 25 | - Extendable fixed-width schemas 26 | 27 | ## Usage 28 | 29 | See the respective READMEs for usage details: 30 | - [`dbn`](rust/dbn/README.md): Rust library crate 31 | - [`dbn-cli`](rust/dbn-cli/README.md): CLI crate providing a `dbn` binary 32 | - [`databento-dbn`](python/README.md): Python package 33 | 34 | ## License 35 | 36 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 37 | -------------------------------------------------------------------------------- /c/src/decode.rs: -------------------------------------------------------------------------------- 1 | // RawFd isn't defined for windows 2 | #![cfg(not(target_os = "windows"))] 3 | 4 | use std::{ 5 | fs::File, 6 | io::BufReader, 7 | os::fd::{FromRawFd, RawFd}, 8 | ptr::{null, null_mut}, 9 | }; 10 | 11 | use dbn::{ 12 | decode::{DbnMetadata, DecodeRecordRef, DynDecoder}, 13 | Compression, Metadata, Record, RecordHeader, VersionUpgradePolicy, 14 | }; 15 | 16 | pub type Decoder = DynDecoder<'static, BufReader>; 17 | 18 | /// Creates a DBN decoder. Returns null in case of error. 19 | /// 20 | /// # Safety 21 | /// `file` must be a valid file descriptor. This function assumes ownership of `file`. 22 | #[no_mangle] 23 | pub unsafe extern "C" fn DbnDecoder_create(file: RawFd, compression: Compression) -> *mut Decoder { 24 | let decoder = match DynDecoder::new( 25 | File::from_raw_fd(file), 26 | compression, 27 | VersionUpgradePolicy::AsIs, 28 | ) { 29 | Ok(d) => d, 30 | Err(_) => { 31 | return null_mut(); 32 | } 33 | }; 34 | Box::into_raw(Box::new(decoder)) 35 | } 36 | 37 | /// Returns a pointer to the decoded DBN metadata. 38 | /// 39 | /// # Safety 40 | /// Verifies `decoder` is not null. 41 | #[no_mangle] 42 | pub unsafe extern "C" fn DbnDecoder_metadata(decoder: *mut Decoder) -> *const Metadata { 43 | if let Some(metadata) = decoder.as_mut().map(|d| d.metadata()) { 44 | metadata 45 | } else { 46 | null() 47 | } 48 | } 49 | 50 | /// Decodes and returns a pointer to the next record. 51 | /// 52 | /// # Safety 53 | /// Verifies `decoder` is not null. 54 | #[no_mangle] 55 | pub unsafe extern "C" fn DbnDecoder_decode(decoder: *mut Decoder) -> *const RecordHeader { 56 | if let Some(Ok(Some(rec))) = decoder.as_mut().map(|d| d.decode_record_ref()) { 57 | rec.header() 58 | } else { 59 | null() 60 | } 61 | } 62 | 63 | /// Frees memory associated with the DBN decoder. 64 | /// 65 | /// # Safety 66 | /// Verifies `decoder` is not null. 67 | #[no_mangle] 68 | pub unsafe extern "C" fn DbnDecoder_free(decoder: *mut Decoder) { 69 | if let Some(decoder) = decoder.as_mut() { 70 | drop(Box::from_raw(decoder)); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /c/src/metadata.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | ffi::{c_char, CStr}, 3 | io, slice, 4 | }; 5 | 6 | use dbn::{ 7 | encode::dbn::MetadataEncoder, 8 | enums::{SType, Schema}, 9 | MetadataBuilder, 10 | }; 11 | 12 | /// The byte offset of the `start` field in DBN-encoded Metadata. 13 | pub const METADATA_START_OFFSET: usize = 26; 14 | /// The minimum buffer size in bytes for encoding DBN Metadata. 15 | pub const METADATA_MIN_ENCODED_SIZE: usize = 128; 16 | 17 | /// Encodes DBN metadata to the given buffer. Returns the number of bytes written. 18 | /// 19 | /// # Errors 20 | /// - Returns -1 if `buffer` is null. 21 | /// - Returns -2 if `dataset` cannot be parsed. 22 | /// - Returns -3 if the metadata cannot be encoded. 23 | /// - Returns -4 if the version is invalid. 24 | /// 25 | /// # Safety 26 | /// This function assumes `dataset` is a valid pointer and `buffer` is of size 27 | /// `length`. 28 | #[no_mangle] 29 | pub unsafe extern "C" fn encode_metadata( 30 | buffer: *mut c_char, 31 | length: libc::size_t, 32 | version: u8, 33 | dataset: *const c_char, 34 | schema: Schema, 35 | start: u64, 36 | ) -> libc::c_int { 37 | let buffer = if let Some(buffer) = (buffer as *mut u8).as_mut() { 38 | slice::from_raw_parts_mut(buffer, length) 39 | } else { 40 | return -1; 41 | }; 42 | let dataset = match CStr::from_ptr(dataset).to_str() { 43 | Ok(dataset) => dataset.to_owned(), 44 | Err(_) => { 45 | return -2; 46 | } 47 | }; 48 | if version == 0 || version > dbn::DBN_VERSION { 49 | return -4; 50 | } 51 | let metadata = MetadataBuilder::new() 52 | .version(version) 53 | .dataset(dataset) 54 | .start(start) 55 | .stype_in(Some(SType::InstrumentId)) 56 | .stype_out(SType::InstrumentId) 57 | .schema(Some(schema)) 58 | .build(); 59 | let mut cursor = io::Cursor::new(buffer); 60 | match MetadataEncoder::new(&mut cursor).encode(&metadata) { 61 | Ok(()) => cursor.position() as i32, 62 | Err(_) => -3, 63 | } 64 | } 65 | 66 | #[cfg(test)] 67 | mod tests { 68 | use super::*; 69 | 70 | // cbindgen doesn't support constants defined with expressions, so we test the equality here 71 | #[test] 72 | fn const_checks() { 73 | assert_eq!( 74 | METADATA_START_OFFSET, 75 | MetadataEncoder::>::START_OFFSET 76 | ); 77 | assert_eq!( 78 | METADATA_MIN_ENCODED_SIZE, 79 | MetadataEncoder::>::MIN_ENCODED_SIZE 80 | ); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /python/src/enums.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use pyo3::{ffi::c_str, prelude::*, types::PyDict, Python}; 4 | use rstest::*; 5 | 6 | use crate::tests::python; 7 | 8 | #[rstest] 9 | #[case("Compression")] 10 | #[case("Encoding")] 11 | #[case("Schema")] 12 | #[case("SType")] 13 | fn test_enum_name_coercion(_python: (), #[case] enum_name: &str) { 14 | Python::attach(|py| { 15 | let globals = PyDict::new(py); 16 | globals.set_item("enum_name", enum_name).unwrap(); 17 | Python::run( 18 | py, 19 | c_str!( 20 | r#"import _lib as db 21 | 22 | enum_type = getattr(db, enum_name) 23 | for variant in enum_type.variants(): 24 | assert variant == enum_type(variant.name) 25 | assert variant == enum_type(variant.name.replace('_', '-')) 26 | assert variant == enum_type(variant.name.lower()) 27 | assert variant == enum_type(variant.name.upper()) 28 | try: 29 | enum_type("bar") # sanity check 30 | assert False, "did not raise an exception" 31 | except db.DBNError: 32 | pass"# 33 | ), 34 | Some(&globals), 35 | None, 36 | ) 37 | .unwrap(); 38 | }); 39 | } 40 | 41 | #[rstest] 42 | fn test_compression_none_coercible(_python: ()) { 43 | Python::attach(|py| { 44 | py.run( 45 | c_str!( 46 | r#"import _lib as db 47 | 48 | assert db.Compression(None) == db.Compression.NONE 49 | "# 50 | ), 51 | // Create an empty `globals` dict to keep tests hermetic 52 | Some(&PyDict::new(py)), 53 | None, 54 | ) 55 | .unwrap() 56 | }); 57 | } 58 | 59 | #[rstest] 60 | #[case("Encoding")] 61 | #[case("Schema")] 62 | #[case("SType")] 63 | fn test_enum_none_not_coercible(_python: (), #[case] enum_name: &str) { 64 | Python::attach(|py| { 65 | let globals = PyDict::new(py); 66 | globals.set_item("enum_name", enum_name).unwrap(); 67 | Python::run( 68 | py, 69 | c_str!( 70 | r#"import _lib as db 71 | 72 | enum_type = getattr(db, enum_name) 73 | try: 74 | enum_type(None) 75 | assert False, "did not raise an exception" 76 | except db.DBNError: 77 | pass"# 78 | ), 79 | Some(&globals), 80 | None, 81 | ) 82 | .unwrap(); 83 | }); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /rust/dbn/src/decode/stream.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | 3 | use fallible_streaming_iterator::FallibleStreamingIterator; 4 | 5 | use super::{DbnMetadata, DecodeStream}; 6 | use crate::{Error, HasRType, Result}; 7 | 8 | /// A consuming iterator wrapping a [`DecodeRecord`](super::DecodeRecord). Lazily 9 | /// decodes the contents of the file or other input stream. 10 | /// 11 | /// Implements [`FallibleStreamingIterator`]. 12 | pub struct StreamIterDecoder { 13 | /// The underlying decoder implementation. 14 | decoder: D, 15 | /// Number of element sthat have been decoded. Used for [`Iterator::size_hint()`]. 16 | /// `None` indicates the end of the stream has been reached. 17 | i: Option, 18 | /// Required to associate this type with a specific record type `T`. 19 | _marker: PhantomData, 20 | } 21 | 22 | impl StreamIterDecoder 23 | where 24 | T: HasRType, 25 | { 26 | /// Creates a new streaming decoder using the given `decoder`. 27 | pub fn new(decoder: D) -> Self { 28 | Self { 29 | decoder, 30 | i: Some(0), 31 | _marker: PhantomData, 32 | } 33 | } 34 | 35 | /// Returns an immutable reference to the inner decoder. 36 | pub fn get_ref(&self) -> &D { 37 | &self.decoder 38 | } 39 | } 40 | 41 | impl FallibleStreamingIterator for StreamIterDecoder 42 | where 43 | D: DecodeStream, 44 | T: HasRType, 45 | { 46 | type Error = Error; 47 | type Item = T; 48 | 49 | fn advance(&mut self) -> Result<()> { 50 | if let Some(i) = self.i.as_mut() { 51 | match self.decoder.decode_record::() { 52 | Ok(Some(_)) => { 53 | *i += 1; 54 | Ok(()) 55 | } 56 | Ok(None) => { 57 | // set error state sentinel 58 | self.i = None; 59 | Ok(()) 60 | } 61 | Err(err) => { 62 | // set error state sentinel 63 | self.i = None; 64 | Err(err) 65 | } 66 | } 67 | } else { 68 | Ok(()) 69 | } 70 | } 71 | 72 | fn get(&self) -> Option<&Self::Item> { 73 | if self.i.is_some() { 74 | self.decoder 75 | .last_record() 76 | // SAFETY: Validated record type in `advance` with call to `decode_record`. 77 | .map(|r| unsafe { r.get_unchecked() }) 78 | } else { 79 | None 80 | } 81 | } 82 | } 83 | 84 | impl DbnMetadata for StreamIterDecoder 85 | where 86 | D: DbnMetadata, 87 | { 88 | fn metadata(&self) -> &crate::Metadata { 89 | self.decoder.metadata() 90 | } 91 | 92 | fn metadata_mut(&mut self) -> &mut crate::Metadata { 93 | self.decoder.metadata_mut() 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /rust/dbn/src/v2/impl_default.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::c_char; 2 | 3 | use crate::{ 4 | rtype, v2, MatchAlgorithm, RecordHeader, SecurityUpdateAction, UNDEF_PRICE, UNDEF_TIMESTAMP, 5 | }; 6 | 7 | use super::InstrumentDefMsg; 8 | 9 | impl Default for InstrumentDefMsg { 10 | fn default() -> Self { 11 | Self { 12 | hd: RecordHeader::default::(rtype::INSTRUMENT_DEF), 13 | ts_recv: UNDEF_TIMESTAMP, 14 | min_price_increment: UNDEF_PRICE, 15 | display_factor: UNDEF_PRICE, 16 | expiration: UNDEF_TIMESTAMP, 17 | activation: UNDEF_TIMESTAMP, 18 | high_limit_price: UNDEF_PRICE, 19 | low_limit_price: UNDEF_PRICE, 20 | max_price_variation: UNDEF_PRICE, 21 | trading_reference_price: UNDEF_PRICE, 22 | unit_of_measure_qty: UNDEF_PRICE, 23 | min_price_increment_amount: UNDEF_PRICE, 24 | price_ratio: UNDEF_PRICE, 25 | strike_price: UNDEF_PRICE, 26 | inst_attrib_value: i32::MAX, 27 | underlying_id: 0, 28 | raw_instrument_id: 0, 29 | market_depth_implied: i32::MAX, 30 | market_depth: i32::MAX, 31 | market_segment_id: u32::MAX, 32 | max_trade_vol: u32::MAX, 33 | min_lot_size: i32::MAX, 34 | min_lot_size_block: i32::MAX, 35 | min_lot_size_round_lot: i32::MAX, 36 | min_trade_vol: u32::MAX, 37 | contract_multiplier: i32::MAX, 38 | decay_quantity: i32::MAX, 39 | original_contract_size: i32::MAX, 40 | trading_reference_date: u16::MAX, 41 | appl_id: i16::MAX, 42 | maturity_year: u16::MAX, 43 | decay_start_date: u16::MAX, 44 | channel_id: u16::MAX, 45 | currency: [0; 4], 46 | settl_currency: [0; 4], 47 | secsubtype: [0; 6], 48 | raw_symbol: [0; v2::SYMBOL_CSTR_LEN], 49 | group: [0; 21], 50 | exchange: [0; 5], 51 | asset: [0; v2::ASSET_CSTR_LEN], 52 | cfi: [0; 7], 53 | security_type: [0; 7], 54 | unit_of_measure: [0; 31], 55 | underlying: [0; 21], 56 | strike_price_currency: [0; 4], 57 | instrument_class: 0, 58 | match_algorithm: MatchAlgorithm::default() as c_char, 59 | md_security_trading_status: u8::MAX, 60 | main_fraction: u8::MAX, 61 | price_display_format: u8::MAX, 62 | settl_price_type: u8::MAX, 63 | sub_fraction: u8::MAX, 64 | underlying_product: u8::MAX, 65 | security_update_action: SecurityUpdateAction::default() as c_char, 66 | maturity_month: u8::MAX, 67 | maturity_day: u8::MAX, 68 | maturity_week: u8::MAX, 69 | user_defined_instrument: Default::default(), 70 | contract_multiplier_unit: i8::MAX, 71 | flow_schedule_type: i8::MAX, 72 | tick_rule: u8::MAX, 73 | _reserved: Default::default(), 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/debug.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::TokenStream; 2 | use quote::quote; 3 | use syn::{parse_macro_input, Field, ItemStruct}; 4 | 5 | use crate::{ 6 | dbn_attr::{ 7 | find_dbn_debug_attr, is_hidden, C_CHAR_ATTR, FIXED_PRICE_ATTR, FMT_BINARY, FMT_METHOD, 8 | }, 9 | utils::crate_name, 10 | }; 11 | 12 | pub fn record_debug_impl(input_struct: &ItemStruct) -> TokenStream { 13 | let record_type = &input_struct.ident; 14 | let field_iter = input_struct 15 | .fields 16 | .iter() 17 | .map(|f| format_field(f).unwrap_or_else(|e| e.into_compile_error())); 18 | quote! { 19 | impl ::std::fmt::Debug for #record_type { 20 | fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { 21 | let mut debug_struct = f.debug_struct(stringify!(#record_type)); 22 | #(#field_iter)* 23 | debug_struct.finish() 24 | } 25 | } 26 | } 27 | } 28 | 29 | pub fn derive_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 30 | // let DeriveInput { ident, data, .. } = parse_macro_input!(input as DeriveInput); 31 | let input_struct = parse_macro_input!(input as ItemStruct); 32 | let record_type = &input_struct.ident; 33 | let field_iter = input_struct 34 | .fields 35 | .iter() 36 | .map(|f| format_field(f).unwrap_or_else(|e| e.into_compile_error())); 37 | quote! { 38 | impl ::std::fmt::Debug for #record_type { 39 | fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { 40 | let mut debug_struct = f.debug_struct(stringify!(#record_type)); 41 | #(#field_iter)* 42 | debug_struct.finish() 43 | } 44 | } 45 | } 46 | .into() 47 | } 48 | 49 | fn format_field(field: &Field) -> syn::Result { 50 | let ident = field.ident.as_ref().unwrap(); 51 | if is_hidden(field) { 52 | return Ok(quote!()); 53 | } 54 | Ok(match find_dbn_debug_attr(field)? { 55 | Some(id) if id == C_CHAR_ATTR => { 56 | quote! { debug_struct.field(stringify!(#ident), &(self.#ident as u8 as char)); } 57 | } 58 | Some(id) if id == FIXED_PRICE_ATTR => { 59 | let crate_name = crate_name(); 60 | quote! { debug_struct.field(stringify!(#ident), &#crate_name::pretty::Px(self.#ident)); } 61 | } 62 | Some(id) if id == FMT_BINARY => { 63 | // format as `0b00101010` 64 | quote! { debug_struct.field(stringify!(#ident), &format_args!("{:#010b}", &self.#ident)); } 65 | } 66 | Some(id) if id == FMT_METHOD => { 67 | // Try to use method to format, otherwise fallback on raw value 68 | return Ok(quote! { 69 | match self.#ident() { 70 | Ok(s) => debug_struct.field(stringify!(#ident), &s), 71 | Err(_) => debug_struct.field(stringify!(#ident), &self.#ident), 72 | }; 73 | }); 74 | } 75 | _ => quote! { debug_struct.field(stringify!(#ident), &self.#ident); }, 76 | }) 77 | } 78 | -------------------------------------------------------------------------------- /rust/dbn/src/record/traits.rs: -------------------------------------------------------------------------------- 1 | use super::ts_to_dt; 2 | use crate::{Publisher, RType, RecordHeader}; 3 | 4 | /// Used for polymorphism around types all beginning with a [`RecordHeader`] where 5 | /// `rtype` is the discriminant used for indicating the type of record. 6 | /// 7 | /// All record types are plain old data held in sequential memory, and therefore 8 | /// implement `AsRef<[u8]>` for simple serialization to bytes. 9 | /// 10 | /// [`RecordRef`](crate::RecordRef) acts similar to a `&dyn Record`. 11 | pub trait Record: AsRef<[u8]> { 12 | /// Returns a reference to the `RecordHeader` that comes at the beginning of all 13 | /// record types. 14 | fn header(&self) -> &RecordHeader; 15 | 16 | /// Returns the size of the record in bytes. 17 | fn record_size(&self) -> usize { 18 | self.header().record_size() 19 | } 20 | 21 | /// Tries to convert the raw record type into an enum which is useful for exhaustive 22 | /// pattern matching. 23 | /// 24 | /// # Errors 25 | /// This function returns an error if the `rtype` field does not 26 | /// contain a valid, known [`RType`]. 27 | fn rtype(&self) -> crate::Result { 28 | self.header().rtype() 29 | } 30 | 31 | /// Tries to convert the raw `publisher_id` into an enum which is useful for 32 | /// exhaustive pattern matching. 33 | /// 34 | /// # Errors 35 | /// This function returns an error if the `publisher_id` does not correspond with 36 | /// any known [`Publisher`]. 37 | fn publisher(&self) -> crate::Result { 38 | self.header().publisher() 39 | } 40 | 41 | /// Returns the raw primary timestamp for the record. 42 | /// 43 | /// This timestamp should be used for sorting records as well as indexing into any 44 | /// symbology data structure. 45 | fn raw_index_ts(&self) -> u64 { 46 | self.header().ts_event 47 | } 48 | 49 | /// Returns the primary timestamp for the record. Returns `None` if the primary 50 | /// timestamp contains the sentinel value for a null timestamp. 51 | /// 52 | /// This timestamp should be used for sorting records as well as indexing into any 53 | /// symbology data structure. 54 | fn index_ts(&self) -> Option { 55 | ts_to_dt(self.raw_index_ts()) 56 | } 57 | 58 | /// Returns the primary date for the record; the date component of the primary 59 | /// timestamp (`index_ts()`). Returns `None` if the primary timestamp contains the 60 | /// sentinel value for a null timestamp. 61 | fn index_date(&self) -> Option { 62 | self.index_ts().map(|dt| dt.date()) 63 | } 64 | } 65 | 66 | /// Used for polymorphism around mutable types beginning with a [`RecordHeader`]. 67 | pub trait RecordMut { 68 | /// Returns a mutable reference to the `RecordHeader` that comes at the beginning of 69 | /// all record types. 70 | fn header_mut(&mut self) -> &mut RecordHeader; 71 | } 72 | 73 | /// An extension of the [`Record`] trait for types with a static [`RType`]. Used for 74 | /// determining if a rtype matches a type. 75 | /// 76 | /// Because of the static function requirement, this trait is implemented by all concrete record 77 | /// types like [`MboMsg`](crate::MboMsg), but not by [`RecordRef`](crate::RecordRef), which can reference a record of 78 | /// dynamic type. 79 | /// 80 | /// While not _dyn compatible_, [`RecordRef`](crate::RecordRef) acts like a `&dyn HasRType`. 81 | pub trait HasRType: Record + RecordMut { 82 | /// Returns `true` if `rtype` matches the value associated with the implementing type. 83 | fn has_rtype(rtype: u8) -> bool; 84 | } 85 | -------------------------------------------------------------------------------- /rust/dbn-cli/src/filter.rs: -------------------------------------------------------------------------------- 1 | use std::num::NonZeroU64; 2 | 3 | use dbn::{ 4 | decode::{DbnMetadata, DecodeRecordRef}, 5 | RType, Record, RecordRef, Schema, 6 | }; 7 | 8 | #[derive(Debug)] 9 | pub struct SchemaFilter { 10 | decoder: D, 11 | rtype: Option, 12 | } 13 | 14 | impl SchemaFilter 15 | where 16 | D: DbnMetadata, 17 | { 18 | pub fn new(mut decoder: D, schema: Option) -> Self { 19 | if let Some(schema) = schema { 20 | decoder.metadata_mut().schema = Some(schema); 21 | } 22 | Self::new_no_metadata(decoder, schema) 23 | } 24 | } 25 | 26 | impl SchemaFilter { 27 | pub fn new_no_metadata(decoder: D, schema: Option) -> Self { 28 | Self { 29 | decoder, 30 | rtype: schema.map(RType::from), 31 | } 32 | } 33 | } 34 | 35 | impl DbnMetadata for SchemaFilter { 36 | fn metadata(&self) -> &dbn::Metadata { 37 | self.decoder.metadata() 38 | } 39 | 40 | fn metadata_mut(&mut self) -> &mut dbn::Metadata { 41 | self.decoder.metadata_mut() 42 | } 43 | } 44 | 45 | impl DecodeRecordRef for SchemaFilter { 46 | fn decode_record_ref(&mut self) -> dbn::Result>> { 47 | while let Some(record) = self.decoder.decode_record_ref()? { 48 | if self 49 | .rtype 50 | .map(|rtype| rtype as u8 == record.header().rtype) 51 | .unwrap_or(true) 52 | { 53 | // Safe: casting reference to pointer so the pointer will always be valid. 54 | // Getting around borrow checker limitation. 55 | return Ok(Some(unsafe { 56 | RecordRef::unchecked_from_header(record.header()) 57 | })); 58 | } 59 | } 60 | Ok(None) 61 | } 62 | } 63 | 64 | #[derive(Debug)] 65 | pub struct LimitFilter { 66 | decoder: D, 67 | limit: Option, 68 | record_count: u64, 69 | } 70 | 71 | impl LimitFilter 72 | where 73 | D: DbnMetadata, 74 | { 75 | pub fn new(mut decoder: D, limit: Option) -> Self { 76 | if let Some(limit) = limit { 77 | let metadata_limit = &mut decoder.metadata_mut().limit; 78 | if let Some(metadata_limit) = metadata_limit { 79 | *metadata_limit = (*metadata_limit).min(limit); 80 | } else { 81 | *metadata_limit = Some(limit); 82 | } 83 | } 84 | Self::new_no_metadata(decoder, limit) 85 | } 86 | } 87 | 88 | impl LimitFilter { 89 | pub fn new_no_metadata(decoder: D, limit: Option) -> Self { 90 | Self { 91 | decoder, 92 | limit, 93 | record_count: 0, 94 | } 95 | } 96 | } 97 | 98 | impl DbnMetadata for LimitFilter { 99 | fn metadata(&self) -> &dbn::Metadata { 100 | self.decoder.metadata() 101 | } 102 | 103 | fn metadata_mut(&mut self) -> &mut dbn::Metadata { 104 | self.decoder.metadata_mut() 105 | } 106 | } 107 | 108 | impl DecodeRecordRef for LimitFilter { 109 | fn decode_record_ref(&mut self) -> dbn::Result>> { 110 | if self 111 | .limit 112 | .map(|limit| self.record_count >= limit.get()) 113 | .unwrap_or(false) 114 | { 115 | return Ok(None); 116 | } 117 | Ok(self.decoder.decode_record_ref()?.inspect(|_| { 118 | self.record_count += 1; 119 | })) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /rust/dbn-cli/README.md: -------------------------------------------------------------------------------- 1 | # dbn-cli 2 | 3 | [![build](https://github.com/databento/dbn/actions/workflows/build.yaml/badge.svg)](https://github.com/databento/dbn/actions/workflows/build.yaml) 4 | ![license](https://img.shields.io/github/license/databento/dbn?color=blue) 5 | [![Current Crates.io Version](https://img.shields.io/crates/v/dbn-cli.svg)](https://crates.io/crates/dbn-cli) 6 | 7 | This crate provides a CLI tool `dbn` for converting [Databento](https://databento.com) 8 | Binary Encoding (DBN) files to text formats, as well as updating legacy DBZ files to 9 | DBN. 10 | 11 | For more information about DBN, read our [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). 12 | 13 | ## Installation 14 | 15 | To install the latest version, run the following command: 16 | ```sh 17 | cargo install dbn-cli 18 | ``` 19 | 20 | ## Usage 21 | 22 | `dbn` currently supports CSV and [JSON lines](https://jsonlines.org/) as output formats. 23 | By default, `dbn` outputs the result to standard output for ease of use with 24 | text-based command-line utilities. 25 | Running 26 | ```sh 27 | dbn some.dbn --csv --limit 5 28 | ``` 29 | will print the header row and the first 5 data rows in `some.dbn` in CSV format to the console. 30 | Similarly, running 31 | ```sh 32 | dbn ohlcv-1d.dbn.zst --json | jq '.high' 33 | ``` 34 | Will extract only the high prices from `ohlcv-1d.dbn.zst`. 35 | `dbn` works with both uncompressed and Zstandard-compressed DBN files. 36 | 37 | You can also save the results directly to another file by running 38 | ```sh 39 | dbn some.dbn.zst --json --output some.json 40 | ``` 41 | `dbn` will create a new file `some.csv` with the data from `some.dbn.zst` 42 | formatted as JSON. 43 | 44 | When the file name passed `--output` or `-o` ends in `.json` or `.csv`, you 45 | can omit the `--json` and `--csv` flags. 46 | ```sh 47 | dbn another.dbn.zst -o data.csv 48 | ``` 49 | This writes the contents of `another.dbn.zst` to `data.json` in CSV format. 50 | 51 | By default, `dbn` will not overwrite an existing file. 52 | To replace the contents of an existing file and allow overwriting files, pass 53 | the `-f` or `--force` flag. 54 | 55 | ### Merging DBN files 56 | You can use `dbn` to merge several DBN files into one, including files of different schemas. 57 | ```sh 58 | # Merge files split by symbols 59 | dbn aapl.trades.dbn tsla.trades.dbn nvda.trades.dbn -o equities.trades.dbn 60 | # Or by date 61 | dbn equs-mini-20250331.dbn equs-mini-20250401.dbn equs-mini-20250402.dbn -o equs-mini-2025W14.dbn 62 | ``` 63 | The only limitation is they must be from the same dataset. 64 | 65 | ### Compressing the output 66 | In addition to reading Zstandard-compressed files, `dbn` can also write compressed JSON and CSV. 67 | 68 | ```sh 69 | dbn ohlcv-1d.dbn -o ohclv-1d.json.zst 70 | ``` 71 | 72 | or explicitly 73 | ``` 74 | dbn ohlcv-1d.dbn --json --zstd -o ohlcv-1d.json.zst 75 | ``` 76 | 77 | ### Converting DBZ files to DBN 78 | 79 | DBN is an evolution of DBZ, which required Zstandard. 80 | To update an old DBZ file to Zstandard-compressed DBN, run 81 | ```sh 82 | dbn 20221212.mbo.dbz -o 20221212.dbn.zst 83 | ``` 84 | or pass `--dbn` to set the output encoding explicitly. 85 | 86 | ### Reading and writing fragments 87 | `dbn` can also read and write DBN files without the metadata header, these are called "DBN fragments". 88 | Pass the `--input-fragment` or `--input-zstd-fragment` flag to read a DBN file without a metadata header. 89 | ``` 90 | dbn 120000_121000.mbo.dbn.frag --input-fragment --json 91 | ``` 92 | You can also write DBN files without the metadata header with the `--fragment` or `-F` flag. 93 | ```sh 94 | dbn ohlcv-1d.dbn -F -o ohlcv-1d.dbn.frag 95 | ``` 96 | 97 | ## License 98 | 99 | Distributed under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). 100 | -------------------------------------------------------------------------------- /rust/dbn/src/record/record_methods_tests.rs: -------------------------------------------------------------------------------- 1 | #![cfg(test)] 2 | 3 | use crate::{SType, StatType}; 4 | 5 | use super::*; 6 | 7 | #[test] 8 | fn invalid_rtype_error() { 9 | let header = RecordHeader::new::(0xE, 1, 2, 3); 10 | assert_eq!( 11 | header.rtype().unwrap_err().to_string(), 12 | "couldn't convert 0x0E to dbn::enums::RType" 13 | ); 14 | } 15 | 16 | #[test] 17 | fn debug_mbo() { 18 | let rec = MboMsg { 19 | hd: RecordHeader::new::( 20 | rtype::MBO, 21 | Publisher::OpraPillarXcbo as u16, 22 | 678, 23 | 1704468548242628731, 24 | ), 25 | flags: FlagSet::empty().set_last().set_bad_ts_recv(), 26 | price: 4_500_500_000_000, 27 | side: b'B' as c_char, 28 | action: b'A' as c_char, 29 | ..Default::default() 30 | }; 31 | assert_eq!( 32 | format!("{rec:?}"), 33 | "MboMsg { hd: RecordHeader { length: 14, rtype: Mbo, publisher_id: OpraPillarXcbo, \ 34 | instrument_id: 678, ts_event: 1704468548242628731 }, order_id: 0, \ 35 | price: 4500.500000000, size: 4294967295, flags: LAST | BAD_TS_RECV (136), channel_id: 255, \ 36 | action: 'A', side: 'B', ts_recv: 18446744073709551615, ts_in_delta: 0, sequence: 0 }" 37 | ); 38 | } 39 | 40 | #[test] 41 | fn debug_stats() { 42 | let rec = StatMsg { 43 | stat_type: StatType::OpenInterest as u16, 44 | update_action: StatUpdateAction::New as u8, 45 | quantity: 5, 46 | stat_flags: 0b00000010, 47 | ..Default::default() 48 | }; 49 | assert_eq!( 50 | format!("{rec:?}"), 51 | "StatMsg { hd: RecordHeader { length: 20, rtype: Statistics, publisher_id: 0, \ 52 | instrument_id: 0, ts_event: 18446744073709551615 }, ts_recv: 18446744073709551615, \ 53 | ts_ref: 18446744073709551615, price: UNDEF_PRICE, quantity: 5, sequence: 0, ts_in_delta: 0, \ 54 | stat_type: OpenInterest, channel_id: 65535, update_action: New, stat_flags: 0b00000010 }" 55 | ); 56 | } 57 | 58 | #[test] 59 | fn debug_instrument_err() { 60 | let rec = ErrorMsg { 61 | err: str_to_c_chars("Missing stype_in").unwrap(), 62 | ..Default::default() 63 | }; 64 | assert_eq!( 65 | format!("{rec:?}"), 66 | "ErrorMsg { hd: RecordHeader { length: 80, rtype: Error, publisher_id: 0, \ 67 | instrument_id: 0, ts_event: 18446744073709551615 }, err: \"Missing stype_in\", code: Unset, is_last: 255 }" 68 | ); 69 | } 70 | 71 | #[test] 72 | fn debug_instrument_sys() { 73 | let rec = SystemMsg::heartbeat(123); 74 | assert_eq!( 75 | format!("{rec:?}"), 76 | "SystemMsg { hd: RecordHeader { length: 80, rtype: System, publisher_id: 0, \ 77 | instrument_id: 0, ts_event: 123 }, msg: \"Heartbeat\", code: Heartbeat }" 78 | ); 79 | } 80 | 81 | #[test] 82 | fn debug_instrument_symbol_mapping() { 83 | let rec = SymbolMappingMsg { 84 | hd: RecordHeader::new::( 85 | rtype::SYMBOL_MAPPING, 86 | 0, 87 | 5602, 88 | 1704466940331347283, 89 | ), 90 | stype_in: SType::RawSymbol as u8, 91 | stype_in_symbol: str_to_c_chars("ESM4").unwrap(), 92 | stype_out: SType::RawSymbol as u8, 93 | stype_out_symbol: str_to_c_chars("ESM4").unwrap(), 94 | ..Default::default() 95 | }; 96 | assert_eq!( 97 | format!("{rec:?}"), 98 | "SymbolMappingMsg { hd: RecordHeader { length: 44, rtype: SymbolMapping, publisher_id: 0, instrument_id: 5602, ts_event: 1704466940331347283 }, stype_in: RawSymbol, stype_in_symbol: \"ESM4\", stype_out: RawSymbol, stype_out_symbol: \"ESM4\", start_ts: 18446744073709551615, end_ts: 18446744073709551615 }" 99 | ); 100 | } 101 | -------------------------------------------------------------------------------- /rust/dbn/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Types for errors that can occur while working with DBN. 2 | use thiserror::Error; 3 | 4 | /// An error that can occur while processing DBN data. 5 | #[derive(Debug, Error)] 6 | #[non_exhaustive] 7 | pub enum Error { 8 | /// An I/O error while reading or writing DBN or another encoding. 9 | #[error("IO error: {source:?} while {context}")] 10 | Io { 11 | /// The original error. 12 | #[source] 13 | source: std::io::Error, 14 | /// The context in which the error occurred. 15 | context: String, 16 | }, 17 | /// An error while decoding from DBN. 18 | #[error("decoding error: {0}")] 19 | Decode(String), 20 | /// An error with text encoding. 21 | #[error("encoding error: {0}")] 22 | Encode(String), 23 | /// An conversion error between types or encodings. 24 | #[error("couldn't convert {input} to {desired_type}")] 25 | Conversion { 26 | /// The input to the conversion. 27 | input: String, 28 | /// The desired type or encoding. 29 | desired_type: &'static str, 30 | }, 31 | /// An error with conversion of bytes to UTF-8. 32 | #[error("UTF-8 error: {source:?} while {context}")] 33 | Utf8 { 34 | /// The original error. 35 | #[source] 36 | source: std::str::Utf8Error, 37 | /// The context in which the error occurred. 38 | context: String, 39 | }, 40 | /// An invalid argument was passed to a function. 41 | #[error("bad argument {param_name}: {desc}")] 42 | BadArgument { 43 | /// The name of the parameter to which the bad argument was passed. 44 | param_name: String, 45 | /// The description of why the argument was invalid. 46 | desc: String, 47 | }, 48 | } 49 | /// An alias for a `Result` with [`dbn::Error`](crate::Error) as the error type. 50 | pub type Result = std::result::Result; 51 | 52 | impl From for Error { 53 | fn from(value: csv::Error) -> Self { 54 | match value.into_kind() { 55 | csv::ErrorKind::Io(io) => Self::io(io, "while writing CSV"), 56 | csv::ErrorKind::Utf8 { pos, err } => { 57 | Self::Encode(format!("UTF-8 error {err:?}{}", Self::opt_pos(&pos))) 58 | } 59 | csv::ErrorKind::UnequalLengths { 60 | pos, 61 | expected_len, 62 | len, 63 | } => Self::Encode(format!( 64 | "unequal CSV row lengths{}: expected {expected_len}, found {len}", 65 | Self::opt_pos(&pos) 66 | )), 67 | e => Self::Encode(format!("{e:?}")), 68 | } 69 | } 70 | } 71 | 72 | impl Error { 73 | /// Creates a new I/O [`dbn::Error`](crate::Error). 74 | pub fn io(error: std::io::Error, context: impl ToString) -> Self { 75 | Self::Io { 76 | source: error, 77 | context: context.to_string(), 78 | } 79 | } 80 | 81 | /// Creates a new decode [`dbn::Error`](crate::Error). 82 | pub fn decode(msg: impl ToString) -> Self { 83 | Self::Decode(msg.to_string()) 84 | } 85 | 86 | /// Creates a new encode [`dbn::Error`](crate::Error). 87 | pub fn encode(msg: impl ToString) -> Self { 88 | Self::Encode(msg.to_string()) 89 | } 90 | 91 | /// Creates a new conversion [`dbn::Error`](crate::Error) where `desired_type` is `T`. 92 | pub fn conversion(input: impl ToString) -> Self { 93 | Self::Conversion { 94 | input: input.to_string(), 95 | desired_type: std::any::type_name::(), 96 | } 97 | } 98 | 99 | /// Creates a new UTF-8 [`dbn::Error`](crate::Error). 100 | pub fn utf8(error: std::str::Utf8Error, context: impl ToString) -> Self { 101 | Self::Utf8 { 102 | source: error, 103 | context: context.to_string(), 104 | } 105 | } 106 | 107 | fn opt_pos(pos: &Option) -> String { 108 | if let Some(pos) = pos.as_ref() { 109 | format!(" at {pos:?}") 110 | } else { 111 | String::default() 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /rust/dbn/src/compat/traits.rs: -------------------------------------------------------------------------------- 1 | use crate::{HasRType, SecurityUpdateAction, StatType, StatUpdateAction}; 2 | 3 | /// A trait for compatibility between different versions of symbol mapping records. 4 | pub trait SymbolMappingRec: HasRType { 5 | /// Returns the input symbol as a `&str`. 6 | /// 7 | /// # Errors 8 | /// This function returns an error if `stype_in_symbol` contains invalid UTF-8. 9 | fn stype_in_symbol(&self) -> crate::Result<&str>; 10 | 11 | /// Returns the output symbol as a `&str`. 12 | /// 13 | /// # Errors 14 | /// This function returns an error if `stype_out_symbol` contains invalid UTF-8. 15 | fn stype_out_symbol(&self) -> crate::Result<&str>; 16 | 17 | /// Parses the raw start of the mapping interval into a datetime. Returns `None` if 18 | /// `start_ts` contains the sentinel for a null timestamp. 19 | fn start_ts(&self) -> Option; 20 | 21 | /// Parses the raw end of the mapping interval into a datetime. Returns `None` if 22 | /// `end_ts` contains the sentinel for a null timestamp. 23 | fn end_ts(&self) -> Option; 24 | } 25 | 26 | /// A trait for compatibility between different versions of definition records. 27 | pub trait InstrumentDefRec: HasRType { 28 | /// Returns the instrument raw symbol assigned by the publisher as a `&str`. 29 | /// 30 | /// # Errors 31 | /// This function returns an error if `raw_symbol` contains invalid UTF-8. 32 | fn raw_symbol(&self) -> crate::Result<&str>; 33 | 34 | /// Returns the underlying asset code (product code) of the instrument as a `&str`. 35 | /// 36 | /// # Errors 37 | /// This function returns an error if `asset` contains invalid UTF-8. 38 | fn asset(&self) -> crate::Result<&str>; 39 | 40 | /// Returns the [Security type](https://databento.com/docs/schemas-and-data-formats/instrument-definitions#security-type) 41 | /// of the instrument, e.g. FUT for future or future spread as a `&str`. 42 | /// 43 | /// # Errors 44 | /// This function returns an error if `security_type` contains invalid UTF-8. 45 | fn security_type(&self) -> crate::Result<&str>; 46 | 47 | /// Returns the action indicating whether the instrument definition has been added, 48 | /// modified, or deleted. 49 | /// 50 | /// # Errors 51 | /// This function returns an error if the `security_update_action` field does not 52 | /// contain a valid [`SecurityUpdateAction`]. 53 | fn security_update_action(&self) -> crate::Result; 54 | 55 | /// The channel ID assigned by Databento as an incrementing integer starting at 56 | /// zero. 57 | fn channel_id(&self) -> u16; 58 | } 59 | 60 | /// A trait for compatibility between different versions of statistics records. 61 | pub trait StatRec: HasRType { 62 | /// The sentinel value for a null `quantity`. 63 | const UNDEF_STAT_QUANTITY: i64; 64 | 65 | /// Tries to convert the raw type of the statistic value to an enum. 66 | /// 67 | /// # Errors 68 | /// This function returns an error if the `stat_type` field does not 69 | /// contain a valid [`StatType`]. 70 | fn stat_type(&self) -> crate::Result; 71 | 72 | /// Parses the raw capture-server-received timestamp into a datetime. Returns `None` 73 | /// if `ts_recv` contains the sentinel for a null timestamp. 74 | fn ts_recv(&self) -> Option; 75 | 76 | /// Parses the raw reference timestamp of the statistic value into a datetime. 77 | /// Returns `None` if `ts_ref` contains the sentinel for a null timestamp. 78 | fn ts_ref(&self) -> Option; 79 | 80 | /// Tries to convert the raw `update_action` to an enum. 81 | /// 82 | /// # Errors 83 | /// This function returns an error if the `update_action` field does not 84 | /// contain a valid [`StatUpdateAction`]. 85 | fn update_action(&self) -> crate::Result; 86 | 87 | /// The value for price statistics expressed as a signed integer where every 1 unit 88 | /// corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001. Will be 89 | /// [`UNDEF_PRICE`](crate::UNDEF_PRICE) when unused. 90 | fn price(&self) -> i64; 91 | 92 | /// The value for quantity statistics. Will be `UNDEF_STAT_QUANTITY` when unused. 93 | fn quantity(&self) -> i64; 94 | } 95 | -------------------------------------------------------------------------------- /rust/dbn-cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::File, 3 | io::{self, BufRead, BufReader}, 4 | path::Path, 5 | }; 6 | 7 | use anyhow::Context; 8 | use clap::Parser; 9 | use dbn::decode::{ 10 | DbnMetadata, DbnRecordDecoder, DecodeRecordRef, DynDecoder, MergeDecoder, MergeRecordDecoder, 11 | }; 12 | use dbn_cli::{ 13 | encode::{encode_from_dbn, encode_from_frag, silence_broken_pipe}, 14 | filter::{LimitFilter, SchemaFilter}, 15 | Args, 16 | }; 17 | 18 | const STDIN_SENTINEL: &str = "-"; 19 | 20 | fn open_input_file(path: &Path) -> anyhow::Result { 21 | File::open(path).with_context(|| format!("opening file to decode at path '{}'", path.display())) 22 | } 23 | 24 | fn wrap_frag(args: &Args, decoder: impl DecodeRecordRef) -> impl DecodeRecordRef { 25 | LimitFilter::new_no_metadata( 26 | SchemaFilter::new_no_metadata(decoder, args.schema_filter), 27 | args.limit, 28 | ) 29 | } 30 | 31 | /// assume no ts_out for fragments 32 | const FRAG_TS_OUT: bool = false; 33 | 34 | fn decode_frag(args: &Args, reader: impl io::Read) -> anyhow::Result { 35 | Ok(wrap_frag( 36 | args, 37 | DbnRecordDecoder::with_version( 38 | reader, 39 | args.input_version(), 40 | args.upgrade_policy(), 41 | FRAG_TS_OUT, 42 | )?, 43 | )) 44 | } 45 | 46 | fn wrap( 47 | args: &Args, 48 | decoder: impl DecodeRecordRef + DbnMetadata, 49 | ) -> impl DecodeRecordRef + DbnMetadata { 50 | LimitFilter::new(SchemaFilter::new(decoder, args.schema_filter), args.limit) 51 | } 52 | 53 | fn with_inputs(args: Args) -> anyhow::Result<()> { 54 | if args.is_input_fragment { 55 | let decoders = args 56 | .input 57 | .iter() 58 | .map(|input| { 59 | Ok(DbnRecordDecoder::with_version( 60 | BufReader::new(open_input_file(input)?), 61 | args.input_version(), 62 | args.upgrade_policy(), 63 | FRAG_TS_OUT, 64 | )?) 65 | }) 66 | .collect::>>>>()?; 67 | encode_from_frag(&args, MergeRecordDecoder::new(decoders)?) 68 | } else if args.is_input_zstd_fragment { 69 | let decoders = args 70 | .input 71 | .iter() 72 | .map(|input| { 73 | Ok(DbnRecordDecoder::with_version( 74 | zstd::stream::Decoder::new(open_input_file(input)?)?, 75 | args.input_version(), 76 | args.upgrade_policy(), 77 | FRAG_TS_OUT, 78 | )?) 79 | }) 80 | .collect::>>>>>()?; 81 | encode_from_frag(&args, MergeRecordDecoder::new(decoders)?) 82 | } else { 83 | let decoders = args 84 | .input 85 | .iter() 86 | .map(|input| DynDecoder::from_file(input, args.upgrade_policy())) 87 | .collect::>>>>()?; 88 | encode_from_dbn(&args, wrap(&args, MergeDecoder::new(decoders)?)) 89 | } 90 | } 91 | 92 | fn with_input(args: Args, reader: impl BufRead) -> anyhow::Result<()> { 93 | if args.is_input_fragment { 94 | encode_from_frag(&args, decode_frag(&args, reader)?) 95 | } else if args.is_input_zstd_fragment { 96 | encode_from_frag( 97 | &args, 98 | decode_frag(&args, zstd::stream::Decoder::with_buffer(reader)?)?, 99 | ) 100 | } else { 101 | encode_from_dbn( 102 | &args, 103 | wrap( 104 | &args, 105 | DynDecoder::inferred_with_buffer(reader, args.upgrade_policy())?, 106 | ), 107 | ) 108 | } 109 | } 110 | 111 | fn main() -> anyhow::Result<()> { 112 | let args = Args::parse(); 113 | if args.input.len() > 1 { 114 | with_inputs(args) 115 | } else if args.input[0].as_os_str() == STDIN_SENTINEL { 116 | with_input(args, io::stdin().lock()) 117 | } else { 118 | let reader = BufReader::new(open_input_file(&args.input[0])?); 119 | with_input(args, reader) 120 | } 121 | .or_else(silence_broken_pipe) 122 | } 123 | -------------------------------------------------------------------------------- /rust/dbn/src/v3.rs: -------------------------------------------------------------------------------- 1 | //! Record data types for encoding different Databento [`Schema`](crate::enums::Schema)s 2 | //! in the upcoming DBN version 3. 3 | 4 | pub use crate::compat::{ 5 | ASSET_CSTR_LEN_V3 as ASSET_CSTR_LEN, SYMBOL_CSTR_LEN_V3 as SYMBOL_CSTR_LEN, 6 | UNDEF_STAT_QUANTITY_V3 as UNDEF_STAT_QUANTITY, 7 | }; 8 | pub use crate::record::{ 9 | Bbo1MMsg, Bbo1SMsg, BboMsg, Cbbo1MMsg, Cbbo1SMsg, CbboMsg, Cmbp1Msg, ErrorMsg, ImbalanceMsg, 10 | InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, StatMsg, StatusMsg, SymbolMappingMsg, 11 | SystemMsg, TbboMsg, TcbboMsg, TradeMsg, WithTsOut, 12 | }; 13 | 14 | use crate::compat::{InstrumentDefRec, StatRec}; 15 | 16 | mod methods; 17 | 18 | /// The DBN version of this module. 19 | pub const DBN_VERSION: u8 = 3; 20 | 21 | impl InstrumentDefRec for InstrumentDefMsg { 22 | fn raw_symbol(&self) -> crate::Result<&str> { 23 | Self::raw_symbol(self) 24 | } 25 | 26 | fn asset(&self) -> crate::Result<&str> { 27 | Self::asset(self) 28 | } 29 | 30 | fn security_type(&self) -> crate::Result<&str> { 31 | Self::security_type(self) 32 | } 33 | 34 | fn security_update_action(&self) -> crate::Result { 35 | Self::security_update_action(self) 36 | } 37 | 38 | fn channel_id(&self) -> u16 { 39 | self.channel_id 40 | } 41 | } 42 | 43 | impl StatRec for StatMsg { 44 | const UNDEF_STAT_QUANTITY: i64 = UNDEF_STAT_QUANTITY; 45 | 46 | fn stat_type(&self) -> crate::Result { 47 | Self::stat_type(self) 48 | } 49 | 50 | fn ts_recv(&self) -> Option { 51 | Self::ts_recv(self) 52 | } 53 | 54 | fn ts_ref(&self) -> Option { 55 | Self::ts_ref(self) 56 | } 57 | 58 | fn update_action(&self) -> crate::Result { 59 | Self::update_action(self) 60 | } 61 | 62 | fn price(&self) -> i64 { 63 | self.price 64 | } 65 | 66 | fn quantity(&self) -> i64 { 67 | self.quantity 68 | } 69 | } 70 | 71 | #[cfg(test)] 72 | mod tests { 73 | use std::mem; 74 | 75 | use rstest::*; 76 | use type_layout::{Field, TypeLayout}; 77 | 78 | use super::*; 79 | 80 | #[cfg(feature = "python")] 81 | #[test] 82 | fn test_consistent_field_order_and_leg_fields_last() { 83 | use std::ops::Not; 84 | 85 | use crate::{python::PyFieldDesc, v2}; 86 | 87 | let v3_fields = InstrumentDefMsg::ordered_fields(""); 88 | let mut v2_fields = v2::InstrumentDefMsg::ordered_fields("") 89 | .into_iter() 90 | .filter(|f| { 91 | matches!( 92 | f.as_str(), 93 | "trading_reference_date" 94 | | "trading_reference_price" 95 | | "settl_price_type" 96 | | "md_security_trading_status" 97 | ) 98 | .not() 99 | }); 100 | let mut has_reached_leg_fields = false; 101 | for (i, field) in v3_fields.into_iter().enumerate() { 102 | if has_reached_leg_fields { 103 | assert!(field.starts_with("leg_"), "{i}"); 104 | } else if field.starts_with("leg_") { 105 | has_reached_leg_fields = true; 106 | assert!(v2_fields.next().is_none(), "{i}"); 107 | } else { 108 | assert_eq!(field, v2_fields.next().unwrap(), "{i}"); 109 | } 110 | } 111 | } 112 | 113 | #[rstest] 114 | #[case::definition(InstrumentDefMsg::default(), 520)] 115 | #[case::definition(StatMsg::default(), 80)] 116 | fn test_sizes(#[case] _rec: R, #[case] exp: usize) { 117 | assert_eq!(mem::size_of::(), exp); 118 | assert!(mem::size_of::() <= crate::MAX_RECORD_LEN); 119 | } 120 | 121 | #[rstest] 122 | #[case::definition(InstrumentDefMsg::default())] 123 | fn test_alignment_and_no_padding(#[case] _rec: R) { 124 | let layout = R::type_layout(); 125 | assert_eq!(layout.alignment, 8, "Unexpected alignment: {layout}"); 126 | for field in layout.fields.iter() { 127 | assert!( 128 | matches!(field, Field::Field { .. }), 129 | "Detected padding: {layout}" 130 | ); 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /rust/dbn/src/python/conversions.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::c_char; 2 | 3 | use pyo3::{ 4 | conversion::IntoPyObjectExt, 5 | intern, 6 | prelude::*, 7 | types::{PyDateTime, PyDict, PyTzInfo}, 8 | }; 9 | 10 | use crate::{ 11 | python::PyFieldDesc, BidAskPair, ConsolidatedBidAskPair, HasRType, WithTsOut, UNDEF_TIMESTAMP, 12 | }; 13 | 14 | pub fn char_to_c_char(c: char) -> crate::Result { 15 | if c.is_ascii() { 16 | Ok(c as c_char) 17 | } else { 18 | Err(crate::Error::Conversion { 19 | input: c.to_string(), 20 | desired_type: "c_char", 21 | }) 22 | } 23 | } 24 | 25 | impl PyFieldDesc for [BidAskPair; N] { 26 | fn field_dtypes(_field_name: &str) -> Vec<(String, String)> { 27 | let mut res = Vec::new(); 28 | let field_dtypes = BidAskPair::field_dtypes(""); 29 | for level in 0..N { 30 | let mut dtypes = field_dtypes.clone(); 31 | for dtype in dtypes.iter_mut() { 32 | dtype.0.push_str(&format!("_{level:02}")); 33 | } 34 | res.extend(dtypes); 35 | } 36 | res 37 | } 38 | 39 | fn price_fields(_field_name: &str) -> Vec { 40 | append_level_suffix::(BidAskPair::price_fields("")) 41 | } 42 | 43 | fn ordered_fields(_field_name: &str) -> Vec { 44 | append_level_suffix::(BidAskPair::ordered_fields("")) 45 | } 46 | } 47 | 48 | impl PyFieldDesc for [ConsolidatedBidAskPair; N] { 49 | fn field_dtypes(_field_name: &str) -> Vec<(String, String)> { 50 | let mut res = Vec::new(); 51 | let field_dtypes = ConsolidatedBidAskPair::field_dtypes(""); 52 | for level in 0..N { 53 | let mut dtypes = field_dtypes.clone(); 54 | for dtype in dtypes.iter_mut() { 55 | dtype.0.push_str(&format!("_{level:02}")); 56 | } 57 | res.extend(dtypes); 58 | } 59 | res 60 | } 61 | 62 | fn price_fields(_field_name: &str) -> Vec { 63 | append_level_suffix::(ConsolidatedBidAskPair::price_fields("")) 64 | } 65 | 66 | fn ordered_fields(_field_name: &str) -> Vec { 67 | append_level_suffix::(ConsolidatedBidAskPair::ordered_fields("")) 68 | } 69 | 70 | fn hidden_fields(_field_name: &str) -> Vec { 71 | append_level_suffix::(ConsolidatedBidAskPair::hidden_fields("")) 72 | } 73 | } 74 | 75 | pub fn append_level_suffix(fields: Vec) -> Vec { 76 | let mut res = Vec::new(); 77 | for level in 0..N { 78 | let mut fields = fields.clone(); 79 | for field in fields.iter_mut() { 80 | field.push_str(&format!("_{level:02}")); 81 | } 82 | res.extend(fields); 83 | } 84 | res 85 | } 86 | 87 | /// `WithTsOut` adds a `ts_out` field to the main record when converted to Python. 88 | impl<'py, R> IntoPyObject<'py> for WithTsOut 89 | where 90 | R: HasRType + IntoPyObject<'py>, 91 | { 92 | type Target = PyAny; 93 | type Output = Bound<'py, PyAny>; 94 | type Error = PyErr; 95 | 96 | fn into_pyobject(self, py: Python<'py>) -> Result { 97 | let obj = self.rec.into_bound_py_any(py)?; 98 | obj.setattr(intern!(py, "ts_out"), self.ts_out).unwrap(); 99 | Ok(obj) 100 | } 101 | } 102 | 103 | pub fn new_py_timestamp_or_datetime( 104 | py: Python<'_>, 105 | timestamp: u64, 106 | ) -> PyResult>> { 107 | if timestamp == UNDEF_TIMESTAMP { 108 | return Ok(None); 109 | } 110 | if let Ok(pandas) = PyModule::import(py, intern!(py, "pandas")) { 111 | let kwargs = PyDict::new(py); 112 | if kwargs.set_item(intern!(py, "utc"), true).is_ok() 113 | && kwargs 114 | .set_item(intern!(py, "errors"), intern!(py, "coerce")) 115 | .is_ok() 116 | && kwargs 117 | .set_item(intern!(py, "unit"), intern!(py, "ns")) 118 | .is_ok() 119 | { 120 | return pandas 121 | .call_method(intern!(py, "to_datetime"), (timestamp,), Some(&kwargs)) 122 | .map(|o| Some(o.into_pyobject(py).unwrap())); 123 | } 124 | } 125 | let utc_tz = PyTzInfo::utc(py)?; 126 | let timestamp_ms = timestamp as f64 / 1_000_000.0; 127 | PyDateTime::from_timestamp(py, timestamp_ms, Some(&utc_tz)) 128 | .map(|o| Some(o.into_pyobject(py).unwrap().into_any())) 129 | } 130 | -------------------------------------------------------------------------------- /rust/dbn-cli/src/encode.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use dbn::{ 4 | decode::{DbnMetadata, DecodeRecordRef}, 5 | encode::{ 6 | json, DbnEncodable, DbnRecordEncoder, DynEncoder, DynWriter, EncodeDbn, EncodeRecordRef, 7 | EncodeRecordTextExt, 8 | }, 9 | rtype_dispatch, Compression, Encoding, MetadataBuilder, SType, SymbolIndex, 10 | }; 11 | 12 | use crate::{infer_encoding, output_from_args, Args}; 13 | 14 | pub fn silence_broken_pipe(err: anyhow::Error) -> anyhow::Result<()> { 15 | // Handle broken pipe as a non-error. 16 | if let Some(err) = err.downcast_ref::() { 17 | if matches!(err, dbn::Error::Io { source, .. } if source.kind() == std::io::ErrorKind::BrokenPipe) 18 | { 19 | return Ok(()); 20 | } 21 | } 22 | Err(err) 23 | } 24 | 25 | pub fn encode_from_dbn(args: &Args, mut decoder: D) -> anyhow::Result<()> 26 | where 27 | D: DecodeRecordRef + DbnMetadata, 28 | { 29 | let writer = output_from_args(args)?; 30 | let (encoding, compression, delimiter) = infer_encoding(args)?; 31 | if args.should_output_metadata { 32 | if encoding != Encoding::Json { 33 | return Err(anyhow::format_err!( 34 | "Metadata flag is only valid with JSON encoding" 35 | )); 36 | } 37 | json::Encoder::new( 38 | writer, 39 | args.should_pretty_print, 40 | args.should_pretty_print, 41 | args.should_pretty_print, 42 | ) 43 | .encode_metadata(decoder.metadata())?; 44 | } else if args.fragment { 45 | encode_fragment(decoder, writer, compression)?; 46 | } else { 47 | let mut encoder = DynEncoder::builder(writer, encoding, compression, decoder.metadata()) 48 | .delimiter(delimiter) 49 | .write_header(args.write_header) 50 | .all_pretty(args.should_pretty_print) 51 | .with_symbol(args.map_symbols) 52 | .build()?; 53 | if args.map_symbols { 54 | let symbol_map = decoder.metadata().symbol_map()?; 55 | let ts_out = decoder.metadata().ts_out; 56 | while let Some(rec) = decoder.decode_record_ref()? { 57 | let sym = symbol_map.get_for_rec(&rec).map(String::as_str); 58 | // SAFETY: `ts_out` is accurate because it's sourced from the metadata 59 | unsafe { 60 | encoder.encode_ref_ts_out_with_sym(rec, ts_out, sym)?; 61 | } 62 | } 63 | } else { 64 | encoder.encode_decoded(decoder)?; 65 | } 66 | } 67 | Ok(()) 68 | } 69 | 70 | pub fn encode_from_frag(args: &Args, mut decoder: D) -> anyhow::Result<()> 71 | where 72 | D: DecodeRecordRef, 73 | { 74 | let writer = output_from_args(args)?; 75 | let (encoding, compression, delimiter) = infer_encoding(args)?; 76 | if args.fragment { 77 | encode_fragment(decoder, writer, compression)?; 78 | return Ok(()); 79 | } 80 | assert!(!args.should_output_metadata); 81 | 82 | let mut encoder = DynEncoder::builder( 83 | writer, 84 | encoding, 85 | compression, 86 | // dummy metadata won't be encoded 87 | &MetadataBuilder::new() 88 | .dataset(String::new()) 89 | .schema(None) 90 | .start(0) 91 | .stype_in(None) 92 | .stype_out(SType::InstrumentId) 93 | .build(), 94 | ) 95 | .delimiter(delimiter) 96 | // Can't write header until we know the record type 97 | .write_header(false) 98 | .all_pretty(args.should_pretty_print) 99 | .build()?; 100 | let mut has_written_header = (encoding != Encoding::Csv) || !args.write_header; 101 | fn write_header( 102 | _record: &T, 103 | encoder: &mut DynEncoder>, 104 | ) -> dbn::Result<()> { 105 | encoder.encode_header::(false) 106 | } 107 | while let Some(record) = decoder.decode_record_ref()? { 108 | if !has_written_header { 109 | rtype_dispatch!(record, write_header(&mut encoder))??; 110 | has_written_header = true; 111 | } 112 | encoder.encode_record_ref(record)?; 113 | } 114 | Ok(()) 115 | } 116 | 117 | fn encode_fragment( 118 | mut decoder: D, 119 | writer: Box, 120 | compression: Compression, 121 | ) -> dbn::Result<()> { 122 | let mut encoder = DbnRecordEncoder::new(DynWriter::new(writer, compression)?); 123 | while let Some(record) = decoder.decode_record_ref()? { 124 | encoder.encode_record_ref(record)?; 125 | } 126 | Ok(()) 127 | } 128 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/py_field_desc.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | 3 | use proc_macro2::TokenStream; 4 | use quote::quote; 5 | use syn::{parse_macro_input, Data, DeriveInput, Field}; 6 | 7 | use crate::dbn_attr::{ 8 | find_dbn_serialize_attr, get_sorted_fields, is_hidden, C_CHAR_ATTR, FIXED_PRICE_ATTR, 9 | UNIX_NANOS_ATTR, 10 | }; 11 | 12 | pub fn derive_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 13 | let DeriveInput { ident, data, .. } = parse_macro_input!(input as DeriveInput); 14 | let Data::Struct(data_struct) = data else { 15 | return syn::Error::new(ident.span(), "Can only derive PyFieldDesc for structs") 16 | .into_compile_error() 17 | .into(); 18 | }; 19 | let syn::Fields::Named(fields) = data_struct.fields else { 20 | return syn::Error::new(ident.span(), "Cannot derive PyFieldDesc for tuple struct") 21 | .into_compile_error() 22 | .into(); 23 | }; 24 | let sorted_fields = match get_sorted_fields(fields.clone()) { 25 | Ok(fields) => fields, 26 | Err(ts) => { 27 | return ts.into_compile_error().into(); 28 | } 29 | }; 30 | let raw_fields: VecDeque<_> = fields.named.into_iter().collect(); 31 | let dtype_iter = raw_fields.iter().map(|f| { 32 | let ident = f.ident.as_ref().unwrap(); 33 | let f_type = &f.ty; 34 | 35 | match find_dbn_serialize_attr(f) { 36 | Ok(attr) => { 37 | if matches!(attr, Some(id) if id == C_CHAR_ATTR) { 38 | quote! { 39 | res.push(( 40 | stringify!(#ident).to_owned(), 41 | "S1".to_owned(), 42 | )); 43 | } 44 | } else { 45 | quote! { res.extend(<#f_type>::field_dtypes(stringify!(#ident))); } 46 | } 47 | } 48 | Err(e) => e.into_compile_error(), 49 | } 50 | }); 51 | let price_fields = fields_with_attr(&raw_fields, FIXED_PRICE_ATTR, quote!(price_fields)); 52 | let hidden_fields = hidden_fields(&raw_fields); 53 | let timestamp_fields = fields_with_attr(&raw_fields, UNIX_NANOS_ATTR, quote!(timestamp_fields)); 54 | let ordered_fields = sorted_fields.iter().filter(|f| !is_hidden(f)).map(|f| { 55 | let ident = f.ident.as_ref().unwrap(); 56 | let f_type = &f.ty; 57 | quote! { 58 | res.extend(<#f_type>::ordered_fields(stringify!(#ident))); 59 | } 60 | }); 61 | 62 | quote! { 63 | impl crate::python::PyFieldDesc for #ident { 64 | fn field_dtypes(_field_name: &str) -> Vec<(String, String)> { 65 | let mut res = Vec::new(); 66 | #(#dtype_iter)* 67 | res 68 | } 69 | fn price_fields(_field_name: &str) -> Vec { 70 | let mut res = Vec::new(); 71 | #price_fields 72 | res 73 | } 74 | fn hidden_fields(_field_name: &str) -> Vec { 75 | let mut res = Vec::new(); 76 | #hidden_fields 77 | res 78 | } 79 | fn timestamp_fields(_field_name: &str) -> Vec { 80 | let mut res = Vec::new(); 81 | #timestamp_fields 82 | res 83 | } 84 | fn ordered_fields(_field_name: &str) -> Vec { 85 | let mut res = Vec::new(); 86 | #(#ordered_fields)* 87 | res 88 | } 89 | } 90 | } 91 | .into() 92 | } 93 | 94 | fn fields_with_attr(fields: &VecDeque, attr: &str, method: TokenStream) -> TokenStream { 95 | let fields_iter = fields.iter().map(|f| { 96 | let ident = f.ident.as_ref().unwrap(); 97 | if matches!(find_dbn_serialize_attr(f).unwrap_or(None), Some(id) if id == attr) { 98 | quote! { res.push(stringify!(#ident).to_owned()); } 99 | } else { 100 | let f_type = &f.ty; 101 | quote! { res.extend(<#f_type>::#method(stringify!(#ident))); } 102 | } 103 | }); 104 | quote! { 105 | #(#fields_iter)* 106 | } 107 | } 108 | 109 | fn hidden_fields(fields: &VecDeque) -> TokenStream { 110 | let fields_iter = fields.iter().map(|f| { 111 | let ident = f.ident.as_ref().unwrap(); 112 | if is_hidden(f) { 113 | quote! { res.push(stringify!(#ident).to_owned()); } 114 | } else { 115 | let f_type = &f.ty; 116 | quote! { res.extend(<#f_type>::hidden_fields(stringify!(#ident))); } 117 | } 118 | }); 119 | quote! { 120 | #(#fields_iter)* 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/has_rtype.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::{Span, TokenStream}; 2 | use quote::quote; 3 | use syn::{ 4 | parse::{Parse, ParseStream}, 5 | parse_macro_input, 6 | punctuated::Punctuated, 7 | spanned::Spanned, 8 | ExprPath, ItemStruct, Token, 9 | }; 10 | 11 | use crate::dbn_attr::{find_dbn_attr_args, INDEX_TS_ATTR}; 12 | 13 | pub fn attribute_macro_impl( 14 | attr: proc_macro::TokenStream, 15 | input: proc_macro::TokenStream, 16 | ) -> proc_macro::TokenStream { 17 | let args = parse_macro_input!(attr as Args); 18 | if args.args.is_empty() { 19 | return syn::Error::new( 20 | args.span, 21 | "Need to specify at least one rtype to match against", 22 | ) 23 | .into_compile_error() 24 | .into(); 25 | } 26 | let input_struct = parse_macro_input!(input as ItemStruct); 27 | let record_type = &input_struct.ident; 28 | let raw_index_ts = get_raw_index_ts(&input_struct).unwrap_or_else(|e| e.into_compile_error()); 29 | let rtypes = args.args.iter(); 30 | let crate_name = crate::utils::crate_name(); 31 | let impl_debug = crate::debug::record_debug_impl(&input_struct); 32 | quote! ( 33 | #input_struct 34 | 35 | impl #crate_name::record::Record for #record_type { 36 | fn header(&self) -> &#crate_name::record::RecordHeader { 37 | &self.hd 38 | } 39 | #raw_index_ts 40 | } 41 | 42 | impl #crate_name::record::RecordMut for #record_type { 43 | fn header_mut(&mut self) -> &mut #crate_name::record::RecordHeader { 44 | &mut self.hd 45 | } 46 | } 47 | 48 | impl #crate_name::record::HasRType for #record_type { 49 | #[allow(deprecated)] 50 | fn has_rtype(rtype: u8) -> bool { 51 | matches!(rtype, #(#rtypes)|*) 52 | } 53 | } 54 | 55 | impl AsRef<[u8]> for #record_type { 56 | fn as_ref(&self) -> &[u8] { 57 | unsafe { ::std::slice::from_raw_parts(self as *const #record_type as *const u8, ::std::mem::size_of::<#record_type>()) } 58 | } 59 | } 60 | 61 | impl std::cmp::PartialOrd for #record_type { 62 | fn partial_cmp(&self, other: &Self) -> Option { 63 | use #crate_name::record::Record; 64 | if self.raw_index_ts() == #crate_name::UNDEF_TIMESTAMP || other.raw_index_ts() == #crate_name::UNDEF_TIMESTAMP { 65 | None 66 | } else { 67 | Some(self.raw_index_ts().cmp(&other.raw_index_ts())) 68 | } 69 | } 70 | } 71 | 72 | #impl_debug 73 | ) 74 | .into() 75 | } 76 | 77 | pub(crate) struct Args { 78 | args: Vec, 79 | span: Span, 80 | } 81 | 82 | impl Parse for Args { 83 | fn parse(input: ParseStream) -> syn::Result { 84 | let args = Punctuated::::parse_terminated(input)?; 85 | Ok(Args { 86 | args: args.into_iter().collect(), 87 | span: input.span(), 88 | }) 89 | } 90 | } 91 | 92 | fn get_raw_index_ts(input_struct: &ItemStruct) -> syn::Result { 93 | let mut index_ts_fields = Vec::new(); 94 | for field in input_struct.fields.iter() { 95 | if find_dbn_attr_args(field)? 96 | .iter() 97 | .any(|id| id == INDEX_TS_ATTR) 98 | { 99 | index_ts_fields.push(field.ident.as_ref().unwrap()) 100 | } 101 | } 102 | match index_ts_fields.len() { 103 | 0 => Ok(quote!()), 104 | 1 => { 105 | let index_ts = index_ts_fields[0]; 106 | Ok(quote!( 107 | fn raw_index_ts(&self) -> u64 { 108 | self.#index_ts 109 | } 110 | )) 111 | } 112 | _ => Err(syn::Error::new( 113 | input_struct.span(), 114 | "Only one field can be marked index_ts", 115 | )), 116 | } 117 | } 118 | 119 | #[cfg(test)] 120 | mod tests { 121 | use super::*; 122 | 123 | #[test] 124 | fn parse_args_single() { 125 | let input = quote!(rtype::MBO); 126 | let args = syn::parse2::(input).unwrap(); 127 | assert_eq!(args.args.len(), 1); 128 | } 129 | 130 | #[test] 131 | fn parse_args_multiple() { 132 | let input = quote!(rtype::MBO, rtype::OHLC); 133 | let args = syn::parse2::(input).unwrap(); 134 | assert_eq!(args.args.len(), 2); 135 | } 136 | 137 | #[test] 138 | fn parse_args_empty() { 139 | let input = quote!(); 140 | let args = syn::parse2::(input).unwrap(); 141 | assert!(args.args.is_empty()); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /rust/dbn/src/v1/impl_default.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::c_char; 2 | 3 | use crate::{ 4 | rtype, v1, MatchAlgorithm, RecordHeader, StatUpdateAction, UNDEF_PRICE, UNDEF_TIMESTAMP, 5 | }; 6 | 7 | use super::{ 8 | ErrorMsg, InstrumentDefMsg, StatMsg, SymbolMappingMsg, SystemMsg, UNDEF_STAT_QUANTITY, 9 | }; 10 | 11 | impl Default for ErrorMsg { 12 | fn default() -> Self { 13 | Self { 14 | hd: RecordHeader::default::(rtype::ERROR), 15 | err: [0; 64], 16 | } 17 | } 18 | } 19 | 20 | impl Default for InstrumentDefMsg { 21 | fn default() -> Self { 22 | Self { 23 | hd: RecordHeader::default::(rtype::INSTRUMENT_DEF), 24 | ts_recv: UNDEF_TIMESTAMP, 25 | min_price_increment: UNDEF_PRICE, 26 | display_factor: UNDEF_PRICE, 27 | expiration: UNDEF_TIMESTAMP, 28 | activation: UNDEF_TIMESTAMP, 29 | high_limit_price: UNDEF_PRICE, 30 | low_limit_price: UNDEF_PRICE, 31 | max_price_variation: UNDEF_PRICE, 32 | trading_reference_price: UNDEF_PRICE, 33 | unit_of_measure_qty: UNDEF_PRICE, 34 | min_price_increment_amount: UNDEF_PRICE, 35 | price_ratio: UNDEF_PRICE, 36 | inst_attrib_value: i32::MAX, 37 | underlying_id: 0, 38 | raw_instrument_id: 0, 39 | market_depth_implied: i32::MAX, 40 | market_depth: i32::MAX, 41 | market_segment_id: u32::MAX, 42 | max_trade_vol: u32::MAX, 43 | min_lot_size: i32::MAX, 44 | min_lot_size_block: i32::MAX, 45 | min_lot_size_round_lot: i32::MAX, 46 | min_trade_vol: u32::MAX, 47 | _reserved2: Default::default(), 48 | contract_multiplier: i32::MAX, 49 | decay_quantity: i32::MAX, 50 | original_contract_size: i32::MAX, 51 | _reserved3: Default::default(), 52 | trading_reference_date: u16::MAX, 53 | appl_id: i16::MAX, 54 | maturity_year: u16::MAX, 55 | decay_start_date: u16::MAX, 56 | channel_id: u16::MAX, 57 | currency: [0; 4], 58 | settl_currency: [0; 4], 59 | secsubtype: [0; 6], 60 | raw_symbol: [0; v1::SYMBOL_CSTR_LEN], 61 | group: [0; 21], 62 | exchange: [0; 5], 63 | asset: [0; v1::ASSET_CSTR_LEN], 64 | cfi: [0; 7], 65 | security_type: [0; 7], 66 | unit_of_measure: [0; 31], 67 | underlying: [0; 21], 68 | strike_price_currency: [0; 4], 69 | instrument_class: 0, 70 | _reserved4: Default::default(), 71 | strike_price: UNDEF_PRICE, 72 | _reserved5: Default::default(), 73 | match_algorithm: MatchAlgorithm::default() as c_char, 74 | md_security_trading_status: u8::MAX, 75 | main_fraction: u8::MAX, 76 | price_display_format: u8::MAX, 77 | settl_price_type: u8::MAX, 78 | sub_fraction: u8::MAX, 79 | underlying_product: u8::MAX, 80 | security_update_action: Default::default(), 81 | maturity_month: u8::MAX, 82 | maturity_day: u8::MAX, 83 | maturity_week: u8::MAX, 84 | user_defined_instrument: Default::default(), 85 | contract_multiplier_unit: i8::MAX, 86 | flow_schedule_type: i8::MAX, 87 | tick_rule: u8::MAX, 88 | _dummy: Default::default(), 89 | } 90 | } 91 | } 92 | 93 | impl Default for StatMsg { 94 | fn default() -> Self { 95 | Self { 96 | hd: RecordHeader::default::(rtype::STATISTICS), 97 | ts_recv: UNDEF_TIMESTAMP, 98 | ts_ref: UNDEF_TIMESTAMP, 99 | price: UNDEF_PRICE, 100 | quantity: UNDEF_STAT_QUANTITY, 101 | sequence: 0, 102 | ts_in_delta: 0, 103 | stat_type: 0, 104 | channel_id: u16::MAX, 105 | update_action: StatUpdateAction::default() as u8, 106 | stat_flags: 0, 107 | _reserved: Default::default(), 108 | } 109 | } 110 | } 111 | 112 | impl Default for SymbolMappingMsg { 113 | fn default() -> Self { 114 | Self { 115 | hd: RecordHeader::default::(rtype::SYMBOL_MAPPING), 116 | stype_in_symbol: [0; v1::SYMBOL_CSTR_LEN], 117 | stype_out_symbol: [0; v1::SYMBOL_CSTR_LEN], 118 | _dummy: Default::default(), 119 | start_ts: UNDEF_TIMESTAMP, 120 | end_ts: UNDEF_TIMESTAMP, 121 | } 122 | } 123 | } 124 | 125 | impl Default for SystemMsg { 126 | fn default() -> Self { 127 | Self { 128 | hd: RecordHeader::default::(rtype::SYSTEM), 129 | msg: [0; 64], 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | # Build and test dbn 4 | 5 | on: 6 | pull_request: 7 | push: 8 | 9 | jobs: 10 | x86_64-build: 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | os: [ubuntu-latest, windows-latest] 15 | python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] 16 | name: build - Python ${{ matrix.python-version }} (x86_64 ${{ matrix.os }}) 17 | runs-on: ${{ matrix.os }} 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v5 22 | 23 | - name: Set up Rust 24 | run: rustup toolchain add --profile minimal stable --component clippy,rustfmt 25 | 26 | # Cargo setup 27 | - name: Set up Cargo cache 28 | uses: actions/cache@v4 29 | with: 30 | path: | 31 | ~/.cargo/registry 32 | ~/.cargo/git 33 | target 34 | key: ${{ runner.os }}-x86_64-cargo-${{ hashFiles('Cargo.lock') }} 35 | 36 | # Python setup 37 | - name: Set up Python environment 38 | uses: actions/setup-python@v5 39 | with: 40 | python-version: ${{ matrix.python-version }} 41 | architecture: ${{ matrix.arch }} 42 | 43 | - name: Build wheels 44 | uses: messense/maturin-action@v1 45 | with: 46 | target: x86_64 47 | args: --release --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} 48 | 49 | - name: Format 50 | run: scripts/format.sh 51 | shell: bash 52 | - name: Build 53 | run: scripts/build.sh 54 | shell: bash 55 | - name: Lint 56 | run: scripts/lint.sh 57 | shell: bash 58 | - name: Test 59 | run: scripts/test.sh 60 | shell: bash 61 | 62 | aarch64-build: 63 | strategy: 64 | fail-fast: false 65 | matrix: 66 | python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] 67 | name: build - Python ${{ matrix.python-version }} (aarch64 linux) 68 | runs-on: ubuntu-latest 69 | 70 | steps: 71 | - name: Checkout repository 72 | uses: actions/checkout@v5 73 | 74 | - name: Set up Rust 75 | run: rustup toolchain add --profile minimal stable --component clippy,rustfmt 76 | 77 | # Cargo setup 78 | - name: Set up Cargo cache 79 | uses: actions/cache@v4 80 | with: 81 | path: | 82 | ~/.cargo/registry 83 | ~/.cargo/git 84 | target 85 | key: ${{ runner.os }}-aarch64-cargo-${{ hashFiles('Cargo.lock') }} 86 | 87 | # Python setup 88 | - name: Set up Python environment 89 | uses: actions/setup-python@v5 90 | with: 91 | python-version: ${{ matrix.python-version }} 92 | 93 | - name: Build wheels 94 | uses: messense/maturin-action@v1 95 | with: 96 | target: aarch64 97 | manylinux: auto 98 | args: --release --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} 99 | 100 | - name: Format 101 | run: scripts/format.sh 102 | - name: Build 103 | run: scripts/build.sh 104 | - name: Lint 105 | run: scripts/lint.sh 106 | - name: Test 107 | run: scripts/test.sh 108 | 109 | macos-build: 110 | strategy: 111 | fail-fast: false 112 | matrix: 113 | python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] 114 | name: build - Python ${{ matrix.python-version }} (macOS) 115 | runs-on: macos-latest 116 | 117 | steps: 118 | - name: Checkout repository 119 | uses: actions/checkout@v5 120 | 121 | - name: Set up Rust 122 | run: rustup toolchain add --profile minimal stable --component clippy,rustfmt 123 | 124 | # Cargo setup 125 | - name: Set up Cargo cache 126 | uses: actions/cache@v4 127 | with: 128 | path: | 129 | ~/.cargo/registry 130 | ~/.cargo/git 131 | target 132 | key: ${{ runner.os }}-x86_64-cargo-${{ hashFiles('Cargo.lock') }} 133 | 134 | # Python setup 135 | - name: Set up Python environment 136 | uses: actions/setup-python@v5 137 | with: 138 | python-version: ${{ matrix.python-version }} 139 | 140 | - name: Build wheels - x86_64 141 | uses: messense/maturin-action@v1 142 | with: 143 | target: x86_64 144 | args: --release --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} 145 | 146 | - name: Build wheels - universal2 147 | uses: messense/maturin-action@v1 148 | with: 149 | args: --release --target universal2-apple-darwin --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} 150 | 151 | - name: Format 152 | run: scripts/format.sh 153 | - name: Build 154 | run: scripts/build.sh 155 | - name: Lint 156 | run: scripts/lint.sh 157 | - name: Test 158 | run: scripts/test.sh 159 | -------------------------------------------------------------------------------- /rust/dbn/src/v1.rs: -------------------------------------------------------------------------------- 1 | //! Record data types for encoding different Databento [`Schema`](crate::enums::Schema)s 2 | //! in DBN version 1. 3 | 4 | pub(crate) use crate::compat::METADATA_RESERVED_LEN_V1 as METADATA_RESERVED_LEN; 5 | pub use crate::compat::{ 6 | ErrorMsgV1 as ErrorMsg, InstrumentDefMsgV1 as InstrumentDefMsg, StatMsgV1 as StatMsg, 7 | SymbolMappingMsgV1 as SymbolMappingMsg, SystemMsgV1 as SystemMsg, 8 | ASSET_CSTR_LEN_V1 as ASSET_CSTR_LEN, SYMBOL_CSTR_LEN_V1 as SYMBOL_CSTR_LEN, 9 | UNDEF_STAT_QUANTITY_V1 as UNDEF_STAT_QUANTITY, 10 | }; 11 | pub use crate::record::{ 12 | Bbo1MMsg, Bbo1SMsg, BboMsg, Cbbo1MMsg, Cbbo1SMsg, CbboMsg, Cmbp1Msg, ImbalanceMsg, MboMsg, 13 | Mbp10Msg, Mbp1Msg, OhlcvMsg, StatusMsg, TbboMsg, TcbboMsg, TradeMsg, WithTsOut, 14 | }; 15 | 16 | mod impl_default; 17 | mod methods; 18 | 19 | use crate::compat::{InstrumentDefRec, StatRec, SymbolMappingRec}; 20 | 21 | /// The DBN version of this module. 22 | pub const DBN_VERSION: u8 = 1; 23 | 24 | impl SymbolMappingRec for SymbolMappingMsg { 25 | fn stype_in_symbol(&self) -> crate::Result<&str> { 26 | Self::stype_in_symbol(self) 27 | } 28 | 29 | fn stype_out_symbol(&self) -> crate::Result<&str> { 30 | Self::stype_out_symbol(self) 31 | } 32 | 33 | fn start_ts(&self) -> Option { 34 | Self::start_ts(self) 35 | } 36 | 37 | fn end_ts(&self) -> Option { 38 | Self::end_ts(self) 39 | } 40 | } 41 | 42 | impl InstrumentDefRec for InstrumentDefMsg { 43 | fn raw_symbol(&self) -> crate::Result<&str> { 44 | Self::raw_symbol(self) 45 | } 46 | 47 | fn asset(&self) -> crate::Result<&str> { 48 | Self::asset(self) 49 | } 50 | 51 | fn security_type(&self) -> crate::Result<&str> { 52 | Self::security_type(self) 53 | } 54 | 55 | fn security_update_action(&self) -> crate::Result { 56 | Ok(self.security_update_action) 57 | } 58 | 59 | fn channel_id(&self) -> u16 { 60 | self.channel_id 61 | } 62 | } 63 | 64 | impl StatRec for StatMsg { 65 | const UNDEF_STAT_QUANTITY: i64 = UNDEF_STAT_QUANTITY as i64; 66 | 67 | fn stat_type(&self) -> crate::Result { 68 | Self::stat_type(self) 69 | } 70 | 71 | fn ts_recv(&self) -> Option { 72 | Self::ts_recv(self) 73 | } 74 | 75 | fn ts_ref(&self) -> Option { 76 | Self::ts_ref(self) 77 | } 78 | 79 | fn update_action(&self) -> crate::Result { 80 | Self::update_action(self) 81 | } 82 | 83 | fn price(&self) -> i64 { 84 | self.price 85 | } 86 | 87 | fn quantity(&self) -> i64 { 88 | self.quantity as i64 89 | } 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | use std::mem; 95 | 96 | use rstest::*; 97 | use type_layout::{Field, TypeLayout}; 98 | 99 | use crate::{v2, v3}; 100 | 101 | use super::*; 102 | 103 | #[test] 104 | fn test_default_equivalency() { 105 | assert_eq!( 106 | v2::InstrumentDefMsg::from(&InstrumentDefMsg::default()), 107 | v2::InstrumentDefMsg::default() 108 | ); 109 | assert_eq!( 110 | v3::InstrumentDefMsg::from(&InstrumentDefMsg::default()), 111 | v3::InstrumentDefMsg::default() 112 | ); 113 | assert_eq!( 114 | v3::StatMsg::from(&StatMsg::default()), 115 | v3::StatMsg::default() 116 | ); 117 | } 118 | 119 | #[cfg(feature = "python")] 120 | #[test] 121 | fn test_strike_price_order_didnt_change() { 122 | use crate::python::PyFieldDesc; 123 | 124 | assert_eq!( 125 | InstrumentDefMsg::ordered_fields(""), 126 | v2::InstrumentDefMsg::ordered_fields("") 127 | ); 128 | assert_eq!(StatMsg::ordered_fields(""), v3::StatMsg::ordered_fields("")); 129 | } 130 | 131 | #[rstest] 132 | #[case::definition(InstrumentDefMsg::default(), 360)] 133 | #[case::stat(StatMsg::default(), 64)] 134 | #[case::error(ErrorMsg::default(), 80)] 135 | #[case::symbol_mapping(SymbolMappingMsg::default(), 80)] 136 | #[case::system(SystemMsg::default(), 80)] 137 | fn test_sizes(#[case] _rec: R, #[case] exp: usize) { 138 | assert_eq!(mem::size_of::(), exp); 139 | assert!(mem::size_of::() <= crate::MAX_RECORD_LEN); 140 | } 141 | 142 | #[rstest] 143 | #[case::definition(InstrumentDefMsg::default())] 144 | #[case::stat(StatMsg::default())] 145 | #[case::error(ErrorMsg::default())] 146 | #[case::symbol_mapping(SymbolMappingMsg::default())] 147 | #[case::system(SystemMsg::default())] 148 | fn test_alignment_and_no_padding(#[case] _rec: R) { 149 | let layout = R::type_layout(); 150 | assert_eq!(layout.alignment, 8, "Unexpected alignment: {layout}"); 151 | for field in layout.fields.iter() { 152 | assert!( 153 | matches!(field, Field::Field { .. }), 154 | "Detected padding: {layout}" 155 | ); 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/lib.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | 3 | mod dbn_attr; 4 | mod debug; 5 | mod has_rtype; 6 | mod py_field_desc; 7 | mod serialize; 8 | mod utils; 9 | 10 | /// Dummy derive macro to get around `cfg_attr` incompatibility of several 11 | /// of pyo3's attribute macros. See . 12 | /// 13 | /// `MockPyo3` is an invented trait. 14 | #[proc_macro_derive(MockPyo3, attributes(pyo3))] 15 | pub fn derive_mock_pyo3(_item: TokenStream) -> TokenStream { 16 | TokenStream::new() 17 | } 18 | 19 | /// Dummy derive macro to enable the `dbn` helper attribute for record types 20 | /// using the `dbn_record` proc macro but neither `CsvSerialize` nor `JsonSerialize` as 21 | /// helper attributes aren't supported for proc macros alone. See 22 | /// . 23 | #[proc_macro_derive(DbnAttr, attributes(dbn))] 24 | pub fn dbn_attr(_item: TokenStream) -> TokenStream { 25 | TokenStream::new() 26 | } 27 | 28 | /// Derive macro for CSV serialization. Supports the following `dbn` attributes: 29 | /// - `c_char`: serializes the field as a `char` 30 | /// - `encode_order`: overrides the position of the field in the CSV table 31 | /// - `fixed_price`: serializes the field as fixed-price, with the output format 32 | /// depending on `PRETTY_PX` 33 | /// - `skip`: does not serialize the field 34 | /// - `unix_nanos`: serializes the field as a UNIX timestamp, with the output format 35 | /// depending on `PRETTY_TS` 36 | /// 37 | /// Note: fields beginning with `_` will automatically be skipped, e.g. `_reserved` 38 | /// isn't serialized. 39 | #[proc_macro_derive(CsvSerialize, attributes(dbn))] 40 | pub fn derive_csv_serialize(input: TokenStream) -> TokenStream { 41 | serialize::derive_csv_macro_impl(input) 42 | } 43 | 44 | /// Derive macro for JSON serialization. 45 | /// 46 | /// Supports the following `dbn` attributes: 47 | /// - `c_char`: serializes the field as a `char` 48 | /// - `fixed_price`: serializes the field as fixed-price, with the output format 49 | /// depending on `PRETTY_PX` 50 | /// - `skip`: does not serialize the field 51 | /// - `unix_nanos`: serializes the field as a UNIX timestamp, with the output format 52 | /// depending on `PRETTY_TS` 53 | /// 54 | /// Note: fields beginning with `_` will automatically be skipped, e.g. `_reserved` 55 | /// isn't serialized. 56 | #[proc_macro_derive(JsonSerialize, attributes(dbn))] 57 | pub fn derive_json_serialize(input: TokenStream) -> TokenStream { 58 | serialize::derive_json_macro_impl(input) 59 | } 60 | 61 | /// Derive macro for field descriptions exposed to Python. 62 | /// 63 | /// Supports the following `dbn` attributes: 64 | /// - `c_char`: indicates the field dtype should be a single-character string rather 65 | /// than an integer 66 | /// - `encode_order`: overrides the position of the field in the ordered list 67 | /// - `fixed_price`: indicates this is a fixed-precision field 68 | /// - `skip`: indicates this field should be hidden 69 | /// - `unix_nanos`: indicates this is a UNIX nanosecond timestamp field 70 | #[proc_macro_derive(PyFieldDesc, attributes(dbn))] 71 | pub fn derive_py_field_desc(input: TokenStream) -> TokenStream { 72 | py_field_desc::derive_impl(input) 73 | } 74 | 75 | /// Attribute macro that acts like a derive macro for `Debug` (with customization), 76 | /// `Record`, `RecordMut`, `HasRType`, `PartialOrd`, and `AsRef<[u8]>`. 77 | /// 78 | /// Expects 1 or more paths to `u8` constants that are the RTypes associated 79 | /// with this record. 80 | /// 81 | /// Supports the following `dbn` attributes: 82 | /// - `c_char`: format the type as a `char` instead of as a numeric 83 | /// - `fixed_price`: format the integer as a fixed-precision decimal 84 | /// - `fmt_binary`: format as a binary 85 | /// - `fmt_method`: try to format by calling the getter method with the same name as the 86 | /// - `index_ts`: indicates this field is the primary timestamp for the record 87 | /// field. If the getter returns an error, the raw field value will be used 88 | /// - `skip`: won't be included in the `Debug` output 89 | /// 90 | /// Note: attribute macros don't support helper attributes on their own. If not deriving 91 | /// `CsvSerialize` or `JsonSerialize`, derive `DbnAttr` to use the `dbn` helper attribute 92 | /// without a compiler error. 93 | #[proc_macro_attribute] 94 | pub fn dbn_record(attr: TokenStream, input: TokenStream) -> TokenStream { 95 | has_rtype::attribute_macro_impl(attr, input) 96 | } 97 | 98 | /// Derive macro for Debug representations with the same extensions for DBN records 99 | /// as `dbn_record`. 100 | /// 101 | /// Supports the following `dbn` attributes: 102 | /// - `c_char`: format the type as a `char` instead of as a numeric 103 | /// - `fixed_price`: format the integer as a fixed-precision decimal 104 | /// - `fmt_binary`: format as a binary 105 | /// - `fmt_method`: try to format by calling the getter method with the same name as the 106 | /// field. If the getter returns an error, the raw field value will be used 107 | /// - `skip`: won't be included in the `Debug` output 108 | /// 109 | /// Note: fields beginning with `_` will automatically be skipped, e.g. `_reserved` 110 | /// isn't included in the `Debug` output. 111 | #[proc_macro_derive(RecordDebug, attributes(dbn))] 112 | pub fn derive_record_debug(input: TokenStream) -> TokenStream { 113 | debug::derive_impl(input) 114 | } 115 | 116 | #[cfg(test)] 117 | mod tests { 118 | #[test] 119 | fn ui() { 120 | let t = trybuild::TestCases::new(); 121 | t.compile_fail("tests/ui/*.rs"); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | info@nautechsystems.io. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /rust/dbn/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! The official crate for working with [**D**atabento](https://databento.com) 2 | //! **B**inary E**n**coding (DBN), an extremely fast message encoding and storage format 3 | //! for normalized market data. The DBN specification includes a simple, self-describing 4 | //! metadata header and a fixed set of struct definitions, which enforce a standardized 5 | //! way to normalize market data. 6 | //! 7 | //! All official Databento client libraries use DBN under the hood, both as a data 8 | //! interchange format and for in-memory representation of data. DBN is also the default 9 | //! encoding for all Databento APIs, including live data streaming, historical data 10 | //! streaming, and batch flat files. For more information about the encoding, read our 11 | //! [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). 12 | //! 13 | //! The crate supports reading and writing DBN files and streams, as well as converting 14 | //! them to other [`Encoding`]s. It can also be used to update legacy 15 | //! DBZ files to DBN. 16 | //! 17 | //! This crate provides: 18 | //! - [Decoders](crate::decode) for DBN and DBZ (the precursor to DBN), both 19 | //! sync and async, with the `async` feature flag 20 | //! - [Encoders](crate::encode) for CSV, DBN, and JSON, both sync and async, 21 | //! with the `async` feature flag 22 | //! - [Normalized market data struct definitions](crate::record) corresponding to the 23 | //! different market data schemas offered by Databento 24 | //! - A [wrapper type](crate::RecordRef) for holding a reference to a record struct of 25 | //! a dynamic type 26 | //! - Helper functions and [macros] for common tasks 27 | //! 28 | //! # Feature flags 29 | //! - `async`: enables async decoding and encoding 30 | //! - `python`: enables `pyo3` bindings 31 | //! - `serde`: enables deriving `serde` traits for types 32 | //! - `trivial_copy`: enables deriving the `Copy` trait for records 33 | 34 | #![cfg_attr(docsrs, feature(doc_cfg))] 35 | #![deny(missing_docs)] 36 | #![deny(rustdoc::broken_intra_doc_links)] 37 | #![deny(clippy::missing_errors_doc)] 38 | 39 | pub mod compat; 40 | pub mod decode; 41 | pub mod encode; 42 | pub mod enums; 43 | pub mod error; 44 | pub mod flags; 45 | mod json_writer; 46 | pub mod macros; 47 | pub mod metadata; 48 | pub mod pretty; 49 | pub mod publishers; 50 | #[cfg(feature = "python")] 51 | pub mod python; 52 | pub mod record; 53 | mod record_enum; 54 | pub mod record_ref; 55 | pub mod symbol_map; 56 | #[cfg(test)] 57 | mod test_utils; 58 | pub mod v1; 59 | pub mod v2; 60 | pub mod v3; 61 | 62 | #[doc(inline)] 63 | pub use crate::{ 64 | enums::{ 65 | rtype, Action, Compression, Encoding, ErrorCode, InstrumentClass, MatchAlgorithm, RType, 66 | SType, Schema, SecurityUpdateAction, Side, StatType, StatUpdateAction, StatusAction, 67 | StatusReason, SystemCode, TradingEvent, TriState, UserDefinedInstrument, 68 | VersionUpgradePolicy, 69 | }, 70 | error::{Error, Result}, 71 | flags::FlagSet, 72 | metadata::{MappingInterval, Metadata, MetadataBuilder, SymbolMapping}, 73 | publishers::{Dataset, Publisher, Venue}, 74 | record::{ 75 | Bbo1MMsg, Bbo1SMsg, BboMsg, BidAskPair, Cbbo1MMsg, Cbbo1SMsg, CbboMsg, Cmbp1Msg, 76 | ConsolidatedBidAskPair, ErrorMsg, HasRType, ImbalanceMsg, InstrumentDefMsg, MboMsg, 77 | Mbp10Msg, Mbp1Msg, OhlcvMsg, Record, RecordHeader, RecordMut, StatMsg, StatusMsg, 78 | SymbolMappingMsg, SystemMsg, TbboMsg, TcbboMsg, TradeMsg, WithTsOut, 79 | }, 80 | record_enum::{RecordEnum, RecordRefEnum}, 81 | record_ref::RecordRef, 82 | symbol_map::{PitSymbolMap, SymbolIndex, TsSymbolMap}, 83 | }; 84 | 85 | /// The current version of the DBN encoding, which is different from the crate version. 86 | pub const DBN_VERSION: u8 = 3; 87 | 88 | /// The length of fixed-length symbol strings. 89 | pub const SYMBOL_CSTR_LEN: usize = v3::SYMBOL_CSTR_LEN; 90 | /// The length of the fixed-length asset string. 91 | pub const ASSET_CSTR_LEN: usize = v3::ASSET_CSTR_LEN; 92 | 93 | const METADATA_DATASET_CSTR_LEN: usize = 16; 94 | const METADATA_RESERVED_LEN: usize = 53; 95 | /// Excludes magic string, version, and length. 96 | const METADATA_FIXED_LEN: usize = 100; 97 | const NULL_LIMIT: u64 = 0; 98 | const NULL_RECORD_COUNT: u64 = u64::MAX; 99 | const NULL_SCHEMA: u16 = u16::MAX; 100 | const NULL_STYPE: u8 = u8::MAX; 101 | 102 | /// The denominator of fixed prices in DBN. 103 | pub const FIXED_PRICE_SCALE: i64 = 1_000_000_000; 104 | /// The sentinel value for an unset or null price. 105 | pub const UNDEF_PRICE: i64 = i64::MAX; 106 | /// The sentinel value for an unset or null order quantity. 107 | pub const UNDEF_ORDER_SIZE: u32 = u32::MAX; 108 | /// The sentinel value for an unset or null stat quantity. 109 | pub const UNDEF_STAT_QUANTITY: i64 = v3::UNDEF_STAT_QUANTITY; 110 | /// The sentinel value for an unset or null timestamp. 111 | pub const UNDEF_TIMESTAMP: u64 = u64::MAX; 112 | /// The length in bytes of the largest record type. 113 | pub const MAX_RECORD_LEN: usize = std::mem::size_of::>(); 114 | 115 | /// New type for validating DBN versions. 116 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 117 | #[repr(transparent)] 118 | pub struct DbnVersion(u8); 119 | 120 | impl TryFrom for DbnVersion { 121 | type Error = crate::Error; 122 | 123 | fn try_from(version: u8) -> crate::Result { 124 | if (1..=DBN_VERSION).contains(&version) { 125 | Ok(Self(version)) 126 | } else { 127 | Err(Error::BadArgument { 128 | param_name: "version".to_owned(), 129 | desc: format!("invalid, must be between 1 and {DBN_VERSION}, inclusive"), 130 | }) 131 | } 132 | } 133 | } 134 | 135 | impl DbnVersion { 136 | /// Returns the version value. 137 | pub fn get(self) -> u8 { 138 | self.0 139 | } 140 | } 141 | 142 | impl PartialEq for DbnVersion { 143 | fn eq(&self, other: &u8) -> bool { 144 | self.0 == *other 145 | } 146 | } 147 | 148 | impl PartialOrd for DbnVersion { 149 | fn partial_cmp(&self, other: &u8) -> Option { 150 | self.0.partial_cmp(other) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /rust/dbn/src/python/metadata.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, io, num::NonZeroU64}; 2 | 3 | use pyo3::{ 4 | intern, 5 | prelude::*, 6 | types::{PyBytes, PyDate, PyDict, PyType}, 7 | Bound, 8 | }; 9 | 10 | use crate::{ 11 | decode::{DbnMetadata, DynDecoder}, 12 | encode::dbn::MetadataEncoder, 13 | enums::{SType, Schema}, 14 | MappingInterval, Metadata, SymbolMapping, VersionUpgradePolicy, 15 | }; 16 | 17 | use super::{py_to_time_date, to_py_err}; 18 | 19 | #[pymethods] 20 | impl Metadata { 21 | #[new] 22 | #[pyo3(signature = ( 23 | dataset, 24 | start, 25 | stype_in, 26 | stype_out, 27 | schema, 28 | symbols=None, 29 | partial=None, 30 | not_found=None, 31 | mappings=None, 32 | end=None, 33 | limit=None, 34 | ts_out=None, 35 | version=crate::DBN_VERSION, 36 | ))] 37 | fn py_new( 38 | dataset: String, 39 | start: u64, 40 | stype_in: Option, 41 | stype_out: SType, 42 | schema: Option, 43 | symbols: Option>, 44 | partial: Option>, 45 | not_found: Option>, 46 | mappings: Option>, 47 | end: Option, 48 | limit: Option, 49 | ts_out: Option, 50 | version: u8, 51 | ) -> Metadata { 52 | Metadata::builder() 53 | .dataset(dataset) 54 | .start(start) 55 | .stype_out(stype_out) 56 | .symbols(symbols.unwrap_or_default()) 57 | .partial(partial.unwrap_or_default()) 58 | .not_found(not_found.unwrap_or_default()) 59 | .mappings(mappings.unwrap_or_default()) 60 | .schema(schema) 61 | .stype_in(stype_in) 62 | .end(NonZeroU64::new(end.unwrap_or_default())) 63 | .limit(NonZeroU64::new(limit.unwrap_or_default())) 64 | .ts_out(ts_out.unwrap_or_default()) 65 | .version(version) 66 | .build() 67 | } 68 | 69 | fn __repr__(&self) -> String { 70 | format!("{self:?}") 71 | } 72 | 73 | /// Encodes Metadata back into DBN format. 74 | fn __bytes__<'py>(&self, py: Python<'py>) -> PyResult> { 75 | self.py_encode(py) 76 | } 77 | 78 | #[getter] 79 | fn get_mappings<'py>(&self, py: Python<'py>) -> PyResult>> { 80 | let mut res = HashMap::new(); 81 | for mapping in self.mappings.iter() { 82 | res.insert( 83 | mapping.raw_symbol.clone(), 84 | mapping.intervals.into_pyobject(py)?, 85 | ); 86 | } 87 | Ok(res) 88 | } 89 | 90 | #[pyo3(name = "decode", signature = (data, upgrade_policy = VersionUpgradePolicy::default()))] 91 | #[classmethod] 92 | fn py_decode( 93 | _cls: &Bound, 94 | data: &Bound, 95 | upgrade_policy: VersionUpgradePolicy, 96 | ) -> PyResult { 97 | let reader = io::BufReader::new(data.as_bytes()); 98 | let mut metadata = DynDecoder::inferred_with_buffer(reader, upgrade_policy)? 99 | .metadata() 100 | .clone(); 101 | metadata.upgrade(upgrade_policy); 102 | Ok(metadata) 103 | } 104 | 105 | #[pyo3(name = "encode")] 106 | fn py_encode<'py>(&self, py: Python<'py>) -> PyResult> { 107 | let mut buffer = Vec::new(); 108 | let mut encoder = MetadataEncoder::new(&mut buffer); 109 | encoder.encode(self)?; 110 | Ok(PyBytes::new(py, buffer.as_slice())) 111 | } 112 | } 113 | 114 | impl<'py> IntoPyObject<'py> for SymbolMapping { 115 | type Target = PyDict; 116 | type Output = Bound<'py, PyDict>; 117 | type Error = PyErr; 118 | 119 | fn into_pyobject(self, py: Python<'py>) -> Result { 120 | let dict = PyDict::new(py); 121 | dict.set_item(intern!(py, "raw_symbol"), &self.raw_symbol)?; 122 | dict.set_item(intern!(py, "intervals"), &self.intervals)?; 123 | Ok(dict) 124 | } 125 | } 126 | 127 | impl<'py> FromPyObject<'py> for MappingInterval { 128 | fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { 129 | let start_date = ob 130 | .getattr(intern!(ob.py(), "start_date")) 131 | .map_err(|_| to_py_err("Missing start_date".to_owned())) 132 | .and_then(extract_date)?; 133 | let end_date = ob 134 | .getattr(intern!(ob.py(), "end_date")) 135 | .map_err(|_| to_py_err("Missing end_date".to_owned())) 136 | .and_then(extract_date)?; 137 | let symbol = ob 138 | .getattr(intern!(ob.py(), "symbol")) 139 | .map_err(|_| to_py_err("Missing symbol".to_owned())) 140 | .and_then(|d| d.extract::())?; 141 | Ok(Self { 142 | start_date, 143 | end_date, 144 | symbol, 145 | }) 146 | } 147 | } 148 | 149 | impl<'py> IntoPyObject<'py> for &MappingInterval { 150 | type Target = PyDict; 151 | type Output = Bound<'py, PyDict>; 152 | type Error = PyErr; 153 | 154 | fn into_pyobject(self, py: Python<'py>) -> Result { 155 | let dict = PyDict::new(py); 156 | dict.set_item( 157 | intern!(py, "start_date"), 158 | PyDate::new( 159 | py, 160 | self.start_date.year(), 161 | self.start_date.month() as u8, 162 | self.start_date.day(), 163 | )?, 164 | )?; 165 | dict.set_item( 166 | intern!(py, "end_date"), 167 | PyDate::new( 168 | py, 169 | self.end_date.year(), 170 | self.end_date.month() as u8, 171 | self.end_date.day(), 172 | )?, 173 | )?; 174 | dict.set_item(intern!(py, "symbol"), &self.symbol)?; 175 | Ok(dict) 176 | } 177 | } 178 | 179 | fn extract_date(any: Bound<'_, PyAny>) -> PyResult { 180 | let py_date = any.downcast::().map_err(PyErr::from)?; 181 | py_to_time_date(py_date) 182 | } 183 | -------------------------------------------------------------------------------- /rust/dbn/src/record/layout_tests.rs: -------------------------------------------------------------------------------- 1 | #![cfg(test)] 2 | 3 | use mem::offset_of; 4 | use rstest::rstest; 5 | use type_layout::{Field, TypeLayout}; 6 | 7 | use crate::Schema; 8 | use crate::UNDEF_TIMESTAMP; 9 | 10 | use super::*; 11 | 12 | const OHLCV_MSG: OhlcvMsg = OhlcvMsg { 13 | hd: RecordHeader { 14 | length: 56, 15 | rtype: rtype::OHLCV_1S, 16 | publisher_id: 1, 17 | instrument_id: 5482, 18 | ts_event: 1609160400000000000, 19 | }, 20 | open: 372025000000000, 21 | high: 372050000000000, 22 | low: 372025000000000, 23 | close: 372050000000000, 24 | volume: 57, 25 | }; 26 | 27 | #[test] 28 | fn test_transmute_record_bytes() { 29 | unsafe { 30 | let ohlcv_bytes = std::slice::from_raw_parts( 31 | &OHLCV_MSG as *const OhlcvMsg as *const u8, 32 | mem::size_of::(), 33 | ) 34 | .to_vec(); 35 | let ohlcv = transmute_record_bytes::(ohlcv_bytes.as_slice()).unwrap(); 36 | assert_eq!(*ohlcv, OHLCV_MSG); 37 | }; 38 | } 39 | 40 | #[test] 41 | #[should_panic] 42 | fn test_transmute_record_bytes_small_buffer() { 43 | let source = OHLCV_MSG; 44 | unsafe { 45 | let slice = std::slice::from_raw_parts( 46 | &source as *const OhlcvMsg as *const u8, 47 | mem::size_of::() - 5, 48 | ); 49 | transmute_record_bytes::(slice); 50 | }; 51 | } 52 | 53 | #[test] 54 | fn test_transmute_record() { 55 | let source = Box::new(OHLCV_MSG); 56 | let ohlcv_ref: &OhlcvMsg = unsafe { transmute_record(&source.hd) }.unwrap(); 57 | assert_eq!(*ohlcv_ref, OHLCV_MSG); 58 | } 59 | 60 | #[test] 61 | fn test_transmute_record_mut() { 62 | let mut source = Box::new(OHLCV_MSG); 63 | let ohlcv_ref: &OhlcvMsg = unsafe { transmute_record_mut(&mut source.hd) }.unwrap(); 64 | assert_eq!(*ohlcv_ref, OHLCV_MSG); 65 | } 66 | 67 | #[rstest] 68 | #[case::header(RecordHeader::default::(rtype::MBO), 16)] 69 | #[case::mbo(MboMsg::default(), 56)] 70 | #[case::ba_pair(BidAskPair::default(), 32)] 71 | #[case::cba_pair(ConsolidatedBidAskPair::default(), mem::size_of::())] 72 | #[case::trade(TradeMsg::default(), 48)] 73 | #[case::mbp1(Mbp1Msg::default(), mem::size_of::() + mem::size_of::())] 74 | #[case::mbp10(Mbp10Msg::default(), mem::size_of::() + mem::size_of::() * 10)] 75 | #[case::bbo(BboMsg::default_for_schema(Schema::Bbo1S), mem::size_of::())] 76 | #[case::cmbp1(Cmbp1Msg::default_for_schema(Schema::Cmbp1), mem::size_of::())] 77 | #[case::cbbo(CbboMsg::default_for_schema(Schema::Cbbo1S), mem::size_of::())] 78 | #[case::ohlcv(OhlcvMsg::default_for_schema(Schema::Ohlcv1S), 56)] 79 | #[case::status(StatusMsg::default(), 40)] 80 | #[case::definition(InstrumentDefMsg::default(), 520)] 81 | #[case::imbalance(ImbalanceMsg::default(), 112)] 82 | #[case::stat(StatMsg::default(), 80)] 83 | #[case::error(ErrorMsg::default(), 320)] 84 | #[case::symbol_mapping(SymbolMappingMsg::default(), 176)] 85 | #[case::system(SystemMsg::default(), 320)] 86 | #[case::with_ts_out(WithTsOut::new(SystemMsg::default(), 0), mem::size_of::() + 8)] 87 | fn test_sizes(#[case] _rec: R, #[case] exp: usize) { 88 | assert_eq!(mem::size_of::(), exp); 89 | assert!(mem::size_of::() <= crate::MAX_RECORD_LEN); 90 | } 91 | 92 | #[rstest] 93 | #[case::header(RecordHeader::default::(rtype::MBO))] 94 | #[case::mbo(MboMsg::default())] 95 | #[case::ba_pair(BidAskPair::default())] 96 | #[case::cba_pair(ConsolidatedBidAskPair::default())] 97 | #[case::trade(TradeMsg::default())] 98 | #[case::mbp1(Mbp1Msg::default())] 99 | #[case::mbp10(Mbp10Msg::default())] 100 | #[case::bbo(BboMsg::default_for_schema(Schema::Bbo1S))] 101 | #[case::cmbp1(Cmbp1Msg::default_for_schema(Schema::Cmbp1))] 102 | #[case::cbbo(CbboMsg::default_for_schema(Schema::Cbbo1S))] 103 | #[case::ohlcv(OhlcvMsg::default_for_schema(Schema::Ohlcv1S))] 104 | #[case::status(StatusMsg::default())] 105 | #[case::definition(InstrumentDefMsg::default())] 106 | #[case::imbalance(ImbalanceMsg::default())] 107 | #[case::stat(StatMsg::default())] 108 | #[case::error(ErrorMsg::default())] 109 | #[case::symbol_mapping(SymbolMappingMsg::default())] 110 | #[case::system(SystemMsg::default())] 111 | fn test_alignment_and_no_padding(#[case] _rec: R) { 112 | let layout = R::type_layout(); 113 | assert_eq!(layout.alignment, 8, "Unexpected alignment: {layout}"); 114 | for field in layout.fields.iter() { 115 | assert!( 116 | matches!(field, Field::Field { .. }), 117 | "Detected padding: {layout}" 118 | ); 119 | } 120 | } 121 | 122 | #[test] 123 | fn test_bbo_alignment_matches_mbp1() { 124 | assert_eq!(offset_of!(BboMsg, hd), offset_of!(Mbp1Msg, hd)); 125 | assert_eq!(offset_of!(BboMsg, price), offset_of!(Mbp1Msg, price)); 126 | assert_eq!(offset_of!(BboMsg, size), offset_of!(Mbp1Msg, size)); 127 | assert_eq!(offset_of!(BboMsg, side), offset_of!(Mbp1Msg, side)); 128 | assert_eq!(offset_of!(BboMsg, flags), offset_of!(Mbp1Msg, flags)); 129 | assert_eq!(offset_of!(BboMsg, ts_recv), offset_of!(Mbp1Msg, ts_recv)); 130 | assert_eq!(offset_of!(BboMsg, sequence), offset_of!(Mbp1Msg, sequence)); 131 | assert_eq!(offset_of!(BboMsg, levels), offset_of!(Mbp1Msg, levels)); 132 | } 133 | 134 | #[test] 135 | fn test_mbo_index_ts() { 136 | let rec = MboMsg { 137 | ts_recv: 1, 138 | ..Default::default() 139 | }; 140 | assert_eq!(rec.raw_index_ts(), 1); 141 | } 142 | 143 | #[test] 144 | fn test_def_index_ts() { 145 | let rec = InstrumentDefMsg { 146 | ts_recv: 1, 147 | ..Default::default() 148 | }; 149 | assert_eq!(rec.raw_index_ts(), 1); 150 | } 151 | 152 | #[test] 153 | fn test_db_ts_always_valid_time_offsetdatetime() { 154 | assert!(time::OffsetDateTime::from_unix_timestamp_nanos(0).is_ok()); 155 | assert!(time::OffsetDateTime::from_unix_timestamp_nanos((u64::MAX - 1) as i128).is_ok()); 156 | assert!(time::OffsetDateTime::from_unix_timestamp_nanos(UNDEF_TIMESTAMP as i128).is_ok()); 157 | } 158 | 159 | #[test] 160 | fn test_record_object_safe() { 161 | let _record: Box = Box::new(ErrorMsg::new(1, None, "Boxed record", true)); 162 | } 163 | -------------------------------------------------------------------------------- /rust/dbn/src/enums/methods.rs: -------------------------------------------------------------------------------- 1 | use crate::{InstrumentClass, RType, Schema, TriState, VersionUpgradePolicy}; 2 | 3 | impl InstrumentClass { 4 | /// Returns `true` if the instrument class is a type of option. 5 | /// 6 | /// Note: excludes [`Self::MixedSpread`], which *may* include options. 7 | pub fn is_option(&self) -> bool { 8 | matches!(self, Self::Call | Self::Put | Self::OptionSpread) 9 | } 10 | 11 | /// Returns `true` if the instrument class is a type of future. 12 | /// 13 | /// Note: excludes [`Self::MixedSpread`], which *may* include futures. 14 | pub fn is_future(&self) -> bool { 15 | matches!(self, Self::Future | Self::FutureSpread) 16 | } 17 | 18 | /// Returns `true` if the instrument class is a type of spread, i.e. composed of two 19 | /// or more instrument legs. 20 | pub fn is_spread(&self) -> bool { 21 | matches!( 22 | self, 23 | Self::FutureSpread | Self::OptionSpread | Self::MixedSpread 24 | ) 25 | } 26 | } 27 | 28 | /// Get the corresponding `rtype` for the given `schema`. 29 | impl From for RType { 30 | fn from(schema: Schema) -> Self { 31 | match schema { 32 | Schema::Mbo => RType::Mbo, 33 | Schema::Mbp1 | Schema::Tbbo => RType::Mbp1, 34 | Schema::Mbp10 => RType::Mbp10, 35 | Schema::Trades => RType::Mbp0, 36 | Schema::Ohlcv1S => RType::Ohlcv1S, 37 | Schema::Ohlcv1M => RType::Ohlcv1M, 38 | Schema::Ohlcv1H => RType::Ohlcv1H, 39 | Schema::Ohlcv1D => RType::Ohlcv1D, 40 | Schema::OhlcvEod => RType::OhlcvEod, 41 | Schema::Definition => RType::InstrumentDef, 42 | Schema::Statistics => RType::Statistics, 43 | Schema::Status => RType::Status, 44 | Schema::Imbalance => RType::Imbalance, 45 | Schema::Cmbp1 => RType::Cmbp1, 46 | Schema::Cbbo1S => RType::Cbbo1S, 47 | Schema::Cbbo1M => RType::Cbbo1M, 48 | Schema::Tcbbo => RType::Tcbbo, 49 | Schema::Bbo1S => RType::Bbo1S, 50 | Schema::Bbo1M => RType::Bbo1M, 51 | } 52 | } 53 | } 54 | 55 | impl RType { 56 | /// Tries to convert the given rtype to a [`Schema`]. 57 | /// 58 | /// Returns `None` if there's no corresponding `Schema` for the given rtype or 59 | /// in the case of `OHLCV_DEPRECATED`, it doesn't map to a single `Schema`. 60 | pub fn try_into_schema(rtype: u8) -> Option { 61 | use crate::enums::rtype::*; 62 | match rtype { 63 | MBP_0 => Some(Schema::Trades), 64 | MBP_1 => Some(Schema::Mbp1), 65 | MBP_10 => Some(Schema::Mbp10), 66 | OHLCV_1S => Some(Schema::Ohlcv1S), 67 | OHLCV_1M => Some(Schema::Ohlcv1M), 68 | OHLCV_1H => Some(Schema::Ohlcv1H), 69 | OHLCV_1D => Some(Schema::Ohlcv1D), 70 | OHLCV_EOD => Some(Schema::OhlcvEod), 71 | STATUS => Some(Schema::Status), 72 | INSTRUMENT_DEF => Some(Schema::Definition), 73 | IMBALANCE => Some(Schema::Imbalance), 74 | STATISTICS => Some(Schema::Statistics), 75 | MBO => Some(Schema::Mbo), 76 | CMBP_1 => Some(Schema::Cmbp1), 77 | CBBO_1S => Some(Schema::Cbbo1S), 78 | CBBO_1M => Some(Schema::Cbbo1M), 79 | TCBBO => Some(Schema::Tcbbo), 80 | BBO_1S => Some(Schema::Bbo1S), 81 | BBO_1M => Some(Schema::Bbo1M), 82 | _ => None, 83 | } 84 | } 85 | 86 | /// Returns the interval associated with the `RType` if it's a subsampled 87 | /// record type, otherwise `None`. 88 | pub const fn interval(self) -> Option { 89 | match self { 90 | RType::Ohlcv1S | RType::Cbbo1S | RType::Bbo1S => Some(time::Duration::SECOND), 91 | RType::Ohlcv1M | RType::Cbbo1M | RType::Bbo1M => Some(time::Duration::MINUTE), 92 | RType::Ohlcv1H => Some(time::Duration::HOUR), 93 | RType::Ohlcv1D | RType::OhlcvEod => Some(time::Duration::DAY), 94 | _ => None, 95 | } 96 | } 97 | } 98 | 99 | impl Schema { 100 | /// Returns the interval associated with the `Schema` if it's a subsampled 101 | /// schema, otherwise `None`. 102 | pub fn interval(self) -> Option { 103 | RType::from(self).interval() 104 | } 105 | } 106 | 107 | impl From for Option { 108 | fn from(value: TriState) -> Self { 109 | match value { 110 | TriState::NotAvailable => None, 111 | TriState::No => Some(false), 112 | TriState::Yes => Some(true), 113 | } 114 | } 115 | } 116 | 117 | impl From> for TriState { 118 | fn from(value: Option) -> Self { 119 | match value { 120 | Some(true) => Self::Yes, 121 | Some(false) => Self::No, 122 | None => Self::NotAvailable, 123 | } 124 | } 125 | } 126 | 127 | impl VersionUpgradePolicy { 128 | /// Validates a given DBN `version` is compatible with the upgrade policy. 129 | /// 130 | /// # Errors 131 | /// This function returns an error if the version and upgrade policy are 132 | /// incompatible. 133 | pub fn validate_compatibility(self, version: u8) -> crate::Result<()> { 134 | if version > 2 && self == Self::UpgradeToV2 { 135 | Err(crate::Error::decode("Invalid combination of `VersionUpgradePolicy::UpgradeToV2` and input version 3. Choose either `AsIs` and `UpgradeToV3` as an upgrade policy")) 136 | } else { 137 | Ok(()) 138 | } 139 | } 140 | 141 | pub(crate) fn is_upgrade_situation(self, version: u8) -> bool { 142 | match (self, version) { 143 | (Self::AsIs, _) => false, 144 | (Self::UpgradeToV2, v) if v < 2 => true, 145 | (Self::UpgradeToV2, _) => false, 146 | (Self::UpgradeToV3, v) if v < 3 => true, 147 | (Self::UpgradeToV3, _) => false, 148 | } 149 | } 150 | 151 | /// Returns the output DBN version given the input version and upgrade policy. 152 | pub fn output_version(self, input_version: u8) -> u8 { 153 | match self { 154 | Self::AsIs => input_version, 155 | Self::UpgradeToV2 => 2, 156 | Self::UpgradeToV3 => 3, 157 | } 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /python/src/encode.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | io::{self, Read, Seek}, 3 | num::NonZeroU64, 4 | sync::Mutex, 5 | }; 6 | 7 | use dbn::encode::dbn::MetadataEncoder; 8 | use pyo3::{exceptions::PyTypeError, intern, prelude::*, types::PyBytes, IntoPyObjectExt}; 9 | 10 | /// Updates existing fields that have already been written to the given file. 11 | #[pyfunction] 12 | #[pyo3(signature = (file, start, end = None, limit = None))] 13 | pub fn update_encoded_metadata( 14 | _py: Python<'_>, 15 | mut file: PyFileLike, 16 | start: u64, 17 | end: Option, 18 | limit: Option, 19 | ) -> PyResult<()> { 20 | file.seek(io::SeekFrom::Start(0))?; 21 | let mut buf = [0; 4]; 22 | file.read_exact(&mut buf)?; 23 | let version = buf[3]; 24 | Ok(MetadataEncoder::new(file).update_encoded( 25 | version, 26 | start, 27 | end.and_then(NonZeroU64::new), 28 | limit.and_then(NonZeroU64::new), 29 | )?) 30 | } 31 | 32 | /// A Python object that implements the Python file interface. 33 | pub struct PyFileLike { 34 | inner: Mutex>, 35 | } 36 | 37 | impl<'py> FromPyObject<'py> for PyFileLike { 38 | fn extract_bound(any: &Bound<'py, pyo3::PyAny>) -> PyResult { 39 | Python::attach(|py| { 40 | let obj: Py = any.extract()?; 41 | if obj.getattr(py, intern!(py, "read")).is_err() { 42 | return Err(PyTypeError::new_err( 43 | "object is missing a `read()` method".to_owned(), 44 | )); 45 | } 46 | if obj.getattr(py, intern!(py, "write")).is_err() { 47 | return Err(PyTypeError::new_err( 48 | "object is missing a `write()` method".to_owned(), 49 | )); 50 | } 51 | if obj.getattr(py, intern!(py, "seek")).is_err() { 52 | return Err(PyTypeError::new_err( 53 | "object is missing a `seek()` method".to_owned(), 54 | )); 55 | } 56 | Ok(PyFileLike { 57 | inner: Mutex::new(obj), 58 | }) 59 | }) 60 | } 61 | } 62 | 63 | impl io::Read for PyFileLike { 64 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 65 | Python::attach(|py| { 66 | let bytes: Vec = self 67 | .inner 68 | .lock() 69 | .unwrap() 70 | .call_method(py, intern!(py, "read"), (buf.len(),), None) 71 | .map_err(py_to_rs_io_err)? 72 | .extract(py)?; 73 | buf[..bytes.len()].clone_from_slice(&bytes); 74 | Ok(bytes.len()) 75 | }) 76 | } 77 | } 78 | 79 | impl io::Write for PyFileLike { 80 | fn write(&mut self, buf: &[u8]) -> Result { 81 | Python::attach(|py| { 82 | let bytes = PyBytes::new(py, buf); 83 | let number_bytes_written = self 84 | .inner 85 | .lock() 86 | .unwrap() 87 | .call_method(py, intern!(py, "write"), (bytes,), None) 88 | .map_err(py_to_rs_io_err)?; 89 | 90 | number_bytes_written.extract(py).map_err(py_to_rs_io_err) 91 | }) 92 | } 93 | 94 | fn flush(&mut self) -> Result<(), io::Error> { 95 | Python::attach(|py| { 96 | self.inner 97 | .lock() 98 | .unwrap() 99 | .call_method(py, intern!(py, "flush"), (), None) 100 | .map_err(py_to_rs_io_err)?; 101 | 102 | Ok(()) 103 | }) 104 | } 105 | } 106 | 107 | impl io::Seek for PyFileLike { 108 | fn seek(&mut self, pos: io::SeekFrom) -> Result { 109 | Python::attach(|py| { 110 | let (whence, offset) = match pos { 111 | io::SeekFrom::Start(i) => (0, i as i64), 112 | io::SeekFrom::Current(i) => (1, i), 113 | io::SeekFrom::End(i) => (2, i), 114 | }; 115 | 116 | let new_position = self 117 | .inner 118 | .lock() 119 | .unwrap() 120 | .call_method(py, intern!(py, "seek"), (offset, whence), None) 121 | .map_err(py_to_rs_io_err)?; 122 | 123 | new_position.extract(py).map_err(py_to_rs_io_err) 124 | }) 125 | } 126 | } 127 | 128 | fn py_to_rs_io_err(e: PyErr) -> io::Error { 129 | Python::attach(|py| { 130 | let e_as_object = e.into_bound_py_any(py).unwrap(); 131 | 132 | match e_as_object.call_method(intern!(py, "__str__"), (), None) { 133 | Ok(repr) => match repr.extract::() { 134 | Ok(s) => io::Error::other(s), 135 | Err(_e) => io::Error::other("An unknown error has occurred"), 136 | }, 137 | Err(_) => io::Error::other("Err doesn't have __str__"), 138 | } 139 | }) 140 | } 141 | 142 | #[cfg(test)] 143 | pub mod tests { 144 | use std::{ 145 | io::{Cursor, Seek, Write}, 146 | sync::{Arc, Mutex}, 147 | }; 148 | 149 | use super::*; 150 | 151 | #[pyclass] 152 | #[derive(Default)] 153 | pub struct MockPyFile { 154 | buf: Arc>>>, 155 | } 156 | 157 | #[pymethods] 158 | impl MockPyFile { 159 | fn read(&self) { 160 | unimplemented!(); 161 | } 162 | 163 | fn write(&mut self, bytes: &[u8]) -> usize { 164 | self.buf.lock().unwrap().write_all(bytes).unwrap(); 165 | bytes.len() 166 | } 167 | 168 | fn flush(&mut self) { 169 | self.buf.lock().unwrap().flush().unwrap(); 170 | } 171 | 172 | fn seek(&self, offset: i64, whence: i32) -> u64 { 173 | self.buf 174 | .lock() 175 | .unwrap() 176 | .seek(match whence { 177 | 0 => io::SeekFrom::Start(offset as u64), 178 | 1 => io::SeekFrom::Current(offset), 179 | 2 => io::SeekFrom::End(offset), 180 | _ => unimplemented!("whence value"), 181 | }) 182 | .unwrap() 183 | } 184 | } 185 | 186 | impl MockPyFile { 187 | pub fn new() -> Self { 188 | Self::default() 189 | } 190 | 191 | pub fn inner(&self) -> Arc>>> { 192 | self.buf.clone() 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /rust/dbn/src/flags.rs: -------------------------------------------------------------------------------- 1 | //! Bit set flags used in Databento market data. 2 | 3 | use std::fmt; 4 | 5 | #[cfg(feature = "python")] 6 | use pyo3::prelude::*; 7 | 8 | /// Indicates it's the last record in the event from the venue for a given 9 | /// `instrument_id`. 10 | pub const LAST: u8 = 1 << 7; 11 | /// Indicates a top-of-book record, not an individual order. 12 | pub const TOB: u8 = 1 << 6; 13 | /// Indicates the record was sourced from a replay, such as a snapshot server. 14 | pub const SNAPSHOT: u8 = 1 << 5; 15 | /// Indicates an aggregated price level record, not an individual order. 16 | pub const MBP: u8 = 1 << 4; 17 | /// Indicates the `ts_recv` value is inaccurate due to clock issues or packet 18 | /// reordering. 19 | pub const BAD_TS_RECV: u8 = 1 << 3; 20 | /// Indicates an unrecoverable gap was detected in the channel. 21 | pub const MAYBE_BAD_BOOK: u8 = 1 << 2; 22 | /// Used to indicate a publisher-specific event. 23 | pub const PUBLISHER_SPECIFIC: u8 = 1 << 1; 24 | 25 | /// A transparent wrapper around the bit field used in several DBN record types, 26 | /// namely [`MboMsg`](crate::MboMsg) and record types derived from it. 27 | #[repr(transparent)] 28 | #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Default, Hash)] 29 | #[cfg_attr(feature = "python", derive(FromPyObject), pyo3(transparent))] 30 | #[cfg_attr( 31 | feature = "serde", 32 | derive(serde::Serialize, serde::Deserialize), 33 | serde(transparent) 34 | )] 35 | pub struct FlagSet { 36 | raw: u8, 37 | } 38 | 39 | impl fmt::Debug for FlagSet { 40 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 41 | let mut has_written_flag = false; 42 | for (flag, name) in [ 43 | (LAST, stringify!(LAST)), 44 | (TOB, stringify!(TOB)), 45 | (SNAPSHOT, stringify!(SNAPSHOT)), 46 | (MBP, stringify!(MBP)), 47 | (BAD_TS_RECV, stringify!(BAD_TS_RECV)), 48 | (MAYBE_BAD_BOOK, stringify!(MAYBE_BAD_BOOK)), 49 | (PUBLISHER_SPECIFIC, stringify!(PUBLISHER_SPECIFIC)), 50 | ] { 51 | if (self.raw() & flag) > 0 { 52 | if has_written_flag { 53 | write!(f, " | {name}")?; 54 | } else { 55 | write!(f, "{name}")?; 56 | has_written_flag = true; 57 | } 58 | } 59 | } 60 | if has_written_flag { 61 | write!(f, " ({})", self.raw()) 62 | } else { 63 | write!(f, "{}", self.raw()) 64 | } 65 | } 66 | } 67 | 68 | impl From for FlagSet { 69 | fn from(raw: u8) -> Self { 70 | Self { raw } 71 | } 72 | } 73 | 74 | impl FlagSet { 75 | /// Returns an empty [`FlagSet`]: one with no flags set. 76 | pub const fn empty() -> Self { 77 | Self { raw: 0 } 78 | } 79 | 80 | /// Creates a new flag set from `raw`. 81 | pub const fn new(raw: u8) -> Self { 82 | Self { raw } 83 | } 84 | 85 | /// Turns all flags off, i.e. to `false`. 86 | pub fn clear(&mut self) -> &mut Self { 87 | self.raw = 0; 88 | self 89 | } 90 | 91 | /// Returns the raw value. 92 | pub const fn raw(&self) -> u8 { 93 | self.raw 94 | } 95 | 96 | /// Sets the flags directly with a raw `u8`. 97 | pub fn set_raw(&mut self, raw: u8) { 98 | self.raw = raw; 99 | } 100 | 101 | /// Returns `true` if any of the flags are on or set to true. 102 | pub const fn any(&self) -> bool { 103 | self.raw > 0 104 | } 105 | 106 | /// Returns `true` if all flags are unset/false. 107 | pub fn is_empty(&self) -> bool { 108 | self.raw == 0 109 | } 110 | 111 | /// Returns `true` if it's the last record in the event from the venue for a given 112 | /// `instrument_id`. 113 | pub const fn is_last(&self) -> bool { 114 | (self.raw & LAST) > 0 115 | } 116 | 117 | /// Sets the `LAST` bit flag to `true` to indicate this is the last record in the 118 | /// event for a given instrument. 119 | pub fn set_last(&mut self) -> Self { 120 | self.raw |= LAST; 121 | *self 122 | } 123 | 124 | /// Returns `true` if it's a top-of-book record, not an individual order. 125 | pub const fn is_tob(&self) -> bool { 126 | (self.raw & TOB) > 0 127 | } 128 | 129 | /// Sets the `TOB` bit flag to `true` to indicate this is a top-of-book record. 130 | pub fn set_tob(&mut self) -> Self { 131 | self.raw |= TOB; 132 | *self 133 | } 134 | 135 | /// Returns `true` if this record was sourced from a replay, such as a snapshot 136 | /// server. 137 | pub const fn is_snapshot(&self) -> bool { 138 | (self.raw & SNAPSHOT) > 0 139 | } 140 | 141 | /// Sets the `SNAPSHOT` bit flag to `true` to indicate this record was sourced from 142 | /// a replay. 143 | pub fn set_snapshot(&mut self) -> Self { 144 | self.raw |= SNAPSHOT; 145 | *self 146 | } 147 | 148 | /// Returns `true` if this record is an aggregated price level record, not an 149 | /// individual order. 150 | pub const fn is_mbp(&self) -> bool { 151 | (self.raw & MBP) > 0 152 | } 153 | 154 | /// Sets the `MBP` bit flag to `true` to indicate this record is an aggregated price 155 | /// level record. 156 | pub fn set_mbp(&mut self) -> Self { 157 | self.raw |= MBP; 158 | *self 159 | } 160 | 161 | /// Returns `true` if this record has an inaccurate `ts_recv` value due to clock 162 | /// issues or packet reordering. 163 | pub const fn is_bad_ts_recv(&self) -> bool { 164 | (self.raw & BAD_TS_RECV) > 0 165 | } 166 | 167 | /// Sets the `BAD_TS_RECV` bit flag to `true` to indicate this record has an 168 | /// inaccurate `ts_recv` value. 169 | pub fn set_bad_ts_recv(&mut self) -> Self { 170 | self.raw |= BAD_TS_RECV; 171 | *self 172 | } 173 | 174 | /// Returns `true` if this record is from a channel where an unrecoverable gap was 175 | /// detected. 176 | pub const fn is_maybe_bad_book(&self) -> bool { 177 | (self.raw & MAYBE_BAD_BOOK) > 0 178 | } 179 | 180 | /// Sets the `MAYBE_BAD_BOOK` bit flag to `true` to indicate this record is from a 181 | /// channel where an unrecoverable gap was detected. 182 | pub fn set_maybe_bad_book(&mut self) -> Self { 183 | self.raw |= MAYBE_BAD_BOOK; 184 | *self 185 | } 186 | 187 | /// Returns `true` if this record has the publisher-specific flag set. 188 | pub const fn is_publisher_specific(&self) -> bool { 189 | (self.raw & PUBLISHER_SPECIFIC) > 0 190 | } 191 | 192 | /// Sets the `PUBLISHER_SPECIFIC` bit flag to `true`. 193 | pub fn set_publisher_specific(&mut self) -> Self { 194 | self.raw |= PUBLISHER_SPECIFIC; 195 | *self 196 | } 197 | } 198 | 199 | #[cfg(test)] 200 | mod tests { 201 | use super::*; 202 | 203 | use rstest::*; 204 | 205 | #[rstest] 206 | #[case::empty(FlagSet::empty(), "0")] 207 | #[case::one_set(FlagSet::empty().set_mbp(), "MBP (16)")] 208 | #[case::three_set(FlagSet::empty().set_tob().set_snapshot().set_maybe_bad_book(), "TOB | SNAPSHOT | MAYBE_BAD_BOOK (100)")] 209 | #[case::reserved_set( 210 | FlagSet::new(255), 211 | "LAST | TOB | SNAPSHOT | MBP | BAD_TS_RECV | MAYBE_BAD_BOOK | PUBLISHER_SPECIFIC (255)" 212 | )] 213 | fn dbg(#[case] target: FlagSet, #[case] exp: &str) { 214 | assert_eq!(format!("{target:?}"), exp); 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /rust/dbn/src/record/conv.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | 3 | /// Provides a _relatively safe_ method for converting a reference to 4 | /// [`RecordHeader`] to a struct beginning with the header. Because it accepts a 5 | /// reference, the lifetime of the returned reference is tied to the input. This 6 | /// function checks `rtype` before casting to ensure `bytes` contains a `T`. 7 | /// 8 | /// # Safety 9 | /// `raw` must contain at least `std::mem::size_of::()` bytes and a valid 10 | /// [`RecordHeader`] instance. 11 | pub unsafe fn transmute_record_bytes(bytes: &[u8]) -> Option<&T> { 12 | assert!( 13 | bytes.len() >= mem::size_of::(), 14 | "Passing a slice smaller than `{}` to `transmute_record_bytes` is invalid", 15 | std::any::type_name::() 16 | ); 17 | let non_null = NonNull::new_unchecked(bytes.as_ptr().cast_mut()); 18 | if T::has_rtype(non_null.cast::().as_ref().rtype) { 19 | Some(non_null.cast::().as_ref()) 20 | } else { 21 | None 22 | } 23 | } 24 | 25 | /// Provides a _relatively safe_ method for converting a view on bytes into a 26 | /// a [`RecordHeader`]. 27 | /// Because it accepts a reference, the lifetime of the returned reference is 28 | /// tied to the input. 29 | /// 30 | /// # Safety 31 | /// `bytes` must contain a complete record (not only the header). This is so that 32 | /// the header can be subsequently passed to `transmute_record`. 33 | /// 34 | /// # Panics 35 | /// This function will panic if `bytes` is shorter the length of [`RecordHeader`], the 36 | /// minimum length a record can have. 37 | pub unsafe fn transmute_header_bytes(bytes: &[u8]) -> Option<&RecordHeader> { 38 | assert!( 39 | bytes.len() >= mem::size_of::(), 40 | concat!( 41 | "Passing a slice smaller than `", 42 | stringify!(RecordHeader), 43 | "` to `transmute_header_bytes` is invalid" 44 | ) 45 | ); 46 | let non_null = NonNull::new_unchecked(bytes.as_ptr().cast_mut()); 47 | let header = non_null.cast::().as_ref(); 48 | if header.record_size() > bytes.len() { 49 | None 50 | } else { 51 | Some(header) 52 | } 53 | } 54 | 55 | /// Provides a _relatively safe_ method for converting a reference to a 56 | /// [`RecordHeader`] to a struct beginning with the header. Because it accepts a reference, 57 | /// the lifetime of the returned reference is tied to the input. 58 | /// 59 | /// # Safety 60 | /// Although this function accepts a reference to a [`RecordHeader`], it's assumed this is 61 | /// part of a larger `T` struct. 62 | pub unsafe fn transmute_record(header: &RecordHeader) -> Option<&T> { 63 | if T::has_rtype(header.rtype) { 64 | // Safety: because it comes from a reference, `header` must not be null. It's ok 65 | // to cast to `mut` because it's never mutated. 66 | let non_null = NonNull::from(header); 67 | Some(non_null.cast::().as_ref()) 68 | } else { 69 | None 70 | } 71 | } 72 | 73 | /// Aliases `data` as a slice of raw bytes. 74 | /// 75 | /// # Safety 76 | /// `data` must be sized and plain old data (POD), i.e. no pointers. 77 | pub(crate) unsafe fn as_u8_slice(data: &T) -> &[u8] { 78 | slice::from_raw_parts((data as *const T).cast(), mem::size_of::()) 79 | } 80 | 81 | /// Provides a _relatively safe_ method for converting a mut reference to a 82 | /// [`RecordHeader`] to a struct beginning with the header. Because it accepts a reference, 83 | /// the lifetime of the returned reference is tied to the input. 84 | /// 85 | /// # Safety 86 | /// Although this function accepts a reference to a [`RecordHeader`], it's assumed this is 87 | /// part of a larger `T` struct. 88 | pub unsafe fn transmute_record_mut(header: &mut RecordHeader) -> Option<&mut T> { 89 | if T::has_rtype(header.rtype) { 90 | // Safety: because it comes from a reference, `header` must not be null. 91 | let non_null = NonNull::from(header); 92 | Some(non_null.cast::().as_mut()) 93 | } else { 94 | None 95 | } 96 | } 97 | 98 | /// Tries to convert a str slice to fixed-length null-terminated C char array. 99 | /// 100 | /// # Errors 101 | /// This function returns an error if `s` contains more than N - 1 characters. The last 102 | /// character is reserved for the null byte. 103 | pub fn str_to_c_chars(s: &str) -> Result<[c_char; N]> { 104 | let s = s.as_bytes(); 105 | if s.len() > (N - 1) { 106 | return Err(Error::encode(format!( 107 | "string cannot be longer than {}; received str of length {}", 108 | N - 1, 109 | s.len(), 110 | ))); 111 | } 112 | let mut res = [0; N]; 113 | res[..s.len()].copy_from_slice( 114 | // Safety: checked length of string and okay to interpret `u8` as `c_char`. 115 | unsafe { std::mem::transmute::<&[u8], &[c_char]>(s) }, 116 | ); 117 | Ok(res) 118 | } 119 | 120 | /// Tries to convert a slice of `c_char`s to a UTF-8 `str`. 121 | /// 122 | /// # Safety 123 | /// This should always be safe. 124 | /// 125 | /// # Preconditions 126 | /// None. 127 | /// 128 | /// # Errors 129 | /// This function returns an error if `chars` contains invalid UTF-8 or is not null-terminated. 130 | pub fn c_chars_to_str(chars: &[c_char; N]) -> Result<&str> { 131 | // Safety: Casting from i8 to u8 slice should be safe 132 | let bytes = unsafe { as_u8_slice(chars) }; 133 | let cstr = CStr::from_bytes_until_nul(bytes).map_err(|_| Error::Conversion { 134 | input: format!("{chars:?}"), 135 | desired_type: "CStr (null-terminated)", 136 | })?; 137 | 138 | cstr.to_str() 139 | .map_err(|e| Error::utf8(e, format!("converting c_char array: {chars:?}"))) 140 | } 141 | 142 | /// Parses a raw nanosecond-precision UNIX timestamp to an `OffsetDateTime`. Returns 143 | /// `None` if `ts` contains the sentinel for a null timestamp. 144 | pub fn ts_to_dt(ts: u64) -> Option { 145 | if ts == crate::UNDEF_TIMESTAMP { 146 | None 147 | } else { 148 | // u64::MAX is within maximum allowable range 149 | Some(time::OffsetDateTime::from_unix_timestamp_nanos(ts as i128).unwrap()) 150 | } 151 | } 152 | 153 | #[cfg(feature = "serde")] 154 | pub(crate) mod cstr_serde { 155 | use std::ffi::c_char; 156 | 157 | use serde::{de, ser, Deserialize, Deserializer, Serializer}; 158 | 159 | use super::{c_chars_to_str, str_to_c_chars}; 160 | 161 | pub fn serialize( 162 | chars: &[c_char; N], 163 | serializer: S, 164 | ) -> Result 165 | where 166 | S: Serializer, 167 | { 168 | serializer.serialize_str(c_chars_to_str(chars).map_err(ser::Error::custom)?) 169 | } 170 | 171 | pub fn deserialize<'de, D, const N: usize>(deserializer: D) -> Result<[c_char; N], D::Error> 172 | where 173 | D: Deserializer<'de>, 174 | { 175 | let str = String::deserialize(deserializer)?; 176 | str_to_c_chars(&str).map_err(de::Error::custom) 177 | } 178 | } 179 | 180 | #[cfg(test)] 181 | mod tests { 182 | use super::*; 183 | use std::os::raw::c_char; 184 | 185 | #[test] 186 | fn test_c_chars_to_str_success() { 187 | let null_terminated: [c_char; 5] = [ 188 | 'A' as c_char, 189 | 'A' as c_char, 190 | 'A' as c_char, 191 | 'A' as c_char, 192 | 0, 193 | ]; 194 | let result = c_chars_to_str(&null_terminated); 195 | assert_eq!(result.unwrap(), "AAAA"); 196 | } 197 | 198 | #[test] 199 | fn test_c_chars_to_str_failure_on_missing_null_terminator() { 200 | let non_null_terminated: [c_char; 5] = ['A' as c_char; 5]; 201 | let err = c_chars_to_str(&non_null_terminated) 202 | .expect_err("Expected failure on non-null terminated string"); 203 | 204 | assert!(matches!( 205 | err, 206 | Error::Conversion { 207 | input: _, 208 | desired_type: "CStr (null-terminated)", 209 | } 210 | )); 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /rust/dbn/src/v3/methods.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::c_char; 2 | 3 | use crate::{rtype, v1, v2, RecordHeader}; 4 | 5 | use super::{InstrumentDefMsg, StatMsg, UNDEF_STAT_QUANTITY}; 6 | 7 | impl From<&v1::InstrumentDefMsg> for InstrumentDefMsg { 8 | fn from(old: &v1::InstrumentDefMsg) -> Self { 9 | let mut res = Self { 10 | // recalculate length 11 | hd: RecordHeader::new::( 12 | rtype::INSTRUMENT_DEF, 13 | old.hd.publisher_id, 14 | old.hd.instrument_id, 15 | old.hd.ts_event, 16 | ), 17 | ts_recv: old.ts_recv, 18 | min_price_increment: old.min_price_increment, 19 | display_factor: old.display_factor, 20 | expiration: old.expiration, 21 | activation: old.activation, 22 | high_limit_price: old.high_limit_price, 23 | low_limit_price: old.low_limit_price, 24 | max_price_variation: old.max_price_variation, 25 | unit_of_measure_qty: old.unit_of_measure_qty, 26 | min_price_increment_amount: old.min_price_increment_amount, 27 | price_ratio: old.price_ratio, 28 | inst_attrib_value: old.inst_attrib_value, 29 | underlying_id: old.underlying_id, 30 | raw_instrument_id: u64::from(old.raw_instrument_id), 31 | market_depth_implied: old.market_depth_implied, 32 | market_depth: old.market_depth, 33 | market_segment_id: old.market_segment_id, 34 | max_trade_vol: old.max_trade_vol, 35 | min_lot_size: old.min_lot_size, 36 | min_lot_size_block: old.min_lot_size_block, 37 | min_lot_size_round_lot: old.min_lot_size_round_lot, 38 | min_trade_vol: old.min_trade_vol, 39 | contract_multiplier: old.contract_multiplier, 40 | decay_quantity: old.decay_quantity, 41 | original_contract_size: old.original_contract_size, 42 | appl_id: old.appl_id, 43 | maturity_year: old.maturity_year, 44 | decay_start_date: old.decay_start_date, 45 | channel_id: old.channel_id, 46 | currency: old.currency, 47 | settl_currency: old.settl_currency, 48 | secsubtype: old.secsubtype, 49 | group: old.group, 50 | exchange: old.exchange, 51 | cfi: old.cfi, 52 | security_type: old.security_type, 53 | unit_of_measure: old.unit_of_measure, 54 | underlying: old.underlying, 55 | strike_price_currency: old.strike_price_currency, 56 | instrument_class: old.instrument_class, 57 | strike_price: old.strike_price, 58 | match_algorithm: old.match_algorithm, 59 | main_fraction: old.main_fraction, 60 | price_display_format: old.price_display_format, 61 | sub_fraction: old.sub_fraction, 62 | underlying_product: old.underlying_product, 63 | security_update_action: old.security_update_action as c_char, 64 | maturity_month: old.maturity_month, 65 | maturity_day: old.maturity_day, 66 | maturity_week: old.maturity_week, 67 | user_defined_instrument: old.user_defined_instrument as c_char, 68 | contract_multiplier_unit: old.contract_multiplier_unit, 69 | flow_schedule_type: old.flow_schedule_type, 70 | tick_rule: old.tick_rule, 71 | ..Default::default() 72 | }; 73 | res.asset[..v1::ASSET_CSTR_LEN].copy_from_slice(old.asset.as_slice()); 74 | res.raw_symbol[..v1::SYMBOL_CSTR_LEN].copy_from_slice(old.raw_symbol.as_slice()); 75 | res 76 | } 77 | } 78 | 79 | impl From<&v2::InstrumentDefMsg> for InstrumentDefMsg { 80 | fn from(old: &v2::InstrumentDefMsg) -> Self { 81 | let mut res = Self { 82 | // recalculate length 83 | hd: RecordHeader::new::( 84 | rtype::INSTRUMENT_DEF, 85 | old.hd.publisher_id, 86 | old.hd.instrument_id, 87 | old.hd.ts_event, 88 | ), 89 | ts_recv: old.ts_recv, 90 | min_price_increment: old.min_price_increment, 91 | display_factor: old.display_factor, 92 | expiration: old.expiration, 93 | activation: old.activation, 94 | high_limit_price: old.high_limit_price, 95 | low_limit_price: old.low_limit_price, 96 | max_price_variation: old.max_price_variation, 97 | unit_of_measure_qty: old.unit_of_measure_qty, 98 | min_price_increment_amount: old.min_price_increment_amount, 99 | price_ratio: old.price_ratio, 100 | inst_attrib_value: old.inst_attrib_value, 101 | underlying_id: old.underlying_id, 102 | raw_instrument_id: u64::from(old.raw_instrument_id), 103 | market_depth_implied: old.market_depth_implied, 104 | market_depth: old.market_depth, 105 | market_segment_id: old.market_segment_id, 106 | max_trade_vol: old.max_trade_vol, 107 | min_lot_size: old.min_lot_size, 108 | min_lot_size_block: old.min_lot_size_block, 109 | min_lot_size_round_lot: old.min_lot_size_round_lot, 110 | min_trade_vol: old.min_trade_vol, 111 | contract_multiplier: old.contract_multiplier, 112 | decay_quantity: old.decay_quantity, 113 | original_contract_size: old.original_contract_size, 114 | appl_id: old.appl_id, 115 | maturity_year: old.maturity_year, 116 | decay_start_date: old.decay_start_date, 117 | channel_id: old.channel_id, 118 | currency: old.currency, 119 | settl_currency: old.settl_currency, 120 | secsubtype: old.secsubtype, 121 | group: old.group, 122 | exchange: old.exchange, 123 | cfi: old.cfi, 124 | security_type: old.security_type, 125 | unit_of_measure: old.unit_of_measure, 126 | underlying: old.underlying, 127 | strike_price_currency: old.strike_price_currency, 128 | instrument_class: old.instrument_class, 129 | strike_price: old.strike_price, 130 | match_algorithm: old.match_algorithm, 131 | main_fraction: old.main_fraction, 132 | price_display_format: old.price_display_format, 133 | sub_fraction: old.sub_fraction, 134 | underlying_product: old.underlying_product, 135 | security_update_action: old.security_update_action as c_char, 136 | maturity_month: old.maturity_month, 137 | maturity_day: old.maturity_day, 138 | maturity_week: old.maturity_week, 139 | user_defined_instrument: old.user_defined_instrument as c_char, 140 | contract_multiplier_unit: old.contract_multiplier_unit, 141 | flow_schedule_type: old.flow_schedule_type, 142 | tick_rule: old.tick_rule, 143 | raw_symbol: old.raw_symbol, 144 | ..Default::default() 145 | }; 146 | res.asset[..v2::ASSET_CSTR_LEN].copy_from_slice(old.asset.as_slice()); 147 | res 148 | } 149 | } 150 | 151 | impl From<&v1::StatMsg> for StatMsg { 152 | fn from(old: &v1::StatMsg) -> Self { 153 | Self { 154 | // recalculate length 155 | hd: RecordHeader::new::( 156 | rtype::STATISTICS, 157 | old.hd.publisher_id, 158 | old.hd.instrument_id, 159 | old.hd.ts_event, 160 | ), 161 | ts_recv: old.ts_recv, 162 | ts_ref: old.ts_ref, 163 | price: old.price, 164 | quantity: if old.quantity == v1::UNDEF_STAT_QUANTITY { 165 | UNDEF_STAT_QUANTITY 166 | } else { 167 | old.quantity as i64 168 | }, 169 | sequence: old.sequence, 170 | ts_in_delta: old.ts_in_delta, 171 | stat_type: old.stat_type, 172 | channel_id: old.channel_id, 173 | update_action: old.update_action, 174 | stat_flags: old.stat_flags, 175 | _reserved: Default::default(), 176 | } 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /rust/dbn-macros/src/serialize.rs: -------------------------------------------------------------------------------- 1 | use proc_macro2::TokenStream; 2 | use quote::quote; 3 | use syn::{parse_macro_input, Data, DeriveInput, Field}; 4 | 5 | use crate::{ 6 | dbn_attr::{ 7 | find_dbn_serialize_attr, get_sorted_fields, is_hidden, C_CHAR_ATTR, FIXED_PRICE_ATTR, 8 | UNIX_NANOS_ATTR, 9 | }, 10 | utils::crate_name, 11 | }; 12 | 13 | pub fn derive_csv_macro_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 14 | let DeriveInput { ident, data, .. } = parse_macro_input!(input as DeriveInput); 15 | 16 | if let Data::Struct(data_struct) = data { 17 | if let syn::Fields::Named(fields) = data_struct.fields { 18 | let crate_name = crate_name(); 19 | let fields = match get_sorted_fields(fields) { 20 | Ok(fields) => fields, 21 | Err(ts) => { 22 | return ts.into_compile_error().into(); 23 | } 24 | }; 25 | let serialize_header_iter = fields.iter().map(write_csv_header_token_stream); 26 | let serialize_fields = fields 27 | .iter() 28 | .map(write_csv_field_token_stream) 29 | .collect::>>() 30 | .unwrap_or_else(|e| vec![syn::Error::to_compile_error(&e)]); 31 | return quote! { 32 | impl #crate_name::encode::csv::serialize::CsvSerialize for #ident { 33 | fn serialize_header(writer: &mut ::csv::Writer) -> ::csv::Result<()> { 34 | use #crate_name::encode::csv::serialize::WriteField; 35 | 36 | #(#serialize_header_iter)* 37 | Ok(()) 38 | } 39 | 40 | fn serialize_to( 41 | &self, 42 | writer: &mut ::csv::Writer 43 | ) -> ::csv::Result<()> { 44 | use #crate_name::encode::csv::serialize::WriteField; 45 | 46 | #(#serialize_fields)* 47 | Ok(()) 48 | } 49 | } 50 | } 51 | .into(); 52 | } 53 | } 54 | syn::Error::new(ident.span(), "Can only derive CsvSerialize for structs") 55 | .into_compile_error() 56 | .into() 57 | } 58 | 59 | pub fn derive_json_macro_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { 60 | let DeriveInput { ident, data, .. } = parse_macro_input!(input as DeriveInput); 61 | 62 | if let Data::Struct(data_struct) = data { 63 | if let syn::Fields::Named(fields) = data_struct.fields { 64 | let crate_name = crate_name(); 65 | let fields = match get_sorted_fields(fields) { 66 | Ok(fields) => fields, 67 | Err(ts) => { 68 | return ts.into_compile_error().into(); 69 | } 70 | }; 71 | let serialize_fields = fields 72 | .iter() 73 | .map(write_json_field_token_stream) 74 | .collect::>>() 75 | .unwrap_or_else(|e| vec![syn::Error::to_compile_error(&e)]); 76 | return quote! { 77 | impl crate::encode::json::serialize::JsonSerialize for #ident { 78 | fn to_json( 79 | &self, 80 | writer: &mut #crate_name::json_writer::JsonObjectWriter, 81 | ) { 82 | use #crate_name::encode::json::serialize::WriteField; 83 | 84 | #(#serialize_fields)* 85 | } 86 | } 87 | } 88 | .into(); 89 | } 90 | } 91 | syn::Error::new(ident.span(), "Can only derive JsonSerialize for structs") 92 | .into_compile_error() 93 | .into() 94 | } 95 | 96 | fn write_csv_header_token_stream(field: &Field) -> TokenStream { 97 | let ident = field.ident.as_ref().unwrap(); 98 | let field_type = &field.ty; 99 | // ignore dummy and skipped fields 100 | if is_hidden(field) { 101 | return TokenStream::new(); 102 | } 103 | quote! { 104 | <#field_type>::write_header(writer, stringify!(#ident))?; 105 | } 106 | } 107 | 108 | fn write_csv_field_token_stream(field: &Field) -> syn::Result { 109 | let ident = field.ident.as_ref().unwrap(); 110 | // ignore dummy fields 111 | if is_hidden(field) { 112 | return Ok(quote! {}); 113 | } 114 | if let Some(dbn_attr_id) = find_dbn_serialize_attr(field)? { 115 | if dbn_attr_id == UNIX_NANOS_ATTR { 116 | Ok(quote! { 117 | crate::encode::csv::serialize::write_ts_field::<_, PRETTY_TS>(writer, self.#ident)?; 118 | }) 119 | } else if dbn_attr_id == FIXED_PRICE_ATTR { 120 | Ok(quote! { 121 | crate::encode::csv::serialize::write_px_field::<_, PRETTY_PX>(writer, self.#ident)?; 122 | }) 123 | } else if dbn_attr_id == C_CHAR_ATTR { 124 | Ok(quote! { 125 | crate::encode::csv::serialize::write_c_char_field(writer, self.#ident)?; 126 | }) 127 | } else { 128 | Err(syn::Error::new( 129 | dbn_attr_id.span(), 130 | format!("Invalid attr `{dbn_attr_id}` passed to `#[dbn]`"), 131 | )) 132 | } 133 | } else { 134 | Ok(quote! { 135 | self.#ident.write_field::<_, PRETTY_PX, PRETTY_TS>(writer)?; 136 | }) 137 | } 138 | } 139 | 140 | fn write_json_field_token_stream(field: &Field) -> syn::Result { 141 | let ident = field.ident.as_ref().unwrap(); 142 | // ignore dummy fields 143 | if is_hidden(field) { 144 | return Ok(quote! {}); 145 | } 146 | if let Some(dbn_attr_id) = find_dbn_serialize_attr(field)? { 147 | if dbn_attr_id == UNIX_NANOS_ATTR { 148 | Ok(quote! { 149 | crate::encode::json::serialize::write_ts_field::<_, PRETTY_TS>(writer, stringify!(#ident), self.#ident); 150 | }) 151 | } else if dbn_attr_id == FIXED_PRICE_ATTR { 152 | Ok(quote! { 153 | crate::encode::json::serialize::write_px_field::<_, PRETTY_PX>(writer, stringify!(#ident), self.#ident); 154 | }) 155 | } else if dbn_attr_id == C_CHAR_ATTR { 156 | Ok(quote! { 157 | crate::encode::json::serialize::write_c_char_field(writer, stringify!(#ident), self.#ident); 158 | }) 159 | } else { 160 | Err(syn::Error::new( 161 | dbn_attr_id.span(), 162 | format!("Invalid attr `{dbn_attr_id}` passed to `#[dbn]`"), 163 | )) 164 | } 165 | } else { 166 | Ok(quote! { 167 | self.#ident.write_field::<_, PRETTY_PX, PRETTY_TS>(writer, stringify!(#ident)); 168 | }) 169 | } 170 | } 171 | 172 | #[cfg(test)] 173 | mod tests { 174 | use syn::FieldsNamed; 175 | 176 | use super::*; 177 | 178 | #[test] 179 | fn skip_field() { 180 | let input = quote!({ 181 | #[dbn(skip)] 182 | pub b: bool, 183 | }); 184 | let fields = syn::parse2::(input).unwrap(); 185 | assert_eq!(fields.named.len(), 1); 186 | let csv_generated = write_csv_field_token_stream(fields.named.first().unwrap()).unwrap(); 187 | let json_generated = write_json_field_token_stream(fields.named.first().unwrap()).unwrap(); 188 | assert!(csv_generated.is_empty()); 189 | assert!(json_generated.is_empty()); 190 | } 191 | 192 | #[test] 193 | fn skip_underscore_field() { 194 | let input = quote!({ 195 | pub _a: bool, 196 | }); 197 | let fields = syn::parse2::(input).unwrap(); 198 | assert_eq!(fields.named.len(), 1); 199 | let csv_generated = write_csv_field_token_stream(fields.named.first().unwrap()).unwrap(); 200 | let json_generated = write_json_field_token_stream(fields.named.first().unwrap()).unwrap(); 201 | assert!(csv_generated.is_empty()); 202 | assert!(json_generated.is_empty()); 203 | } 204 | } 205 | --------------------------------------------------------------------------------