├── .coveragerc ├── .github ├── copilot-instructions.md └── workflows │ ├── docs.yml │ ├── publish.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .python-version ├── CLAUDE.md ├── LICENSE ├── README.md ├── codecov.yml ├── docs ├── codecov-setup.md ├── contributing.md ├── guide │ ├── 01_listing_symbols.ipynb │ ├── 02_extracting_symbols.ipynb │ ├── 03_top_of_book_snapshots.ipynb │ ├── 04_full_lob_snapshots.ipynb │ └── getting-started.md ├── images │ └── meatpy.svg ├── index.md └── installation.md ├── mkdocs.yml ├── pyproject.toml ├── pytest.ini ├── rules └── python.mdc ├── samples ├── itch41 │ ├── Step0_ExtractSymbols.py │ ├── Step1_Parsing.py │ └── Step2_Processing.py └── itch50 │ ├── 01_listing_symbols.py │ ├── 02_extracting_symbols.py │ ├── 03_top_of_book_snapshots.py │ └── 04_full_lob_snapshots.py ├── src └── meatpy │ ├── __init__.py │ ├── event_handlers │ ├── __init__.py │ ├── lob_event_recorder.py │ ├── lob_recorder.py │ ├── ofi_recorder.py │ └── spot_measures_recorder.py │ ├── events.py │ ├── itch41 │ ├── __init__.py │ ├── itch41_exec_trade_recorder.py │ ├── itch41_market_message.py │ ├── itch41_market_processor.py │ ├── itch41_message_reader.py │ ├── itch41_ofi_recorder.py │ ├── itch41_order_event_recorder.py │ ├── itch41_top_of_book_message_recorder.py │ └── itch41_writer.py │ ├── itch50 │ ├── __init__.py │ ├── itch50_exec_trade_recorder.py │ ├── itch50_market_message.py │ ├── itch50_market_processor.py │ ├── itch50_message_reader.py │ ├── itch50_ofi_recorder.py │ ├── itch50_order_event_recorder.py │ ├── itch50_top_of_book_message_recorder.py │ └── itch50_writer.py │ ├── level.py │ ├── lob.py │ ├── market_event_handler.py │ ├── market_processor.py │ ├── message_reader.py │ ├── timestamp.py │ ├── trading_status.py │ ├── types.py │ └── writers │ ├── __init__.py │ ├── base_writer.py │ ├── csv_writer.py │ └── parquet_writer.py ├── tests ├── __init__.py ├── test_csv_writer.py ├── test_events.py ├── test_itch41_messages.py ├── test_itch41_processor.py ├── test_itch41_reader.py ├── test_itch50_json.py ├── test_itch50_reader_writer.py ├── test_itch50_validation.py ├── test_level.py ├── test_lob.py ├── test_market_event_handler.py ├── test_market_processor.py ├── test_parquet_writer.py ├── test_recorders.py ├── test_timestamp.py ├── test_trading_status.py ├── test_types.py └── test_writers.py └── uv.lock /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = src/meatpy 3 | branch = true 4 | omit = 5 | */tests/* 6 | */test_* 7 | */venv/* 8 | */.venv/* 9 | */site-packages/* 10 | setup.py 11 | 12 | [report] 13 | # Regexes for lines to exclude from consideration 14 | exclude_lines = 15 | # Have to re-enable the standard pragma 16 | pragma: no cover 17 | 18 | # Don't complain about missing debug-only code: 19 | def __repr__ 20 | if self\.debug 21 | 22 | # Don't complain if tests don't hit defensive assertion code: 23 | raise AssertionError 24 | raise NotImplementedError 25 | 26 | # Don't complain if non-runnable code isn't run: 27 | if 0: 28 | if __name__ .__main__.: 29 | 30 | # Don't complain about abstract methods, they aren't run: 31 | @(abc\.)?abstractmethod 32 | 33 | # Don't complain about type checking imports 34 | if TYPE_CHECKING: 35 | 36 | ignore_errors = true 37 | show_missing = true 38 | precision = 2 39 | 40 | [html] 41 | directory = htmlcov 42 | -------------------------------------------------------------------------------- /.github/copilot-instructions.md: -------------------------------------------------------------------------------- 1 | You are an AI assistant specialized in Python development. Your approach emphasizes: 2 | 3 | 1. Clear project structure with separate directories for source code, tests, docs, and config. 4 | 2. Modular design with distinct files for models, services, controllers, and utilities. 5 | 3. Use f-strings for formatting strings. 6 | 4. Configuration management using environment variables. 7 | 5. Robust error handling and logging, including context capture. 8 | 6. Comprehensive testing with pytest. 9 | 7. Detailed documentation using docstrings and README files. Use Google style docstrings. 10 | 8. Dependency management using uv. 11 | 9. Code style consistency using Ruff. 12 | 10. CI/CD implementation with GitHub Actions or GitLab CI. 13 | 11. AI-friendly coding practices: 14 | - Descriptive variable and function names 15 | - Type hints 16 | - Detailed comments for complex logic 17 | - Rich error context for debugging 18 | 19 | You provide code snippets and explanations tailored to these principles, optimizing for clarity and AI-assisted development. 20 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: read 13 | pages: write 14 | id-token: write 15 | 16 | concurrency: 17 | group: "pages" 18 | cancel-in-progress: false 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | 27 | - name: Set up Python 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: '3.11' 31 | 32 | - name: Install uv 33 | uses: astral-sh/setup-uv@v3 34 | with: 35 | version: "latest" 36 | 37 | - name: Install dependencies 38 | run: | 39 | uv sync --group docs 40 | 41 | - name: Build documentation 42 | run: | 43 | uv run mkdocs build --strict 44 | 45 | - name: Upload artifact 46 | if: github.ref == 'refs/heads/main' 47 | uses: actions/upload-pages-artifact@v3 48 | with: 49 | path: ./site 50 | 51 | deploy: 52 | if: github.ref == 'refs/heads/main' 53 | environment: 54 | name: github-pages 55 | url: ${{ steps.deployment.outputs.page_url }} 56 | runs-on: ubuntu-latest 57 | needs: build 58 | steps: 59 | - name: Deploy to GitHub Pages 60 | id: deployment 61 | uses: actions/deploy-pages@v4 62 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Manual Publish to PyPI 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | inputs: 8 | version: 9 | description: 'Version to publish (e.g., 1.0.0)' 10 | required: true 11 | type: string 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | test: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up Python 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: '3.11' 27 | 28 | - name: Install uv 29 | uses: astral-sh/setup-uv@v3 30 | with: 31 | version: "latest" 32 | 33 | - name: Install dependencies 34 | run: | 35 | uv sync --group dev 36 | 37 | - name: Run tests 38 | continue-on-error: true 39 | run: | 40 | uv run pytest 41 | 42 | - name: Run linting 43 | continue-on-error: true 44 | run: | 45 | uv run ruff check 46 | 47 | - name: Run formatting check 48 | continue-on-error: true 49 | run: | 50 | uv run ruff format --check 51 | 52 | build: 53 | needs: test 54 | runs-on: ubuntu-latest 55 | steps: 56 | - name: Checkout 57 | uses: actions/checkout@v4 58 | 59 | - name: Set up Python 60 | uses: actions/setup-python@v4 61 | with: 62 | python-version: '3.11' 63 | 64 | - name: Install uv 65 | uses: astral-sh/setup-uv@v3 66 | with: 67 | version: "latest" 68 | 69 | - name: Build package 70 | run: | 71 | uv build 72 | 73 | - name: Upload artifacts 74 | uses: actions/upload-artifact@v4 75 | with: 76 | name: dist 77 | path: dist/ 78 | 79 | publish: 80 | needs: build 81 | runs-on: ubuntu-latest 82 | environment: release 83 | permissions: 84 | id-token: write 85 | steps: 86 | - name: Download artifacts 87 | uses: actions/download-artifact@v4 88 | with: 89 | name: dist 90 | path: dist/ 91 | 92 | - name: Publish to PyPI 93 | uses: pypa/gh-action-pypi-publish@release/v1 94 | with: 95 | skip-existing: true 96 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Automated Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | 12 | jobs: 13 | release: 14 | runs-on: ubuntu-latest 15 | outputs: 16 | released: ${{ steps.release.outputs.released }} 17 | version: ${{ steps.release.outputs.version }} 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 23 | token: ${{ secrets.BYPASS_TOKEN }} 24 | 25 | - name: Set up Python 26 | uses: actions/setup-python@v4 27 | with: 28 | python-version: '3.11' 29 | 30 | - name: Install uv 31 | uses: astral-sh/setup-uv@v3 32 | with: 33 | version: "latest" 34 | 35 | - name: Install dependencies 36 | run: | 37 | uv sync --group dev 38 | 39 | - name: Run tests 40 | continue-on-error: true 41 | run: | 42 | uv run pytest 43 | 44 | - name: Run linting 45 | continue-on-error: true 46 | run: | 47 | uv run ruff check 48 | 49 | - name: Run formatting check 50 | continue-on-error: true 51 | run: | 52 | uv run ruff format --check 53 | 54 | - name: Determine version bump 55 | id: version_bump 56 | run: | 57 | # Get the latest tag 58 | LATEST_TAG=$(git tag -l 'v*' --sort=-version:refname | head -n1) 59 | if [ -z "$LATEST_TAG" ]; then 60 | LATEST_TAG="v0.0.0" 61 | fi 62 | 63 | echo "Latest tag: $LATEST_TAG" 64 | 65 | # Get commits since last tag 66 | if [ "$LATEST_TAG" = "v0.0.0" ]; then 67 | # No tags exist, get all commits 68 | COMMITS=$(git log --oneline --pretty=format:"%s") 69 | else 70 | # Get commits since last tag 71 | COMMITS=$(git log ${LATEST_TAG}..HEAD --oneline --pretty=format:"%s") 72 | fi 73 | 74 | echo "Commits since last tag:" 75 | echo "$COMMITS" 76 | 77 | # Check if there are any commits since last tag 78 | if [ -z "$COMMITS" ]; then 79 | echo "No commits since last tag, skipping release" 80 | echo "should_release=false" >> $GITHUB_OUTPUT 81 | exit 0 82 | fi 83 | 84 | # Determine version bump type based on commit messages 85 | BUMP_TYPE="patch" 86 | 87 | if echo "$COMMITS" | grep -qi "BREAKING CHANGE\|breaking:"; then 88 | BUMP_TYPE="major" 89 | elif echo "$COMMITS" | grep -qi "feat\|feature:"; then 90 | BUMP_TYPE="minor" 91 | elif echo "$COMMITS" | grep -qi "fix\|bug\|patch:"; then 92 | BUMP_TYPE="patch" 93 | fi 94 | 95 | echo "Version bump type: $BUMP_TYPE" 96 | echo "bump_type=$BUMP_TYPE" >> $GITHUB_OUTPUT 97 | 98 | # Calculate new version 99 | CURRENT_VERSION=$(echo $LATEST_TAG | sed 's/v//') 100 | if [ "$CURRENT_VERSION" = "0.0.0" ]; then 101 | # No existing tags, get current version from pyproject.toml 102 | CURRENT_VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\([^"]*\)"/\1/') 103 | echo "Current version from pyproject.toml: $CURRENT_VERSION" 104 | 105 | # Bump based on commit type 106 | IFS='.' read -r major minor patch <<< "$CURRENT_VERSION" 107 | case $BUMP_TYPE in 108 | major) 109 | NEW_VERSION="$((major + 1)).0.0" 110 | ;; 111 | minor) 112 | NEW_VERSION="$major.$((minor + 1)).0" 113 | ;; 114 | patch) 115 | NEW_VERSION="$major.$minor.$((patch + 1))" 116 | ;; 117 | esac 118 | else 119 | IFS='.' read -r major minor patch <<< "$CURRENT_VERSION" 120 | case $BUMP_TYPE in 121 | major) 122 | NEW_VERSION="$((major + 1)).0.0" 123 | ;; 124 | minor) 125 | NEW_VERSION="$major.$((minor + 1)).0" 126 | ;; 127 | patch) 128 | NEW_VERSION="$major.$minor.$((patch + 1))" 129 | ;; 130 | esac 131 | fi 132 | 133 | echo "New version: $NEW_VERSION" 134 | echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT 135 | echo "should_release=true" >> $GITHUB_OUTPUT 136 | 137 | - name: Update version in pyproject.toml 138 | id: update_version 139 | if: steps.version_bump.outputs.should_release == 'true' 140 | run: | 141 | NEW_VERSION="${{ steps.version_bump.outputs.new_version }}" 142 | sed -i "s/version = \"[^\"]*\"/version = \"$NEW_VERSION\"/" pyproject.toml 143 | 144 | echo "Updated pyproject.toml version to $NEW_VERSION" 145 | 146 | # Check if there are any changes to commit 147 | if git diff --quiet pyproject.toml; then 148 | echo "No version changes to commit" 149 | else 150 | git config user.name "github-actions[bot]" 151 | git config user.email "github-actions[bot]@users.noreply.github.com" 152 | git add pyproject.toml 153 | git commit -m "chore: bump version to $NEW_VERSION" 154 | git push 155 | fi 156 | 157 | - name: Create GitHub Release 158 | id: release 159 | if: steps.version_bump.outputs.should_release == 'true' 160 | env: 161 | GITHUB_TOKEN: ${{ secrets.BYPASS_TOKEN }} 162 | run: | 163 | NEW_VERSION="${{ steps.version_bump.outputs.new_version }}" 164 | BUMP_TYPE="${{ steps.version_bump.outputs.bump_type }}" 165 | 166 | # Create tag 167 | git tag "v$NEW_VERSION" 168 | git push origin "v$NEW_VERSION" 169 | 170 | # Create release 171 | gh release create "v$NEW_VERSION" \ 172 | --title "Release v$NEW_VERSION" \ 173 | --notes "## Changes in this release 174 | 175 | This release was automatically generated based on commit messages. 176 | 177 | Version bump type: $BUMP_TYPE 178 | 179 | ### Recent commits: 180 | $(git log --oneline --since="1 day ago" --pretty=format:"- %s")" \ 181 | --latest 182 | 183 | echo "released=true" >> $GITHUB_OUTPUT 184 | echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT 185 | 186 | publish: 187 | needs: release 188 | if: needs.release.outputs.released == 'true' 189 | runs-on: ubuntu-latest 190 | environment: release 191 | permissions: 192 | id-token: write 193 | contents: read 194 | steps: 195 | - name: Checkout 196 | uses: actions/checkout@v4 197 | 198 | - name: Set up Python 199 | uses: actions/setup-python@v4 200 | with: 201 | python-version: '3.11' 202 | 203 | - name: Install uv 204 | uses: astral-sh/setup-uv@v3 205 | with: 206 | version: "latest" 207 | 208 | - name: Update version for PyPI build 209 | run: | 210 | NEW_VERSION="${{ needs.release.outputs.version }}" 211 | sed -i "s/version = \"[^\"]*\"/version = \"$NEW_VERSION\"/" pyproject.toml 212 | echo "Updated pyproject.toml version to $NEW_VERSION for PyPI build" 213 | grep "version = " pyproject.toml 214 | 215 | - name: Build package 216 | run: | 217 | uv build 218 | 219 | - name: Publish to PyPI 220 | uses: pypa/gh-action-pypi-publish@release/v1 221 | with: 222 | skip-existing: true 223 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests and Coverage 2 | 3 | on: 4 | push: 5 | branches: [ main, develop ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: ['3.11', '3.12', '3.13'] 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Install uv 29 | uses: astral-sh/setup-uv@v3 30 | with: 31 | version: "latest" 32 | 33 | - name: Install dependencies 34 | run: | 35 | uv sync --group dev 36 | 37 | - name: Run linting 38 | run: | 39 | uv run ruff check 40 | 41 | - name: Run formatting check 42 | run: | 43 | uv run ruff format --check 44 | 45 | - name: Run tests with coverage 46 | run: | 47 | uv run pytest --cov=src/meatpy --cov-report=xml --cov-report=term-missing 48 | 49 | - name: Upload coverage to Codecov 50 | if: matrix.python-version == '3.11' 51 | uses: codecov/codecov-action@v4 52 | with: 53 | token: ${{ secrets.CODECOV_TOKEN }} 54 | file: ./coverage.xml 55 | flags: unittests 56 | name: codecov-umbrella 57 | fail_ci_if_error: false 58 | verbose: true 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python,macos 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,macos 3 | 4 | ### macOS ### 5 | # General 6 | .DS_Store 7 | .AppleDouble 8 | .LSOverride 9 | 10 | # Icon must end with two \r 11 | Icon 12 | 13 | 14 | # Thumbnails 15 | ._* 16 | 17 | # Files that might appear in the root of a volume 18 | .DocumentRevisions-V100 19 | .fseventsd 20 | .Spotlight-V100 21 | .TemporaryItems 22 | .Trashes 23 | .VolumeIcon.icns 24 | .com.apple.timemachine.donotpresent 25 | 26 | # Directories potentially created on remote AFP share 27 | .AppleDB 28 | .AppleDesktop 29 | Network Trash Folder 30 | Temporary Items 31 | .apdisk 32 | 33 | ### macOS Patch ### 34 | # iCloud generated files 35 | *.icloud 36 | 37 | ### Python ### 38 | # Byte-compiled / optimized / DLL files 39 | __pycache__/ 40 | *.py[cod] 41 | *$py.class 42 | 43 | # C extensions 44 | *.so 45 | 46 | # Distribution / packaging 47 | .Python 48 | build/ 49 | develop-eggs/ 50 | dist/ 51 | downloads/ 52 | eggs/ 53 | .eggs/ 54 | lib/ 55 | lib64/ 56 | parts/ 57 | sdist/ 58 | var/ 59 | wheels/ 60 | share/python-wheels/ 61 | *.egg-info/ 62 | .installed.cfg 63 | *.egg 64 | MANIFEST 65 | 66 | # PyInstaller 67 | # Usually these files are written by a python script from a template 68 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 69 | *.manifest 70 | *.spec 71 | 72 | # Installer logs 73 | pip-log.txt 74 | pip-delete-this-directory.txt 75 | 76 | # Unit test / coverage reports 77 | htmlcov/ 78 | .tox/ 79 | .nox/ 80 | .coverage 81 | .coverage.* 82 | .cache 83 | nosetests.xml 84 | coverage.xml 85 | *.cover 86 | *.py,cover 87 | .hypothesis/ 88 | .pytest_cache/ 89 | cover/ 90 | 91 | # Translations 92 | *.mo 93 | *.pot 94 | 95 | # Django stuff: 96 | *.log 97 | local_settings.py 98 | db.sqlite3 99 | db.sqlite3-journal 100 | 101 | # Flask stuff: 102 | instance/ 103 | .webassets-cache 104 | 105 | # Scrapy stuff: 106 | .scrapy 107 | 108 | # Sphinx documentation 109 | docs/_build/ 110 | 111 | # PyBuilder 112 | .pybuilder/ 113 | target/ 114 | 115 | # Jupyter Notebook 116 | .ipynb_checkpoints 117 | 118 | # IPython 119 | profile_default/ 120 | ipython_config.py 121 | 122 | # pyenv 123 | # For a library or package, you might want to ignore these files since the code is 124 | # intended to run in multiple environments; otherwise, check them in: 125 | # .python-version 126 | 127 | # pipenv 128 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 129 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 130 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 131 | # install all needed dependencies. 132 | #Pipfile.lock 133 | 134 | # poetry 135 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 136 | # This is especially recommended for binary packages to ensure reproducibility, and is more 137 | # commonly ignored for libraries. 138 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 139 | #poetry.lock 140 | 141 | # pdm 142 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 143 | #pdm.lock 144 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 145 | # in version control. 146 | # https://pdm.fming.dev/#use-with-ide 147 | .pdm.toml 148 | 149 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 150 | __pypackages__/ 151 | 152 | # Celery stuff 153 | celerybeat-schedule 154 | celerybeat.pid 155 | 156 | # SageMath parsed files 157 | *.sage.py 158 | 159 | # Environments 160 | .env 161 | .venv 162 | env/ 163 | venv/ 164 | ENV/ 165 | env.bak/ 166 | venv.bak/ 167 | 168 | # Spyder project settings 169 | .spyderproject 170 | .spyproject 171 | 172 | # Rope project settings 173 | .ropeproject 174 | 175 | # mkdocs documentation 176 | /site 177 | 178 | # mypy 179 | .mypy_cache/ 180 | .dmypy.json 181 | dmypy.json 182 | 183 | # Pyre type checker 184 | .pyre/ 185 | 186 | # pytype static type analyzer 187 | .pytype/ 188 | 189 | # Cython debug symbols 190 | cython_debug/ 191 | 192 | # PyCharm 193 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 194 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 195 | # and can be added to the global gitignore or merged into this file. For a more nuclear 196 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 197 | #.idea/ 198 | 199 | ### Python Patch ### 200 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 201 | poetry.toml 202 | 203 | # ruff 204 | .ruff_cache/ 205 | 206 | # LSP config files 207 | pyrightconfig.json 208 | 209 | # End of https://www.toptal.com/developers/gitignore/api/python,macos 210 | poetry.lock 211 | 212 | docs/guide/data/ 213 | samples/*/data/ 214 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-added-large-files 8 | - id: check-merge-conflict 9 | - id: check-case-conflict 10 | - id: check-docstring-first 11 | - id: debug-statements 12 | 13 | - repo: https://github.com/astral-sh/ruff-pre-commit 14 | rev: v0.12.0 15 | hooks: 16 | - id: ruff 17 | args: [--fix, --exit-non-zero-on-fix] 18 | additional_dependencies: [ruff] 19 | exclude: \.ipynb$ 20 | - id: ruff-format 21 | additional_dependencies: [ruff] 22 | exclude: \.ipynb$ 23 | 24 | - repo: local 25 | hooks: 26 | - id: pytest 27 | name: pytest 28 | entry: pytest 29 | language: system 30 | pass_filenames: false 31 | always_run: true 32 | stages: [manual] 33 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CLAUDE.md 2 | 3 | This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. 4 | 5 | ## Project Overview 6 | 7 | MeatPy is a Python framework for processing and analyzing high-frequency financial data, specifically designed for Nasdaq ITCH 5.0 format. It provides tools for parsing market messages, reconstructing limit order books, and analyzing market microstructure. 8 | 9 | ## Development Commands 10 | 11 | ### Environment Setup 12 | ```bash 13 | # Install dependencies using uv (recommended) 14 | uv sync 15 | 16 | # Install pre-commit hooks 17 | uv run pre-commit install 18 | ``` 19 | 20 | ### Testing 21 | ```bash 22 | # Run all tests with coverage 23 | uv run pytest 24 | 25 | # Run specific test markers 26 | uv run pytest -m "not slow" # Skip slow tests 27 | uv run pytest -m unit # Only unit tests 28 | uv run pytest -m integration # Only integration tests 29 | ``` 30 | 31 | ### Code Quality 32 | ```bash 33 | # Format code 34 | uv run ruff format 35 | 36 | # Check linting issues 37 | uv run ruff check 38 | 39 | # Fix auto-fixable linting issues 40 | uv run ruff check --fix 41 | ``` 42 | 43 | ### Documentation 44 | ```bash 45 | # Build documentation 46 | mkdocs build 47 | 48 | # Serve documentation locally 49 | mkdocs serve 50 | ``` 51 | 52 | ## Architecture Overview 53 | 54 | ### Core Components 55 | 56 | **Market Processing Pipeline**: The library follows an event-driven architecture centered around `MarketProcessor` classes that process market messages and maintain limit order book state. 57 | 58 | **Key Classes**: 59 | - `MarketProcessor`: Abstract base class for processing market messages (src/meatpy/market_processor.py:19) 60 | - `ITCH50MarketProcessor`: ITCH 5.0 specific implementation (src/meatpy/itch50/itch50_market_processor.py:87) 61 | - `LimitOrderBook`: Maintains the current state of the order book (src/meatpy/lob.py) 62 | - `MarketEventHandler`: Handles events during processing (src/meatpy/market_event_handler.py) 63 | 64 | ### Message Processing Flow 65 | 66 | 1. **Message Reading**: `MessageReader` classes parse binary market data into `MarketMessage` objects 67 | 2. **Event Processing**: `MarketProcessor` processes messages, updates the LOB, and notifies handlers 68 | 3. **Data Recording**: Event handlers record market events (trades, quotes, etc.) to various output formats 69 | 70 | ### Package Structure 71 | 72 | - `src/meatpy/` - Core library code 73 | - `itch50/` - ITCH 5.0 specific implementations 74 | - `event_handlers/` - Event recording and processing handlers 75 | - `writers/` - Output format writers (CSV, Parquet) 76 | - `tests/` - Test suite 77 | - `docs/` - Documentation (MkDocs format) 78 | - `samples/` - Example scripts showing typical usage patterns 79 | 80 | ### Generic Type System 81 | 82 | The codebase uses extensive generic typing for market data types: 83 | - `Price`, `Volume`, `OrderID`, `TradeRef`, `Qualifiers` are generic type parameters 84 | - ITCH 5.0 implementation uses concrete types: `int` for prices/volumes/IDs, `dict[str, str]` for qualifiers 85 | 86 | ## Development Guidelines 87 | 88 | ### Code Style 89 | - Use Google-style docstrings for all public APIs 90 | - Type hints are required for all function signatures 91 | - Follow Ruff formatting standards 92 | - Maintain test coverage above 80% 93 | 94 | ### Testing Approach 95 | - Tests use pytest with custom markers (unit, integration, slow, performance) 96 | - Coverage reporting configured in pytest.ini 97 | - Test data available in `tests/` and `docs/guide/data/` 98 | 99 | ### Configuration Files 100 | - `pyproject.toml` - Project metadata, dependencies, and tool configuration 101 | - `pytest.ini` - Test configuration with coverage requirements 102 | - `mkdocs.yml` - Documentation configuration 103 | - `rules/python.mdc` - Development guidelines for cursor/AI tools 104 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2025, Vincent Grégoire and Charles Martineau 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MeatPy 2 | 3 | 4 | [![PyPI version](https://badge.fury.io/py/meatpy.svg)](https://badge.fury.io/py/meatpy) 5 | [![License](https://img.shields.io/pypi/l/meatpy.svg)](https://github.com/vgreg/MeatPy/blob/main/LICENSE) 6 | [![Documentation Status](https://img.shields.io/badge/docs-online-brightgreen.svg)](https://www.vincentgregoire.com/MeatPy) 7 | [![codecov](https://codecov.io/gh/vgreg/MeatPy/branch/main/graph/badge.svg)](https://codecov.io/gh/vgreg/MeatPy) 8 | 9 | MeatPy Logo 10 | 11 | MeatPy is a Python framework for processing and analyzing high-frequency financial market data, specifically designed for working with NASDAQ ITCH protocol data feeds. It provides robust tools for reconstructing limit order books and extracting key market events from historical market data files. 12 | 13 | ## 🎯 Key Features 14 | 15 | - **📊 Limit Order Book Reconstruction**: Complete order book state tracking with proper handling of all order types and modifications 16 | - **🏛️ NASDAQ ITCH Support**: Full implementation for ITCH 5.0 and 4.1 protocols with native message parsing 17 | - **⚡ Event-Driven Architecture**: Flexible observer pattern for real-time event processing and analysis 18 | - **🔒 Type Safety**: Modern Python with comprehensive type hints and generic interfaces for robust data handling 19 | - **📁 Multiple Output Formats**: Export to CSV, Parquet, or implement custom output formats 20 | - **🚀 Performance Optimized**: Efficiently process multi-gigabyte ITCH files with streaming capabilities 21 | - **🔧 Extensible Design**: Easy to adapt for other market data formats and custom analysis needs 22 | 23 | ## 📊 Common Use Cases 24 | 25 | MeatPy is designed for market microstructure research and analysis: 26 | 27 | - **Order Book Reconstruction**: Rebuild complete limit order book state at any point in time 28 | - **Market Event Analysis**: Extract and analyze trades, quotes, and order modifications 29 | - **Top-of-Book Sampling**: Generate regular snapshots of best bid/ask prices and sizes 30 | 31 | ## 📦 Installation 32 | 33 | ### Quick Install 34 | 35 | ```bash 36 | pip install meatpy 37 | ``` 38 | 39 | ### With Optional Dependencies 40 | 41 | ```bash 42 | # For Parquet file support 43 | pip install meatpy[parquet] 44 | ``` 45 | 46 | 47 | ## 🚀 Quick Start 48 | 49 | Complete documentation is available at [https://www.vincentgregoire.com/MeatPy](https://www.vincentgregoire.com/MeatPy) 50 | 51 | 52 | ### Basic Message Reading 53 | 54 | ```python 55 | from pathlib import Path 56 | from meatpy.itch50 import ITCH50MessageReader 57 | 58 | # Define the path to our sample data file 59 | data_dir = Path("data") 60 | file_path = data_dir / "S081321-v50.txt.gz" 61 | 62 | # Read ITCH messages from a file 63 | with ITCH50MessageReader(file_path) as reader: 64 | for i, message in enumerate(reader): 65 | print(f"Message {i}: {message.type} - {message}") 66 | if i >= 10: # Just show first 10 messages 67 | break 68 | ``` 69 | 70 | ### List Available Symbols 71 | 72 | ```python 73 | symbols = set() 74 | message_count = 0 75 | 76 | with ITCH50MessageReader(file_path) as reader: 77 | for message in reader: 78 | message_count += 1 79 | 80 | # Stock Directory messages (type 'R') contain symbol information 81 | if message.type == b"R": 82 | symbol = message.stock.decode().strip() 83 | symbols.add(symbol) 84 | 85 | if message_count >= 100000: 86 | break 87 | ``` 88 | 89 | ### Extract all Messages for Specific Symbols 90 | 91 | ```python 92 | from pathlib import Path 93 | from meatpy.itch50 import ITCH50MessageReader, ITCH50Writer 94 | 95 | # Define paths 96 | data_dir = Path("data") 97 | input_file = data_dir / "S081321-v50.txt.gz" 98 | output_file = data_dir / "S081321-v50-AAPL-SPY.itch50.gz" 99 | 100 | # Symbols we want to extract 101 | target_symbols = ["AAPL", "SPY"] 102 | 103 | message_count = 0 104 | with ITCH50MessageReader(input_file) as reader: 105 | with ITCH50Writer(output_file, symbols=target_symbols) as writer: 106 | for message in reader: 107 | message_count += 1 108 | writer.process_message(message) 109 | ``` 110 | 111 | 112 | ### Extract Full LOB at 1-Minute Intervals 113 | 114 | ```python 115 | from pathlib import Path 116 | import datetime 117 | from meatpy.itch50 import ITCH50MessageReader, ITCH50MarketProcessor 118 | from meatpy.event_handlers.lob_recorder import LOBRecorder 119 | from meatpy.writers.parquet_writer import ParquetWriter 120 | 121 | # Define paths and parameters 122 | data_dir = Path("data") 123 | 124 | file_path = data_dir / "S081321-v50-AAPL-SPY.itch50.gz" 125 | outfile_path = data_dir / "spy_lob.parquet" 126 | book_date = datetime.datetime(2021, 8, 13) 127 | 128 | with ITCH50MessageReader(file_path) as reader, ParquetWriter(outfile_path) as writer: 129 | processor = ITCH50MarketProcessor("SPY", book_date) 130 | 131 | # We only care about the top of book 132 | lob_recorder = LOBRecorder(writer=writer, collapse_orders=False) 133 | # Generate a list of timedeltas from 9:30 to 16:00 (inclusive) in 30-minute increments 134 | market_open = book_date + datetime.timedelta(hours=9, minutes=30) 135 | market_close = book_date + datetime.timedelta(hours=16, minutes=0) 136 | record_timestamps = [market_open + datetime.timedelta(minutes=i) 137 | for i in range(int((market_close - market_open).total_seconds() // (30*60)) + 1)] 138 | lob_recorder.record_timestamps = record_timestamps 139 | 140 | # Attach the recorders to the processor 141 | processor.handlers.append(lob_recorder) 142 | 143 | for message in reader: 144 | processor.process_message(message) 145 | ``` 146 | 147 | 148 | ## 📊 Common Use Cases 149 | 150 | MeatPy is designed for market microstructure research and analysis: 151 | 152 | - **Order Book Reconstruction**: Rebuild complete limit order book state at any point in time 153 | - **Market Event Analysis**: Extract and analyze trades, quotes, and order modifications 154 | - **Top-of-Book Sampling**: Generate regular snapshots of best bid/ask prices and sizes 155 | - **Market Quality Metrics**: Calculate spreads, depth, and other liquidity measures 156 | - **Academic Research**: Analyze market microstructure for research papers and studies 157 | 158 | MeatPy is **not** suitable for real-time applications or for production use where money is at stake. 159 | 160 | ## 🎓 Academic Use 161 | 162 | MeatPy has been used in several academic publications, including: 163 | 164 | - Grégoire, V. and Martineau, C. (2022), [How is Earnings News Transmitted to Stock Prices?](https://doi.org/10.1111/1475-679X.12394). Journal of Accounting Research, 60: 261-297. 165 | 166 | - Comerton-Forde, C., Grégoire, V., & Zhong, Z. (2019). [Inverted fee structures, tick size, and market quality](https://doi.org/10.1016/j.jfineco.2019.03.005). Journal of Financial Economics, 134(1), 141-164. 167 | 168 | - Yaali, J., Grégoire, V., & Hurtut, T. (2022). [HFTViz: Visualization for the exploration of high frequency trading data](https://journals.sagepub.com/doi/full/10.1177/14738716211064921). Information Visualization, 21(2), 182-193. 169 | 170 | 171 | ## 🤝 Contributing 172 | 173 | We welcome contributions! Please see our [Contributing Guide](https://www.vincentgregoire.com/MeatPy/contributing/) for details. 174 | 175 | ## 📄 License 176 | 177 | MeatPy is released under the permissive BSD 3-Clause License. See [LICENSE](LICENSE) file for details. 178 | 179 | ## 👥 Credits 180 | 181 | MeatPy was created by [Vincent Grégoire](https://www.vincentgregoire.com/) and [Charles Martineau](https://www.charlesmartineau.com/). Seoin Kim and Javad YaAli provided valuable research assistance on the project. 182 | 183 | 184 | **Acknowledgments**: MeatPy development benefited from the financial support of [IVADO](https://ivado.ca/) 185 | 186 | ## 📞 Support 187 | 188 | - **Bug Reports**: Please use [GitHub Issues](https://github.com/vgreg/MeatPy/issues) 189 | - **Questions & Discussions**: Use [GitHub Discussions](https://github.com/vgreg/MeatPy/discussions) 190 | 191 | --- 192 | 193 | Made with ❤️ for the market microstructure research community 194 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | # Require 80% coverage 6 | target: 45% 7 | # Allow 5% drop from base 8 | threshold: 5% 9 | # Only run on patch changes 10 | informational: false 11 | patch: 12 | default: 13 | # Require 80% coverage on new code 14 | target: 45% 15 | # Allow 5% drop from base 16 | threshold: 5% 17 | # Only run on patch changes 18 | informational: false 19 | 20 | ignore: 21 | # Ignore test files 22 | - "tests/" 23 | # Ignore documentation 24 | - "docs/" 25 | # Ignore setup/build files 26 | - "setup.py" 27 | - "pyproject.toml" 28 | # Ignore sample files 29 | - "samples/" 30 | 31 | round: down 32 | precision: 2 33 | 34 | comment: 35 | layout: "reach,diff,flags,tree" 36 | behavior: default 37 | require_changes: false 38 | require_base: false 39 | require_head: true 40 | 41 | flags: 42 | unittests: 43 | paths: 44 | - src/meatpy/ 45 | -------------------------------------------------------------------------------- /docs/codecov-setup.md: -------------------------------------------------------------------------------- 1 | # Codecov Configuration Guide 2 | 3 | This guide explains how to set up code coverage reporting with Codecov for the MeatPy project. 4 | 5 | ## Overview 6 | 7 | The project is configured to: 8 | - Generate coverage reports during testing 9 | - Upload coverage data to Codecov via GitHub Actions 10 | - Display coverage badges and detailed reports 11 | - Track coverage trends over time 12 | 13 | ## Configuration Files 14 | 15 | ### 1. Test Workflow (`.github/workflows/test.yml`) 16 | - Runs tests on multiple Python versions (3.11, 3.12, 3.13) 17 | - Generates coverage reports in XML format 18 | - Uploads coverage to Codecov using the official action 19 | 20 | ### 2. Codecov Configuration (`codecov.yml`) 21 | - Sets coverage targets (80% for project and patches) 22 | - Configures which files to ignore 23 | - Sets up coverage comment format for PRs 24 | 25 | ### 3. Pytest Configuration (`pytest.ini`) 26 | - Enables branch coverage tracking 27 | - Outputs coverage in multiple formats (terminal, HTML, XML) 28 | - Sets minimum coverage threshold to 80% 29 | 30 | ### 4. Coverage Configuration (`.coveragerc`) 31 | - Excludes test files and debug code from coverage 32 | - Configures coverage exclusion patterns 33 | - Sets up HTML report generation 34 | 35 | ## Setup Steps 36 | 37 | ### 1. Repository Setup on Codecov 38 | 1. Go to [codecov.io](https://codecov.io) 39 | 2. Sign in with your GitHub account 40 | 3. Find and enable the MeatPy repository 41 | 4. Copy the repository upload token 42 | 43 | ### 2. GitHub Secrets Configuration 44 | 1. Go to your GitHub repository settings 45 | 2. Navigate to Secrets and Variables → Actions 46 | 3. Add a new repository secret: 47 | - Name: `CODECOV_TOKEN` 48 | - Value: [Your Codecov upload token] 49 | 50 | ### 3. Badge Setup (Optional) 51 | Add coverage badge to README.md: 52 | 53 | ```markdown 54 | [![codecov](https://codecov.io/gh/vgreg/MeatPy/branch/main/graph/badge.svg)](https://codecov.io/gh/vgreg/MeatPy) 55 | ``` 56 | 57 | ## Local Testing 58 | 59 | Test coverage generation locally: 60 | 61 | ```bash 62 | # Run tests with coverage 63 | uv run pytest 64 | 65 | # Generate coverage report 66 | uv run pytest --cov=src/meatpy --cov-report=html 67 | 68 | # View HTML report 69 | open htmlcov/index.html 70 | ``` 71 | 72 | ## Coverage Targets 73 | 74 | - **Project Coverage**: 80% minimum 75 | - **Patch Coverage**: 80% minimum for new code 76 | - **Branch Coverage**: Enabled for better accuracy 77 | 78 | ## Excluded Files 79 | 80 | The following are excluded from coverage: 81 | - Test files (`tests/`) 82 | - Documentation (`docs/`) 83 | - Sample files (`samples/`) 84 | - Setup/build files 85 | 86 | ## Troubleshooting 87 | 88 | ### Common Issues 89 | 90 | 1. **Coverage not uploading**: Check that `CODECOV_TOKEN` is correctly set in GitHub secrets 91 | 2. **Low coverage**: Review the coverage report to identify untested code 92 | 3. **CI failing**: Ensure all tests pass before coverage upload 93 | 94 | ### Debugging Coverage 95 | 96 | ```bash 97 | # Run with coverage debug info 98 | uv run pytest --cov=src/meatpy --cov-report=term-missing --cov-branch -v 99 | 100 | # Check coverage configuration 101 | uv run coverage config --help 102 | ``` 103 | 104 | ## Resources 105 | 106 | - [Codecov Documentation](https://docs.codecov.com/) 107 | - [pytest-cov Documentation](https://pytest-cov.readthedocs.io/) 108 | - [Coverage.py Documentation](https://coverage.readthedocs.io/) 109 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to MeatPy 2 | 3 | We welcome contributions to MeatPy! This document provides guidelines for contributing to the project. 4 | 5 | ## Development Setup 6 | 7 | ### Prerequisites 8 | 9 | - Python 3.11 or higher 10 | - Git 11 | - `uv` package manager 12 | 13 | ### Setting Up Development Environment 14 | 15 | 1. **Clone the repository:** 16 | ```bash 17 | git clone https://github.com/vgreg/MeatPy.git 18 | cd MeatPy 19 | ``` 20 | 21 | 2. **Install dependencies using uv (recommended):** 22 | ```bash 23 | uv sync 24 | ``` 25 | 26 | Or using pip: 27 | ```bash 28 | pip install -e ".[dev]" 29 | ``` 30 | 31 | 3. **Install pre-commit hooks:** 32 | ```bash 33 | uv run pre-commit install 34 | ``` 35 | 36 | ## Development Workflow 37 | 38 | ### Code Quality 39 | 40 | We use several tools to maintain code quality: 41 | 42 | - **Ruff**: For linting and formatting 43 | - **Pre-commit**: For automated code checks 44 | - **Pytest**: For testing 45 | 46 | ### Running Tests 47 | 48 | It is best to run tests locally before submitting changes to ensure everything works as expected. Tests are also executed automatically in the CI pipeline when you create a pull request. 49 | 50 | ```bash 51 | # Run all tests 52 | uv run pytest 53 | ``` 54 | 55 | ### Code Formatting and Linting 56 | 57 | ```bash 58 | # Format code 59 | uv run ruff format 60 | 61 | # Check for linting issues 62 | uv run ruff check 63 | 64 | # Fix auto-fixable linting issues 65 | uv run ruff check --fix 66 | ``` 67 | 68 | ## Contribution Guidelines 69 | 70 | ### Code Style 71 | 72 | - Follow Ruff (black) style guidelines 73 | - Use type hints for all public APIs 74 | - Write descriptive variable and function names 75 | - Keep functions focused and modular 76 | - Add Google-style docstrings to all public classes and methods 77 | 78 | Example: 79 | ```python 80 | def process_order_message( 81 | message: AddOrderMessage, 82 | lob: LimitOrderBook[Price, Volume, OrderID] 83 | ) -> None: 84 | """Process an add order message and update the limit order book. 85 | 86 | Args: 87 | message: The order message to process 88 | lob: The limit order book to update 89 | 90 | Raises: 91 | InvalidMessageError: If the message is malformed 92 | """ 93 | # Implementation here 94 | ``` 95 | 96 | ### Testing 97 | 98 | - Write unit tests for all new functionality 99 | - Maintain test coverage above 80% 100 | - Use descriptive test names that explain what is being tested 101 | - Group related tests in test classes 102 | - Use pytest fixtures for common test setup 103 | 104 | Example test structure: 105 | ```python 106 | class TestLimitOrderBook: 107 | """Tests for the LimitOrderBook class.""" 108 | 109 | def test_add_bid_order_updates_best_bid(self, empty_lob): 110 | """Test that adding a bid order correctly updates the best bid.""" 111 | # Test implementation 112 | 113 | def test_cancel_order_removes_from_book(self, lob_with_orders): 114 | """Test that canceling an order removes it from the book.""" 115 | # Test implementation 116 | ``` 117 | 118 | ### Documentation 119 | 120 | - Update documentation for any API changes 121 | - Add examples for new features 122 | - Use clear, concise language 123 | - Include code examples where helpful 124 | 125 | ### Commit Messages 126 | 127 | This project uses **automated semantic versioning** based on conventional commit messages. When code is merged to `main`, the system automatically: 128 | 129 | 1. Analyzes commit messages since the last release 130 | 2. Determines the version bump type (major, minor, or patch) 131 | 3. Updates the version in `pyproject.toml` 132 | 4. Creates a GitHub release with a new tag 133 | 5. Publishes the package to PyPI 134 | 135 | #### Conventional Commit Format 136 | 137 | Use these commit message patterns to trigger appropriate version bumps: 138 | 139 | **Major Version Bump (Breaking Changes):** 140 | - `BREAKING CHANGE:` in commit body 141 | - `breaking:` prefix in commit message 142 | 143 | **Minor Version Bump (New Features):** 144 | - `feat:` prefix for new features 145 | - `feature:` prefix for new features 146 | 147 | **Patch Version Bump (Bug Fixes):** 148 | - `fix:` prefix for bug fixes 149 | - `bug:` prefix for bug fixes 150 | - `patch:` prefix for patches 151 | 152 | **Examples:** 153 | 154 | ```bash 155 | # Patch version bump (0.1.0 -> 0.1.1) 156 | git commit -m "fix: resolve issue with order book reconstruction" 157 | 158 | # Minor version bump (0.1.0 -> 0.2.0) 159 | git commit -m "feat: add support for new message types" 160 | 161 | # Major version bump (0.1.0 -> 1.0.0) 162 | git commit -m "refactor: redesign API for better performance 163 | 164 | BREAKING CHANGE: MarketProcessor constructor now requires different parameters" 165 | ``` 166 | 167 | #### Manual Releases 168 | 169 | If you need to publish a release manually: 170 | 171 | 1. Go to the GitHub repository 172 | 2. Navigate to Actions → Manual Publish to PyPI 173 | 3. Click "Run workflow" and specify the version 174 | 175 | ## Types of Contributions 176 | 177 | ### Bug Reports 178 | 179 | When reporting bugs, please include: 180 | 181 | - Python version and platform 182 | - MeatPy version 183 | - Minimal code example that reproduces the issue 184 | - Expected vs actual behavior 185 | - Error messages or stack traces 186 | 187 | ### Feature Requests 188 | 189 | For new features: 190 | 191 | - Describe the use case and motivation 192 | - Provide examples of how the feature would be used 193 | - Consider backward compatibility 194 | - Discuss performance implications for large datasets 195 | 196 | ## Getting Help 197 | 198 | - **Discussions**: Use GitHub Discussions for questions 199 | - **Issues**: Create issues for bugs and feature requests 200 | - **Email**: Contact maintainers for sensitive issues 201 | 202 | ## Code of Conduct 203 | 204 | - Be respectful and inclusive 205 | - Focus on constructive feedback 206 | - Help maintain a welcoming environment 207 | - Follow the project's code of conduct 208 | 209 | ## Recognition 210 | 211 | Contributors will be acknowledged in: 212 | 213 | - Release notes for significant contributions 214 | - GitHub contributors list 215 | 216 | Thank you for contributing to MeatPy! 217 | -------------------------------------------------------------------------------- /docs/guide/01_listing_symbols.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Listing Symbols in an ITCH 5.0 File\n", 8 | "\n", 9 | "This notebook shows how to extract all available symbols from an ITCH 5.0 file. To process ITCH 4.1 files, use the `itch41` module instead.\n", 10 | "\n", 11 | "The example uses a sample ITCH 5.0 file that should be placed in the `data/` subdirectory. You can download a [sample file](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/) from Nasdaq." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 6, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "✅ Found sample file: data/S081321-v50.txt.gz\n", 24 | "File size: 4.55 GB\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "from pathlib import Path\n", 30 | "from meatpy.itch50 import ITCH50MessageReader\n", 31 | "\n", 32 | "# Define the path to our sample data file\n", 33 | "data_dir = Path(\"data\")\n", 34 | "file_path = data_dir / \"S081321-v50.txt.gz\"\n", 35 | "print(f\"✅ Found sample file: {file_path}\")\n", 36 | "print(f\"File size: {file_path.stat().st_size / (1024**3):.2f} GB\")" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 7, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "Reading ITCH file to extract symbols...\n", 49 | "Found 11096 symbols after processing 100,000 messages\n" 50 | ] 51 | } 52 | ], 53 | "source": [ 54 | "symbols = set()\n", 55 | "message_count = 0\n", 56 | "\n", 57 | "print(\"Reading ITCH file to extract symbols...\")\n", 58 | "\n", 59 | "with ITCH50MessageReader(file_path) as reader:\n", 60 | " for message in reader:\n", 61 | " message_count += 1\n", 62 | "\n", 63 | " # Stock Directory messages (type 'R') contain symbol information\n", 64 | " if message.type == b\"R\":\n", 65 | " symbol = message.stock.decode().strip()\n", 66 | " symbols.add(symbol)\n", 67 | "\n", 68 | " if message_count >= 100000:\n", 69 | " break\n", 70 | "\n", 71 | "print(f\"Found {len(symbols)} symbols after processing {message_count:,} messages\")\n", 72 | "\n", 73 | "symbols = sorted(symbols)" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 8, 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "name": "stdout", 83 | "output_type": "stream", 84 | "text": [ 85 | "First 20 symbols:\n", 86 | "A\n", 87 | "AA\n", 88 | "AAA\n", 89 | "AAAU\n", 90 | "AAC\n", 91 | "AAC+\n", 92 | "AAC=\n", 93 | "AACG\n", 94 | "AACIU\n", 95 | "AACOU\n", 96 | "AADR\n", 97 | "AAIC\n", 98 | "AAIC-B\n", 99 | "AAIC-C\n", 100 | "AAIN\n", 101 | "AAL\n", 102 | "AAMC\n", 103 | "AAME\n", 104 | "AAN\n", 105 | "AAOI\n" 106 | ] 107 | } 108 | ], 109 | "source": [ 110 | "print(\"First 20 symbols:\")\n", 111 | "for symbol in symbols[:20]:\n", 112 | " print(symbol)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## Key Points\n", 120 | "\n", 121 | "- **Stock Directory Messages**: ITCH files begin with Stock Directory messages (type 'R') that contain symbol information\n", 122 | "- **Early Termination**: Since these messages appear at the beginning, we can stop reading after processing a reasonable number of messages (e.g., 100,000) to avoid unnecessary processing. _Note: This is not guaranteed in the specification._ \n", 123 | "- **Memory Efficiency**: This approach is memory-efficient for large files since we don't need to process the entire file\n", 124 | "- **Symbol Format**: ITCH symbols are 8-byte fields, padded with spaces, which we strip for display\n", 125 | "\n", 126 | "## Next Steps\n", 127 | "\n", 128 | "Once you have the list of symbols, you can:\n", 129 | "1. Filter the file to extract data for specific symbols of interest\n", 130 | "2. Process order book data for particular symbols\n", 131 | "3. Generate reports or visualizations for selected symbols" 132 | ] 133 | } 134 | ], 135 | "metadata": { 136 | "kernelspec": { 137 | "display_name": "meatpy", 138 | "language": "python", 139 | "name": "python3" 140 | }, 141 | "language_info": { 142 | "codemirror_mode": { 143 | "name": "ipython", 144 | "version": 3 145 | }, 146 | "file_extension": ".py", 147 | "mimetype": "text/x-python", 148 | "name": "python", 149 | "nbconvert_exporter": "python", 150 | "pygments_lexer": "ipython3", 151 | "version": "3.13.0" 152 | } 153 | }, 154 | "nbformat": 4, 155 | "nbformat_minor": 4 156 | } 157 | -------------------------------------------------------------------------------- /docs/guide/02_extracting_symbols.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Extracting Specific Symbols\n", 8 | "\n", 9 | "This notebook demonstrates how to create a new ITCH file containing only data for specific symbols of interest.\n", 10 | "\n", 11 | "Filtering large ITCH files to specific symbols can significantly reduce file size and processing time for analysis focused on particular securities. It is also useful for parallel processing, where each process can handle a subset of symbols." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [ 19 | { 20 | "name": "stdout", 21 | "output_type": "stream", 22 | "text": [ 23 | "Input file size: 4.55 GB\n" 24 | ] 25 | } 26 | ], 27 | "source": [ 28 | "from pathlib import Path\n", 29 | "from meatpy.itch50 import ITCH50MessageReader, ITCH50Writer\n", 30 | "\n", 31 | "# Define paths\n", 32 | "data_dir = Path(\"data\")\n", 33 | "input_file = data_dir / \"S081321-v50.txt.gz\"\n", 34 | "output_file = data_dir / \"S081321-v50-AAPL-SPY.itch50.gz\"\n", 35 | "\n", 36 | "# Symbols we want to extract\n", 37 | "target_symbols = [\"AAPL\", \"SPY\"]\n", 38 | "\n", 39 | "input_size_gb = input_file.stat().st_size / (1024**3)\n", 40 | "print(f\"Input file size: {input_size_gb:.2f} GB\")" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 5, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Total messages processed: 367,986,583\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "# This takes about 10 minutes on a MacBook Pro M3 Max\n", 58 | "message_count = 0\n", 59 | "with ITCH50MessageReader(input_file) as reader:\n", 60 | " with ITCH50Writer(output_file, symbols=target_symbols) as writer:\n", 61 | " for message in reader:\n", 62 | " message_count += 1\n", 63 | " writer.process_message(message)\n", 64 | "\n", 65 | "print(f\"Total messages processed: {message_count:,}\")" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 6, 71 | "metadata": {}, 72 | "outputs": [ 73 | { 74 | "name": "stdout", 75 | "output_type": "stream", 76 | "text": [ 77 | "Total messages in filtered file: 4,503,791\n", 78 | "Output file size: 0.13 GB\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "new_message_count = 0\n", 84 | "with ITCH50MessageReader(output_file) as reader:\n", 85 | " for message in reader:\n", 86 | " new_message_count += 1\n", 87 | "\n", 88 | "print(f\"Total messages in filtered file: {new_message_count:,}\")\n", 89 | "output_size_gb = output_file.stat().st_size / (1024**3)\n", 90 | "print(f\"Output file size: {output_size_gb:.2f} GB\")" 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": {}, 96 | "source": [ 97 | "## Key Points\n", 98 | "\n", 99 | "- **Processing Speed**: Smaller filtered files process much faster for subsequent analysis. If your analysis only requires data for a few symbols, filtering out the rest can save significant time for downstream tasks.\n", 100 | "- **Output Format**: The output is a valid ITCH 5.0 file that can be processed by any ITCH-compatible tool\n", 101 | "\n", 102 | "## Performance Tips\n", 103 | "\n", 104 | "- **Early Filtering**: Filter as early as possible in your data pipeline to reduce downstream processing time\n", 105 | "- **Multiple Symbols**: You can filter for multiple symbols in a single pass\n", 106 | "- **Memory Usage**: The ITCH50Writer buffers data efficiently to minimize memory usage during filtering\n", 107 | "\n", 108 | "## Next Steps\n", 109 | "\n", 110 | "With your filtered file, you can now:\n", 111 | "1. Process order book data much faster\n", 112 | "2. Generate snapshots at regular intervals\n", 113 | "3. Calculate trading metrics and statistics\n", 114 | "4. Create visualizations and reports" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [] 121 | } 122 | ], 123 | "metadata": { 124 | "kernelspec": { 125 | "display_name": "meatpy", 126 | "language": "python", 127 | "name": "python3" 128 | }, 129 | "language_info": { 130 | "codemirror_mode": { 131 | "name": "ipython", 132 | "version": 3 133 | }, 134 | "file_extension": ".py", 135 | "mimetype": "text/x-python", 136 | "name": "python", 137 | "nbconvert_exporter": "python", 138 | "pygments_lexer": "ipython3", 139 | "version": "3.13.0" 140 | } 141 | }, 142 | "nbformat": 4, 143 | "nbformat_minor": 4 144 | } 145 | -------------------------------------------------------------------------------- /docs/guide/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started with MeatPy 2 | 3 | This guide will help you get started with MeatPy for processing financial market data. 4 | 5 | ## Basic Concepts 6 | 7 | ### Core Components 8 | 9 | - **MarketProcessor**: Processes market messages and maintains order book state 10 | - **MessageReader**: Reads market data from files in various formats 11 | - **LimitOrderBook (LOB)**: Represents the current state of buy and sell orders 12 | - **Event Handlers**: Process and record market events as they occur 13 | 14 | ### Supported Data Formats 15 | 16 | MeatPy currently supports: 17 | 18 | - **ITCH 5.0**: NASDAQ's binary market data format 19 | 20 | ## Basic Usage 21 | 22 | The simplest way to read ITCH 5.0 data: 23 | 24 | ```python 25 | from meatpy.itch50 import ITCH50MessageReader 26 | 27 | # Read messages from an ITCH file 28 | with ITCH50MessageReader("market_data.txt.gz") as reader: 29 | for i, message in enumerate(reader): 30 | print(f"Message {i}: {message}") 31 | if i >= 10: # Just show first 10 messages 32 | break 33 | ``` 34 | 35 | 36 | Other common tasks include: 37 | 38 | - **Listing Symbols**: Extracting unique stock symbols from ITCH files 39 | - **Extracting Specific Symbols**: Creating new ITCH files with only specific symbols 40 | - **Top of Book Snapshots**: Generating snapshots of the top of book state for analysis 41 | - **Order Book Snapshots**: Creating snapshots of the full limit order book state 42 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ![MeatPy Logo](images/meatpy.svg){width=200} 2 | 3 | # MeatPy 4 | 5 | MeatPy is a Python framework for processing financial market data, specifically designed for limit order book reconstruction and analysis. It provides efficient tools for handling high-frequency trading data formats like NASDAQ ITCH. 6 | 7 | ## Key Features 8 | 9 | - **Limit Order Book Processing**: Complete reconstruction and analysis of limit order books 10 | - **ITCH Support**: Full implementation for NASDAQ ITCH 4.1 and 5.0 formats 11 | - **Event-Driven Architecture**: Extensible framework for real-time market data processing 12 | - **Type Safety**: Modern Python typing for robust financial data handling 13 | - **Multiple Output Formats**: Support for CSV, Parquet, and custom formats 14 | 15 | ## Who is it for? 16 | 17 | MeatPy was originally developed for academic research in high-frequency trading. It is also suitable for backtesting trading strategies and learning about market microstructure. The framework is designed to be flexible and extensible, making it a good fit for both researchers and developers working with other limit order book data formats. 18 | 19 | ## What is it not for? 20 | 21 | MeatPy is not a trading platform or a high-frequency trading engine. Instead, it focuses on the processing and analysis of historical market data. Also, it is not intended for use in production systems without further development and testing. In any case, you probably would not want to use it for real-time application as it is not fast enough for that purpose. 22 | 23 | ## Architecture Overview 24 | 25 | MeatPy is built around several core components: 26 | 27 | - **MarketProcessor**: Abstract base class for processing market messages 28 | - **LimitOrderBook**: Core data structure representing the order book state 29 | - **MessageReader**: Interface for reading different market data formats 30 | - **Event Handlers**: Observer pattern for handling market events and recording data 31 | 32 | 33 | ## What's Next? 34 | 35 | - Check out the [Installation Guide](installation.md) to get started 36 | - Read the [User Guide](guide/getting-started.md) for usage instructions 37 | - See [Contributing](contributing.md) if you want to contribute to the project 38 | 39 | ## Academic Papers using MeatPy 40 | 41 | - Grégoire, V. and Martineau, C. (2022), [How is Earnings News Transmitted to Stock Prices?](https://doi.org/10.1111/1475-679X.12394). Journal of Accounting Research, 60: 261-297. 42 | 43 | - Comerton-Forde, C., Grégoire, V., & Zhong, Z. (2019). [Inverted fee structures, tick size, and market quality](https://doi.org/10.1016/j.jfineco.2019.03.005). Journal of Financial Economics, 134(1), 141-164. 44 | 45 | - Yaali, J., Grégoire, V., & Hurtut, T. (2022). [HFTViz: Visualization for the exploration of high frequency trading data](https://journals.sagepub.com/doi/full/10.1177/14738716211064921). Information Visualization, 21(2), 182-193. 46 | 47 | 48 | ## Credits 49 | 50 | MeatPy was created by [Vincent Grégoire](https://www.vincentgregoire.com/) and [Charles Martineau](https://www.charlesmartineau.com/). Seoin Kim and Javad YaAli provided valuable research assistance on the project. 51 | 52 | ## Funding 53 | MeatPy development benefited from the financial support of [IVADO](https://ivado.ca/) 54 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Requirements 4 | 5 | MeatPy requires Python 3.11 or higher. 6 | 7 | ## Install from PyPI 8 | 9 | The easiest way to install MeatPy is using pip: 10 | 11 | ```bash 12 | pip install meatpy 13 | ``` 14 | 15 | For Parquet output support, install with the parquet extra: 16 | 17 | ```bash 18 | pip install meatpy[parquet] 19 | ``` 20 | 21 | ## Install from Source 22 | 23 | To install the latest development version from GitHub: 24 | 25 | ```bash 26 | git clone https://github.com/vgreg/MeatPy.git 27 | cd MeatPy 28 | pip install -e . 29 | ``` 30 | 31 | ## Development Installation 32 | 33 | For development, we recommend using `uv` for dependency management: 34 | 35 | ```bash 36 | git clone https://github.com/vgreg/MeatPy.git 37 | cd MeatPy 38 | uv sync 39 | ``` 40 | 41 | This will create a virtual environment and install all dependencies including development tools. 42 | 43 | ## Verify Installation 44 | 45 | You can verify your installation by running: 46 | 47 | ```python 48 | import meatpy 49 | print(meatpy.__version__) 50 | ``` 51 | 52 | ## Optional Dependencies 53 | 54 | - **pyarrow**: Required for Parquet output format support 55 | - **jupyter** and **pandas**: For running example notebooks 56 | - **pytest**: For running tests if you're contributing 57 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: MeatPy 2 | site_description: A framework for processing high-frequency financial market data 3 | site_author: Vincent Grégoire, Charles Martineau 4 | site_url: https://www.vincentgregoire.com/MeatPy 5 | 6 | repo_name: vgreg/MeatPy 7 | repo_url: https://github.com/vgreg/MeatPy 8 | 9 | theme: 10 | name: material 11 | logo: images/meatpy.svg 12 | favicon: images/meatpy.svg 13 | palette: 14 | - scheme: default 15 | primary: green 16 | accent: green 17 | toggle: 18 | icon: material/brightness-7 19 | name: Switch to dark mode 20 | - scheme: slate 21 | primary: green 22 | accent: green 23 | toggle: 24 | icon: material/brightness-4 25 | name: Switch to light mode 26 | features: 27 | - navigation.tabs 28 | - navigation.sections 29 | - navigation.expand 30 | - navigation.top 31 | - search.highlight 32 | - search.share 33 | - content.code.copy 34 | 35 | markdown_extensions: 36 | - pymdownx.highlight: 37 | anchor_linenums: true 38 | - pymdownx.inlinehilite 39 | - pymdownx.snippets 40 | - pymdownx.superfences 41 | - admonition 42 | - pymdownx.details 43 | - pymdownx.tabbed: 44 | alternate_style: true 45 | - def_list 46 | - pymdownx.tasklist: 47 | custom_checkbox: true 48 | - attr_list 49 | 50 | nav: 51 | - Home: index.md 52 | - Installation: installation.md 53 | - User Guide: 54 | - Getting Started: guide/getting-started.md 55 | - Listing Symbols: guide/01_listing_symbols.ipynb 56 | - Extracting Symbols: guide/02_extracting_symbols.ipynb 57 | - Top of Book Snapshots: guide/03_top_of_book_snapshots.ipynb 58 | - Full LOB Snapshots: guide/04_full_lob_snapshots.ipynb 59 | - Contributing: contributing.md 60 | 61 | plugins: 62 | - search 63 | - autorefs 64 | - mkdocs-jupyter: 65 | execute: false 66 | allow_errors: false 67 | include_source: true 68 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "meatpy" 3 | version = "0.2.8" 4 | description = "Read and process limit order book data" 5 | authors = [ 6 | { name = "Vincent Grégoire", email = "vincent.gregoire@hec.ca" }, 7 | { name = "Charles Martineau", email = "charles.martineau@utoronto.ca" }, 8 | ] 9 | maintainers = [{ name = "Vincent Grégoire", email = "vincent.gregoire@hec.ca" }] 10 | readme = "README.md" 11 | license = { text = "BSD-3-Clause" } 12 | requires-python = ">=3.11" 13 | dependencies = ["pyarrow>=18.0.0"] 14 | classifiers = [ 15 | "Intended Audience :: Education", 16 | "Intended Audience :: Financial and Insurance Industry", 17 | "Topic :: Office/Business :: Financial", 18 | "Topic :: Scientific/Engineering :: Information Analysis", 19 | ] 20 | 21 | [project.optional-dependencies] 22 | parquet = ["pyarrow>=10.0.0"] 23 | 24 | [project.urls] 25 | Repository = "https://github.com/vgreg/MeatPy" 26 | 27 | [build-system] 28 | requires = ["hatchling"] 29 | build-backend = "hatchling.build" 30 | 31 | [dependency-groups] 32 | dev = [ 33 | "jupyter>=1.1.1", 34 | "pre-commit>=4.2.0", 35 | "pytest>=8.4.1", 36 | "pytest-cov>=6.2.1", 37 | "ruff>=0.12.2", 38 | ] 39 | docs = [ 40 | "mkdocs>=1.5.0", 41 | "mkdocs-material>=9.0.0", 42 | "mkdocs-autorefs>=0.5.0", 43 | "mkdocs-jupyter>=0.24.0", 44 | "pandas>=2.0.0", 45 | ] 46 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | testpaths = tests 3 | python_files = test_*.py 4 | python_classes = Test* 5 | python_functions = test_* 6 | addopts = 7 | -v 8 | --tb=short 9 | --strict-markers 10 | --disable-warnings 11 | --cov=src/meatpy 12 | --cov-report=term-missing 13 | --cov-report=html:htmlcov 14 | --cov-report=xml:coverage.xml 15 | --cov-fail-under=80 16 | --cov-branch 17 | markers = 18 | slow: marks tests as slow (deselect with '-m "not slow"') 19 | integration: marks tests as integration tests 20 | unit: marks tests as unit tests 21 | performance: marks tests as performance tests 22 | filterwarnings = 23 | ignore::DeprecationWarning 24 | ignore::PendingDeprecationWarning 25 | -------------------------------------------------------------------------------- /rules/python.mdc: -------------------------------------------------------------------------------- 1 | --- 2 | description: All Python projects 3 | globs: *.py 4 | alwaysApply: false 5 | --- 6 | You are an AI assistant specialized in Python development. Your approach emphasizes: 7 | 8 | 1. Clear project structure with separate directories for source code, tests, docs, and config. 9 | 2. Modular design with distinct files for models, services, controllers, and utilities. 10 | 3. Use f-strings for formatting strings. 11 | 4. Configuration management using environment variables. 12 | 5. Robust error handling and logging, including context capture. Use Google style docstrings. 13 | 6. Comprehensive testing with pytest. 14 | 7. Detailed documentation using docstrings and README files. Use Google style docstrings. 15 | 8. Dependency management using uv. 16 | 9. Code style consistency using Ruff. 17 | 10. CI/CD implementation with GitHub Actions or GitLab CI. 18 | 11. AI-friendly coding practices: 19 | - Descriptive variable and function names 20 | - Type hints. Assume Python 3.10 or higher. 21 | - Detailed comments for complex logic 22 | - Rich error context for debugging 23 | 24 | You provide code snippets and explanations tailored to these principles, optimizing for clarity and AI-assisted development. 25 | -------------------------------------------------------------------------------- /samples/itch41/Step0_ExtractSymbols.py: -------------------------------------------------------------------------------- 1 | """Listing Symbols in an ITCH 4.1 File 2 | 3 | This script shows how to extract all available symbols from an ITCH 4.1 file. 4 | Based on the pattern from the MeatPy documentation notebooks. 5 | """ 6 | 7 | from pathlib import Path 8 | from meatpy.itch41 import ITCH41MessageReader 9 | 10 | # Define the path to our sample data file 11 | script_dir = Path(__file__).parent 12 | data_dir = script_dir / "data" 13 | file_path = data_dir / "S083012-v41.txt.gz" 14 | 15 | print(f"Reading ITCH 4.1 file: {file_path}") 16 | if file_path.exists(): 17 | file_size_mb = file_path.stat().st_size / (1024**2) 18 | print(f"File size: {file_size_mb:.2f} MB") 19 | else: 20 | print("⚠️ Sample file not found - this is expected in most setups") 21 | print("You can download ITCH 4.1 sample files or use your own data") 22 | exit(1) 23 | 24 | symbols = set() 25 | message_count = 0 26 | 27 | print("Reading ITCH 4.1 file to extract symbols...") 28 | 29 | with ITCH41MessageReader(file_path) as reader: 30 | for message in reader: 31 | message_count += 1 32 | 33 | # Stock Directory messages (type 'R') contain symbol information 34 | if message.type == b"R": 35 | symbol = message.stock.decode().strip() 36 | symbols.add(symbol) 37 | 38 | # For ITCH 4.1, we can break early since stock directory messages 39 | # typically appear at the beginning of the file 40 | if message_count >= 50000: 41 | break 42 | 43 | print(f"Found {len(symbols)} symbols after processing {message_count:,} messages") 44 | 45 | symbols = sorted(symbols) 46 | 47 | # Display first 20 symbols 48 | print("\nFirst 20 symbols:") 49 | for symbol in symbols[:20]: 50 | print(f" {symbol}") 51 | 52 | if len(symbols) > 20: 53 | print(f" ... and {len(symbols) - 20} more") 54 | 55 | # Save symbols to file 56 | output_file = data_dir / "itch41_symbols.txt" 57 | with open(output_file, "w") as f: 58 | for symbol in symbols: 59 | f.write(f"{symbol}\n") 60 | 61 | print(f"\n✅ Symbols saved to: {output_file}") 62 | -------------------------------------------------------------------------------- /samples/itch41/Step1_Parsing.py: -------------------------------------------------------------------------------- 1 | """Extracting Specific Symbols from ITCH 4.1 File 2 | 3 | This script demonstrates how to create a new ITCH 4.1 file containing only data 4 | for specific symbols of interest. Based on the pattern from MeatPy documentation. 5 | """ 6 | 7 | from pathlib import Path 8 | 9 | from meatpy.itch41 import ITCH41MessageReader, ITCH41Writer 10 | 11 | # Define paths 12 | data_dir = Path("data") 13 | input_file = data_dir / "S083012-v41.txt.gz" 14 | output_file = data_dir / "S083012-v41-AAPL-SPY.itch41.gz" 15 | 16 | # Symbols we want to extract 17 | target_symbols = ["AAPL", "SPY"] 18 | 19 | print(f"Input file: {input_file}") 20 | if input_file.exists(): 21 | input_size_mb = input_file.stat().st_size / (1024**2) 22 | print(f"Input file size: {input_size_mb:.2f} MB") 23 | else: 24 | print("⚠️ Sample file not found - this is expected in most setups") 25 | print("You can download ITCH 4.1 sample files or use your own data") 26 | exit(1) 27 | 28 | print(f"Extracting symbols: {target_symbols}") 29 | print(f"Output file: {output_file}") 30 | 31 | # Process the file and filter for target symbols 32 | message_count = 0 33 | with ( 34 | ITCH41MessageReader(input_file) as reader, 35 | ITCH41Writer(output_file, symbols=target_symbols) as writer, 36 | ): 37 | for message in reader: 38 | message_count += 1 39 | writer.process_message(message) 40 | 41 | # Progress update every 10,000 messages 42 | if message_count % 10000 == 0: 43 | print(f"Processed {message_count:,} messages...") 44 | 45 | print(f"Total messages processed: {message_count:,}") 46 | 47 | # # Check the filtered file 48 | # if output_file.exists(): 49 | # new_message_count = 0 50 | # with ITCH41MessageReader(output_file) as reader: 51 | # for message in reader: 52 | # print(message) 53 | # new_message_count += 1 54 | 55 | # print(f"Total messages in filtered file: {new_message_count:,}") 56 | # output_size_mb = output_file.stat().st_size / (1024**2) 57 | # print(f"Output file size: {output_size_mb:.2f} MB") 58 | 59 | # size_reduction = (1 - output_size_mb / input_size_mb) * 100 60 | # print(f"Size reduction: {size_reduction:.1f}%") 61 | 62 | # print(f"\n✅ Filtered file created: {output_file}") 63 | # else: 64 | # print("❌ Failed to create output file") 65 | -------------------------------------------------------------------------------- /samples/itch41/Step2_Processing.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from pathlib import Path 3 | 4 | from meatpy.event_handlers.lob_recorder import LOBRecorder 5 | from meatpy.itch41 import ITCH41MarketProcessor, ITCH41MessageReader 6 | from meatpy.lob import ExecutionPriorityExceptionList 7 | from meatpy.writers.parquet_writer import ParquetWriter 8 | 9 | # Define paths and parameters 10 | data_dir = Path("data") 11 | 12 | file_path = data_dir / "S083012-v41-AAPL-SPY.itch41.gz" 13 | outfile_path = data_dir / "spy_tob.parquet" 14 | book_date = datetime.datetime(2012, 8, 30) 15 | 16 | 17 | with ITCH41MessageReader(file_path) as reader, ParquetWriter(outfile_path) as writer: 18 | processor = ITCH41MarketProcessor("SPY", book_date) 19 | 20 | # We only care about the top of book 21 | tob_recorder = LOBRecorder(writer=writer, max_depth=1) 22 | # Generate a list of timedeltas from 9:30 to 16:00 (inclusive) in 1-minute increments 23 | market_open = book_date + datetime.timedelta(hours=9, minutes=30) 24 | market_close = book_date + datetime.timedelta(hours=16, minutes=0) 25 | record_timestamps = [ 26 | market_open + datetime.timedelta(minutes=i) 27 | for i in range(int((market_close - market_open).total_seconds() // 60) + 1) 28 | ] 29 | tob_recorder.record_timestamps = record_timestamps 30 | 31 | # Attach the recorders to the processor 32 | processor.handlers.append(tob_recorder) 33 | 34 | message_types = set() 35 | 36 | for i, message in enumerate(reader): 37 | # if i % 10_000 == 0: 38 | # print(f"Processing message {i:,}...") 39 | if message.type != b"T": 40 | print(message) 41 | try: 42 | processor.process_message(message) 43 | except ExecutionPriorityExceptionList as e: 44 | print(f"Execution priority exception: {e}") 45 | message_types.add(message.type) 46 | 47 | print(f"Processed {i + 1:,} messages.") 48 | print(f"Message types: {message_types}") 49 | -------------------------------------------------------------------------------- /samples/itch50/01_listing_symbols.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from meatpy.itch50 import ITCH50MessageReader 4 | 5 | # Define the path to our sample data file 6 | data_dir = Path("data") 7 | file_path = data_dir / "S081321-v50.txt.gz" 8 | print(f"✅ Found sample file: {file_path}") 9 | print(f"File size: {file_path.stat().st_size / (1024**3):.2f} GB") 10 | 11 | symbols = set() 12 | message_count = 0 13 | 14 | print("Reading ITCH file to extract symbols...") 15 | 16 | with ITCH50MessageReader(file_path) as reader: 17 | for message in reader: 18 | message_count += 1 19 | 20 | # Stock Directory messages (type 'R') contain symbol information 21 | if message.type == b"R": 22 | symbol = message.stock.decode().strip() 23 | symbols.add(symbol) 24 | 25 | if message_count >= 100000: 26 | break 27 | 28 | print(f"Found {len(symbols)} symbols after processing {message_count:,} messages") 29 | 30 | symbols = sorted(symbols) 31 | 32 | print("First 20 symbols:") 33 | for symbol in symbols[:20]: 34 | print(symbol) 35 | -------------------------------------------------------------------------------- /samples/itch50/02_extracting_symbols.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from meatpy.itch50 import ITCH50MessageReader, ITCH50Writer 4 | 5 | # Define paths 6 | data_dir = Path("data") 7 | input_file = data_dir / "S081321-v50.txt.gz" 8 | output_file = data_dir / "S081321-v50-AAPL-SPY.itch50.gz" 9 | 10 | # Symbols we want to extract 11 | target_symbols = ["AAPL", "SPY"] 12 | 13 | input_size_gb = input_file.stat().st_size / (1024**3) 14 | print(f"Input file size: {input_size_gb:.2f} GB") 15 | 16 | # This takes about 10 minutes on a MacBook Pro M3 Max 17 | message_count = 0 18 | with ITCH50MessageReader(input_file) as reader: 19 | with ITCH50Writer(output_file, symbols=target_symbols) as writer: 20 | for message in reader: 21 | message_count += 1 22 | writer.process_message(message) 23 | 24 | print(f"Total messages processed: {message_count:,}") 25 | 26 | 27 | new_message_count = 0 28 | with ITCH50MessageReader(output_file) as reader: 29 | for message in reader: 30 | new_message_count += 1 31 | 32 | print(f"Total messages in filtered file: {new_message_count:,}") 33 | output_size_gb = output_file.stat().st_size / (1024**3) 34 | print(f"Output file size: {output_size_gb:.2f} GB") 35 | -------------------------------------------------------------------------------- /samples/itch50/03_top_of_book_snapshots.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from pathlib import Path 3 | 4 | from meatpy.event_handlers.lob_recorder import LOBRecorder 5 | from meatpy.itch50 import ITCH50MarketProcessor, ITCH50MessageReader 6 | from meatpy.writers.parquet_writer import ParquetWriter 7 | 8 | # Define paths and parameters 9 | data_dir = Path("data") 10 | 11 | file_path = data_dir / "S081321-v50-AAPL-SPY.itch50.gz" 12 | outfile_path = data_dir / "spy_tob.parquet" 13 | book_date = datetime.datetime(2021, 8, 13) 14 | 15 | 16 | with ITCH50MessageReader(file_path) as reader, ParquetWriter(outfile_path) as writer: 17 | processor = ITCH50MarketProcessor("SPY", book_date) 18 | 19 | # We only care about the top of book 20 | tob_recorder = LOBRecorder(writer=writer, max_depth=1) 21 | # Generate a list of timedeltas from 9:30 to 16:00 (inclusive) in 1-minute increments 22 | market_open = book_date + datetime.timedelta(hours=9, minutes=30) 23 | market_close = book_date + datetime.timedelta(hours=16, minutes=0) 24 | record_timestamps = [ 25 | market_open + datetime.timedelta(minutes=i) 26 | for i in range(int((market_close - market_open).total_seconds() // 60) + 1) 27 | ] 28 | tob_recorder.record_timestamps = record_timestamps 29 | 30 | # Attach the recorders to the processor 31 | processor.handlers.append(tob_recorder) 32 | 33 | for message in reader: 34 | processor.process_message(message) 35 | -------------------------------------------------------------------------------- /samples/itch50/04_full_lob_snapshots.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from pathlib import Path 3 | 4 | from meatpy.event_handlers.lob_recorder import LOBRecorder 5 | from meatpy.itch50 import ITCH50MarketProcessor, ITCH50MessageReader 6 | from meatpy.writers.parquet_writer import ParquetWriter 7 | 8 | # Define paths and parameters 9 | data_dir = Path("data") 10 | 11 | file_path = data_dir / "S081321-v50-AAPL-SPY.itch50.gz" 12 | outfile_path = data_dir / "spy_lob.parquet" 13 | book_date = datetime.datetime(2021, 8, 13) 14 | 15 | with ITCH50MessageReader(file_path) as reader, ParquetWriter(outfile_path) as writer: 16 | processor = ITCH50MarketProcessor("SPY", book_date) 17 | 18 | # We only care about the top of book 19 | lob_recorder = LOBRecorder(writer=writer, collapse_orders=False) 20 | # Generate a list of timedeltas from 9:30 to 16:00 (inclusive) in 30-minute increments 21 | market_open = book_date + datetime.timedelta(hours=9, minutes=30) 22 | market_close = book_date + datetime.timedelta(hours=16, minutes=0) 23 | record_timestamps = [ 24 | market_open + datetime.timedelta(minutes=i) 25 | for i in range( 26 | int((market_close - market_open).total_seconds() // (30 * 60)) + 1 27 | ) 28 | ] 29 | lob_recorder.record_timestamps = record_timestamps 30 | 31 | # Attach the recorders to the processor 32 | processor.handlers.append(lob_recorder) 33 | 34 | for message in reader: 35 | processor.process_message(message) 36 | -------------------------------------------------------------------------------- /src/meatpy/__init__.py: -------------------------------------------------------------------------------- 1 | from .events import BaseEventHandler 2 | from .level import ( 3 | ExecutionPriorityException, 4 | Level, 5 | OrderOnBook, 6 | VolumeInconsistencyException, 7 | ) 8 | from .lob import ( 9 | ExecutionPriorityExceptionList, 10 | InexistantValueException, 11 | LimitOrderBook, 12 | ) 13 | from .market_event_handler import MarketEventHandler 14 | from .market_processor import MarketProcessor 15 | from .message_reader import MarketMessage, MessageReader 16 | from .trading_status import ( 17 | ClosedTradingStatus, 18 | ClosingAuctionTradingStatus, 19 | HaltedTradingStatus, 20 | PostTradeTradingStatus, 21 | PreTradeTradingStatus, 22 | QuoteOnlyTradingStatus, 23 | TradeTradingStatus, 24 | ) 25 | from .types import OrderID, Price, Qualifiers, TradeRef, Volume 26 | 27 | __all__ = [ 28 | # Core classes 29 | "ExecutionPriorityException", 30 | "VolumeInconsistencyException", 31 | "OrderOnBook", 32 | "Level", 33 | "InexistantValueException", 34 | "ExecutionPriorityExceptionList", 35 | "LimitOrderBook", 36 | "MarketProcessor", 37 | "MarketMessage", 38 | # Trading status 39 | "TradeTradingStatus", 40 | "HaltedTradingStatus", 41 | "PreTradeTradingStatus", 42 | "PostTradeTradingStatus", 43 | "QuoteOnlyTradingStatus", 44 | "ClosingAuctionTradingStatus", 45 | "ClosedTradingStatus", 46 | # New systems 47 | "MarketEventHandler", 48 | "BaseEventHandler", 49 | "FormatRegistry", 50 | "registry", 51 | "MeatPyConfig", 52 | "default_config", 53 | "MarketDataProcessor", 54 | "create_processor", 55 | "create_parser", 56 | "list_available_formats", 57 | "MessageReader", 58 | # Types 59 | "Price", 60 | "Volume", 61 | "OrderID", 62 | "TradeRef", 63 | "Qualifiers", 64 | ] 65 | 66 | # ITCH format imports (available when format-specific modules are imported) 67 | try: 68 | from . import itch41 # noqa: F401 69 | 70 | __all__.extend(["itch41"]) 71 | except ImportError: 72 | pass 73 | 74 | try: 75 | from . import itch50 # noqa: F401 76 | 77 | __all__.extend(["itch50"]) 78 | except ImportError: 79 | pass 80 | -------------------------------------------------------------------------------- /src/meatpy/event_handlers/__init__.py: -------------------------------------------------------------------------------- 1 | """Event handler classes for market data processing. 2 | 3 | This package provides various event recorder and handler classes for use with 4 | limit order book and market event processing in MeatPy. 5 | """ 6 | 7 | from .lob_event_recorder import LOBEventRecorder 8 | from .lob_recorder import LOBRecorder 9 | from .ofi_recorder import OFIRecorder 10 | from .spot_measures_recorder import SpotMeasuresRecorder 11 | 12 | __all__ = [ 13 | "LOBEventRecorder", 14 | "LOBRecorder", 15 | "OFIRecorder", 16 | "SpotMeasuresRecorder", 17 | ] 18 | -------------------------------------------------------------------------------- /src/meatpy/event_handlers/lob_recorder.py: -------------------------------------------------------------------------------- 1 | """Recorder for limit order book (LOB) snapshots and CSV export. 2 | 3 | This module provides the LOBRecorder class, which records LOB snapshots and 4 | exports them to CSV files, supporting both aggregate and detailed order data. 5 | """ 6 | 7 | from typing import Optional 8 | 9 | from ..lob import LimitOrderBook 10 | from .lob_event_recorder import LOBEventRecorder 11 | 12 | 13 | class LOBRecorder(LOBEventRecorder): 14 | """Records limit order book snapshots and exports to CSV. 15 | 16 | Attributes: 17 | max_depth: Maximum depth of the book to record 18 | collapse_orders: Whether to aggregate orders by level 19 | show_age: Whether to include order age in output 20 | """ 21 | 22 | def __init__( 23 | self, 24 | writer, 25 | max_depth: Optional[int] = None, 26 | collapse_orders: bool = True, 27 | show_age: bool = False, 28 | ): 29 | """Initialize the LOBRecorder. 30 | 31 | Args: 32 | writer: DataWriter instance for output 33 | max_depth: Maximum depth of the book to record (None for all) 34 | collapse_orders: Whether to aggregate orders by level 35 | show_age: Whether to include order age in output 36 | """ 37 | self._max_depth: int | None = max_depth 38 | self._collapse_orders: bool = collapse_orders 39 | self._show_age: bool = show_age 40 | 41 | LOBEventRecorder.__init__(self, writer=writer) 42 | 43 | # Set the schema based on current parameters 44 | self._set_writer_schema() 45 | 46 | @property 47 | def max_depth(self) -> int | None: 48 | """Maximum depth of the book to record.""" 49 | return self._max_depth 50 | 51 | @max_depth.setter 52 | def max_depth(self, value: int | None) -> None: 53 | """Set maximum depth of the book to record.""" 54 | if self._max_depth != value: 55 | self._max_depth = value 56 | self._update_writer_schema() 57 | 58 | @property 59 | def collapse_orders(self) -> bool: 60 | """Whether to aggregate orders by level.""" 61 | return self._collapse_orders 62 | 63 | @collapse_orders.setter 64 | def collapse_orders(self, value: bool) -> None: 65 | """Set whether to aggregate orders by level.""" 66 | if self._collapse_orders != value: 67 | self._collapse_orders = value 68 | self._update_writer_schema() 69 | 70 | @property 71 | def show_age(self) -> bool: 72 | """Whether to include order age in output.""" 73 | return self._show_age 74 | 75 | @show_age.setter 76 | def show_age(self, value: bool) -> None: 77 | """Set whether to include order age in output.""" 78 | if self._show_age != value: 79 | self._show_age = value 80 | self._update_writer_schema() 81 | 82 | def _set_writer_schema(self) -> None: 83 | """Set the writer schema based on current recording parameters.""" 84 | schema = self.get_schema() 85 | self.writer.set_schema(schema) 86 | 87 | def _update_writer_schema(self) -> None: 88 | """Update the writer schema if not locked, otherwise raise warning.""" 89 | if not self.writer.is_schema_locked(): 90 | self._set_writer_schema() 91 | else: 92 | import warnings 93 | 94 | warnings.warn( 95 | "Cannot update schema after writing has begun. " 96 | "Schema changes will take effect for new writers only.", 97 | UserWarning, 98 | stacklevel=3, 99 | ) 100 | 101 | def record(self, lob: LimitOrderBook, record_timestamp=None): 102 | """Record a snapshot of the limit order book. 103 | 104 | Args: 105 | lob: The current limit order book 106 | record_timestamp: Optional timestamp to override 107 | """ 108 | # Create a copy of the LOB for recording 109 | new_record = lob.copy(max_level=self.max_depth) 110 | if record_timestamp is not None: 111 | new_record.timestamp = record_timestamp 112 | 113 | # Convert to structured records and write directly to the writer 114 | records = new_record.to_records( 115 | collapse_orders=self.collapse_orders, 116 | show_age=self.show_age, 117 | max_depth=self.max_depth, 118 | ) 119 | 120 | # Write each record to the writer 121 | for record in records: 122 | self.writer.buffer_record(record) 123 | 124 | def get_schema(self): 125 | """Get schema definition for the data writer.""" 126 | if self.show_age: 127 | if self.collapse_orders: 128 | return { 129 | "fields": { 130 | "Timestamp": "string", 131 | "Type": "string", 132 | "Level": "int64", 133 | "Price": "float64", 134 | "Volume": "int64", 135 | "N Orders": "int64", 136 | "Volume-Weighted Average Age": "float64", 137 | "Average Age": "float64", 138 | "First Age": "float64", 139 | "Last Age": "float64", 140 | } 141 | } 142 | else: 143 | return { 144 | "fields": { 145 | "Timestamp": "string", 146 | "Type": "string", 147 | "Level": "int64", 148 | "Price": "float64", 149 | "Order ID": "int64", 150 | "Volume": "int64", 151 | "Order Timestamp": "string", 152 | "Age": "float64", 153 | } 154 | } 155 | else: 156 | if self.collapse_orders: 157 | return { 158 | "fields": { 159 | "Timestamp": "string", 160 | "Type": "string", 161 | "Level": "int64", 162 | "Price": "float64", 163 | "Volume": "int64", 164 | "N Orders": "int64", 165 | } 166 | } 167 | else: 168 | return { 169 | "fields": { 170 | "Timestamp": "string", 171 | "Type": "string", 172 | "Level": "int64", 173 | "Price": "float64", 174 | "Order ID": "int64", 175 | "Volume": "int64", 176 | "Order Timestamp": "string", 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/meatpy/event_handlers/ofi_recorder.py: -------------------------------------------------------------------------------- 1 | """Order Flow Imbalance (OFI) event recorder for limit order books. 2 | 3 | This module provides the OFIRecorder class, which records order flow imbalance 4 | metrics as described in Cont et al. (2013) and exports them to CSV files. 5 | 6 | See Equation (10) of 7 | Cont, R., et al. (2013). "The Price Impact of Order Book Events." 8 | Journal of Financial Econometrics 12(1): 47-88. 9 | """ 10 | 11 | from ..event_handlers.lob_event_recorder import LOBEventRecorder 12 | from ..lob import LimitOrderBook 13 | 14 | 15 | class OFIRecorder(LOBEventRecorder): 16 | """Records order flow imbalance (OFI) metrics for limit order books. 17 | 18 | Attributes: 19 | previous_lob: The previous limit order book snapshot 20 | """ 21 | 22 | def __init__(self, writer): 23 | """Initialize the OFIRecorder. 24 | 25 | Args: 26 | writer: DataWriter instance for output 27 | """ 28 | self.previous_lob: LimitOrderBook | None = None 29 | LOBEventRecorder.__init__(self, writer=writer) 30 | 31 | # Set the schema based on OFI requirements 32 | schema = self.get_schema() 33 | self.writer.set_schema(schema) 34 | 35 | def record(self, lob: LimitOrderBook, record_timestamp: bool = None): 36 | """Record the OFI metric for the current LOB state. 37 | 38 | Args: 39 | lob: The current limit order book 40 | record_timestamp: Optional timestamp to record 41 | """ 42 | if record_timestamp is None: 43 | record_timestamp = lob.timestamp 44 | new_lob = lob.copy(max_level=1) 45 | if self.previous_lob is not None: 46 | try: 47 | Pb_new = new_lob.bid_levels[0].price 48 | qb_new = new_lob.bid_levels[0].volume() 49 | except IndexError: 50 | Pb_new = 0 51 | qb_new = 0 52 | try: 53 | Pb_prev = self.previous_lob.bid_levels[0].price 54 | qb_prev = self.previous_lob.bid_levels[0].volume() 55 | except IndexError: 56 | Pb_prev = Pb_new 57 | qb_prev = 0 58 | try: 59 | Ps_prev = self.previous_lob.ask_levels[0].price 60 | qs_prev = self.previous_lob.ask_levels[0].volume() 61 | except IndexError: 62 | Ps_prev = 0 63 | qs_prev = 0 64 | try: 65 | Ps_new = new_lob.ask_levels[0].price 66 | qs_new = new_lob.ask_levels[0].volume() 67 | except IndexError: 68 | Ps_new = Ps_prev 69 | qs_new = 0 70 | e_n = 0 71 | if Pb_new >= Pb_prev: 72 | e_n += qb_new 73 | if Pb_new <= Pb_prev: 74 | e_n -= qb_prev 75 | if Ps_new <= Ps_prev: 76 | e_n -= qs_new 77 | if Ps_new >= Ps_prev: 78 | e_n += qs_prev 79 | # Write record directly to the writer 80 | record = {"Timestamp": str(record_timestamp), "e_n": e_n} 81 | self.writer.buffer_record(record) 82 | self.previous_lob = new_lob 83 | 84 | def get_schema(self): 85 | """Get schema definition for the data writer.""" 86 | return {"fields": {"Timestamp": "string", "e_n": "int64"}} 87 | -------------------------------------------------------------------------------- /src/meatpy/event_handlers/spot_measures_recorder.py: -------------------------------------------------------------------------------- 1 | """Spot measures event recorder for limit order books. 2 | 3 | This module provides the SpotMeasuresRecorder class, which records various spot 4 | measures (e.g., bid-ask spread, mid quote) from the limit order book and exports 5 | them to CSV files. 6 | """ 7 | 8 | from copy import deepcopy 9 | from io import TextIOWrapper 10 | from typing import Any 11 | 12 | from ..lob import InexistantValueException, LimitOrderBook 13 | from .lob_event_recorder import LOBEventRecorder 14 | 15 | 16 | class UnknownMeasureError(Exception): 17 | """Exception raised when an unknown measure is requested. 18 | 19 | This exception is raised when the measure name is not recognized 20 | by the spot measures recorder. 21 | """ 22 | 23 | pass 24 | 25 | 26 | class SpotMeasuresRecorder(LOBEventRecorder): 27 | """Records spot measures from the limit order book and exports to CSV. 28 | 29 | Attributes: 30 | measures: List of measure names to record 31 | """ 32 | 33 | def __init__(self): 34 | """Initialize the SpotMeasuresRecorder.""" 35 | self.measures: list[str] = [] # List of strings (measure names) 36 | LOBEventRecorder.__init__(self) 37 | 38 | def record(self, lob: LimitOrderBook, record_timestamp: bool = None): 39 | """Record spot measures from the limit order book. 40 | 41 | Args: 42 | lob: The current limit order book 43 | record_timestamp: Optional timestamp to record 44 | """ 45 | if record_timestamp is None: 46 | new_record = [deepcopy(lob.timestamp)] 47 | else: 48 | new_record = [record_timestamp] 49 | for x in self.measures: 50 | if x == "Bid-Ask Spread": 51 | try: 52 | spread = lob.bid_ask_spread() 53 | except InexistantValueException: 54 | spread = "" 55 | new_record.append(spread) 56 | elif x == "Mid Quote": 57 | try: 58 | mid_quote = lob.mid_quote() 59 | except InexistantValueException: 60 | mid_quote = "" 61 | new_record.append(mid_quote) 62 | elif x == "Best Ask": 63 | try: 64 | best_ask = lob.best_ask() 65 | except InexistantValueException: 66 | best_ask = "" 67 | new_record.append(best_ask) 68 | elif x == "Best Bid": 69 | try: 70 | best_bid = lob.best_bid() 71 | except InexistantValueException: 72 | best_bid = "" 73 | new_record.append(best_bid) 74 | elif x == "Quote Slope": 75 | try: 76 | quote_slope = lob.quote_slope() 77 | except InexistantValueException: 78 | quote_slope = "" 79 | new_record.append(quote_slope) 80 | elif x == "Log Quote Slope": 81 | try: 82 | log_quote_slope = lob.log_quote_slope() 83 | except InexistantValueException: 84 | log_quote_slope = "" 85 | new_record.append(log_quote_slope) 86 | else: 87 | raise UnknownMeasureError(f"Unknown measure: {x}") 88 | self.records.append(new_record) 89 | 90 | def write_csv(self, file: TextIOWrapper, collapse: bool = False): 91 | """Write recorded spot measures to a CSV file. 92 | 93 | Args: 94 | file: File object to write to 95 | collapse: Whether to collapse records by timestamp 96 | """ 97 | file.write("Timestamp") 98 | for x in self.measures: 99 | file.write("," + x) 100 | file.write("\n") 101 | if collapse: 102 | last_ts = None 103 | next_write = None 104 | for x in self.records: 105 | if last_ts is None: 106 | last_ts = x[0] 107 | next_write = x 108 | elif x[0] == last_ts: 109 | next_write = x 110 | else: 111 | if next_write is not None: 112 | self.__write_record(file, next_write) 113 | last_ts = x[0] 114 | next_write = x 115 | if next_write is not None: 116 | self.__write_record(file, next_write) 117 | else: 118 | for x in self.records: 119 | self.__write_record(file, x) 120 | 121 | def __write_record(self, file: TextIOWrapper, record: list[Any]): 122 | """Write a single record to the CSV file. 123 | 124 | Args: 125 | file: File object to write to 126 | record: List of values to write as a row 127 | """ 128 | first = True 129 | for y in record: 130 | if not first: 131 | file.write(",") 132 | else: 133 | first = False 134 | file.write(str(y)) 135 | file.write("\n") 136 | 137 | def write_csv_header(self, file: TextIOWrapper): 138 | """Write the CSV header row to the file. 139 | 140 | Args: 141 | file: File object to write the header to 142 | """ 143 | file.write("Timestamp") 144 | for x in self.measures: 145 | file.write("," + x) 146 | file.write("\n") 147 | 148 | def append_csv(self, file: TextIOWrapper): 149 | """Append spot measure records to a CSV file. 150 | 151 | Args: 152 | file: File object to append to 153 | """ 154 | for x in self.records: 155 | first = True 156 | for y in x: 157 | if not first: 158 | file.write(",") 159 | else: 160 | first = False 161 | file.write(str(y)) 162 | file.write("\n") 163 | self.records = [] 164 | -------------------------------------------------------------------------------- /src/meatpy/events.py: -------------------------------------------------------------------------------- 1 | """Event system for market data processing. 2 | 3 | This module provides a protocol-based event system for handling market events 4 | in a type-safe and extensible manner. 5 | """ 6 | 7 | from typing import Protocol, runtime_checkable 8 | 9 | from .lob import LimitOrderBook, OrderType 10 | from .market_processor import MarketProcessor 11 | from .message_reader import MarketMessage 12 | from .timestamp import Timestamp 13 | from .types import OrderID, Price, TradeRef, Volume 14 | 15 | 16 | @runtime_checkable 17 | class MarketEventHandler(Protocol): 18 | """Protocol for market event handlers. 19 | 20 | This protocol defines the interface that all market event handlers 21 | must implement. Handlers can choose to implement only the events 22 | they care about. 23 | """ 24 | 25 | def before_lob_update( 26 | self, lob: LimitOrderBook | None, new_timestamp: Timestamp 27 | ) -> None: 28 | """Handle event before a limit order book update.""" 29 | ... 30 | 31 | def message_event( 32 | self, 33 | market_processor: MarketProcessor, 34 | timestamp: Timestamp, 35 | message: MarketMessage, 36 | ) -> None: 37 | """Handle a raw market message event.""" 38 | ... 39 | 40 | def enter_quote_event( 41 | self, 42 | market_processor: MarketProcessor, 43 | timestamp: Timestamp, 44 | price: Price, 45 | volume: Volume, 46 | order_id: OrderID, 47 | order_type: OrderType | None = None, 48 | ) -> None: 49 | """Handle a quote entry event.""" 50 | ... 51 | 52 | def cancel_quote_event( 53 | self, 54 | market_processor: MarketProcessor, 55 | timestamp: Timestamp, 56 | volume: Volume, 57 | order_id: OrderID, 58 | order_type: OrderType | None = None, 59 | ) -> None: 60 | """Handle a quote cancellation event.""" 61 | ... 62 | 63 | def delete_quote_event( 64 | self, 65 | market_processor: MarketProcessor, 66 | timestamp: Timestamp, 67 | order_id: OrderID, 68 | order_type: OrderType | None = None, 69 | ) -> None: 70 | """Handle a quote deletion event.""" 71 | ... 72 | 73 | def replace_quote_event( 74 | self, 75 | market_processor: MarketProcessor, 76 | timestamp: Timestamp, 77 | orig_order_id: OrderID, 78 | new_order_id: OrderID, 79 | price: Price, 80 | volume: Volume, 81 | order_type: OrderType | None = None, 82 | ) -> None: 83 | """Handle a quote replacement event.""" 84 | ... 85 | 86 | def execute_trade_event( 87 | self, 88 | market_processor: MarketProcessor, 89 | timestamp: Timestamp, 90 | volume: Volume, 91 | order_id: OrderID, 92 | trade_ref: TradeRef, 93 | order_type: OrderType | None = None, 94 | ) -> None: 95 | """Handle a trade execution event.""" 96 | ... 97 | 98 | def execute_trade_price_event( 99 | self, 100 | market_processor: MarketProcessor, 101 | timestamp: Timestamp, 102 | volume: Volume, 103 | order_id: OrderID, 104 | trade_ref: TradeRef, 105 | price: Price, 106 | order_type: OrderType | None = None, 107 | ) -> None: 108 | """Handle a trade execution event with price information.""" 109 | ... 110 | 111 | def auction_trade_event( 112 | self, 113 | market_processor: MarketProcessor, 114 | timestamp: Timestamp, 115 | volume: Volume, 116 | price: Price, 117 | bid_id: OrderID, 118 | ask_id: OrderID, 119 | ) -> None: 120 | """Handle an auction trade event.""" 121 | ... 122 | 123 | def crossing_trade_event( 124 | self, 125 | market_processor: MarketProcessor, 126 | timestamp: Timestamp, 127 | volume: Volume, 128 | price: Price, 129 | bid_id: OrderID, 130 | ask_id: OrderID, 131 | ) -> None: 132 | """Handle a crossing trade event.""" 133 | ... 134 | 135 | 136 | class BaseEventHandler: 137 | """Base class for market event handlers with empty implementations. 138 | 139 | This class provides empty implementations for all event methods. 140 | Subclasses can override only the methods they need to handle. 141 | """ 142 | 143 | def before_lob_update( 144 | self, lob: LimitOrderBook | None, new_timestamp: Timestamp 145 | ) -> None: 146 | """Handle event before a limit order book update.""" 147 | pass 148 | 149 | def message_event( 150 | self, 151 | market_processor: MarketProcessor, 152 | timestamp: Timestamp, 153 | message: MarketMessage, 154 | ) -> None: 155 | """Handle a raw market message event.""" 156 | pass 157 | 158 | def enter_quote_event( 159 | self, 160 | market_processor: MarketProcessor, 161 | timestamp: Timestamp, 162 | price: Price, 163 | volume: Volume, 164 | order_id: OrderID, 165 | order_type: OrderType | None = None, 166 | ) -> None: 167 | """Handle a quote entry event.""" 168 | pass 169 | 170 | def cancel_quote_event( 171 | self, 172 | market_processor: MarketProcessor, 173 | timestamp: Timestamp, 174 | volume: Volume, 175 | order_id: OrderID, 176 | order_type: OrderType | None = None, 177 | ) -> None: 178 | """Handle a quote cancellation event.""" 179 | pass 180 | 181 | def delete_quote_event( 182 | self, 183 | market_processor: MarketProcessor, 184 | timestamp: Timestamp, 185 | order_id: OrderID, 186 | order_type: OrderType | None = None, 187 | ) -> None: 188 | """Handle a quote deletion event.""" 189 | pass 190 | 191 | def replace_quote_event( 192 | self, 193 | market_processor: MarketProcessor, 194 | timestamp: Timestamp, 195 | orig_order_id: OrderID, 196 | new_order_id: OrderID, 197 | price: Price, 198 | volume: Volume, 199 | order_type: OrderType | None = None, 200 | ) -> None: 201 | """Handle a quote replacement event.""" 202 | pass 203 | 204 | def execute_trade_event( 205 | self, 206 | market_processor: MarketProcessor, 207 | timestamp: Timestamp, 208 | volume: Volume, 209 | order_id: OrderID, 210 | trade_ref: TradeRef, 211 | order_type: OrderType | None = None, 212 | ) -> None: 213 | """Handle a trade execution event.""" 214 | pass 215 | 216 | def execute_trade_price_event( 217 | self, 218 | market_processor: MarketProcessor, 219 | timestamp: Timestamp, 220 | volume: Volume, 221 | order_id: OrderID, 222 | trade_ref: TradeRef, 223 | price: Price, 224 | order_type: OrderType | None = None, 225 | ) -> None: 226 | """Handle a trade execution event with price information.""" 227 | pass 228 | 229 | def auction_trade_event( 230 | self, 231 | market_processor: MarketProcessor, 232 | timestamp: Timestamp, 233 | volume: Volume, 234 | price: Price, 235 | bid_id: OrderID, 236 | ask_id: OrderID, 237 | ) -> None: 238 | """Handle an auction trade event.""" 239 | pass 240 | 241 | def crossing_trade_event( 242 | self, 243 | market_processor: MarketProcessor, 244 | timestamp: Timestamp, 245 | volume: Volume, 246 | price: Price, 247 | bid_id: OrderID, 248 | ask_id: OrderID, 249 | ) -> None: 250 | """Handle a crossing trade event.""" 251 | pass 252 | -------------------------------------------------------------------------------- /src/meatpy/itch41/__init__.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 market data subpackage. 2 | 3 | This package provides message types, parsers, processors, and recorders for handling ITCH 4.1 market data in MeatPy. 4 | """ 5 | 6 | from .itch41_exec_trade_recorder import ITCH41ExecTradeRecorder 7 | from .itch41_market_message import ( 8 | AddOrderMessage, 9 | AddOrderMPIDMessage, 10 | BrokenTradeMessage, 11 | CrossTradeMessage, 12 | MarketParticipantPositionMessage, 13 | OrderCancelMessage, 14 | OrderDeleteMessage, 15 | OrderExecutedMessage, 16 | OrderExecutedPriceMessage, 17 | OrderReplaceMessage, 18 | RegSHOMessage, 19 | SecondsMessage, 20 | StockDirectoryMessage, 21 | StockTradingActionMessage, 22 | SystemEventMessage, 23 | TradeMessage, 24 | ) 25 | from .itch41_market_processor import ITCH41MarketProcessor 26 | from .itch41_message_reader import ITCH41MessageReader 27 | from .itch41_ofi_recorder import ITCH41OFIRecorder 28 | from .itch41_order_event_recorder import ITCH41OrderEventRecorder 29 | from .itch41_top_of_book_message_recorder import ( 30 | ITCH41TopOfBookMessageRecorder, 31 | ) 32 | from .itch41_writer import ITCH41Writer 33 | 34 | __all__ = [ 35 | "ITCH41ExecTradeRecorder", 36 | "ITCH41MarketMessage", 37 | "ITCH41MarketProcessor", 38 | "ITCH41MessageParser", 39 | "ITCH41MessageReader", 40 | "ITCH41Writer", 41 | "ITCH41OFIRecorder", 42 | "ITCH41OrderEventRecorder", 43 | "ITCH41TopOfBookMessageRecorder", 44 | "AddOrderMessage", 45 | "AddOrderMPIDMessage", 46 | "BrokenTradeMessage", 47 | "CrossTradeMessage", 48 | "MarketParticipantPositionMessage", 49 | "OrderCancelMessage", 50 | "OrderDeleteMessage", 51 | "OrderExecutedMessage", 52 | "OrderExecutedPriceMessage", 53 | "OrderReplaceMessage", 54 | "RegSHOMessage", 55 | "SecondsMessage", 56 | "StockDirectoryMessage", 57 | "StockTradingActionMessage", 58 | "SystemEventMessage", 59 | "TradeMessage", 60 | ] 61 | -------------------------------------------------------------------------------- /src/meatpy/itch41/itch41_exec_trade_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 execution trade recorder for limit order books. 2 | 3 | This module provides the ITCH41ExecTradeRecorder class, which records trade 4 | executions from ITCH 4.1 market data and exports them to CSV files. 5 | """ 6 | 7 | from typing import Any 8 | 9 | from ..market_event_handler import MarketEventHandler 10 | from .itch41_market_message import ( 11 | OrderExecutedMessage, 12 | OrderExecutedPriceMessage, 13 | TradeMessage, 14 | ) 15 | 16 | 17 | class ITCH41ExecTradeRecorder(MarketEventHandler): 18 | """Records trade executions from ITCH 4.1 market data. 19 | 20 | This recorder detects and records trade executions, including both 21 | visible and hidden trades, and exports them to CSV format. 22 | 23 | Attributes: 24 | records: List of recorded trade execution records 25 | """ 26 | 27 | def __init__(self) -> None: 28 | """Initialize the ITCH41ExecTradeRecorder.""" 29 | self.records: list[Any] = [] 30 | 31 | def message_event( 32 | self, 33 | market_processor, 34 | timestamp, 35 | message, 36 | ) -> None: 37 | """Detect messages that represent trade executions and record them. 38 | 39 | Args: 40 | market_processor: The market processor instance 41 | timestamp: The timestamp of the message 42 | message: The market message to process 43 | """ 44 | lob = market_processor.lob 45 | if lob is None: 46 | return 47 | 48 | if isinstance(message, OrderExecutedMessage): 49 | # An executed order will ALWAYS be against top of book 50 | # because of price priority, so record. 51 | if lob.ask_order_on_book(message.order_ref): 52 | record = { 53 | "MessageType": "Exec", 54 | "Volume": message.shares, 55 | "OrderID": message.order_ref, 56 | } 57 | record["Queue"] = "Ask" 58 | record["Price"] = lob.ask_levels[0].price 59 | try: 60 | (queue, i, j) = lob.find_order(message.order_ref) 61 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 62 | except Exception: 63 | record["OrderTimestamp"] = "" 64 | self.records.append((timestamp, record)) 65 | elif lob.bid_order_on_book(message.order_ref): 66 | record = { 67 | "MessageType": "Exec", 68 | "Volume": message.shares, 69 | "OrderID": message.order_ref, 70 | } 71 | record["Queue"] = "Bid" 72 | record["Price"] = lob.bid_levels[0].price 73 | try: 74 | (queue, i, j) = lob.find_order(message.order_ref) 75 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 76 | except Exception: 77 | record["OrderTimestamp"] = "" 78 | self.records.append((timestamp, record)) 79 | elif isinstance(message, TradeMessage): 80 | if message.side == b"S": 81 | record = { 82 | "MessageType": "ExecHid", 83 | "Volume": message.shares, 84 | "OrderID": "", 85 | "OrderTimestamp": "", 86 | } 87 | record["Queue"] = "Ask" 88 | record["Price"] = message.price 89 | self.records.append((timestamp, record)) 90 | elif message.side == b"B": 91 | record = { 92 | "MessageType": "ExecHid", 93 | "Volume": message.shares, 94 | "OrderID": "", 95 | "OrderTimestamp": "", 96 | } 97 | record["Queue"] = "Bid" 98 | record["Price"] = message.price 99 | self.records.append((timestamp, record)) 100 | elif isinstance(message, OrderExecutedPriceMessage): 101 | if len(lob.ask_levels) > 0 and lob.ask_levels[0].order_on_book( 102 | message.order_ref 103 | ): 104 | record = { 105 | "MessageType": "ExecPrice", 106 | "Queue": "Ask", 107 | "Volume": message.shares, 108 | "OrderID": message.order_ref, 109 | "Price": message.price, 110 | } 111 | try: 112 | (queue, i, j) = lob.find_order(message.order_ref) 113 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 114 | except Exception: 115 | record["OrderTimestamp"] = "" 116 | self.records.append((timestamp, record)) 117 | elif len(lob.bid_levels) > 0 and lob.bid_levels[0].order_on_book( 118 | message.order_ref 119 | ): 120 | record = { 121 | "MessageType": "ExecPrice", 122 | "Queue": "Bid", 123 | "Volume": message.shares, 124 | "OrderID": message.order_ref, 125 | "Price": message.price, 126 | } 127 | try: 128 | (queue, i, j) = lob.find_order(message.order_ref) 129 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 130 | except Exception: 131 | record["OrderTimestamp"] = "" 132 | self.records.append((timestamp, record)) 133 | -------------------------------------------------------------------------------- /src/meatpy/itch41/itch41_message_reader.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 file reader with generator interface. 2 | 3 | This module provides the ITCH41MessageReader class, which reads ITCH 4.1 market data files 4 | and yields structured message objects one at a time using a generator interface. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | from pathlib import Path 10 | from typing import Generator, Optional 11 | 12 | from ..message_reader import MessageReader 13 | from .itch41_market_message import ( 14 | ITCH41MarketMessage, 15 | ) 16 | 17 | 18 | class ITCH41MessageReader(MessageReader): 19 | """A market message reader for ITCH 4.1 data with generator interface. 20 | 21 | This reader reads ITCH 4.1 files and yields message objects one at a time, 22 | supporting automatic detection of compressed files (gzip, bzip2, xz, zip). 23 | 24 | Attributes: 25 | file_path: Path to the ITCH file to read 26 | _file_handle: Internal file handle when used as context manager 27 | """ 28 | 29 | def __init__( 30 | self, 31 | file_path: Optional[Path | str] = None, 32 | ) -> None: 33 | """Initialize the ITCH41MessageReader. 34 | 35 | Args: 36 | file_path: Path to the ITCH file to read (optional if using read_file method) 37 | """ 38 | super().__init__(file_path) 39 | 40 | def __iter__(self) -> Generator[ITCH41MarketMessage, None, None]: 41 | """Make the reader iterable when used as a context manager.""" 42 | if self._file_handle is None: 43 | raise RuntimeError( 44 | "Reader must be used as a context manager to be iterable" 45 | ) 46 | yield from self._read_messages(self._file_handle) 47 | 48 | def read_file( 49 | self, file_path: Path | str 50 | ) -> Generator[ITCH41MarketMessage, None, None]: 51 | """Parse an ITCH 4.1 file and yield messages one at a time. 52 | 53 | Args: 54 | file_path: Path to the ITCH file to read 55 | 56 | Yields: 57 | ITCH41MarketMessage objects 58 | """ 59 | file_path = Path(file_path) 60 | with self._open_file(file_path) as file: 61 | yield from self._read_messages(file) 62 | 63 | def _read_messages(self, file) -> Generator[ITCH41MarketMessage, None, None]: 64 | """Internal method to read messages from an open file handle. 65 | 66 | Args: 67 | file: Open file handle to read from 68 | 69 | Yields: 70 | ITCH41MarketMessage objects 71 | """ 72 | from ..message_reader import InvalidMessageFormatError 73 | 74 | cachesize = 1024 * 4 75 | 76 | data_buffer = file.read(cachesize) 77 | data_view = memoryview(data_buffer) 78 | offset = 0 79 | buflen = len(data_view) 80 | eof_reached = False 81 | 82 | while True: 83 | # Check if we need more data 84 | if offset + 2 > buflen: 85 | if eof_reached: 86 | break 87 | new_data = file.read(cachesize) 88 | if not new_data: 89 | eof_reached = True 90 | break 91 | data_buffer = data_view[offset:].tobytes() + new_data 92 | data_view = memoryview(data_buffer) 93 | buflen = len(data_view) 94 | offset = 0 95 | continue 96 | 97 | if data_view[offset] != 0: 98 | raise InvalidMessageFormatError(f"Unexpected byte: {data_view[offset]}") 99 | 100 | message_len = data_view[offset + 1] 101 | message_end = offset + 2 + message_len 102 | 103 | # Check if we have enough data for the complete message 104 | if message_end > buflen: 105 | if eof_reached: 106 | break 107 | new_data = file.read(cachesize) 108 | if not new_data: 109 | eof_reached = True 110 | break 111 | data_buffer = data_view[offset:].tobytes() + new_data 112 | data_view = memoryview(data_buffer) 113 | buflen = len(data_view) 114 | offset = 0 115 | continue 116 | 117 | message = ITCH41MarketMessage.from_bytes( 118 | data_view[offset + 2 : message_end].tobytes() 119 | ) 120 | 121 | yield message 122 | offset = message_end 123 | 124 | # Check if we've reached the end of the buffer and EOF 125 | if offset >= buflen and eof_reached: 126 | break 127 | -------------------------------------------------------------------------------- /src/meatpy/itch41/itch41_ofi_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 Order Flow Imbalance (OFI) recorder for limit order books. 2 | 3 | This module provides the ITCH41OFIRecorder class, which records order flow 4 | imbalance metrics adapted for ITCH 4.1 data, including trades against hidden orders. 5 | 6 | See Equations (4) and (10) of 7 | Cont, R., et al. (2013). "The Price Impact of Order Book Events." 8 | Journal of Financial Econometrics 12(1): 47-88. 9 | 10 | The recorder follows equation (10) but accounts for trades against 11 | hidden orders as well. 12 | """ 13 | 14 | from typing import Any 15 | 16 | from ..event_handlers.ofi_recorder import OFIRecorder 17 | from .itch41_market_message import TradeMessage 18 | 19 | 20 | class ITCH41OFIRecorder(OFIRecorder): 21 | """Records order flow imbalance (OFI) metrics for ITCH 4.1 data. 22 | 23 | This recorder extends the base OFIRecorder to handle ITCH 4.1 specific 24 | features, including trades against hidden orders that are not captured 25 | in the standard limit order book. 26 | 27 | Attributes: 28 | records: List of recorded OFI measures 29 | previous_lob: The previous limit order book snapshot 30 | """ 31 | 32 | def __init__(self) -> None: 33 | """Initialize the ITCH41OFIRecorder.""" 34 | self.records: list[Any] = [] 35 | self.previous_lob = None 36 | OFIRecorder.__init__(self) 37 | 38 | def message_event(self, market_processor, timestamp, message) -> None: 39 | """Detect trades against hidden orders and record OFI metrics. 40 | 41 | Args: 42 | market_processor: The market processor instance 43 | timestamp: The timestamp of the message 44 | message: The market message to process 45 | """ 46 | # For a hidden order to execute, it must be first in line 47 | # (in front of first level), so we record ALL trades against hidden 48 | # orders. 49 | if isinstance(message, TradeMessage): 50 | volume = message.shares 51 | # OFI decreases when a trade is executed against a hidden bid. 52 | if message.side == b"B": 53 | sign = -1 54 | elif message.side == b"S": 55 | sign = 1 56 | e_n = sign * volume 57 | self.records.append((timestamp, e_n)) 58 | -------------------------------------------------------------------------------- /src/meatpy/itch41/itch41_order_event_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 order event recorder for limit order books. 2 | 3 | This module provides the ITCH41OrderEventRecorder class, which records order-related 4 | events from ITCH 4.1 market data and exports them to CSV files. 5 | """ 6 | 7 | from ..market_event_handler import MarketEventHandler 8 | from .itch41_market_message import ( 9 | AddOrderMessage, 10 | AddOrderMPIDMessage, 11 | OrderCancelMessage, 12 | OrderDeleteMessage, 13 | OrderExecutedMessage, 14 | OrderExecutedPriceMessage, 15 | OrderReplaceMessage, 16 | ) 17 | 18 | 19 | class ITCH41OrderEventRecorder(MarketEventHandler): 20 | """Records order-related events from ITCH 4.1 market data. 21 | 22 | This recorder detects and records various order events including additions, 23 | executions, cancellations, deletions, and replacements, along with the 24 | current state of the limit order book. 25 | 26 | Attributes: 27 | records: List of recorded order event records 28 | """ 29 | 30 | def __init__(self) -> None: 31 | """Initialize the ITCH41OrderEventRecorder.""" 32 | self.records = [] 33 | 34 | def message_event(self, market_processor, timestamp, message) -> None: 35 | """Detect messages that involve orders and record them. 36 | 37 | Args: 38 | market_processor: The market processor instance 39 | timestamp: The timestamp of the message 40 | message: The market message to process 41 | """ 42 | if not ( 43 | isinstance(message, AddOrderMessage) 44 | or isinstance(message, AddOrderMPIDMessage) 45 | or isinstance(message, OrderExecutedMessage) 46 | or isinstance(message, OrderExecutedPriceMessage) 47 | or isinstance(message, OrderCancelMessage) 48 | or isinstance(message, OrderDeleteMessage) 49 | or isinstance(message, OrderReplaceMessage) 50 | ): 51 | return 52 | lob = market_processor.lob 53 | ask_price = None 54 | ask_size = None 55 | bid_price = None 56 | bid_size = None 57 | # LOB is only initialised after first event. 58 | if lob is not None: 59 | if len(lob.ask_levels) > 0: 60 | ask_price = lob.ask_levels[0].price 61 | ask_size = lob.ask_levels[0].volume() 62 | if len(lob.bid_levels) > 0: 63 | bid_price = lob.bid_levels[0].price 64 | bid_size = lob.bid_levels[0].volume() 65 | # For OrderExecuted and OrderCancel, find price of corresponding 66 | # limit order. For OrderDelete also find quantity. 67 | if ( 68 | isinstance(message, OrderExecutedMessage) 69 | or isinstance(message, OrderCancelMessage) 70 | or isinstance(message, OrderDeleteMessage) 71 | ): 72 | price = None 73 | shares = None 74 | try: 75 | if lob is not None: 76 | queue, i, j = lob.find_order(message.order_ref) 77 | price = queue[i].price 78 | shares = queue[i].queue[j].volume 79 | except Exception as e: 80 | print( 81 | f"ITCH41OrderEventRecorder ::{e} for order ID {message.order_ref}" 82 | ) 83 | if isinstance(message, AddOrderMessage): 84 | record = { 85 | "order_ref": message.order_ref, 86 | "bsindicator": message.side.decode(), 87 | "shares": message.shares, 88 | "price": message.price, 89 | "neworder_ref": "", 90 | "MessageType": "AddOrder", 91 | } 92 | elif isinstance(message, AddOrderMPIDMessage): 93 | record = { 94 | "order_ref": message.order_ref, 95 | "bsindicator": message.side.decode(), 96 | "shares": message.shares, 97 | "price": message.price, 98 | "neworder_ref": "", 99 | "MessageType": "AddOrderMPID", 100 | } 101 | elif isinstance(message, OrderExecutedMessage): 102 | record = { 103 | "order_ref": message.order_ref, 104 | "bsindicator": "", 105 | "shares": message.shares, 106 | "price": price, 107 | "neworder_ref": "", 108 | "MessageType": "OrderExecuted", 109 | } 110 | elif isinstance(message, OrderExecutedPriceMessage): 111 | record = { 112 | "order_ref": message.order_ref, 113 | "bsindicator": "", 114 | "shares": message.shares, 115 | "price": message.price, 116 | "neworder_ref": "", 117 | "MessageType": "OrderExecutedPrice", 118 | } 119 | elif isinstance(message, OrderCancelMessage): 120 | record = { 121 | "order_ref": message.order_ref, 122 | "bsindicator": "", 123 | "shares": message.shares, 124 | "price": price, 125 | "neworder_ref": "", 126 | "MessageType": "OrderCancel", 127 | } 128 | elif isinstance(message, OrderDeleteMessage): 129 | record = { 130 | "order_ref": message.order_ref, 131 | "bsindicator": price, 132 | "shares": shares, 133 | "price": price, 134 | "neworder_ref": "", 135 | "MessageType": "OrderDelete", 136 | } 137 | elif isinstance(message, OrderReplaceMessage): 138 | record = { 139 | "order_ref": message.original_ref, 140 | "bsindicator": "", 141 | "shares": message.shares, 142 | "price": message.price, 143 | "neworder_ref": message.new_ref, 144 | "MessageType": "OrderReplace", 145 | } 146 | record["ask_price"] = ask_price 147 | record["ask_size"] = ask_size 148 | record["bid_price"] = bid_price 149 | record["bid_size"] = bid_size 150 | self.records.append((timestamp, record)) 151 | -------------------------------------------------------------------------------- /src/meatpy/itch41/itch41_writer.py: -------------------------------------------------------------------------------- 1 | """ITCH 4.1 file writer with buffering and compression support. 2 | 3 | This module provides the ITCH41Writer class, which writes ITCH 4.1 market data 4 | messages to files with support for buffering and compression. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import bz2 10 | import gzip 11 | import lzma 12 | from pathlib import Path 13 | from typing import Optional 14 | 15 | from .itch41_market_message import ITCH41MarketMessage 16 | 17 | 18 | class ITCH41Writer: 19 | """A writer for ITCH 4.1 market data files. 20 | 21 | This writer keeps track of messages relevant to specified symbols and 22 | writes them to files in ITCH format with support for buffering and compression. 23 | 24 | Attributes: 25 | symbols: List of symbols to track (None for all) 26 | output_path: Path to the output file 27 | message_buffer: Number of messages to buffer before writing 28 | compress: Whether to compress the output file 29 | compression_type: Type of compression to use ('gzip', 'bzip2', 'xz') 30 | """ 31 | 32 | def __init__( 33 | self, 34 | output_path: Optional[Path | str] = None, 35 | symbols: Optional[list[bytes | str]] = None, 36 | message_buffer: int = 2000, 37 | compress: bool = False, 38 | compression_type: str = "gzip", 39 | ) -> None: 40 | """Initialize the ITCH41Writer. 41 | 42 | Args: 43 | symbols: List of symbols to track (None for all) 44 | output_path: Path to the output file 45 | message_buffer: Number of messages to buffer before writing 46 | compress: Whether to compress the output file 47 | compression_type: Type of compression to use ('gzip', 'bzip2', 'xz') 48 | """ 49 | self._symbols = ( 50 | [ 51 | f"{symbol:<8}".encode("ascii") if isinstance(symbol, str) else symbol 52 | for symbol in symbols 53 | ] 54 | if symbols 55 | else None 56 | ) 57 | self.output_path = Path(output_path) if output_path else None 58 | self.message_buffer = message_buffer 59 | self.compress = compress 60 | self.compression_type = compression_type 61 | 62 | # Internal state 63 | self._order_refs: set[int] = set() 64 | self._matches: set[int] = set() 65 | self._buffer: list[ITCH41MarketMessage] = [] 66 | self._message_count = 0 67 | 68 | def _get_file_handle(self, mode: str = "ab"): 69 | """Get a file handle with appropriate compression if needed. 70 | 71 | Args: 72 | mode: File open mode 73 | 74 | Returns: 75 | File-like object for writing 76 | """ 77 | if not self.output_path: 78 | raise ValueError("No output path specified") 79 | 80 | if not self.compress: 81 | return open(self.output_path, mode) 82 | elif self.compression_type == "gzip": 83 | return gzip.open(self.output_path, mode) 84 | elif self.compression_type == "bzip2": 85 | return bz2.open(self.output_path, mode) 86 | elif self.compression_type == "xz": 87 | return lzma.open(self.output_path, mode) 88 | else: 89 | raise ValueError(f"Unsupported compression type: {self.compression_type}") 90 | 91 | def _write_message(self, file_handle, message: ITCH41MarketMessage) -> None: 92 | """Write a single message to the file in ITCH format. 93 | 94 | Args: 95 | file_handle: File handle to write to 96 | message: Message to write 97 | """ 98 | file_handle.write(b"\x00") 99 | file_handle.write(bytes([message.message_size])) 100 | file_handle.write(message.to_bytes()) 101 | 102 | def _write_messages(self, force: bool = False) -> None: 103 | """Write buffered messages to the file. 104 | 105 | Args: 106 | force: Whether to write regardless of buffer size 107 | """ 108 | if len(self._buffer) > self.message_buffer or force: 109 | with self._get_file_handle() as file_handle: 110 | for message in self._buffer: 111 | self._write_message(file_handle, message) 112 | self._buffer = [] 113 | 114 | def _append_message(self, message: ITCH41MarketMessage) -> None: 115 | """Append a message to the stock message buffer. 116 | 117 | Args: 118 | message: Message to append 119 | """ 120 | self._buffer.append(message) 121 | self._write_messages() 122 | 123 | def _validate_symbol(self, symbol: bytes) -> bool: 124 | """Validate a symbol. 125 | 126 | Args: 127 | symbol: Symbol to validate 128 | 129 | Returns: 130 | True if the symbol is valid, False otherwise 131 | """ 132 | if self._symbols is None: 133 | return True 134 | return symbol in self._symbols 135 | 136 | def process_message(self, message: ITCH41MarketMessage) -> None: 137 | """Process a message and add it to the appropriate buffers. 138 | 139 | Args: 140 | message: Message to process 141 | """ 142 | self._message_count += 1 143 | 144 | # Get message type 145 | message_type = getattr(message.__class__, "type", None) 146 | if message_type is None: 147 | return 148 | 149 | # Handle different message types 150 | if message_type == b"R": # Stock Directory 151 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 152 | self._append_message(message) 153 | 154 | elif message_type in b"ST": # System messages (Seconds, System Event) 155 | # Add to all stock message buffers 156 | self._append_message(message) 157 | 158 | elif ( 159 | message_type in b"HYL" 160 | ): # Stock-specific messages (Trading Action, RegSHO, Market Participant Position) 161 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 162 | self._append_message(message) 163 | 164 | elif message_type in b"AF": # Add order messages 165 | if ( 166 | hasattr(message, "stock") 167 | and hasattr(message, "order_ref") 168 | and self._validate_symbol(message.stock) 169 | ): 170 | self._order_refs.add(message.order_ref) 171 | self._append_message(message) 172 | 173 | elif message_type in b"ECXD": # Order execution/cancel/delete 174 | if hasattr(message, "order_ref") and message.order_ref in self._order_refs: 175 | self._append_message(message) 176 | if message_type == b"D": # Order delete 177 | self._order_refs.remove(message.order_ref) 178 | elif message_type in b"EC": # Order executed 179 | if hasattr(message, "match"): 180 | self._matches.add(message.match) 181 | 182 | elif message_type == b"U": # Order replace 183 | if ( 184 | hasattr(message, "original_ref") 185 | and message.original_ref in self._order_refs 186 | ): 187 | self._append_message(message) 188 | self._order_refs.remove(message.original_ref) 189 | if hasattr(message, "new_ref"): 190 | self._order_refs.add(message.new_ref) 191 | 192 | elif message_type == b"B": # Broken trade 193 | if hasattr(message, "match") and message.match in self._matches: 194 | self._append_message(message) 195 | 196 | elif message_type in b"PQ": # Trade messages (Trade, Cross Trade) 197 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 198 | self._append_message(message) 199 | if hasattr(message, "match"): 200 | self._matches.add(message.match) 201 | 202 | elif message_type in b"IN": # NOII and RPII messages 203 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 204 | self._append_message(message) 205 | 206 | def flush(self) -> None: 207 | """Flush all buffered messages to the file.""" 208 | self._write_messages(force=True) 209 | 210 | def close(self) -> None: 211 | """Close the writer and flush any remaining messages.""" 212 | self.flush() 213 | 214 | def __enter__(self): 215 | """Context manager entry.""" 216 | return self 217 | 218 | def __exit__(self, exc_type, exc_val, exc_tb): 219 | """Context manager exit.""" 220 | self.close() 221 | -------------------------------------------------------------------------------- /src/meatpy/itch50/__init__.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 market data subpackage. 2 | 3 | This package provides message types, parsers, processors, and recorders for handling ITCH 5.0 market data in MeatPy. 4 | """ 5 | 6 | from .itch50_exec_trade_recorder import ITCH50ExecTradeRecorder 7 | from .itch50_market_message import ( 8 | AddOrderMessage, 9 | AddOrderMPIDMessage, 10 | BrokenTradeMessage, 11 | CrossTradeMessage, 12 | IPOQuotingPeriodUpdateMessage, 13 | LULDAuctionCollarMessage, 14 | MarketParticipantPositionMessage, 15 | MWCBBreachMessage, 16 | MWCBDeclineLevelMessage, 17 | NoiiMessage, 18 | OperationalHaltMessage, 19 | OrderCancelMessage, 20 | OrderDeleteMessage, 21 | OrderExecutedMessage, 22 | OrderExecutedPriceMessage, 23 | OrderReplaceMessage, 24 | RegSHOMessage, 25 | RpiiMessage, 26 | StockDirectoryMessage, 27 | StockTradingActionMessage, 28 | SystemEventMessage, 29 | TradeMessage, 30 | ) 31 | from .itch50_market_processor import ITCH50MarketProcessor 32 | from .itch50_message_reader import ITCH50MessageReader 33 | from .itch50_ofi_recorder import ITCH50OFIRecorder 34 | from .itch50_order_event_recorder import ITCH50OrderEventRecorder 35 | from .itch50_top_of_book_message_recorder import ( 36 | ITCH50TopOfBookMessageRecorder, 37 | ) 38 | from .itch50_writer import ITCH50Writer 39 | 40 | __all__ = [ 41 | "ITCH50ExecTradeRecorder", 42 | "ITCH50MarketMessage", 43 | "ITCH50MarketProcessor", 44 | "ITCH50MessageParser", 45 | "ITCH50MessageReader", 46 | "ITCH50Writer", 47 | "ITCH50OFIRecorder", 48 | "ITCH50OrderEventRecorder", 49 | "ITCH50TopOfBookMessageRecorder", 50 | "AddOrderMessage", 51 | "AddOrderMPIDMessage", 52 | "BrokenTradeMessage", 53 | "CrossTradeMessage", 54 | "IPOQuotingPeriodUpdateMessage", 55 | "LULDAuctionCollarMessage", 56 | "MarketParticipantPositionMessage", 57 | "MWCBBreachMessage", 58 | "MWCBDeclineLevelMessage", 59 | "NoiiMessage", 60 | "OperationalHaltMessage", 61 | "OrderCancelMessage", 62 | "OrderDeleteMessage", 63 | "OrderExecutedMessage", 64 | "OrderExecutedPriceMessage", 65 | "OrderReplaceMessage", 66 | "RegSHOMessage", 67 | "RpiiMessage", 68 | "StockDirectoryMessage", 69 | "StockTradingActionMessage", 70 | "SystemEventMessage", 71 | "TradeMessage", 72 | ] 73 | -------------------------------------------------------------------------------- /src/meatpy/itch50/itch50_exec_trade_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 execution trade recorder for limit order books. 2 | 3 | This module provides the ITCH50ExecTradeRecorder class, which records trade 4 | executions from ITCH 5.0 market data and exports them to CSV files. 5 | """ 6 | 7 | from typing import Any 8 | 9 | from ..market_event_handler import MarketEventHandler 10 | from .itch50_market_message import ( 11 | OrderExecutedMessage, 12 | OrderExecutedPriceMessage, 13 | TradeMessage, 14 | ) 15 | 16 | 17 | class ITCH50ExecTradeRecorder(MarketEventHandler): 18 | """Records trade executions from ITCH 5.0 market data. 19 | 20 | This recorder detects and records trade executions, including both 21 | visible and hidden trades, and exports them to CSV format. 22 | 23 | Attributes: 24 | records: List of recorded trade execution records 25 | """ 26 | 27 | def __init__(self) -> None: 28 | """Initialize the ITCH50ExecTradeRecorder.""" 29 | self.records: list[Any] = [] 30 | 31 | def message_event( 32 | self, 33 | market_processor, 34 | timestamp, 35 | message, 36 | ) -> None: 37 | """Detect messages that represent trade executions and record them. 38 | 39 | Args: 40 | market_processor: The market processor instance 41 | timestamp: The timestamp of the message 42 | message: The market message to process 43 | """ 44 | lob = market_processor.current_lob 45 | if isinstance(message, OrderExecutedMessage): 46 | # An executed order will ALWAYS be against top of book 47 | # because of price priority, so record. 48 | if lob.ask_order_on_book(message.order_ref): 49 | record = { 50 | "MessageType": "Exec", 51 | "Volume": message.shares, 52 | "OrderID": message.order_ref, 53 | } 54 | record["Queue"] = "Ask" 55 | record["Price"] = lob.ask_levels[0].price 56 | (queue, i, j) = lob.find_order(message.order_ref) 57 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 58 | self.records.append((timestamp, record)) 59 | elif lob.bid_order_on_book(message.order_ref): 60 | record = { 61 | "MessageType": "Exec", 62 | "Volume": message.shares, 63 | "OrderID": message.order_ref, 64 | } 65 | record["Queue"] = "Bid" 66 | record["Price"] = lob.bid_levels[0].price 67 | (queue, i, j) = lob.find_order(message.order_ref) 68 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 69 | self.records.append((timestamp, record)) 70 | elif isinstance(message, TradeMessage): 71 | if message.bsindicator == b"S": 72 | record = { 73 | "MessageType": "ExecHid", 74 | "Volume": message.shares, 75 | "OrderID": "", 76 | "OrderTimestamp": "", 77 | } 78 | record["Queue"] = "Ask" 79 | record["Price"] = message.price 80 | self.records.append((timestamp, record)) 81 | elif message.bsindicator == b"B": 82 | record = { 83 | "MessageType": "ExecHid", 84 | "Volume": message.shares, 85 | "OrderID": "", 86 | "OrderTimestamp": "", 87 | } 88 | record["Queue"] = "Bid" 89 | record["Price"] = message.price 90 | self.records.append((timestamp, record)) 91 | elif isinstance(message, OrderExecutedPriceMessage): 92 | if len(lob.ask_levels) > 0 and lob.ask_levels[0].order_on_book( 93 | message.order_ref 94 | ): 95 | record = { 96 | "MessageType": "ExecPrice", 97 | "Queue": "Ask", 98 | "Volume": message.shares, 99 | "OrderID": message.order_ref, 100 | "Price": message.price, 101 | } 102 | (queue, i, j) = lob.find_order(message.order_ref) 103 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 104 | self.records.append((timestamp, record)) 105 | elif len(lob.bid_levels) > 0 and lob.bid_levels[0].order_on_book( 106 | message.order_ref 107 | ): 108 | record = { 109 | "MessageType": "ExecPrice", 110 | "Queue": "Bid", 111 | "Volume": message.shares, 112 | "OrderID": message.order_ref, 113 | "Price": message.price, 114 | } 115 | (queue, i, j) = lob.find_order(message.order_ref) 116 | record["OrderTimestamp"] = queue[i].queue[j].timestamp 117 | self.records.append((timestamp, record)) 118 | 119 | def write_csv(self, file) -> None: 120 | """Write recorded trade executions to a CSV file. 121 | 122 | Args: 123 | file: File object to write to 124 | """ 125 | file.write( 126 | "Timestamp,MessageType,Queue,Price,Volume,OrderID,OrderTimestamp\n".encode() 127 | ) 128 | for x in self.records: 129 | row = f"{x[0]},{x[1]['MessageType']},{x[1]['Queue']},{x[1]['Price']},{x[1]['Volume']},{x[1]['OrderID']},{x[1]['OrderTimestamp']}\n" 130 | file.write(row.encode()) 131 | -------------------------------------------------------------------------------- /src/meatpy/itch50/itch50_message_reader.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 file reader with generator interface. 2 | 3 | This module provides the ITCH50MessageReader class, which reads ITCH 5.0 market data files 4 | and yields structured message objects one at a time using a generator interface. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | from pathlib import Path 10 | from typing import Generator, Optional 11 | 12 | from ..message_reader import MessageReader 13 | from .itch50_market_message import ( 14 | ITCH50MarketMessage, 15 | ) 16 | 17 | 18 | class ITCH50MessageReader(MessageReader): 19 | """A market message reader for ITCH 5.0 data with generator interface. 20 | 21 | This reader reads ITCH 5.0 files and yields message objects one at a time, 22 | supporting automatic detection of compressed files (gzip, bzip2, xz, zip). 23 | 24 | Attributes: 25 | file_path: Path to the ITCH file to read 26 | _file_handle: Internal file handle when used as context manager 27 | """ 28 | 29 | def __init__( 30 | self, 31 | file_path: Optional[Path | str] = None, 32 | ) -> None: 33 | """Initialize the ITCH50MessageReader. 34 | 35 | Args: 36 | file_path: Path to the ITCH file to read (optional if using read_file method) 37 | """ 38 | super().__init__(file_path) 39 | 40 | def __iter__(self) -> Generator[ITCH50MarketMessage, None, None]: 41 | """Make the reader iterable when used as a context manager.""" 42 | if self._file_handle is None: 43 | raise RuntimeError( 44 | "Reader must be used as a context manager to be iterable" 45 | ) 46 | yield from self._read_messages(self._file_handle) 47 | 48 | def read_file( 49 | self, file_path: Path | str 50 | ) -> Generator[ITCH50MarketMessage, None, None]: 51 | """Parse an ITCH 5.0 file and yield messages one at a time. 52 | 53 | Args: 54 | file_path: Path to the ITCH file to read 55 | 56 | Yields: 57 | ITCH50MarketMessage objects 58 | """ 59 | file_path = Path(file_path) 60 | with self._open_file(file_path) as file: 61 | yield from self._read_messages(file) 62 | 63 | def _read_messages(self, file) -> Generator[ITCH50MarketMessage, None, None]: 64 | """Internal method to read messages from an open file handle. 65 | 66 | Args: 67 | file: Open file handle to read from 68 | 69 | Yields: 70 | ITCH50MarketMessage objects 71 | """ 72 | from ..message_reader import InvalidMessageFormatError 73 | 74 | cachesize = 1024 * 4 75 | 76 | data_buffer = file.read(cachesize) 77 | data_view = memoryview(data_buffer) 78 | offset = 0 79 | buflen = len(data_view) 80 | eof_reached = False 81 | 82 | while True: 83 | # Check if we need more data 84 | if offset + 2 > buflen: 85 | if eof_reached: 86 | break 87 | new_data = file.read(cachesize) 88 | if not new_data: 89 | eof_reached = True 90 | break 91 | data_buffer = data_view[offset:].tobytes() + new_data 92 | data_view = memoryview(data_buffer) 93 | buflen = len(data_view) 94 | offset = 0 95 | continue 96 | 97 | if data_view[offset] != 0: 98 | raise InvalidMessageFormatError(f"Unexpected byte: {data_view[offset]}") 99 | 100 | message_len = data_view[offset + 1] 101 | message_end = offset + 2 + message_len 102 | 103 | # Check if we have enough data for the complete message 104 | if message_end > buflen: 105 | if eof_reached: 106 | break 107 | new_data = file.read(cachesize) 108 | if not new_data: 109 | eof_reached = True 110 | break 111 | data_buffer = data_view[offset:].tobytes() + new_data 112 | data_view = memoryview(data_buffer) 113 | buflen = len(data_view) 114 | offset = 0 115 | continue 116 | 117 | message = ITCH50MarketMessage.from_bytes( 118 | data_view[offset + 2 : message_end].tobytes() 119 | ) 120 | 121 | yield message 122 | offset = message_end 123 | 124 | # Check if we've reached the end of the buffer and EOF 125 | if offset >= buflen and eof_reached: 126 | break 127 | -------------------------------------------------------------------------------- /src/meatpy/itch50/itch50_ofi_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 Order Flow Imbalance (OFI) recorder for limit order books. 2 | 3 | This module provides the ITCH50OFIRecorder class, which records order flow 4 | imbalance metrics adapted for ITCH 5.0 data, including trades against hidden orders. 5 | 6 | See Equations (4) and (10) of 7 | Cont, R., et al. (2013). "The Price Impact of Order Book Events." 8 | Journal of Financial Econometrics 12(1): 47-88. 9 | 10 | The recorder follows equation (10) but accounts for trades against 11 | hidden orders as well. 12 | """ 13 | 14 | from typing import Any 15 | 16 | from ..event_handlers.ofi_recorder import OFIRecorder 17 | from .itch50_market_message import TradeMessage 18 | 19 | 20 | class ITCH50OFIRecorder(OFIRecorder): 21 | """Records order flow imbalance (OFI) metrics for ITCH 5.0 data. 22 | 23 | This recorder extends the base OFIRecorder to handle ITCH 5.0 specific 24 | features, including trades against hidden orders that are not captured 25 | in the standard limit order book. 26 | 27 | Attributes: 28 | records: List of recorded OFI measures 29 | previous_lob: The previous limit order book snapshot 30 | """ 31 | 32 | def __init__(self) -> None: 33 | """Initialize the ITCH50OFIRecorder.""" 34 | self.records: list[Any] = [] 35 | self.previous_lob = None 36 | OFIRecorder.__init__(self) 37 | 38 | def message_event(self, market_processor, timestamp, message) -> None: 39 | """Detect trades against hidden orders and record OFI metrics. 40 | 41 | Args: 42 | market_processor: The market processor instance 43 | timestamp: The timestamp of the message 44 | message: The market message to process 45 | """ 46 | # For a hidden order to execute, it must be first in line 47 | # (in front of first level), so we record ALL trades against hidden 48 | # orders. 49 | if isinstance(message, TradeMessage): 50 | volume = message.shares 51 | # OFI decreases when a trade is executed against a hidden bid. 52 | if message.bsindicator == b"B": 53 | sign = -1 54 | elif message.bsindicator == b"S": 55 | sign = 1 56 | e_n = sign * volume 57 | self.records.append((timestamp, e_n)) 58 | -------------------------------------------------------------------------------- /src/meatpy/itch50/itch50_order_event_recorder.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 order event recorder for limit order books. 2 | 3 | This module provides the ITCH50OrderEventRecorder class, which records order-related 4 | events from ITCH 5.0 market data and exports them to CSV files. 5 | """ 6 | 7 | from ..market_event_handler import MarketEventHandler 8 | from .itch50_market_message import ( 9 | AddOrderMessage, 10 | AddOrderMPIDMessage, 11 | OrderCancelMessage, 12 | OrderDeleteMessage, 13 | OrderExecutedMessage, 14 | OrderExecutedPriceMessage, 15 | OrderReplaceMessage, 16 | ) 17 | 18 | 19 | class ITCH50OrderEventRecorder(MarketEventHandler): 20 | """Records order-related events from ITCH 5.0 market data. 21 | 22 | This recorder detects and records various order events including additions, 23 | executions, cancellations, deletions, and replacements, along with the 24 | current state of the limit order book. 25 | 26 | Attributes: 27 | records: List of recorded order event records 28 | """ 29 | 30 | def __init__(self) -> None: 31 | """Initialize the ITCH50OrderEventRecorder.""" 32 | self.records = [] 33 | 34 | def message_event(self, market_processor, timestamp, message) -> None: 35 | """Detect messages that involve orders and record them. 36 | 37 | Args: 38 | market_processor: The market processor instance 39 | timestamp: The timestamp of the message 40 | message: The market message to process 41 | """ 42 | if not ( 43 | isinstance(message, AddOrderMessage) 44 | or isinstance(message, AddOrderMPIDMessage) 45 | or isinstance(message, OrderExecutedMessage) 46 | or isinstance(message, OrderExecutedPriceMessage) 47 | or isinstance(message, OrderCancelMessage) 48 | or isinstance(message, OrderDeleteMessage) 49 | or isinstance(message, OrderReplaceMessage) 50 | ): 51 | return 52 | lob = market_processor.current_lob 53 | ask_price = None 54 | ask_size = None 55 | bid_price = None 56 | bid_size = None 57 | # LOB is only initialised after first event. 58 | if lob is not None: 59 | if len(lob.ask_levels) > 0: 60 | ask_price = lob.ask_levels[0].price 61 | ask_size = lob.ask_levels[0].volume() 62 | if len(lob.bid_levels) > 0: 63 | bid_price = lob.bid_levels[0].price 64 | bid_size = lob.bid_levels[0].volume() 65 | # For OrderExecuted and OrderCancel, find price of corresponding 66 | # limit order. For OrderDelete also find quantity. 67 | if ( 68 | isinstance(message, OrderExecutedMessage) 69 | or isinstance(message, OrderCancelMessage) 70 | or isinstance(message, OrderDeleteMessage) 71 | ): 72 | price = None 73 | shares = None 74 | try: 75 | queue, i, j = lob.find_order(message.order_ref) 76 | price = queue[i].price 77 | shares = queue[i].queue[j].volume 78 | except Exception as e: 79 | print( 80 | f"ITCH50OrderEventRecorder ::{e} for order ID {message.order_ref}" 81 | ) 82 | if isinstance(message, AddOrderMessage): 83 | record = { 84 | "order_ref": message.order_ref, 85 | "bsindicator": message.bsindicator.decode(), 86 | "shares": message.shares, 87 | "price": message.price, 88 | "neworder_ref": "", 89 | "MessageType": "AddOrder", 90 | } 91 | elif isinstance(message, AddOrderMPIDMessage): 92 | record = { 93 | "order_ref": message.order_ref, 94 | "bsindicator": message.bsindicator.decode(), 95 | "shares": message.shares, 96 | "price": message.price, 97 | "neworder_ref": "", 98 | "MessageType": "AddOrderMPID", 99 | } 100 | elif isinstance(message, OrderExecutedMessage): 101 | record = { 102 | "order_ref": message.order_ref, 103 | "bsindicator": "", 104 | "shares": message.shares, 105 | "price": price, 106 | "neworder_ref": "", 107 | "MessageType": "OrderExecuted", 108 | } 109 | elif isinstance(message, OrderExecutedPriceMessage): 110 | record = { 111 | "order_ref": message.order_ref, 112 | "bsindicator": "", 113 | "shares": message.shares, 114 | "price": message.price, 115 | "neworder_ref": "", 116 | "MessageType": "OrderExecutedPrice", 117 | } 118 | elif isinstance(message, OrderCancelMessage): 119 | record = { 120 | "order_ref": message.order_ref, 121 | "bsindicator": "", 122 | "shares": message.cancelShares, 123 | "price": price, 124 | "neworder_ref": "", 125 | "MessageType": "OrderCancel", 126 | } 127 | elif isinstance(message, OrderDeleteMessage): 128 | record = { 129 | "order_ref": message.order_ref, 130 | "bsindicator": price, 131 | "shares": shares, 132 | "price": price, 133 | "neworder_ref": "", 134 | "MessageType": "OrderDelete", 135 | } 136 | elif isinstance(message, OrderReplaceMessage): 137 | record = { 138 | "order_ref": message.origorder_ref, 139 | "bsindicator": "", 140 | "shares": message.shares, 141 | "price": message.price, 142 | "neworder_ref": message.neworder_ref, 143 | "MessageType": "OrderReplace", 144 | } 145 | record["ask_price"] = ask_price 146 | record["ask_size"] = ask_size 147 | record["bid_price"] = bid_price 148 | record["bid_size"] = bid_size 149 | self.records.append((timestamp, record)) 150 | 151 | def write_csv(self, file) -> None: 152 | """Write recorded order events to a CSV file. 153 | 154 | Args: 155 | file: File object to write to 156 | """ 157 | file.write( 158 | "Timestamp,MessageType,BuySellIndicator,Price,Volume,OrderID,NewOrderID,AskPrice,AskSize,BidPrice,BidSize\n".encode() 159 | ) 160 | for x in self.records: 161 | row = f"{x[0]},{x[1]['MessageType']},{x[1]['bsindicator']},{x[1]['price']},{x[1]['shares']},{x[1]['order_ref']},{x[1]['neworder_ref']},{x[1]['ask_price']},{x[1]['ask_size']},{x[1]['bid_price']},{x[1]['bid_size']}\n" 162 | file.write(row.encode()) 163 | -------------------------------------------------------------------------------- /src/meatpy/itch50/itch50_writer.py: -------------------------------------------------------------------------------- 1 | """ITCH 5.0 file writer with buffering and compression support. 2 | 3 | This module provides the ITCH50Writer class, which writes ITCH 5.0 market data 4 | messages to files with support for buffering and compression. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import bz2 10 | import gzip 11 | import lzma 12 | from pathlib import Path 13 | from typing import Optional 14 | 15 | from .itch50_market_message import ITCH50MarketMessage 16 | 17 | 18 | class ITCH50Writer: 19 | """A writer for ITCH 5.0 market data files. 20 | 21 | This writer keeps track of messages relevant to specified symbols and 22 | writes them to files in ITCH format with support for buffering and compression. 23 | 24 | Attributes: 25 | symbols: List of symbols to track (None for all) 26 | output_path: Path to the output file 27 | message_buffer: Number of messages to buffer before writing 28 | compress: Whether to compress the output file 29 | compression_type: Type of compression to use ('gzip', 'bzip2', 'xz') 30 | """ 31 | 32 | def __init__( 33 | self, 34 | output_path: Optional[Path | str] = None, 35 | symbols: Optional[list[bytes | str]] = None, 36 | message_buffer: int = 2000, 37 | compress: bool = False, 38 | compression_type: str = "gzip", 39 | ) -> None: 40 | """Initialize the ITCH50Writer. 41 | 42 | Args: 43 | symbols: List of symbols to track (None for all) 44 | output_path: Path to the output file 45 | message_buffer: Number of messages to buffer before writing 46 | compress: Whether to compress the output file 47 | compression_type: Type of compression to use ('gzip', 'bzip2', 'xz') 48 | """ 49 | self._symbols = ( 50 | [ 51 | f"{symbol:<8}".encode("ascii") if isinstance(symbol, str) else symbol 52 | for symbol in symbols 53 | ] 54 | if symbols 55 | else None 56 | ) 57 | self.output_path = Path(output_path) if output_path else None 58 | self.message_buffer = message_buffer 59 | self.compress = compress 60 | self.compression_type = compression_type 61 | 62 | # Internal state 63 | self._order_refs: set[int] = set() 64 | self._matches: set[int] = set() 65 | self._buffer: list[ITCH50MarketMessage] = [] 66 | self._message_count = 0 67 | 68 | def _get_file_handle(self, mode: str = "ab"): 69 | """Get a file handle with appropriate compression if needed. 70 | 71 | Args: 72 | mode: File open mode 73 | 74 | Returns: 75 | File-like object for writing 76 | """ 77 | if not self.output_path: 78 | raise ValueError("No output path specified") 79 | 80 | if not self.compress: 81 | return open(self.output_path, mode) 82 | elif self.compression_type == "gzip": 83 | return gzip.open(self.output_path, mode) 84 | elif self.compression_type == "bzip2": 85 | return bz2.open(self.output_path, mode) 86 | elif self.compression_type == "xz": 87 | return lzma.open(self.output_path, mode) 88 | else: 89 | raise ValueError(f"Unsupported compression type: {self.compression_type}") 90 | 91 | def _write_message(self, file_handle, message: ITCH50MarketMessage) -> None: 92 | """Write a single message to the file in ITCH format. 93 | 94 | Args: 95 | file_handle: File handle to write to 96 | message: Message to write 97 | """ 98 | file_handle.write(b"\x00") 99 | file_handle.write(bytes([message.message_size])) 100 | file_handle.write(message.to_bytes()) 101 | 102 | def _write_messages(self, force: bool = False) -> None: 103 | """Write buffered messages to the file. 104 | 105 | Args: 106 | force: Whether to write regardless of buffer size 107 | """ 108 | if len(self._buffer) > self.message_buffer or force: 109 | with self._get_file_handle() as file_handle: 110 | for message in self._buffer: 111 | self._write_message(file_handle, message) 112 | self._buffer = [] 113 | 114 | def _append_message(self, message: ITCH50MarketMessage) -> None: 115 | """Append a message to the stock message buffer. 116 | 117 | Args: 118 | message: Message to append 119 | """ 120 | self._buffer.append(message) 121 | self._write_messages() 122 | 123 | def _validate_symbol(self, symbol: bytes) -> bool: 124 | """Validate a symbol. 125 | 126 | Args: 127 | symbol: Symbol to validate 128 | 129 | Returns: 130 | True if the symbol is valid, False otherwise 131 | """ 132 | if self._symbols is None: 133 | return True 134 | return symbol in self._symbols 135 | 136 | def process_message(self, message: ITCH50MarketMessage) -> None: 137 | """Process a message and add it to the appropriate buffers. 138 | 139 | Args: 140 | message: Message to process 141 | """ 142 | self._message_count += 1 143 | 144 | # Get message type 145 | message_type = getattr(message.__class__, "type", None) 146 | if message_type is None: 147 | return 148 | 149 | # Handle different message types 150 | if message_type == b"R": # Stock Directory 151 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 152 | self._append_message(message) 153 | 154 | elif message_type in b"SVW": # System messages 155 | # Add to all stock message buffers 156 | self._append_message(message) 157 | 158 | elif message_type in b"HYQINKLJh": # Stock-specific messages 159 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 160 | self._append_message(message) 161 | 162 | elif message_type in b"AF": # Add order messages 163 | if ( 164 | hasattr(message, "stock") 165 | and hasattr(message, "order_ref") 166 | and self._validate_symbol(message.stock) 167 | ): 168 | self._order_refs.add(message.order_ref) 169 | self._append_message(message) 170 | 171 | elif message_type in b"ECXD": # Order execution/cancel/delete 172 | if hasattr(message, "order_ref") and message.order_ref in self._order_refs: 173 | self._append_message(message) 174 | if message_type == b"D": # Order delete 175 | self._order_refs.remove(message.order_ref) 176 | elif message_type in b"EC": # Order executed 177 | if hasattr(message, "match"): 178 | self._matches.add(message.match) 179 | 180 | elif message_type == b"U": # Order replace 181 | if ( 182 | hasattr(message, "original_ref") 183 | and message.original_ref in self._order_refs 184 | ): 185 | self._append_message(message) 186 | self._order_refs.remove(message.original_ref) 187 | if hasattr(message, "new_ref"): 188 | self._order_refs.add(message.new_ref) 189 | 190 | elif message_type == b"B": # Broken trade 191 | if hasattr(message, "match") and message.match in self._matches: 192 | self._append_message(message) 193 | 194 | elif message_type == b"P": # Trade 195 | if hasattr(message, "stock") and self._validate_symbol(message.stock): 196 | self._append_message(message) 197 | if hasattr(message, "match"): 198 | self._matches.add(message.match) 199 | 200 | def flush(self) -> None: 201 | """Flush all buffered messages to the file.""" 202 | self._write_messages(force=True) 203 | 204 | def close(self) -> None: 205 | """Close the writer and flush any remaining messages.""" 206 | self.flush() 207 | 208 | def __enter__(self): 209 | """Context manager entry.""" 210 | return self 211 | 212 | def __exit__(self, exc_type, exc_val, exc_tb): 213 | """Context manager exit.""" 214 | self.close() 215 | -------------------------------------------------------------------------------- /src/meatpy/market_event_handler.py: -------------------------------------------------------------------------------- 1 | """Market event handler framework. 2 | 3 | This module provides the MarketEventHandler base class that defines the interface 4 | for handling various market events during processing. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | from typing import TYPE_CHECKING 10 | 11 | from .lob import OrderType 12 | from .message_reader import MarketMessage 13 | from .timestamp import Timestamp 14 | from .types import OrderID, Price, TradeRef, Volume 15 | 16 | if TYPE_CHECKING: 17 | from .market_processor import MarketProcessor 18 | 19 | 20 | class MarketEventHandler: 21 | """A handler for market events. 22 | 23 | The handler gets triggered whenever there is a market event and 24 | handles the event accordingly. This is a base class that provides 25 | empty implementations for all event methods. Subclasses should 26 | override the methods they want to handle. 27 | """ 28 | 29 | def before_lob_update( 30 | self, market_processor: MarketProcessor, new_timestamp: Timestamp 31 | ) -> None: 32 | """Handle event before a limit order book update. 33 | 34 | This method is called before the LOB is updated with a new timestamp. 35 | 36 | Args: 37 | market_processor: The market processor instance 38 | new_timestamp: The new timestamp for the upcoming update 39 | """ 40 | pass 41 | 42 | def message_event( 43 | self, 44 | market_processor: MarketProcessor, 45 | timestamp: Timestamp, 46 | message: MarketMessage, 47 | ) -> None: 48 | """Handle a raw market message event. 49 | 50 | Args: 51 | market_processor: The market processor that received the message 52 | timestamp: The timestamp of the message 53 | message: The parsed market message 54 | """ 55 | pass 56 | 57 | def enter_quote_event( 58 | self, 59 | market_processor: MarketProcessor, 60 | timestamp: Timestamp, 61 | price: Price, 62 | volume: Volume, 63 | order_id: OrderID, 64 | order_type: OrderType | None = None, 65 | ) -> None: 66 | """Handle a quote entry event. 67 | 68 | Args: 69 | market_processor: The market processor that processed the event 70 | timestamp: The timestamp of the event 71 | price: The price of the quote 72 | volume: The volume of the quote 73 | order_id: The unique identifier for the order 74 | order_type: The type of order (bid/ask), if known 75 | """ 76 | pass 77 | 78 | def cancel_quote_event( 79 | self, 80 | market_processor: MarketProcessor, 81 | timestamp: Timestamp, 82 | volume: Volume, 83 | order_id: OrderID, 84 | order_type: OrderType | None = None, 85 | ) -> None: 86 | """Handle a quote cancellation event. 87 | 88 | Args: 89 | market_processor: The market processor that processed the event 90 | timestamp: The timestamp of the event 91 | volume: The volume being cancelled 92 | order_id: The unique identifier for the order 93 | order_type: The type of order (bid/ask), if known 94 | """ 95 | pass 96 | 97 | def delete_quote_event( 98 | self, 99 | market_processor: MarketProcessor, 100 | timestamp: Timestamp, 101 | order_id: OrderID, 102 | order_type: OrderType | None = None, 103 | ) -> None: 104 | """Handle a quote deletion event. 105 | 106 | Args: 107 | market_processor: The market processor that processed the event 108 | timestamp: The timestamp of the event 109 | order_id: The unique identifier for the order being deleted 110 | order_type: The type of order (bid/ask), if known 111 | """ 112 | pass 113 | 114 | def replace_quote_event( 115 | self, 116 | market_processor: MarketProcessor, 117 | timestamp: Timestamp, 118 | orig_order_id: OrderID, 119 | new_order_id: OrderID, 120 | price: Price, 121 | volume: Volume, 122 | order_type: OrderType | None = None, 123 | ) -> None: 124 | """Handle a quote replacement event. 125 | 126 | Args: 127 | market_processor: The market processor that processed the event 128 | timestamp: The timestamp of the event 129 | orig_order_id: The original order identifier 130 | new_order_id: The new order identifier 131 | price: The new price of the quote 132 | volume: The new volume of the quote 133 | order_type: The type of order (bid/ask), if known 134 | """ 135 | pass 136 | 137 | def execute_trade_event( 138 | self, 139 | market_processor: MarketProcessor, 140 | timestamp: Timestamp, 141 | volume: Volume, 142 | order_id: OrderID, 143 | trade_ref: TradeRef, 144 | order_type: OrderType | None = None, 145 | ) -> None: 146 | """Handle a trade execution event. 147 | 148 | Args: 149 | market_processor: The market processor that processed the event 150 | timestamp: The timestamp of the event 151 | volume: The volume traded 152 | order_id: The unique identifier for the order 153 | trade_ref: The trade reference identifier 154 | order_type: The type of order (bid/ask), if known 155 | """ 156 | pass 157 | 158 | def execute_trade_price_event( 159 | self, 160 | market_processor: MarketProcessor, 161 | timestamp: Timestamp, 162 | volume: Volume, 163 | order_id: OrderID, 164 | trade_ref: TradeRef, 165 | price: Price, 166 | order_type: OrderType | None = None, 167 | ) -> None: 168 | """Handle a trade execution event with price information. 169 | 170 | Args: 171 | market_processor: The market processor that processed the event 172 | timestamp: The timestamp of the event 173 | volume: The volume traded 174 | order_id: The unique identifier for the order 175 | trade_ref: The trade reference identifier 176 | price: The execution price 177 | order_type: The type of order (bid/ask), if known 178 | """ 179 | pass 180 | 181 | def auction_trade_event( 182 | self, 183 | market_processor: MarketProcessor, 184 | timestamp: Timestamp, 185 | volume: Volume, 186 | price: Price, 187 | bid_id: OrderID, 188 | ask_id: OrderID, 189 | ) -> None: 190 | """Handle an auction trade event. 191 | 192 | Args: 193 | market_processor: The market processor that processed the event 194 | timestamp: The timestamp of the event 195 | volume: The volume traded 196 | price: The auction price 197 | bid_id: The bid order identifier 198 | ask_id: The ask order identifier 199 | """ 200 | pass 201 | 202 | def crossing_trade_event( 203 | self, 204 | market_processor: MarketProcessor, 205 | timestamp: Timestamp, 206 | volume: Volume, 207 | price: Price, 208 | bid_id: OrderID, 209 | ask_id: OrderID, 210 | ) -> None: 211 | """Handle a crossing trade event. 212 | 213 | Args: 214 | market_processor: The market processor that processed the event 215 | timestamp: The timestamp of the event 216 | volume: The volume traded 217 | price: The crossing price 218 | bid_id: The bid order identifier 219 | ask_id: The ask order identifier 220 | """ 221 | pass 222 | -------------------------------------------------------------------------------- /src/meatpy/message_reader.py: -------------------------------------------------------------------------------- 1 | """Abstract base class for market message readers. 2 | 3 | This module provides the MessageReader abstract base class, which defines 4 | the interface and common functionality for reading market data files 5 | and yielding structured message objects. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | import abc 11 | import bz2 12 | import gzip 13 | import lzma 14 | import zipfile 15 | from pathlib import Path 16 | from typing import Generator, Optional 17 | 18 | 19 | class MarketMessage: 20 | """A message that has been parsed and is ready to be processed by a market processor. 21 | 22 | This is an abstract class that should be overloaded for specific 23 | exchanges. Subclasses should contain the parsed data from raw market 24 | messages in a structured format. 25 | """ 26 | 27 | __metaclass__ = abc.ABCMeta 28 | pass 29 | 30 | 31 | class InvalidMessageFormatError(Exception): 32 | """Exception raised when a message has an invalid format. 33 | 34 | This exception is raised when the message format does not match 35 | the expected message structure. 36 | """ 37 | 38 | pass 39 | 40 | 41 | class UnknownMessageTypeError(Exception): 42 | """Exception raised when an unknown message type is encountered. 43 | 44 | This exception is raised when the message type is not recognized 45 | by the message reader. 46 | """ 47 | 48 | pass 49 | 50 | 51 | class MessageReader(abc.ABC): 52 | """Abstract base class for market message readers with generator interface. 53 | 54 | This abstract class provides the foundation for reading market data files 55 | and yielding message objects one at a time, supporting automatic detection 56 | of compressed files (gzip, bzip2, xz, zip). 57 | 58 | Attributes: 59 | file_path: Path to the market data file to read 60 | _file_handle: Internal file handle when used as context manager 61 | """ 62 | 63 | def __init__( 64 | self, 65 | file_path: Optional[Path | str] = None, 66 | ) -> None: 67 | """Initialize the MessageReader. 68 | 69 | Args: 70 | file_path: Path to the market data file to read (optional if using read_file method) 71 | """ 72 | self.file_path = Path(file_path) if file_path else None 73 | self._file_handle = None 74 | 75 | def __enter__(self): 76 | """Context manager entry. Opens the file if file_path was provided.""" 77 | if self.file_path is None: 78 | raise ValueError("No file_path provided. Use read_file() method instead.") 79 | self._file_handle = self._open_file(self.file_path) 80 | return self 81 | 82 | def __exit__(self, exc_type, exc_val, exc_tb): 83 | """Context manager exit. Closes the file handle.""" 84 | if self._file_handle is not None: 85 | self._file_handle.close() 86 | self._file_handle = None 87 | 88 | def __iter__(self) -> Generator[MarketMessage, None, None]: 89 | """Make the reader iterable when used as a context manager.""" 90 | if self._file_handle is None: 91 | raise RuntimeError( 92 | "Reader must be used as a context manager to be iterable" 93 | ) 94 | yield from self._read_messages(self._file_handle) 95 | 96 | def read_file(self, file_path: Path | str) -> Generator[MarketMessage, None, None]: 97 | """Parse a market data file and yield messages one at a time. 98 | 99 | Args: 100 | file_path: Path to the market data file to read 101 | 102 | Yields: 103 | MarketMessage objects 104 | """ 105 | file_path = Path(file_path) 106 | with self._open_file(file_path) as file: 107 | yield from self._read_messages(file) 108 | 109 | @abc.abstractmethod 110 | def _read_messages(self, file) -> Generator[MarketMessage, None, None]: 111 | """Internal method to read messages from an open file handle. 112 | 113 | This is an abstract method that must be implemented by subclasses 114 | to handle their specific message format and parsing logic. 115 | 116 | Args: 117 | file: Open file handle to read from 118 | 119 | Yields: 120 | MarketMessage objects 121 | """ 122 | pass 123 | 124 | def _detect_compression(self, file_path: Path) -> tuple[bool, str]: 125 | """Detect if a file is compressed and determine the compression type. 126 | 127 | Args: 128 | file_path: Path to the file to check 129 | 130 | Returns: 131 | Tuple of (is_compressed, compression_type) 132 | """ 133 | with open(file_path, "rb") as f: 134 | magic_bytes = f.read(4) 135 | 136 | if magic_bytes.startswith(b"\x1f\x8b"): 137 | return True, "gzip" 138 | elif magic_bytes.startswith(b"BZ"): 139 | return True, "bzip2" 140 | elif magic_bytes.startswith(b"\xfd7zXZ"): 141 | return True, "xz" 142 | elif magic_bytes.startswith(b"PK"): 143 | return True, "zip" 144 | else: 145 | return False, "none" 146 | 147 | def _open_file(self, file_path: Path): 148 | """Open a file with appropriate decompression if needed. 149 | 150 | Args: 151 | file_path: Path to the file to open 152 | 153 | Returns: 154 | File-like object for reading 155 | """ 156 | is_compressed, compression_type = self._detect_compression(file_path) 157 | 158 | if not is_compressed: 159 | return open(file_path, "rb") 160 | elif compression_type == "gzip": 161 | return gzip.open(file_path, "rb") 162 | elif compression_type == "bzip2": 163 | return bz2.open(file_path, "rb") 164 | elif compression_type == "xz": 165 | return lzma.open(file_path, "rb") 166 | elif compression_type == "zip": 167 | # For zip files, we need to handle them differently 168 | # Assume single file in zip or use the first file 169 | with zipfile.ZipFile(file_path, "r") as zip_file: 170 | if len(zip_file.namelist()) == 0: 171 | raise ValueError("Zip file is empty") 172 | # Return a file-like object for the first file in the zip 173 | return zip_file.open(zip_file.namelist()[0], "r") 174 | else: 175 | raise ValueError(f"Unsupported compression type: {compression_type}") 176 | -------------------------------------------------------------------------------- /src/meatpy/timestamp.py: -------------------------------------------------------------------------------- 1 | """Timestamp utilities for market data processing. 2 | 3 | This module provides a Timestamp class that extends datetime with 4 | microsecond precision and standardized formatting for market data. 5 | """ 6 | 7 | from copy import deepcopy 8 | from datetime import datetime 9 | 10 | TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S.%f" 11 | 12 | 13 | class Timestamp(datetime): 14 | """A timestamp class derived from datetime with microsecond precision. 15 | 16 | The timestamp is derived from the standard datetime class with the main 17 | feature that precision is limited to microseconds. Timestamps need to be 18 | comparable (<, <=, ==, !=, >=, >) and have a __str__() method. 19 | 20 | Attributes: 21 | year: Year of the timestamp 22 | month: Month of the timestamp 23 | day: Day of the timestamp 24 | hour: Hour of the timestamp 25 | minute: Minute of the timestamp 26 | second: Second of the timestamp 27 | microsecond: Microsecond of the timestamp 28 | nanoseconds: Original nanosecond timestamp (if available) 29 | """ 30 | 31 | def __new__( 32 | cls, 33 | year=None, 34 | month=None, 35 | day=None, 36 | hour=0, 37 | minute=0, 38 | second=0, 39 | microsecond=0, 40 | tzinfo=None, 41 | *, 42 | fold=0, 43 | nanoseconds=None, 44 | ): 45 | """Create a new Timestamp instance with optional nanoseconds storage.""" 46 | instance = super().__new__( 47 | cls, year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold 48 | ) 49 | instance._nanoseconds = nanoseconds 50 | return instance 51 | 52 | def __str__(self) -> str: 53 | """Return string representation in standard format. 54 | 55 | Returns: 56 | str: Timestamp formatted as 'YYYY-MM-DD HH:MM:SS.microseconds' 57 | """ 58 | return self.strftime(TIMESTAMP_FORMAT) 59 | 60 | def __repr__(self) -> str: 61 | """Return detailed string representation. 62 | 63 | Returns: 64 | str: Timestamp with 'Timestamp:' prefix 65 | """ 66 | return "Timestamp: " + self.__str__() 67 | 68 | @property 69 | def nanoseconds(self) -> int | None: 70 | """Get the original nanosecond timestamp if available. 71 | 72 | Returns: 73 | int: The original nanosecond timestamp, or None if not set 74 | """ 75 | return getattr(self, "_nanoseconds", None) 76 | 77 | def __reduce_ex__(self, protocol): 78 | """Custom pickling support for Timestamp. 79 | 80 | This ensures that the Timestamp can be properly pickled/unpickled 81 | with its custom _nanoseconds attribute. 82 | """ 83 | # Get the constructor arguments 84 | args = ( 85 | self.year, 86 | self.month, 87 | self.day, 88 | self.hour, 89 | self.minute, 90 | self.second, 91 | self.microsecond, 92 | self.tzinfo, 93 | ) 94 | # Get the state (custom attributes) 95 | state = {"_nanoseconds": getattr(self, "_nanoseconds", None), "fold": self.fold} 96 | # Return (callable, args, state) 97 | return (self.__class__, args, state) 98 | 99 | def __setstate__(self, state): 100 | """Set state for unpickling.""" 101 | if isinstance(state, dict): 102 | self._nanoseconds = state.get("_nanoseconds") 103 | if "fold" in state: 104 | # Can't set fold directly, it's read-only 105 | pass 106 | 107 | def copy(self) -> "Timestamp": 108 | """Create a deep copy of this timestamp. 109 | 110 | Returns: 111 | Timestamp: A new Timestamp instance with identical values 112 | """ 113 | return deepcopy(self) 114 | 115 | @classmethod 116 | def from_datetime(cls, dt: datetime, nanoseconds: int | None = None) -> "Timestamp": 117 | """Create a Timestamp from a datetime object. 118 | 119 | Args: 120 | dt: The datetime object to convert 121 | nanoseconds: Optional original nanosecond timestamp to preserve 122 | 123 | Returns: 124 | Timestamp: A new Timestamp instance with the datetime values 125 | """ 126 | return Timestamp( 127 | year=dt.year, 128 | month=dt.month, 129 | day=dt.day, 130 | hour=dt.hour, 131 | minute=dt.minute, 132 | microsecond=dt.microsecond, 133 | nanoseconds=nanoseconds, 134 | ) 135 | -------------------------------------------------------------------------------- /src/meatpy/types.py: -------------------------------------------------------------------------------- 1 | """Type definitions for the MeatPy library. 2 | 3 | This module defines the core types used throughout the MeatPy library 4 | for representing market data with proper type safety. 5 | """ 6 | 7 | from decimal import Decimal 8 | from typing import TYPE_CHECKING, TypeVar 9 | 10 | if TYPE_CHECKING: 11 | pass 12 | 13 | # Generic type variables for market data 14 | # These ensure type consistency within a single MarketProcessor instance 15 | Price = TypeVar("Price", int, Decimal) 16 | Volume = TypeVar("Volume", int, Decimal) 17 | OrderID = TypeVar("OrderID", int, str, bytes) 18 | TradeRef = TypeVar("TradeRef", int, str, bytes) 19 | Qualifiers = TypeVar("Qualifiers", dict[str, str], dict[str, int]) 20 | 21 | 22 | # Note: Event classes are not defined here to avoid complicating the generic type system. 23 | # Instead, use the existing event handler system in market_event_handler.py for type-safe event handling. 24 | # The MarketProcessor class already provides type-safe event methods with proper generic typing. 25 | -------------------------------------------------------------------------------- /src/meatpy/writers/__init__.py: -------------------------------------------------------------------------------- 1 | """Data writers for exporting market data in various formats. 2 | 3 | This module provides a common interface for writing market data to different 4 | file formats including CSV and Parquet. 5 | """ 6 | 7 | from .base_writer import DataWriter 8 | from .csv_writer import CSVWriter 9 | 10 | __all__ = ["DataWriter", "CSVWriter"] 11 | 12 | try: 13 | from .parquet_writer import ParquetWriter 14 | 15 | __all__.append("ParquetWriter") 16 | except ImportError: 17 | # ParquetWriter requires pyarrow which is an optional dependency 18 | ParquetWriter = None 19 | -------------------------------------------------------------------------------- /src/meatpy/writers/base_writer.py: -------------------------------------------------------------------------------- 1 | """Base writer interface for exporting market data. 2 | 3 | This module provides the abstract base class for all data writers, defining 4 | a common interface for writing market data to various file formats. 5 | """ 6 | 7 | from abc import ABC, abstractmethod 8 | from pathlib import Path 9 | from typing import Any, Dict, List, Optional, Union 10 | 11 | 12 | class SchemaLockedException(Exception): 13 | """Exception raised when attempting to modify a locked schema. 14 | 15 | This exception is raised when a DataWriter's schema has been locked 16 | (typically after writing has begun) and an attempt is made to change it. 17 | """ 18 | 19 | pass 20 | 21 | 22 | class DataWriter(ABC): 23 | """Abstract base class for data writers. 24 | 25 | This class defines the common interface that all data writers must implement 26 | for writing market data to various file formats. 27 | 28 | Attributes: 29 | output_path: Path to the output file 30 | buffer_size: Number of records to buffer before writing 31 | compression: Compression type to use (format-specific) 32 | """ 33 | 34 | def __init__( 35 | self, 36 | output_path: Union[Path, str], 37 | buffer_size: int = 1000, 38 | compression: Optional[str] = None, 39 | ): 40 | """Initialize the data writer. 41 | 42 | Args: 43 | output_path: Path to the output file 44 | buffer_size: Number of records to buffer before writing 45 | compression: Compression type to use (format-specific) 46 | """ 47 | self.output_path = Path(output_path) 48 | self.buffer_size = buffer_size 49 | self.compression = compression 50 | self._buffer: List[Any] = [] 51 | self._schema: Optional[Dict[str, Any]] = None 52 | self._header_written = False 53 | self._schema_locked = False 54 | 55 | @abstractmethod 56 | def write_header(self, schema: Dict[str, Any]) -> None: 57 | """Write the header/schema information to the file. 58 | 59 | Args: 60 | schema: Schema definition for the data 61 | """ 62 | pass 63 | 64 | @abstractmethod 65 | def write_records(self, records: List[Any]) -> None: 66 | """Write a batch of records to the file. 67 | 68 | Args: 69 | records: List of records to write 70 | """ 71 | pass 72 | 73 | @abstractmethod 74 | def append_records(self, records: List[Any]) -> None: 75 | """Append records to an existing file. 76 | 77 | Args: 78 | records: List of records to append 79 | """ 80 | pass 81 | 82 | def buffer_record(self, record: Any) -> None: 83 | """Add a record to the buffer and write if buffer is full. 84 | 85 | Args: 86 | record: Record to buffer 87 | """ 88 | self._buffer.append(record) 89 | if len(self._buffer) >= self.buffer_size: 90 | self.flush() 91 | 92 | def flush(self) -> None: 93 | """Flush all buffered records to the file.""" 94 | if self._buffer: 95 | if self._header_written: 96 | self.append_records(self._buffer) 97 | else: 98 | self.write_records(self._buffer) 99 | self._buffer.clear() 100 | 101 | def close(self) -> None: 102 | """Close the writer and flush any remaining records.""" 103 | self.flush() 104 | 105 | def __enter__(self): 106 | """Context manager entry.""" 107 | return self 108 | 109 | def __exit__(self, exc_type, exc_val, exc_tb): 110 | """Context manager exit.""" 111 | self.close() 112 | 113 | def set_schema(self, schema: Dict[str, Any]) -> None: 114 | """Set the schema for the writer. 115 | 116 | Args: 117 | schema: Schema definition for the data 118 | 119 | Raises: 120 | SchemaLockedException: If the schema is locked and cannot be changed 121 | """ 122 | if self._schema_locked: 123 | raise SchemaLockedException( 124 | "Cannot modify schema after it has been locked. " 125 | "Schema is typically locked after writing begins." 126 | ) 127 | 128 | self._schema = schema 129 | if not self._header_written: 130 | self.write_header(schema) 131 | self._header_written = True 132 | # Lock schema after first write 133 | self._schema_locked = True 134 | 135 | def lock_schema(self) -> None: 136 | """Lock the schema to prevent further modifications.""" 137 | self._schema_locked = True 138 | 139 | def is_schema_locked(self) -> bool: 140 | """Check if the schema is locked. 141 | 142 | Returns: 143 | True if schema is locked, False otherwise 144 | """ 145 | return self._schema_locked 146 | -------------------------------------------------------------------------------- /src/meatpy/writers/csv_writer.py: -------------------------------------------------------------------------------- 1 | """CSV writer for exporting market data in CSV format. 2 | 3 | This module provides the CSVWriter class for writing market data to 4 | CSV files with support for custom delimiters, headers, and compression. 5 | """ 6 | 7 | import csv 8 | import gzip 9 | from pathlib import Path 10 | from typing import Any, Dict, List, Optional, Union, TextIO 11 | 12 | from .base_writer import DataWriter 13 | 14 | 15 | class CSVWriter(DataWriter): 16 | """Writer for CSV format files. 17 | 18 | This writer converts market data records to CSV format with support 19 | for custom delimiters, headers, and compression. 20 | 21 | Attributes: 22 | output_path: Path to the output CSV file 23 | buffer_size: Number of records to buffer before writing 24 | compression: Whether to use gzip compression 25 | delimiter: Field delimiter character 26 | quotechar: Character used to quote fields 27 | lineterminator: Line terminator string 28 | """ 29 | 30 | def __init__( 31 | self, 32 | output_path: Union[Path, str], 33 | buffer_size: int = 1000, 34 | compression: Optional[str] = None, 35 | delimiter: str = ",", 36 | quotechar: str = '"', 37 | lineterminator: str = "\n", 38 | ): 39 | """Initialize the CSVWriter. 40 | 41 | Args: 42 | output_path: Path to the output CSV file 43 | buffer_size: Number of records to buffer before writing 44 | compression: Compression type ('gzip' or None) 45 | delimiter: Field delimiter character 46 | quotechar: Character used to quote fields 47 | lineterminator: Line terminator string 48 | """ 49 | super().__init__(output_path, buffer_size, compression) 50 | self.delimiter = delimiter 51 | self.quotechar = quotechar 52 | self.lineterminator = lineterminator 53 | self._fieldnames: Optional[List[str]] = None 54 | 55 | def _get_file_handle(self, mode: str = "w") -> TextIO: 56 | """Get a file handle with appropriate compression if needed. 57 | 58 | Args: 59 | mode: File open mode 60 | 61 | Returns: 62 | File-like object for writing 63 | """ 64 | if self.compression == "gzip": 65 | return gzip.open(self.output_path, mode + "t", encoding="utf-8") 66 | else: 67 | return open(self.output_path, mode, encoding="utf-8") 68 | 69 | def _extract_fieldnames(self, records: List[Any]) -> List[str]: 70 | """Extract field names from sample records. 71 | 72 | Args: 73 | records: Sample records to extract field names from 74 | 75 | Returns: 76 | List of field names 77 | """ 78 | if not records: 79 | return [] 80 | 81 | sample_record = records[0] 82 | 83 | if isinstance(sample_record, dict): 84 | return list(sample_record.keys()) 85 | elif isinstance(sample_record, (list, tuple)): 86 | return [f"column_{i}" for i in range(len(sample_record))] 87 | else: 88 | return ["value"] 89 | 90 | def _record_to_dict(self, record: Any) -> Dict[str, Any]: 91 | """Convert a record to dictionary format. 92 | 93 | Args: 94 | record: Record to convert 95 | 96 | Returns: 97 | Dictionary representation of the record 98 | """ 99 | if isinstance(record, dict): 100 | return record 101 | elif isinstance(record, (list, tuple)): 102 | if self._fieldnames is None: 103 | raise ValueError("Field names not set for sequence records") 104 | return dict(zip(self._fieldnames, record)) 105 | else: 106 | if self._fieldnames is None: 107 | return {"value": record} 108 | else: 109 | return {self._fieldnames[0]: record} 110 | 111 | def write_header(self, schema: Dict[str, Any]) -> None: 112 | """Write the header/schema information to the file. 113 | 114 | Args: 115 | schema: Schema definition for the data 116 | """ 117 | if "fieldnames" in schema: 118 | self._fieldnames = schema["fieldnames"] 119 | elif "fields" in schema: 120 | self._fieldnames = list(schema["fields"].keys()) 121 | else: 122 | raise ValueError("Schema must contain 'fieldnames' or 'fields'") 123 | 124 | self._schema = schema 125 | 126 | with self._get_file_handle("w") as file: 127 | writer = csv.DictWriter( 128 | file, 129 | fieldnames=self._fieldnames, 130 | delimiter=self.delimiter, 131 | quotechar=self.quotechar, 132 | lineterminator=self.lineterminator, 133 | ) 134 | writer.writeheader() 135 | 136 | def write_records(self, records: List[Any]) -> None: 137 | """Write a batch of records to the file. 138 | 139 | Args: 140 | records: List of records to write 141 | """ 142 | if not records: 143 | return 144 | 145 | if self._fieldnames is None: 146 | self._fieldnames = self._extract_fieldnames(records) 147 | 148 | with self._get_file_handle("w") as file: 149 | writer = csv.DictWriter( 150 | file, 151 | fieldnames=self._fieldnames, 152 | delimiter=self.delimiter, 153 | quotechar=self.quotechar, 154 | lineterminator=self.lineterminator, 155 | ) 156 | 157 | if not self._header_written: 158 | writer.writeheader() 159 | self._header_written = True 160 | 161 | for record in records: 162 | dict_record = self._record_to_dict(record) 163 | writer.writerow(dict_record) 164 | 165 | def append_records(self, records: List[Any]) -> None: 166 | """Append records to an existing file. 167 | 168 | Args: 169 | records: List of records to append 170 | """ 171 | if not records: 172 | return 173 | 174 | if self._fieldnames is None: 175 | raise RuntimeError("Field names not set. Call write_records first.") 176 | 177 | with self._get_file_handle("a") as file: 178 | writer = csv.DictWriter( 179 | file, 180 | fieldnames=self._fieldnames, 181 | delimiter=self.delimiter, 182 | quotechar=self.quotechar, 183 | lineterminator=self.lineterminator, 184 | ) 185 | 186 | for record in records: 187 | dict_record = self._record_to_dict(record) 188 | writer.writerow(dict_record) 189 | 190 | def write_csv_header_string(self) -> str: 191 | """Get the CSV header as a string. 192 | 193 | Returns: 194 | CSV header string 195 | """ 196 | if self._fieldnames is None: 197 | raise RuntimeError("Field names not set") 198 | 199 | return self.delimiter.join(self._fieldnames) + self.lineterminator 200 | 201 | def record_to_csv_string(self, record: Any) -> str: 202 | """Convert a record to CSV string format. 203 | 204 | Args: 205 | record: Record to convert 206 | 207 | Returns: 208 | CSV string representation of the record 209 | """ 210 | dict_record = self._record_to_dict(record) 211 | 212 | if self._fieldnames is None: 213 | raise RuntimeError("Field names not set") 214 | 215 | values = [] 216 | for fieldname in self._fieldnames: 217 | value = dict_record.get(fieldname, "") 218 | # Simple CSV escaping 219 | if isinstance(value, str) and ( 220 | self.delimiter in value or self.quotechar in value or "\n" in value 221 | ): 222 | value = ( 223 | self.quotechar 224 | + value.replace(self.quotechar, self.quotechar + self.quotechar) 225 | + self.quotechar 226 | ) 227 | values.append(str(value)) 228 | 229 | return self.delimiter.join(values) + self.lineterminator 230 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Test package for MeatPy.""" 2 | -------------------------------------------------------------------------------- /tests/test_itch41_reader.py: -------------------------------------------------------------------------------- 1 | """Tests for ITCH 4.1 message reader functionality.""" 2 | 3 | import gzip 4 | import tempfile 5 | import pytest 6 | from pathlib import Path 7 | 8 | from meatpy.itch41.itch41_message_reader import ITCH41MessageReader 9 | from meatpy.itch41.itch41_market_message import SystemEventMessage, AddOrderMessage 10 | 11 | 12 | class TestITCH41MessageReader: 13 | """Test the ITCH41MessageReader class.""" 14 | 15 | def create_test_data(self) -> bytes: 16 | """Create test ITCH data with a few messages.""" 17 | messages_data = b"" 18 | 19 | # Create a system event message 20 | system_msg = SystemEventMessage() 21 | system_msg.timestamp = 12345 # ITCH 4.1 timestamps are seconds 22 | system_msg.event_code = b"O" 23 | system_bytes = system_msg.to_bytes() 24 | 25 | # Add ITCH frame (length byte + message) 26 | messages_data += b"\x00" # Start byte 27 | messages_data += bytes([len(system_bytes)]) # Length 28 | messages_data += system_bytes 29 | 30 | # Create an add order message 31 | order_msg = AddOrderMessage() 32 | order_msg.timestamp = 12346 # ITCH 4.1 timestamps are seconds 33 | order_msg.order_ref = 999 34 | order_msg.side = b"B" 35 | order_msg.shares = 100 36 | order_msg.stock = b"AAPL " 37 | order_msg.price = 15000 38 | order_bytes = order_msg.to_bytes() 39 | 40 | # Add ITCH frame 41 | messages_data += b"\x00" # Start byte 42 | messages_data += bytes([len(order_bytes)]) # Length 43 | messages_data += order_bytes 44 | 45 | return messages_data 46 | 47 | def test_read_file_uncompressed(self): 48 | """Test reading uncompressed ITCH file.""" 49 | test_data = self.create_test_data() 50 | 51 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 52 | temp_file.write(test_data) 53 | temp_file.flush() 54 | 55 | reader = ITCH41MessageReader() 56 | messages = list(reader.read_file(temp_file.name)) 57 | 58 | assert len(messages) == 2 59 | assert isinstance(messages[0], SystemEventMessage) 60 | assert isinstance(messages[1], AddOrderMessage) 61 | 62 | # Check first message details 63 | assert messages[0].event_code == b"O" 64 | assert messages[0].timestamp == 12345 65 | 66 | # Check second message details 67 | assert messages[1].side == b"B" 68 | assert messages[1].shares == 100 69 | assert messages[1].price == 15000 70 | assert messages[1].timestamp == 12346 71 | 72 | def test_read_file_gzip_compressed(self): 73 | """Test reading gzip compressed ITCH file.""" 74 | test_data = self.create_test_data() 75 | 76 | with tempfile.NamedTemporaryFile(suffix=".gz", delete=False) as temp_file: 77 | with gzip.open(temp_file.name, "wb") as gz_file: 78 | gz_file.write(test_data) 79 | 80 | reader = ITCH41MessageReader() 81 | messages = list(reader.read_file(temp_file.name)) 82 | 83 | assert len(messages) == 2 84 | assert isinstance(messages[0], SystemEventMessage) 85 | assert isinstance(messages[1], AddOrderMessage) 86 | 87 | def test_context_manager_usage(self): 88 | """Test using the reader as a context manager.""" 89 | test_data = self.create_test_data() 90 | 91 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 92 | temp_file.write(test_data) 93 | temp_file.flush() 94 | 95 | messages = [] 96 | with ITCH41MessageReader(temp_file.name) as reader: 97 | for message in reader: 98 | messages.append(message) 99 | 100 | assert len(messages) == 2 101 | assert isinstance(messages[0], SystemEventMessage) 102 | assert isinstance(messages[1], AddOrderMessage) 103 | 104 | def test_context_manager_without_file_path(self): 105 | """Test context manager error when no file path provided.""" 106 | reader = ITCH41MessageReader() 107 | 108 | with pytest.raises(ValueError, match="No file_path provided"): 109 | with reader: 110 | pass 111 | 112 | def test_iterator_without_context_manager(self): 113 | """Test iterator error when not used as context manager.""" 114 | reader = ITCH41MessageReader() 115 | 116 | with pytest.raises( 117 | RuntimeError, match="Reader must be used as a context manager" 118 | ): 119 | next(iter(reader)) 120 | 121 | def test_empty_file(self): 122 | """Test reading an empty file.""" 123 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 124 | # File is empty 125 | pass 126 | 127 | reader = ITCH41MessageReader() 128 | messages = list(reader.read_file(temp_file.name)) 129 | 130 | assert len(messages) == 0 131 | 132 | def test_invalid_message_format(self): 133 | """Test handling of invalid message format.""" 134 | # Create data with invalid start byte 135 | invalid_data = b"\x01\x10" + b"A" * 16 # Start byte should be 0x00 136 | 137 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 138 | temp_file.write(invalid_data) 139 | temp_file.flush() 140 | 141 | reader = ITCH41MessageReader() 142 | 143 | with pytest.raises(Exception): # Should raise InvalidMessageFormatError 144 | list(reader.read_file(temp_file.name)) 145 | 146 | def test_large_buffer_handling(self): 147 | """Test reading with large amounts of data to test buffer management.""" 148 | # Create many messages to test buffer refilling 149 | messages_data = b"" 150 | 151 | for i in range(100): # Create 100 messages 152 | msg = SystemEventMessage() 153 | msg.timestamp = 12345 + i # ITCH 4.1 timestamps are seconds 154 | msg.event_code = b"O" 155 | msg_bytes = msg.to_bytes() 156 | 157 | messages_data += b"\x00" 158 | messages_data += bytes([len(msg_bytes)]) 159 | messages_data += msg_bytes 160 | 161 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 162 | temp_file.write(messages_data) 163 | temp_file.flush() 164 | 165 | reader = ITCH41MessageReader() 166 | messages = list(reader.read_file(temp_file.name)) 167 | 168 | assert len(messages) == 100 169 | 170 | # Check that all messages are correct 171 | for i, message in enumerate(messages): 172 | assert isinstance(message, SystemEventMessage) 173 | assert message.timestamp == 12345 + i 174 | assert message.event_code == b"O" 175 | 176 | def test_incomplete_message_at_end(self): 177 | """Test handling of incomplete message at end of file.""" 178 | # Create one complete message followed by incomplete data 179 | msg = SystemEventMessage() 180 | msg.timestamp = 12345 # ITCH 4.1 timestamps are seconds 181 | msg.event_code = b"O" 182 | msg_bytes = msg.to_bytes() 183 | 184 | complete_data = b"\x00" + bytes([len(msg_bytes)]) + msg_bytes 185 | incomplete_data = b"\x00\x10" + b"A" * 5 # Claims 16 bytes but only has 5 186 | 187 | test_data = complete_data + incomplete_data 188 | 189 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 190 | temp_file.write(test_data) 191 | temp_file.flush() 192 | 193 | reader = ITCH41MessageReader() 194 | messages = list(reader.read_file(temp_file.name)) 195 | 196 | # Should only get the complete message 197 | assert len(messages) == 1 198 | assert isinstance(messages[0], SystemEventMessage) 199 | 200 | def test_pathlib_path_support(self): 201 | """Test that Path objects are supported.""" 202 | test_data = self.create_test_data() 203 | 204 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: 205 | temp_file.write(test_data) 206 | temp_file.flush() 207 | 208 | # Use Path object instead of string 209 | file_path = Path(temp_file.name) 210 | 211 | reader = ITCH41MessageReader() 212 | messages = list(reader.read_file(file_path)) 213 | 214 | assert len(messages) == 2 215 | assert isinstance(messages[0], SystemEventMessage) 216 | assert isinstance(messages[1], AddOrderMessage) 217 | -------------------------------------------------------------------------------- /tests/test_itch50_reader_writer.py: -------------------------------------------------------------------------------- 1 | """Tests for ITCH50MessageReader and ITCH50Writer classes.""" 2 | 3 | import struct 4 | import tempfile 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | from meatpy.itch50 import ITCH50MessageReader, ITCH50Writer, SystemEventMessage 10 | 11 | 12 | def test_itch50_parser_initialization(): 13 | """Test ITCH50MessageReader initialization.""" 14 | parser = ITCH50MessageReader() 15 | assert parser.file_path is None 16 | 17 | parser = ITCH50MessageReader("test_file.itch") 18 | assert parser.file_path == Path("test_file.itch") 19 | 20 | 21 | def test_itch50_writer_initialization(): 22 | """Test ITCH50Writer initialization.""" 23 | with tempfile.NamedTemporaryFile() as tmp: 24 | writer = ITCH50Writer(output_path=tmp.name) 25 | assert writer._symbols is None 26 | assert writer.output_path == Path(tmp.name) 27 | assert writer.message_buffer == 2000 28 | assert writer.compress is False 29 | assert writer.compression_type == "gzip" 30 | 31 | writer = ITCH50Writer( 32 | symbols=[b"AAPL "], 33 | output_path=tmp.name, 34 | message_buffer=1000, 35 | compress=True, 36 | compression_type="bzip2", 37 | ) 38 | assert writer._symbols == [b"AAPL "] 39 | assert writer.message_buffer == 1000 40 | assert writer.compress is True 41 | assert writer.compression_type == "bzip2" 42 | 43 | 44 | def test_itch50_writer_context_manager(): 45 | """Test ITCH50Writer as context manager.""" 46 | with tempfile.NamedTemporaryFile() as tmp: 47 | with ITCH50Writer(output_path=tmp.name) as writer: 48 | assert writer.output_path == Path(tmp.name) 49 | # Should not raise any exceptions 50 | 51 | 52 | def test_itch50_writer_process_message(): 53 | """Test ITCH50Writer message processing.""" 54 | with tempfile.NamedTemporaryFile() as tmp: 55 | writer = ITCH50Writer(output_path=tmp.name) 56 | 57 | # Create a proper system event message 58 | # Format: type(1) + stock_locate(2) + tracking_number(2) + ts1(4) + ts2(4) + code(1) = 12 bytes 59 | payload = struct.pack("!HHHIc", 1, 2, 0, 0, b"C") 60 | message_data = b"S" + payload 61 | message = SystemEventMessage.from_bytes(message_data) 62 | 63 | # Process the message 64 | writer.process_message(message) 65 | 66 | # Flush and close 67 | writer.flush() 68 | writer.close() 69 | 70 | 71 | def test_itch50_parser_compression_detection(): 72 | """Test ITCH50MessageReader compression detection.""" 73 | parser = ITCH50MessageReader() 74 | 75 | # Test with a non-existent file (should not crash) 76 | with pytest.raises(FileNotFoundError): 77 | list(parser.read_file("nonexistent_file.bin")) 78 | 79 | 80 | def test_itch50_parser_context_manager(): 81 | """Test ITCH50MessageReader as context manager.""" 82 | parser = ITCH50MessageReader("test_file.itch") 83 | 84 | # Test that context manager methods exist 85 | assert hasattr(parser, "__enter__") 86 | assert hasattr(parser, "__exit__") 87 | 88 | # Test context manager usage (should raise FileNotFoundError for non-existent file) 89 | with pytest.raises(FileNotFoundError): 90 | with ITCH50MessageReader("nonexistent_file.itch") as parser: 91 | pass 92 | 93 | 94 | def test_itch50_parser_natural_interface(): 95 | """Test ITCH50MessageReader natural interface with file_path in constructor.""" 96 | # Test initialization with file path 97 | parser = ITCH50MessageReader("test_file.itch") 98 | assert parser.file_path == Path("test_file.itch") 99 | 100 | # Test initialization with file path (no filters in new interface) 101 | parser = ITCH50MessageReader("test_file.itch") 102 | assert parser.file_path == Path("test_file.itch") 103 | 104 | 105 | def test_itch50_parser_natural_interface_context_manager(): 106 | """Test ITCH50MessageReader natural interface as context manager.""" 107 | # Test that context manager methods exist 108 | parser = ITCH50MessageReader("test_file.itch") 109 | assert hasattr(parser, "__enter__") 110 | assert hasattr(parser, "__exit__") 111 | assert hasattr(parser, "__iter__") 112 | 113 | # Test that it raises error when no file_path is provided 114 | parser_no_path = ITCH50MessageReader() 115 | with pytest.raises(ValueError, match="No file_path provided"): 116 | with parser_no_path: 117 | pass 118 | 119 | 120 | def test_itch50_parser_legacy_interface(): 121 | """Test ITCH50MessageReader legacy interface still works.""" 122 | # Test initialization without file path 123 | parser = ITCH50MessageReader() 124 | assert parser.file_path is None 125 | 126 | # Test that read_file method exists 127 | assert hasattr(parser, "read_file") 128 | -------------------------------------------------------------------------------- /tests/test_timestamp.py: -------------------------------------------------------------------------------- 1 | """Tests for the timestamp module.""" 2 | 3 | from datetime import datetime, timezone 4 | 5 | from meatpy.timestamp import Timestamp 6 | 7 | 8 | class TestTimestampCreation: 9 | """Test timestamp creation and initialization.""" 10 | 11 | def test_create_from_datetime(self): 12 | """Test creating timestamp from datetime object.""" 13 | dt = datetime(2024, 1, 1, 9, 30, 0) 14 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 15 | assert ts.year == dt.year 16 | assert ts.month == dt.month 17 | assert ts.day == dt.day 18 | assert ts.hour == dt.hour 19 | assert ts.minute == dt.minute 20 | 21 | def test_create_from_datetime_object(self): 22 | """Test creating timestamp from datetime object using from_datetime.""" 23 | dt = datetime(2024, 1, 1, 9, 30, 0) 24 | ts = Timestamp.from_datetime(dt) 25 | assert ts.year == dt.year 26 | assert ts.month == dt.month 27 | assert ts.day == dt.day 28 | assert ts.hour == dt.hour 29 | assert ts.minute == dt.minute 30 | 31 | def test_create_with_microseconds(self): 32 | """Test creating timestamp with microseconds.""" 33 | ts = Timestamp(2024, 1, 1, 9, 30, 0, 123456) 34 | assert ts.microsecond == 123456 35 | 36 | def test_create_with_timezone(self): 37 | """Test creating timestamp with timezone.""" 38 | dt = datetime(2024, 1, 1, 9, 30, 0, tzinfo=timezone.utc) 39 | ts = Timestamp.from_datetime(dt) 40 | assert ts.year == dt.year 41 | assert ts.month == dt.month 42 | assert ts.day == dt.day 43 | assert ts.hour == dt.hour 44 | assert ts.minute == dt.minute 45 | # Note: from_datetime doesn't preserve timezone info 46 | assert ts.tzinfo is None 47 | 48 | 49 | class TestTimestampProperties: 50 | """Test timestamp properties and attributes.""" 51 | 52 | def test_datetime_inheritance(self): 53 | """Test that Timestamp inherits from datetime.""" 54 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 55 | assert isinstance(ts, datetime) 56 | assert isinstance(ts, Timestamp) 57 | 58 | def test_year_property(self): 59 | """Test year property.""" 60 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 61 | assert ts.year == 2024 62 | 63 | def test_month_property(self): 64 | """Test month property.""" 65 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 66 | assert ts.month == 1 67 | 68 | def test_day_property(self): 69 | """Test day property.""" 70 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 71 | assert ts.day == 1 72 | 73 | def test_hour_property(self): 74 | """Test hour property.""" 75 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 76 | assert ts.hour == 9 77 | 78 | def test_minute_property(self): 79 | """Test minute property.""" 80 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 81 | assert ts.minute == 30 82 | 83 | def test_second_property(self): 84 | """Test second property.""" 85 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 86 | assert ts.second == 0 87 | 88 | def test_microsecond_property(self): 89 | """Test microsecond property.""" 90 | ts = Timestamp(2024, 1, 1, 9, 30, 0, 123456) 91 | assert ts.microsecond == 123456 92 | 93 | 94 | class TestTimestampComparison: 95 | """Test timestamp comparison operations.""" 96 | 97 | def test_equality(self): 98 | """Test timestamp equality.""" 99 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0) 100 | ts2 = Timestamp(2024, 1, 1, 9, 30, 0) 101 | assert ts1 == ts2 102 | 103 | def test_inequality(self): 104 | """Test timestamp inequality.""" 105 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0) 106 | ts2 = Timestamp(2024, 1, 1, 9, 31, 0) 107 | assert ts1 != ts2 108 | 109 | def test_less_than(self): 110 | """Test timestamp less than comparison.""" 111 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0) 112 | ts2 = Timestamp(2024, 1, 1, 9, 31, 0) 113 | assert ts1 < ts2 114 | 115 | def test_greater_than(self): 116 | """Test timestamp greater than comparison.""" 117 | ts1 = Timestamp(2024, 1, 1, 9, 31, 0) 118 | ts2 = Timestamp(2024, 1, 1, 9, 30, 0) 119 | assert ts1 > ts2 120 | 121 | def test_less_equal(self): 122 | """Test timestamp less than or equal comparison.""" 123 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0) 124 | ts2 = Timestamp(2024, 1, 1, 9, 30, 0) 125 | ts3 = Timestamp(2024, 1, 1, 9, 31, 0) 126 | assert ts1 <= ts2 127 | assert ts1 <= ts3 128 | 129 | def test_greater_equal(self): 130 | """Test timestamp greater than or equal comparison.""" 131 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0) 132 | ts2 = Timestamp(2024, 1, 1, 9, 30, 0) 133 | ts3 = Timestamp(2024, 1, 1, 9, 29, 0) 134 | assert ts1 >= ts2 135 | assert ts1 >= ts3 136 | 137 | 138 | class TestTimestampArithmetic: 139 | """Test timestamp arithmetic operations.""" 140 | 141 | def test_addition_timedelta(self): 142 | """Test adding timedelta to timestamp.""" 143 | from datetime import timedelta 144 | 145 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 146 | delta = timedelta(minutes=30) 147 | result = ts + delta 148 | expected = Timestamp(2024, 1, 1, 10, 0, 0) 149 | assert result == expected 150 | 151 | def test_subtraction_timedelta(self): 152 | """Test subtracting timedelta from timestamp.""" 153 | from datetime import timedelta 154 | 155 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 156 | delta = timedelta(minutes=30) 157 | result = ts - delta 158 | expected = Timestamp(2024, 1, 1, 9, 0, 0) 159 | assert result == expected 160 | 161 | def test_subtraction_timestamp(self): 162 | """Test subtracting timestamp from timestamp.""" 163 | ts1 = Timestamp(2024, 1, 1, 10, 0, 0) 164 | ts2 = Timestamp(2024, 1, 1, 9, 30, 0) 165 | result = ts1 - ts2 166 | from datetime import timedelta 167 | 168 | expected = timedelta(minutes=30) 169 | assert result == expected 170 | 171 | 172 | class TestTimestampStringRepresentation: 173 | """Test timestamp string representation.""" 174 | 175 | def test_str_representation(self): 176 | """Test string representation.""" 177 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 178 | assert str(ts) == "2024-01-01 09:30:00.000000" 179 | 180 | def test_repr_representation(self): 181 | """Test repr representation.""" 182 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 183 | assert "Timestamp: " in repr(ts) 184 | assert "2024-01-01 09:30:00.000000" in repr(ts) 185 | 186 | def test_iso_format(self): 187 | """Test ISO format string.""" 188 | ts = Timestamp(2024, 1, 1, 9, 30, 0) 189 | assert ts.isoformat() == "2024-01-01T09:30:00" 190 | 191 | 192 | class TestTimestampCopy: 193 | """Test timestamp copying functionality.""" 194 | 195 | def test_copy_method(self): 196 | """Test copy method.""" 197 | ts1 = Timestamp(2024, 1, 1, 9, 30, 0, 123456) 198 | ts2 = ts1.copy() 199 | assert ts1 == ts2 200 | assert ts1 is not ts2 201 | assert ts2.microsecond == 123456 202 | 203 | def test_copy_preserves_timezone(self): 204 | """Test copy preserves timezone.""" 205 | dt = datetime(2024, 1, 1, 9, 30, 0, tzinfo=timezone.utc) 206 | ts1 = Timestamp.from_datetime(dt) 207 | ts2 = ts1.copy() 208 | # Note: from_datetime doesn't preserve timezone, so both should be None 209 | assert ts1.tzinfo is None 210 | assert ts2.tzinfo is None 211 | assert ts1 == ts2 212 | 213 | 214 | class TestTimestampEdgeCases: 215 | """Test timestamp edge cases.""" 216 | 217 | def test_leap_year(self): 218 | """Test leap year handling.""" 219 | ts = Timestamp(2024, 2, 29, 12, 0, 0) 220 | assert ts.year == 2024 221 | assert ts.month == 2 222 | assert ts.day == 29 223 | 224 | def test_midnight(self): 225 | """Test midnight time.""" 226 | ts = Timestamp(2024, 1, 1, 0, 0, 0) 227 | assert ts.hour == 0 228 | assert ts.minute == 0 229 | assert ts.second == 0 230 | 231 | def test_end_of_day(self): 232 | """Test end of day time.""" 233 | ts = Timestamp(2024, 1, 1, 23, 59, 59, 999999) 234 | assert ts.hour == 23 235 | assert ts.minute == 59 236 | assert ts.second == 59 237 | assert ts.microsecond == 999999 238 | 239 | def test_microseconds(self): 240 | """Test microsecond precision.""" 241 | ts = Timestamp(2024, 1, 1, 9, 30, 0, 123456) 242 | assert ts.microsecond == 123456 243 | assert str(ts) == "2024-01-01 09:30:00.123456" 244 | -------------------------------------------------------------------------------- /tests/test_types.py: -------------------------------------------------------------------------------- 1 | """Tests for the types module.""" 2 | 3 | from decimal import Decimal 4 | 5 | from meatpy.types import OrderID, Price, Qualifiers, TradeRef, Volume 6 | 7 | 8 | class TestTypeVariables: 9 | """Test that type variables are correctly defined.""" 10 | 11 | def test_price_type_variable(self): 12 | """Test that Price TypeVar has correct constraints.""" 13 | constraints = Price.__constraints__ 14 | assert int in constraints 15 | assert Decimal in constraints 16 | assert len(constraints) == 2 17 | 18 | def test_volume_type_variable(self): 19 | """Test that Volume TypeVar has correct constraints.""" 20 | constraints = Volume.__constraints__ 21 | assert int in constraints 22 | assert Decimal in constraints 23 | assert len(constraints) == 2 24 | 25 | def test_order_id_type_variable(self): 26 | """Test that OrderID TypeVar has correct constraints.""" 27 | constraints = OrderID.__constraints__ 28 | assert int in constraints 29 | assert str in constraints 30 | assert bytes in constraints 31 | assert len(constraints) == 3 32 | 33 | def test_trade_ref_type_variable(self): 34 | """Test that TradeRef TypeVar has correct constraints.""" 35 | constraints = TradeRef.__constraints__ 36 | assert int in constraints 37 | assert str in constraints 38 | assert bytes in constraints 39 | assert len(constraints) == 3 40 | 41 | def test_qualifiers_type_variable(self): 42 | """Test that Qualifiers TypeVar has correct constraints.""" 43 | constraints = Qualifiers.__constraints__ 44 | assert dict[str, str] in constraints 45 | assert dict[str, int] in constraints 46 | assert len(constraints) == 2 47 | 48 | 49 | class TestTypeCompatibility: 50 | """Test that types are compatible with expected values.""" 51 | 52 | def test_price_int_compatibility(self): 53 | """Test that int values are compatible with Price.""" 54 | price: Price = 10000 # Should not raise type error 55 | assert isinstance(price, int) 56 | assert price == 10000 57 | 58 | def test_price_decimal_compatibility(self): 59 | """Test that Decimal values are compatible with Price.""" 60 | price: Price = Decimal("100.00") # Should not raise type error 61 | assert isinstance(price, Decimal) 62 | assert price == Decimal("100.00") 63 | 64 | def test_volume_int_compatibility(self): 65 | """Test that int values are compatible with Volume.""" 66 | volume: Volume = 100 # Should not raise type error 67 | assert isinstance(volume, int) 68 | assert volume == 100 69 | 70 | def test_volume_decimal_compatibility(self): 71 | """Test that Decimal values are compatible with Volume.""" 72 | volume: Volume = Decimal("100.5") # Should not raise type error 73 | assert isinstance(volume, Decimal) 74 | assert volume == Decimal("100.5") 75 | 76 | def test_order_id_int_compatibility(self): 77 | """Test that int values are compatible with OrderID.""" 78 | order_id: OrderID = 12345 # Should not raise type error 79 | assert isinstance(order_id, int) 80 | assert order_id == 12345 81 | 82 | def test_order_id_str_compatibility(self): 83 | """Test that str values are compatible with OrderID.""" 84 | order_id: OrderID = "order-12345" # Should not raise type error 85 | assert isinstance(order_id, str) 86 | assert order_id == "order-12345" 87 | 88 | def test_order_id_bytes_compatibility(self): 89 | """Test that bytes values are compatible with OrderID.""" 90 | order_id: OrderID = b"order-12345" # Should not raise type error 91 | assert isinstance(order_id, bytes) 92 | assert order_id == b"order-12345" 93 | 94 | def test_trade_ref_int_compatibility(self): 95 | """Test that int values are compatible with TradeRef.""" 96 | trade_ref: TradeRef = 67890 # Should not raise type error 97 | assert isinstance(trade_ref, int) 98 | assert trade_ref == 67890 99 | 100 | def test_trade_ref_str_compatibility(self): 101 | """Test that str values are compatible with TradeRef.""" 102 | trade_ref: TradeRef = "trade-67890" # Should not raise type error 103 | assert isinstance(trade_ref, str) 104 | assert trade_ref == "trade-67890" 105 | 106 | def test_qualifiers_str_dict_compatibility(self): 107 | """Test that dict[str, str] values are compatible with Qualifiers.""" 108 | qualifiers: Qualifiers = { 109 | "exchange": "NASDAQ", 110 | "type": "limit", 111 | } # Should not raise type error 112 | assert isinstance(qualifiers, dict) 113 | assert all( 114 | isinstance(k, str) and isinstance(v, str) for k, v in qualifiers.items() 115 | ) 116 | 117 | def test_qualifiers_int_dict_compatibility(self): 118 | """Test that dict[str, int] values are compatible with Qualifiers.""" 119 | qualifiers: Qualifiers = { 120 | "level": 1, 121 | "priority": 2, 122 | } # Should not raise type error 123 | assert isinstance(qualifiers, dict) 124 | assert all( 125 | isinstance(k, str) and isinstance(v, int) for k, v in qualifiers.items() 126 | ) 127 | 128 | 129 | class TestTypeIncompatibility: 130 | """Test that incompatible types are properly rejected by type checkers.""" 131 | 132 | def test_price_float_incompatibility(self): 133 | """Test that float values are not compatible with Price.""" 134 | # This would cause a type error in a type checker 135 | # price: Price = 100.0 # Type error: float not in Price constraints 136 | pass 137 | 138 | def test_volume_str_incompatibility(self): 139 | """Test that str values are not compatible with Volume.""" 140 | # This would cause a type error in a type checker 141 | # volume: Volume = "100" # Type error: str not in Volume constraints 142 | pass 143 | 144 | def test_order_id_float_incompatibility(self): 145 | """Test that float values are not compatible with OrderID.""" 146 | # This would cause a type error in a type checker 147 | # order_id: OrderID = 123.45 # Type error: float not in OrderID constraints 148 | pass 149 | 150 | def test_qualifiers_mixed_dict_incompatibility(self): 151 | """Test that mixed dict values are not compatible with Qualifiers.""" 152 | # This would cause a type error in a type checker 153 | # qualifiers: Qualifiers = {"str": "value", "int": 123} # Type error: mixed types 154 | pass 155 | --------------------------------------------------------------------------------