├── .editorconfig ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE.txt ├── README.md ├── bin ├── check_version.sh └── target_driver.sh ├── neo4j └── _codec │ └── packstream │ └── .keep ├── pyproject.toml ├── requirements-dev.txt ├── src ├── lib.rs ├── v1.rs └── v1 │ ├── pack.rs │ └── unpack.rs ├── testkit ├── .dockerignore ├── Dockerfile ├── _common.py ├── backend.py ├── build.py ├── integration.py ├── stress.py ├── testkit.json └── unittests.py ├── tests ├── benchmarks │ └── test_benchmarks.py ├── requirements.txt └── v1 │ ├── from_driver │ └── test_packstream.py │ └── test_injection.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | charset = utf-8 9 | 10 | [*.sh] 11 | indent_style = space 12 | indent_size = 4 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | *~ 4 | *.py[cod] 5 | __pycache__/ 6 | .pytest_cache/ 7 | *.lprof 8 | *.class 9 | 10 | *.so 11 | 12 | .DS_Store 13 | 14 | .Python 15 | .venv/ 16 | .tox/ 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | include/ 28 | man/ 29 | venv/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | .benchmarks 34 | 35 | # PyCharm 36 | .idea/ 37 | 38 | # VSCode 39 | .vscode/ 40 | 41 | testkit/CAs 42 | testkit/CustomCAs 43 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "driver"] 2 | path = driver 3 | url = https://github.com/neo4j/neo4j-python-driver 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | exclude: ^(driver/.*|tests/(.*/)?from_driver/.*)$ 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v4.1.0 7 | hooks: 8 | - id: check-byte-order-marker 9 | - id: check-case-conflict 10 | - id: check-executables-have-shebangs 11 | - id: check-shebang-scripts-are-executable 12 | - id: check-merge-conflict 13 | - id: check-symlinks 14 | - id: destroyed-symlinks 15 | - id: end-of-file-fixer 16 | - id: mixed-line-ending 17 | args: [ --fix=lf ] 18 | exclude_types: 19 | - batch 20 | - id: trailing-whitespace 21 | args: [ --markdown-linebreak-ext=md ] 22 | - repo: https://github.com/doublify/pre-commit-rust 23 | rev: v1.0 24 | hooks: 25 | - id: fmt 26 | - id: cargo-check 27 | - id: clippy 28 | - repo: local 29 | hooks: 30 | - id: isort 31 | name: isort 32 | entry: isort 33 | types_or: [ python, pyi ] 34 | language: system 35 | - repo: https://github.com/astral-sh/ruff-pre-commit 36 | rev: v0.6.4 37 | hooks: 38 | - id: ruff-format 39 | - id: ruff 40 | args: [ --fix ] 41 | - id: ruff-format 42 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | ## Next Release 5 | * Moved repository: 6 | from https://github.com/neo4j-drivers/neo4j-python-driver-rust-ext 7 | to https://github.com/neo4j/neo4j-python-driver-rust-ext 8 | * Metadata: removed `Beta` tag, added `Production/Stable`. 9 | * Bump MSRV (minimum supported Rust version) to 1.67.0. 10 | * Clarify installation documentation: `neo4j` and `neo4j-rust-ext` can both be installed at the same time. 11 | ℹ️ Make sure to specify matching versions if you do so. 12 | 13 | 14 | ## 5.23.0.0 (2024-07-29) 15 | * Target driver version 5.23.0 16 | 17 | 18 | ## 5.22.0.0 (2024-06-27) 19 | * Target driver version 5.22.0 20 | 21 | 22 | ## 5.21.0.0 (2024-06-11) 23 | * Target driver version 5.21.0 24 | 25 | 26 | ## 5.20.0.0 (2024-04-26) 27 | * Target driver version 5.20.0 28 | 29 | 30 | ## 5.19.0.0 (2024-05-02) 31 | * Target driver version 5.19.0 32 | 33 | 34 | ## 5.18.0.0 (2024-02-29) 35 | * Target driver version 5.18.0 36 | 37 | 38 | ## 5.17.0.0b1 (2024-01-29) 39 | * Target driver version 5.17.0 40 | 41 | 42 | ## 5.16.0.0b1 (2023-12-28) 43 | * Target driver version 5.16.0 44 | 45 | 46 | ## 5.15.0.0b1 (2023-11-28) 47 | * Target driver version 5.15.0 48 | 49 | 50 | ## 5.14.1.0a1 (2023-11-03) 51 | * Target driver version 5.14.1 52 | * Initial release. 53 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to the Neo4j Ecosystem 2 | 3 | At [Neo4j](https://neo4j.com/), we develop our software in the open at GitHub. 4 | This provides transparency for you, our users, and allows you to fork the software to make your own additions and enhancements. 5 | We also provide areas specifically for community contributions, in particular the [neo4j-contrib](https://github.com/neo4j-contrib) space. 6 | 7 | There's an active [Neo4j Online Community](https://community.neo4j.com/) where we work directly with the community. 8 | If you're not already a member, sign up! 9 | 10 | We love our community and wouldn't be where we are without you. 11 | 12 | 13 | ## Need to raise an issue? 14 | 15 | Where you raise an issue depends largely on the nature of the problem. 16 | 17 | Firstly, if you are an Enterprise customer, you might want to head over to our [Customer Support Portal](https://support.neo4j.com/). 18 | 19 | There are plenty of public channels available too, though. 20 | If you simply want to get started or have a question on how to use a particular feature, ask a question in [Neo4j Online Community](https://community.neo4j.com/). 21 | If you think you might have hit a bug in our software (it happens occasionally!) or you have specific feature request then use the issue feature on the relevant GitHub repository. 22 | Check first though as someone else may have already raised something similar. 23 | 24 | [StackOverflow](https://stackoverflow.com/questions/tagged/neo4j) also hosts a ton of questions and might already have a discussion around your problem. 25 | Make sure you have a look there too. 26 | 27 | Include as much information as you can in any request you make: 28 | 29 | - Which versions of our products are you using? 30 | - Which language (and which version of that language) are you developing with? 31 | - What operating system are you on? 32 | - Are you working with a cluster or on a single machine? 33 | - What code are you running? 34 | - What errors are you seeing? 35 | - What solutions have you tried already? 36 | 37 | 38 | ## Want to contribute? 39 | 40 | If you want to contribute a pull request, we have a little bit of process you'll need to follow: 41 | 42 | - Do all your work in a personal fork of the original repository 43 | - [Rebase](https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request), don't merge (we prefer to keep our history clean) 44 | - Create a branch (with a useful name) for your contribution 45 | - Make sure you're familiar with the appropriate coding style (this varies by language so ask if you're in doubt) 46 | - Include unit tests if appropriate (obviously not necessary for documentation changes) 47 | - Take a moment to read and sign our [CLA](https://neo4j.com/developer/cla) 48 | 49 | We can't guarantee that we'll accept pull requests and may ask you to make some changes before they go in. 50 | Occasionally, we might also have logistical, commercial, or legal reasons why we can't accept your work but we'll try to find an alternative way for you to contribute in that case. 51 | Remember that many community members have become regular contributors and some are now even Neo employees! 52 | 53 | 54 | ## Specifically for this project: 55 | 56 | ### Setting up the Development Environment 57 | * Install Python 3.8+ 58 | * Install the requirements 59 | ```bash 60 | $ python3 -m pip install -U pip 61 | $ python3 -m pip install -Ur requirements-dev.txt 62 | ``` 63 | * Install pre-commit hooks to notice mistakes before the CI does it for you ;) 64 | ```bash 65 | $ pre-commit install 66 | ``` 67 | 68 | ### Working with Pre-commit 69 | If you want to run the pre-commit checks manually, you can do so: 70 | ```bash 71 | $ pre-commit run --all-files 72 | # or 73 | $ pre-commit run --file path/to/a/file 74 | ``` 75 | 76 | To commit skipping the pre-commit checks, you can do so: 77 | ```bash 78 | git commit --no-verify ... 79 | ``` 80 | 81 | ### Running Tests 82 | ```bash 83 | # in the project root 84 | pip install . -r tests/requirements.txt 85 | python -m pytest tests 86 | ``` 87 | 88 | ### Running Benchmarks 89 | Go into `tests/benchmarks/test_benchmarks.py` and adjust the connection details to the database you want to benchmark against. 90 | This implies you're having a running database. 91 | Then run the benchmarks with: 92 | ```bash 93 | python -m tox -e py312-test -- --benchmark-only --benchmark-autosave --benchmark-group-by=fullname 94 | # or to compare the results with the previous run 95 | python -m tox -e py312-test -- --benchmark-only --benchmark-autosave --benchmark-group-by=fullname --benchmark-compare 96 | ``` 97 | 98 | 99 | ## Got an idea for a new project? 100 | 101 | If you have an idea for a new tool or library, start by talking to other people in the community. 102 | Chances are that someone has a similar idea or may have already started working on it. 103 | The best software comes from getting like minds together to solve a problem. 104 | And we'll do our best to help you promote and co-ordinate your Neo4j ecosystem projects. 105 | 106 | 107 | ## Further reading 108 | 109 | If you want to find out more about how you can contribute, head over to our website for [more information](https://neo4j.com/developer/contributing-code/). 110 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "autocfg" 7 | version = "1.3.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" 10 | 11 | [[package]] 12 | name = "cfg-if" 13 | version = "1.0.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 16 | 17 | [[package]] 18 | name = "heck" 19 | version = "0.5.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 22 | 23 | [[package]] 24 | name = "indoc" 25 | version = "2.0.5" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" 28 | 29 | [[package]] 30 | name = "libc" 31 | version = "0.2.155" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" 34 | 35 | [[package]] 36 | name = "memoffset" 37 | version = "0.9.1" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" 40 | dependencies = [ 41 | "autocfg", 42 | ] 43 | 44 | [[package]] 45 | name = "neo4j-rust-ext" 46 | version = "0.1.0" 47 | dependencies = [ 48 | "pyo3", 49 | ] 50 | 51 | [[package]] 52 | name = "once_cell" 53 | version = "1.19.0" 54 | source = "registry+https://github.com/rust-lang/crates.io-index" 55 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 56 | 57 | [[package]] 58 | name = "portable-atomic" 59 | version = "1.6.0" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" 62 | 63 | [[package]] 64 | name = "proc-macro2" 65 | version = "1.0.86" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" 68 | dependencies = [ 69 | "unicode-ident", 70 | ] 71 | 72 | [[package]] 73 | name = "pyo3" 74 | version = "0.24.2" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" 77 | dependencies = [ 78 | "cfg-if", 79 | "indoc", 80 | "libc", 81 | "memoffset", 82 | "once_cell", 83 | "portable-atomic", 84 | "pyo3-build-config", 85 | "pyo3-ffi", 86 | "pyo3-macros", 87 | "unindent", 88 | ] 89 | 90 | [[package]] 91 | name = "pyo3-build-config" 92 | version = "0.24.2" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" 95 | dependencies = [ 96 | "once_cell", 97 | "target-lexicon", 98 | ] 99 | 100 | [[package]] 101 | name = "pyo3-ffi" 102 | version = "0.24.2" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" 105 | dependencies = [ 106 | "libc", 107 | "pyo3-build-config", 108 | ] 109 | 110 | [[package]] 111 | name = "pyo3-macros" 112 | version = "0.24.2" 113 | source = "registry+https://github.com/rust-lang/crates.io-index" 114 | checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" 115 | dependencies = [ 116 | "proc-macro2", 117 | "pyo3-macros-backend", 118 | "quote", 119 | "syn", 120 | ] 121 | 122 | [[package]] 123 | name = "pyo3-macros-backend" 124 | version = "0.24.2" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" 127 | dependencies = [ 128 | "heck", 129 | "proc-macro2", 130 | "pyo3-build-config", 131 | "quote", 132 | "syn", 133 | ] 134 | 135 | [[package]] 136 | name = "quote" 137 | version = "1.0.36" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" 140 | dependencies = [ 141 | "proc-macro2", 142 | ] 143 | 144 | [[package]] 145 | name = "syn" 146 | version = "2.0.68" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" 149 | dependencies = [ 150 | "proc-macro2", 151 | "quote", 152 | "unicode-ident", 153 | ] 154 | 155 | [[package]] 156 | name = "target-lexicon" 157 | version = "0.13.2" 158 | source = "registry+https://github.com/rust-lang/crates.io-index" 159 | checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" 160 | 161 | [[package]] 162 | name = "unicode-ident" 163 | version = "1.0.12" 164 | source = "registry+https://github.com/rust-lang/crates.io-index" 165 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 166 | 167 | [[package]] 168 | name = "unindent" 169 | version = "0.2.3" 170 | source = "registry+https://github.com/rust-lang/crates.io-index" 171 | checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" 172 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "neo4j-rust-ext" 3 | version = "0.1.0" 4 | edition = "2021" 5 | rust-version = "1.77" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | [lib] 9 | name = "neo4j_rust_ext" 10 | crate-type = ["cdylib"] 11 | 12 | [dependencies] 13 | pyo3 = "0.24.2" 14 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | https://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rust Extensions for a Faster Neo4j Bolt Driver for Python 2 | 3 | This project contains Rust extensions to speed up the [official Python driver for Neo4j](https://github.com/neo4j/neo4j-python-driver). 4 | 5 | The exact speedup depends on the use-case but has been measured to be up to 10x faster. 6 | Use-cases moving only few but big records out of the DBMS tend to benefit the most. 7 | 8 | 9 | ## Installation 10 | Adjust your dependencies (`requirements.txt`, `pyproject.toml` or similar) like so: 11 | ``` 12 | # remove: 13 | # neo4j == X.Y.Z # needs to be at least 5.14.1 for a matching Rust extension to exist 14 | # add: 15 | neo4j-rust-ext == X.Y.Z.* 16 | ``` 17 | 18 | I.e., install the same version of `neo4j-rust-ext` as you would install of `neo4j` (except for the last segment which is used for patches of this library). 19 | That's it! 20 | You don't have to change your code but can use the driver as you normally would. 21 | This package will install the driver as its dependency and then inject itself in a place where the driver can find it and pick it up. 22 | 23 | N.B., since the driver is a simple Python dependency of this package, you can also manually install/specify both packages at the same time without issues. 24 | However, make sure the versions match if you do so or leave the version of one of the two unspecified to let the package manager pick a compatible version for you (resolution might be slow, however). 25 | 26 | If you experience issues with the driver, consider troubleshooting without the Rust extension first. 27 | For that, simply make sure you haven't installed `neo4j-rust-ext` but *only* `neo4j`. 28 | 29 | 30 | ## Requirements 31 | For many operating systems and architectures, the pre-built wheels will work out of the box. 32 | If they don't, pip (or any other Python packaging front-end) will try to build the extension from source. 33 | Here's what you'll need for this: 34 | * Rust 1.77 or later: 35 | https://www.rust-lang.org/tools/install 36 | * Further build tools (depending on the platform). 37 | E.g., `gcc` on Ubuntu: `sudo apt install gcc` 38 | -------------------------------------------------------------------------------- /bin/check_version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | version="$1"; shift 5 | 6 | if ! grep -q --perl-regexp "(?m)(?=3.7" 28 | keywords = ["neo4j", "graph", "database"] 29 | classifiers = [ 30 | "Development Status :: 5 - Production/Stable", 31 | "Intended Audience :: Developers", 32 | "License :: OSI Approved :: Apache Software License", 33 | "Operating System :: OS Independent", 34 | "Programming Language :: Python :: 3.7", 35 | "Programming Language :: Python :: 3.8", 36 | "Programming Language :: Python :: 3.9", 37 | "Programming Language :: Python :: 3.10", 38 | "Programming Language :: Python :: 3.11", 39 | "Programming Language :: Python :: 3.12", 40 | "Programming Language :: Python :: 3.13", 41 | "Programming Language :: Rust", 42 | "Topic :: Database", 43 | "Topic :: Software Development", 44 | ] 45 | version = "5.28.1.0" 46 | 47 | [project.urls] 48 | Homepage = "https://neo4j.com/" 49 | Repository = "https://github.com/neo4j/neo4j-python-driver-rust-ext" 50 | "Issue Tracker" = "https://github.com/neo4j/neo4j-python-driver-rust-ext/issues" 51 | Changelog = "https://github.com/neo4j/neo4j-python-driver-rust-ext/blob/HEAD/CHANGELOG.md" 52 | Forum = "https://community.neo4j.com/c/drivers-stacks/python/" 53 | Discord = "https://discord.com/invite/neo4j" 54 | 55 | [project.optional-dependencies] 56 | numpy = ["neo4j[numpy]"] 57 | pandas = ["neo4j[pandas]"] 58 | pyarrow = ["neo4j[pyarrow]"] 59 | 60 | [build-system] 61 | requires = ["maturin ~= 1.8.3"] 62 | build-backend = "maturin" 63 | 64 | [tool.maturin] 65 | features = ["pyo3/extension-module", "pyo3/generate-import-lib"] 66 | module-name = "neo4j._codec.packstream._rust" 67 | exclude = [ 68 | "/.editorconfig", 69 | ".gitignore", 70 | ".gitmodules", 71 | ".pre-commit-config.yaml", 72 | "bin/**/*", 73 | "driver/**/*", 74 | "test*/**/*", 75 | "CONTRIBUTING.md", 76 | "requirements*.txt", 77 | "tox.ini", 78 | { path = "neo4j/**/.keep", format = "wheel" } 79 | ] 80 | 81 | [tool.isort] 82 | combine_as_imports = true 83 | ensure_newline_before_comments = true 84 | force_grid_wrap = 2 85 | # breaks order of relative imports 86 | # https://github.com/PyCQA/isort/issues/1944 87 | #force_sort_within_sections = true 88 | include_trailing_comma = true 89 | # currently broken 90 | # https://github.com/PyCQA/isort/issues/1855 91 | #lines_before_imports = 2 92 | lines_after_imports = 2 93 | lines_between_sections = 1 94 | multi_line_output = 3 95 | order_by_type = false 96 | remove_redundant_aliases = true 97 | use_parentheses = true 98 | known_first_party = ["neo4j"] 99 | 100 | [tool.ruff] 101 | line-length = 79 102 | extend-exclude = [ 103 | "driver", 104 | ] 105 | 106 | [tool.ruff.lint] 107 | preview = true # to get CPY lints 108 | extend-ignore = [ 109 | "RUF002", # allow ’ (RIGHT SINGLE QUOTATION MARK) to be used as an apostrophe (e.g. "it’s") 110 | "SIM117", # TODO: when Python 3.10+ is the minimum, 111 | # we can start to use multi-item `with` statements 112 | # pydocstyle 113 | "D1", # disable check for undocumented items (way too noisy) 114 | "D203", # `one-blank-line-before-class` 115 | "D212", # `multi-line-summary-first-line` 116 | 117 | # comprehensions 118 | "C417", # map is ok, no need to rewrite to list comprehension 119 | 120 | # too noisy and opinionated pytest lints 121 | "PT007", 122 | "PT011", 123 | "PT012", 124 | "PT018", 125 | 126 | # too noisy and opinionated pylint lints 127 | "PLC0415", 128 | "PLC1901", 129 | "PLC2401", 130 | "PLC2701", 131 | "PLR09", 132 | "PLR1702", 133 | "PLR1704", 134 | "PLR2004", 135 | "PLR6301", 136 | "PLW2901", 137 | "PLW1641", 138 | 139 | # too noisy and opinionated tryceratops lints 140 | "TRY003", 141 | "TRY300", 142 | "TRY301", 143 | "TRY400", 144 | 145 | # too noisy and opinionated return statement lints 146 | "RET505", 147 | "RET506", 148 | "RET507", 149 | "RET508", 150 | 151 | "PERF203", # try-except within loop is fine. Especially in a retry scenario 152 | 153 | # too noisy and opinionated FURB lints 154 | "FURB113", 155 | "FURB118", 156 | "FURB140", 157 | "FURB154", 158 | # needs fixing in ruff to work with typing.Protocol 159 | # https://github.com/astral-sh/ruff/issues/13307 160 | "FURB180", 161 | ] 162 | select = [ 163 | # ruff 164 | "RUF", 165 | # pycodestyle 166 | "E", 167 | "W", 168 | # Pyflakes 169 | "F", 170 | # pyupgrade 171 | "UP", 172 | # flake8-bugbear 173 | "B", 174 | # flake8-simplify 175 | "SIM", 176 | # pep8-naming 177 | "N", 178 | # pydocstyle 179 | "D", 180 | # pydocstyle: explicit rules not selected by the chosen convention 181 | "D404", 182 | # Does not yet fully support sphinx style docstrings 183 | # https://github.com/astral-sh/ruff/pull/13286 184 | # # pydoclint 185 | # "DOC", 186 | # pylint 187 | "PL", 188 | # tryceratops 189 | "TRY", 190 | # flynt 191 | "FLY", 192 | # Perflint 193 | "PERF", 194 | # refurb 195 | "FURB", 196 | # async checks 197 | "ASYNC", 198 | # check comprehensions 199 | "C4", 200 | # check for left-over debugger calls 201 | "T100", 202 | # check for left-over print calls 203 | "T20", 204 | # qoute styles 205 | "Q", 206 | # check for unnecessary parantheses in raise statements 207 | "RSE", 208 | # check return statements 209 | "RET", 210 | # TODO: 6.0 - enable and add __slots__ to applicable classes 211 | # # check __slots__ usage 212 | # "SLOT", 213 | # check type-checking usage 214 | "TCH", 215 | # copyright notice 216 | "CPY", 217 | # check shebangs 218 | "EXE", 219 | # logging calls + formats 220 | "LOG", 221 | "G", 222 | # flake8-pie 223 | "PIE", 224 | # pytest lints 225 | "PT", 226 | ] 227 | 228 | [tool.ruff.lint.per-file-ignores] 229 | "{testkit,tests}/**" = [ 230 | "T20", # print statements are ok in the testing infrastructure 231 | ] 232 | "tests/**" = [ 233 | "PLW1641", # no need for production grade test code 234 | "FURB152", # don't tell us to use math.pi, when all we need is just some random float 235 | # allow async functions without await to enable type checking, pretending to be async, matching type signatures 236 | "RUF029", 237 | ] 238 | "bin/**" = [ 239 | "T20", # print statements are ok in our helper scripts 240 | ] 241 | 242 | [tool.ruff.lint.pycodestyle] 243 | max-doc-length = 83 # 79 (max line length) + 4 indentation of code blocks 244 | 245 | [tool.ruff.lint.pep8-naming] 246 | extend-ignore-names = ["mcs"] 247 | 248 | [tool.ruff.lint.flake8-copyright] 249 | notice-rgx = "# Copyright \\(c\\) \"Neo4j\"" 250 | 251 | [tool.ruff.lint.pydocstyle] 252 | convention = "pep257" 253 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | .[numpy,pandas,pyarrow] 2 | 3 | # for local development 4 | pre-commit>=2.21.0 # TODO: 6.0 - bump when support for Python 3.7 is dropped 5 | isort>=5.11.5 # TODO: 6.0 - bump when support for Python 3.7 is dropped 6 | 7 | # for unit tests 8 | tox>=4.8.0 # TODO: 6.0 - bump when support for Python 3.7 is dropped 9 | pytest>=7.4.4 # TODO: 6.0 - bump when support for Python 3.7 is dropped 10 | pytest-benchmark>=4.0.0 11 | 12 | # for Python driver's TestKit backend 13 | freezegun>=1.5.1 14 | 15 | # for packaging 16 | maturin>=1.7.8 17 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) "Neo4j" 2 | // Neo4j Sweden AB [https://neo4j.com] 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | pub mod v1; 17 | 18 | use pyo3::basic::CompareOp; 19 | use pyo3::exceptions::PyValueError; 20 | use pyo3::prelude::*; 21 | use pyo3::types::{PyBytes, PyTuple}; 22 | use pyo3::IntoPyObjectExt; 23 | 24 | #[pymodule(gil_used = false)] 25 | #[pyo3(name = "_rust")] 26 | fn packstream(m: &Bound) -> PyResult<()> { 27 | let py = m.py(); 28 | 29 | m.add_class::()?; 30 | 31 | let mod_v1 = PyModule::new(py, "v1")?; 32 | mod_v1.gil_used(false)?; 33 | v1::register(&mod_v1)?; 34 | m.add_submodule(&mod_v1)?; 35 | register_package(&mod_v1, "v1")?; 36 | 37 | Ok(()) 38 | } 39 | 40 | // hack to make python pick up the submodule as a package 41 | // https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021 42 | fn register_package(m: &Bound, name: &str) -> PyResult<()> { 43 | let py = m.py(); 44 | let module_name = format!("neo4j._codec.packstream._rust.{name}").into_pyobject(py)?; 45 | 46 | py.import("sys")? 47 | .getattr("modules")? 48 | .set_item(&module_name, m)?; 49 | m.setattr("__name__", &module_name)?; 50 | 51 | Ok(()) 52 | } 53 | 54 | #[pyclass] 55 | #[derive(Debug)] 56 | pub struct Structure { 57 | tag: u8, 58 | #[pyo3(get)] 59 | fields: Vec, 60 | } 61 | 62 | #[pymethods] 63 | impl Structure { 64 | #[new] 65 | #[pyo3(signature = (tag, *fields))] 66 | #[pyo3(text_signature = "(tag, *fields)")] 67 | fn new(tag: &[u8], fields: Vec) -> PyResult { 68 | if tag.len() != 1 { 69 | return Err(PyErr::new::("tag must be a single byte")); 70 | } 71 | let tag = tag[0]; 72 | Ok(Self { tag, fields }) 73 | } 74 | 75 | #[getter(tag)] 76 | fn read_tag<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { 77 | PyBytes::new(py, &[self.tag]) 78 | } 79 | 80 | #[getter(fields)] 81 | fn read_fields<'py>(&self, py: Python<'py>) -> PyResult> { 82 | PyTuple::new(py, &self.fields) 83 | } 84 | 85 | fn eq(&self, other: &Self, py: Python<'_>) -> PyResult { 86 | if self.tag != other.tag || self.fields.len() != other.fields.len() { 87 | return Ok(false); 88 | } 89 | for (a, b) in self 90 | .fields 91 | .iter() 92 | .map(|e| e.bind(py)) 93 | .zip(other.fields.iter().map(|e| e.bind(py))) 94 | { 95 | if !a.eq(b)? { 96 | return Ok(false); 97 | } 98 | } 99 | Ok(true) 100 | } 101 | 102 | fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> PyResult { 103 | Ok(match op { 104 | CompareOp::Eq => self.eq(other, py)?.into_py_any(py)?, 105 | CompareOp::Ne => (!self.eq(other, py)?).into_py_any(py)?, 106 | _ => py.NotImplemented(), 107 | }) 108 | } 109 | 110 | fn __hash__(&self, py: Python<'_>) -> PyResult { 111 | let mut fields_hash = 0; 112 | for field in &self.fields { 113 | fields_hash += field.bind(py).hash()?; 114 | } 115 | Ok(fields_hash.wrapping_add(self.tag.into())) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/v1.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) "Neo4j" 2 | // Neo4j Sweden AB [https://neo4j.com] 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | mod pack; 17 | mod unpack; 18 | 19 | use pyo3::prelude::*; 20 | use pyo3::wrap_pyfunction; 21 | 22 | const TINY_STRING: u8 = 0x80; 23 | const TINY_LIST: u8 = 0x90; 24 | const TINY_MAP: u8 = 0xA0; 25 | const TINY_STRUCT: u8 = 0xB0; 26 | const NULL: u8 = 0xC0; 27 | const FALSE: u8 = 0xC2; 28 | const TRUE: u8 = 0xC3; 29 | const INT_8: u8 = 0xC8; 30 | const INT_16: u8 = 0xC9; 31 | const INT_32: u8 = 0xCA; 32 | const INT_64: u8 = 0xCB; 33 | const FLOAT_64: u8 = 0xC1; 34 | const STRING_8: u8 = 0xD0; 35 | const STRING_16: u8 = 0xD1; 36 | const STRING_32: u8 = 0xD2; 37 | const LIST_8: u8 = 0xD4; 38 | const LIST_16: u8 = 0xD5; 39 | const LIST_32: u8 = 0xD6; 40 | const MAP_8: u8 = 0xD8; 41 | const MAP_16: u8 = 0xD9; 42 | const MAP_32: u8 = 0xDA; 43 | const BYTES_8: u8 = 0xCC; 44 | const BYTES_16: u8 = 0xCD; 45 | const BYTES_32: u8 = 0xCE; 46 | 47 | pub(crate) fn register(m: &Bound) -> PyResult<()> { 48 | m.add_function(wrap_pyfunction!(unpack::unpack, m)?)?; 49 | m.add_function(wrap_pyfunction!(pack::pack, m)?)?; 50 | 51 | Ok(()) 52 | } 53 | -------------------------------------------------------------------------------- /src/v1/pack.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) "Neo4j" 2 | // Neo4j Sweden AB [https://neo4j.com] 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | use std::borrow::Cow; 17 | use std::sync::atomic::{AtomicBool, Ordering}; 18 | 19 | use pyo3::exceptions::{PyImportError, PyOverflowError, PyTypeError, PyValueError}; 20 | use pyo3::prelude::*; 21 | use pyo3::sync::GILOnceCell; 22 | use pyo3::types::{PyBytes, PyDict, PyString, PyType}; 23 | use pyo3::{intern, IntoPyObjectExt}; 24 | 25 | use super::{ 26 | BYTES_16, BYTES_32, BYTES_8, FALSE, FLOAT_64, INT_16, INT_32, INT_64, INT_8, LIST_16, LIST_32, 27 | LIST_8, MAP_16, MAP_32, MAP_8, NULL, STRING_16, STRING_32, STRING_8, TINY_LIST, TINY_MAP, 28 | TINY_STRING, TINY_STRUCT, TRUE, 29 | }; 30 | use crate::Structure; 31 | 32 | #[derive(Debug)] 33 | struct TypeMappings { 34 | none_values: Vec, 35 | true_values: Vec, 36 | false_values: Vec, 37 | int_types: PyObject, 38 | float_types: PyObject, 39 | sequence_types: PyObject, 40 | mapping_types: PyObject, 41 | bytes_types: PyObject, 42 | } 43 | 44 | impl TypeMappings { 45 | fn new(locals: &Bound) -> PyResult { 46 | let py = locals.py(); 47 | Ok(Self { 48 | none_values: locals 49 | .get_item("NONE_VALUES")? 50 | .ok_or_else(|| { 51 | PyErr::new::("Type mappings are missing NONE_VALUES.") 52 | })? 53 | .extract()?, 54 | true_values: locals 55 | .get_item("TRUE_VALUES")? 56 | .ok_or_else(|| { 57 | PyErr::new::("Type mappings are missing TRUE_VALUES.") 58 | })? 59 | .extract()?, 60 | false_values: locals 61 | .get_item("FALSE_VALUES")? 62 | .ok_or_else(|| { 63 | PyErr::new::("Type mappings are missing FALSE_VALUES.") 64 | })? 65 | .extract()?, 66 | int_types: locals 67 | .get_item("INT_TYPES")? 68 | .ok_or_else(|| { 69 | PyErr::new::("Type mappings are missing INT_TYPES.") 70 | })? 71 | .into_py_any(py)?, 72 | float_types: locals 73 | .get_item("FLOAT_TYPES")? 74 | .ok_or_else(|| { 75 | PyErr::new::("Type mappings are missing FLOAT_TYPES.") 76 | })? 77 | .into_py_any(py)?, 78 | sequence_types: locals 79 | .get_item("SEQUENCE_TYPES")? 80 | .ok_or_else(|| { 81 | PyErr::new::("Type mappings are missing SEQUENCE_TYPES.") 82 | })? 83 | .into_py_any(py)?, 84 | mapping_types: locals 85 | .get_item("MAPPING_TYPES")? 86 | .ok_or_else(|| { 87 | PyErr::new::("Type mappings are missing MAPPING_TYPES.") 88 | })? 89 | .into_py_any(py)?, 90 | bytes_types: locals 91 | .get_item("BYTES_TYPES")? 92 | .ok_or_else(|| { 93 | PyErr::new::("Type mappings are missing BYTES_TYPES.") 94 | })? 95 | .into_py_any(py)?, 96 | }) 97 | } 98 | } 99 | 100 | static TYPE_MAPPINGS: GILOnceCell> = GILOnceCell::new(); 101 | static TYPE_MAPPINGS_INIT: AtomicBool = AtomicBool::new(false); 102 | 103 | fn get_type_mappings(py: Python<'_>) -> PyResult<&'static TypeMappings> { 104 | let mappings = TYPE_MAPPINGS.get_or_try_init(py, || { 105 | fn init(py: Python<'_>) -> PyResult { 106 | let locals = PyDict::new(py); 107 | py.run( 108 | c"from neo4j._codec.packstream.v1.types import *", 109 | None, 110 | Some(&locals), 111 | )?; 112 | TypeMappings::new(&locals) 113 | } 114 | 115 | if TYPE_MAPPINGS_INIT.swap(true, Ordering::SeqCst) { 116 | return Err(PyErr::new::( 117 | "Cannot call _rust.pack while loading `neo4j._codec.packstream.v1.types`", 118 | )); 119 | } 120 | Ok(init(py)) 121 | }); 122 | mappings?.as_ref().map_err(|e| e.clone_ref(py)) 123 | } 124 | 125 | #[pyfunction] 126 | #[pyo3(signature = (value, dehydration_hooks=None))] 127 | pub(super) fn pack<'py>( 128 | value: &Bound<'py, PyAny>, 129 | dehydration_hooks: Option<&Bound<'py, PyAny>>, 130 | ) -> PyResult> { 131 | let py = value.py(); 132 | let type_mappings = get_type_mappings(py)?; 133 | let mut encoder = PackStreamEncoder::new(dehydration_hooks, type_mappings); 134 | encoder.write(value)?; 135 | Ok(PyBytes::new(py, &encoder.buffer)) 136 | } 137 | 138 | struct PackStreamEncoder<'a> { 139 | dehydration_hooks: Option<&'a Bound<'a, PyAny>>, 140 | type_mappings: &'a TypeMappings, 141 | buffer: Vec, 142 | } 143 | 144 | impl<'a> PackStreamEncoder<'a> { 145 | fn new( 146 | dehydration_hooks: Option<&'a Bound<'a, PyAny>>, 147 | type_mappings: &'a TypeMappings, 148 | ) -> Self { 149 | Self { 150 | dehydration_hooks, 151 | type_mappings, 152 | buffer: Default::default(), 153 | } 154 | } 155 | 156 | fn write(&mut self, value: &Bound) -> PyResult<()> { 157 | let py = value.py(); 158 | 159 | if self.write_exact_value(value, &self.type_mappings.none_values, &[NULL])? { 160 | return Ok(()); 161 | } 162 | if self.write_exact_value(value, &self.type_mappings.true_values, &[TRUE])? { 163 | return Ok(()); 164 | } 165 | if self.write_exact_value(value, &self.type_mappings.false_values, &[FALSE])? { 166 | return Ok(()); 167 | } 168 | 169 | if value.is_instance(self.type_mappings.float_types.bind(py))? { 170 | let value = value.extract::()?; 171 | return self.write_float(value); 172 | } 173 | 174 | if value.is_instance(self.type_mappings.int_types.bind(py))? { 175 | let value = value.extract::()?; 176 | return self.write_int(value); 177 | } 178 | 179 | if value.is_instance(&PyType::new::(py))? { 180 | return self.write_string(value.extract::<&str>()?); 181 | } 182 | 183 | if value.is_instance(self.type_mappings.bytes_types.bind(py))? { 184 | return self.write_bytes(value.extract::>()?); 185 | } 186 | 187 | if value.is_instance(self.type_mappings.sequence_types.bind(py))? { 188 | let size = Self::usize_to_u64(value.len()?)?; 189 | self.write_list_header(size)?; 190 | return value.try_iter()?.try_for_each(|item| self.write(&item?)); 191 | } 192 | 193 | if value.is_instance(self.type_mappings.mapping_types.bind(py))? { 194 | let size = Self::usize_to_u64(value.getattr(intern!(py, "keys"))?.call0()?.len()?)?; 195 | self.write_dict_header(size)?; 196 | let items = value.getattr(intern!(py, "items"))?.call0()?; 197 | return items.try_iter()?.try_for_each(|item| { 198 | let (key, value) = item?.extract::<(Bound, Bound)>()?; 199 | let key = match key.extract::<&str>() { 200 | Ok(key) => key, 201 | Err(_) => { 202 | return Err(PyErr::new::(format!( 203 | "Map keys must be strings, not {}", 204 | key.get_type().str()? 205 | ))) 206 | } 207 | }; 208 | self.write_string(key)?; 209 | self.write(&value) 210 | }); 211 | } 212 | 213 | if let Ok(value) = value.extract::>() { 214 | let value_ref = value.borrow(); 215 | let size = value_ref.fields.len().try_into().map_err(|_| { 216 | PyErr::new::("Structure header size out of range") 217 | })?; 218 | self.write_struct_header(value_ref.tag, size)?; 219 | return value_ref 220 | .fields 221 | .iter() 222 | .try_for_each(|item| self.write(item.bind(py))); 223 | } 224 | 225 | if let Some(dehydration_hooks) = self.dehydration_hooks { 226 | let transformer = 227 | dehydration_hooks.call_method1(intern!(py, "get_transformer"), (value,))?; 228 | if !transformer.is_none() { 229 | let value = transformer.call1((value,))?; 230 | return self.write(&value); 231 | } 232 | } 233 | 234 | // raise ValueError("Values of type %s are not supported" % type(value)) 235 | Err(PyErr::new::(format!( 236 | "Values of type {} are not supported", 237 | value.get_type().str()? 238 | ))) 239 | } 240 | 241 | fn write_exact_value( 242 | &mut self, 243 | value: &Bound, 244 | values: &[PyObject], 245 | bytes: &[u8], 246 | ) -> PyResult { 247 | for v in values { 248 | if value.is(v) { 249 | self.buffer.extend(bytes); 250 | return Ok(true); 251 | } 252 | } 253 | Ok(false) 254 | } 255 | 256 | fn write_int(&mut self, i: i64) -> PyResult<()> { 257 | if (-16..=127).contains(&i) { 258 | self.buffer.extend(&i8::to_be_bytes(i as i8)); 259 | } else if (-128..=127).contains(&i) { 260 | self.buffer.extend(&[INT_8]); 261 | self.buffer.extend(&i8::to_be_bytes(i as i8)); 262 | } else if (-32_768..=32_767).contains(&i) { 263 | self.buffer.extend(&[INT_16]); 264 | self.buffer.extend(&i16::to_be_bytes(i as i16)); 265 | } else if (-2_147_483_648..=2_147_483_647).contains(&i) { 266 | self.buffer.extend(&[INT_32]); 267 | self.buffer.extend(&i32::to_be_bytes(i as i32)); 268 | } else { 269 | self.buffer.extend(&[INT_64]); 270 | self.buffer.extend(&i64::to_be_bytes(i)); 271 | } 272 | Ok(()) 273 | } 274 | 275 | fn write_float(&mut self, f: f64) -> PyResult<()> { 276 | self.buffer.extend(&[FLOAT_64]); 277 | self.buffer.extend(&f64::to_be_bytes(f)); 278 | Ok(()) 279 | } 280 | 281 | fn write_bytes(&mut self, b: Cow<[u8]>) -> PyResult<()> { 282 | let size = Self::usize_to_u64(b.len())?; 283 | if size <= 255 { 284 | self.buffer.extend(&[BYTES_8]); 285 | self.buffer.extend(&u8::to_be_bytes(size as u8)); 286 | } else if size <= 65_535 { 287 | self.buffer.extend(&[BYTES_16]); 288 | self.buffer.extend(&u16::to_be_bytes(size as u16)); 289 | } else if size <= 2_147_483_647 { 290 | self.buffer.extend(&[BYTES_32]); 291 | self.buffer.extend(&u32::to_be_bytes(size as u32)); 292 | } else { 293 | return Err(PyErr::new::( 294 | "Bytes header size out of range", 295 | )); 296 | } 297 | self.buffer.extend(b.iter()); 298 | Ok(()) 299 | } 300 | 301 | fn usize_to_u64(size: usize) -> PyResult { 302 | u64::try_from(size).map_err(|e| PyErr::new::(e.to_string())) 303 | } 304 | 305 | fn write_string(&mut self, s: &str) -> PyResult<()> { 306 | let bytes = s.as_bytes(); 307 | let size = Self::usize_to_u64(bytes.len())?; 308 | if size <= 15 { 309 | self.buffer.extend(&[TINY_STRING + size as u8]); 310 | } else if size <= 255 { 311 | self.buffer.extend(&[STRING_8]); 312 | self.buffer.extend(&u8::to_be_bytes(size as u8)); 313 | } else if size <= 65_535 { 314 | self.buffer.extend(&[STRING_16]); 315 | self.buffer.extend(&u16::to_be_bytes(size as u16)); 316 | } else if size <= 2_147_483_647 { 317 | self.buffer.extend(&[STRING_32]); 318 | self.buffer.extend(&u32::to_be_bytes(size as u32)); 319 | } else { 320 | return Err(PyErr::new::( 321 | "String header size out of range", 322 | )); 323 | } 324 | self.buffer.extend(bytes); 325 | Ok(()) 326 | } 327 | 328 | fn write_list_header(&mut self, size: u64) -> PyResult<()> { 329 | if size <= 15 { 330 | self.buffer.extend(&[TINY_LIST + size as u8]); 331 | } else if size <= 255 { 332 | self.buffer.extend(&[LIST_8]); 333 | self.buffer.extend(&u8::to_be_bytes(size as u8)); 334 | } else if size <= 65_535 { 335 | self.buffer.extend(&[LIST_16]); 336 | self.buffer.extend(&u16::to_be_bytes(size as u16)); 337 | } else if size <= 2_147_483_647 { 338 | self.buffer.extend(&[LIST_32]); 339 | self.buffer.extend(&u32::to_be_bytes(size as u32)); 340 | } else { 341 | return Err(PyErr::new::( 342 | "List header size out of range", 343 | )); 344 | } 345 | Ok(()) 346 | } 347 | 348 | fn write_dict_header(&mut self, size: u64) -> PyResult<()> { 349 | if size <= 15 { 350 | self.buffer.extend(&[TINY_MAP + size as u8]); 351 | } else if size <= 255 { 352 | self.buffer.extend(&[MAP_8]); 353 | self.buffer.extend(&u8::to_be_bytes(size as u8)); 354 | } else if size <= 65_535 { 355 | self.buffer.extend(&[MAP_16]); 356 | self.buffer.extend(&u16::to_be_bytes(size as u16)); 357 | } else if size <= 2_147_483_647 { 358 | self.buffer.extend(&[MAP_32]); 359 | self.buffer.extend(&u32::to_be_bytes(size as u32)); 360 | } else { 361 | return Err(PyErr::new::( 362 | "Map header size out of range", 363 | )); 364 | } 365 | Ok(()) 366 | } 367 | 368 | fn write_struct_header(&mut self, tag: u8, size: u8) -> PyResult<()> { 369 | if size > 15 { 370 | return Err(PyErr::new::( 371 | "Structure size out of range", 372 | )); 373 | } 374 | self.buffer.extend(&[TINY_STRUCT + size, tag]); 375 | Ok(()) 376 | } 377 | } 378 | -------------------------------------------------------------------------------- /src/v1/unpack.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) "Neo4j" 2 | // Neo4j Sweden AB [https://neo4j.com] 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // https://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | use pyo3::exceptions::PyValueError; 17 | use pyo3::prelude::*; 18 | use pyo3::sync::with_critical_section; 19 | use pyo3::types::{IntoPyDict, PyByteArray, PyBytes, PyDict, PyList, PyTuple}; 20 | use pyo3::{intern, IntoPyObjectExt}; 21 | 22 | use super::{ 23 | BYTES_16, BYTES_32, BYTES_8, FALSE, FLOAT_64, INT_16, INT_32, INT_64, INT_8, LIST_16, LIST_32, 24 | LIST_8, MAP_16, MAP_32, MAP_8, NULL, STRING_16, STRING_32, STRING_8, TINY_LIST, TINY_MAP, 25 | TINY_STRING, TINY_STRUCT, TRUE, 26 | }; 27 | use crate::Structure; 28 | 29 | #[pyfunction] 30 | #[pyo3(signature = (bytes, idx, hydration_hooks=None))] 31 | pub(super) fn unpack( 32 | bytes: Bound, 33 | idx: usize, 34 | hydration_hooks: Option>, 35 | ) -> PyResult<(PyObject, usize)> { 36 | let py = bytes.py(); 37 | let mut decoder = PackStreamDecoder::new(py, bytes, idx, hydration_hooks); 38 | let result = decoder.read()?; 39 | Ok((result, decoder.index)) 40 | } 41 | 42 | struct PackStreamDecoder<'a> { 43 | py: Python<'a>, 44 | bytes: Bound<'a, PyByteArray>, 45 | index: usize, 46 | hydration_hooks: Option>, 47 | } 48 | 49 | impl<'a> PackStreamDecoder<'a> { 50 | fn new( 51 | py: Python<'a>, 52 | bytes: Bound<'a, PyByteArray>, 53 | idx: usize, 54 | hydration_hooks: Option>, 55 | ) -> Self { 56 | Self { 57 | py, 58 | bytes, 59 | index: idx, 60 | hydration_hooks, 61 | } 62 | } 63 | 64 | fn read(&mut self) -> PyResult { 65 | let marker = self.read_byte()?; 66 | self.read_value(marker) 67 | } 68 | 69 | fn read_value(&mut self, marker: u8) -> PyResult { 70 | let high_nibble = marker & 0xF0; 71 | 72 | Ok(match marker { 73 | // tiny int 74 | _ if marker as i8 >= -16 => (marker as i8).into_py_any(self.py)?, 75 | NULL => self.py.None(), 76 | FLOAT_64 => self.read_f64()?.into_py_any(self.py)?, 77 | FALSE => false.into_py_any(self.py)?, 78 | TRUE => true.into_py_any(self.py)?, 79 | INT_8 => self.read_i8()?.into_py_any(self.py)?, 80 | INT_16 => self.read_i16()?.into_py_any(self.py)?, 81 | INT_32 => self.read_i32()?.into_py_any(self.py)?, 82 | INT_64 => self.read_i64()?.into_py_any(self.py)?, 83 | BYTES_8 => { 84 | let len = self.read_u8()?; 85 | self.read_bytes(len)? 86 | } 87 | BYTES_16 => { 88 | let len = self.read_u16()?; 89 | self.read_bytes(len)? 90 | } 91 | BYTES_32 => { 92 | let len = self.read_u32()?; 93 | self.read_bytes(len)? 94 | } 95 | _ if high_nibble == TINY_STRING => self.read_string((marker & 0x0F).into())?, 96 | STRING_8 => { 97 | let len = self.read_u8()?; 98 | self.read_string(len)? 99 | } 100 | STRING_16 => { 101 | let len = self.read_u16()?; 102 | self.read_string(len)? 103 | } 104 | STRING_32 => { 105 | let len = self.read_u32()?; 106 | self.read_string(len)? 107 | } 108 | _ if high_nibble == TINY_LIST => self.read_list((marker & 0x0F).into())?, 109 | LIST_8 => { 110 | let len = self.read_u8()?; 111 | self.read_list(len)? 112 | } 113 | LIST_16 => { 114 | let len = self.read_u16()?; 115 | self.read_list(len)? 116 | } 117 | LIST_32 => { 118 | let len = self.read_u32()?; 119 | self.read_list(len)? 120 | } 121 | _ if high_nibble == TINY_MAP => self.read_map((marker & 0x0F).into())?, 122 | MAP_8 => { 123 | let len = self.read_u8()?; 124 | self.read_map(len)? 125 | } 126 | MAP_16 => { 127 | let len = self.read_u16()?; 128 | self.read_map(len)? 129 | } 130 | MAP_32 => { 131 | let len = self.read_u32()?; 132 | self.read_map(len)? 133 | } 134 | _ if high_nibble == TINY_STRUCT => self.read_struct((marker & 0x0F).into())?, 135 | _ => { 136 | // raise ValueError("Unknown PackStream marker %02X" % marker) 137 | return Err(PyErr::new::(format!( 138 | "Unknown PackStream marker {:02X}", 139 | marker 140 | ))); 141 | } 142 | }) 143 | } 144 | 145 | fn read_list(&mut self, length: usize) -> PyResult { 146 | if length == 0 { 147 | return Ok(PyList::empty(self.py).into_any().unbind()); 148 | } 149 | let mut items = Vec::with_capacity(length); 150 | for _ in 0..length { 151 | items.push(self.read()?); 152 | } 153 | items.into_py_any(self.py) 154 | } 155 | 156 | fn read_string(&mut self, length: usize) -> PyResult { 157 | if length == 0 { 158 | return "".into_py_any(self.py); 159 | } 160 | let data = with_critical_section(&self.bytes, || { 161 | // Safety: 162 | // * We're using a critical section to avoid other threads mutating the bytes while 163 | // we're reading them. 164 | // * We're not mutating the bytes ourselves. 165 | // * We're not interacting with Python while using the bytes as that might indirectly 166 | // cause the bytes to be mutated. 167 | unsafe { 168 | let data = &self.bytes.as_bytes()[self.index..self.index + length]; 169 | // We have to copy the data to uphold the safety invariant. 170 | String::from_utf8(Vec::from(data)) 171 | } 172 | }); 173 | let data = data.map_err(|e| PyErr::new::(e.to_string()))?; 174 | self.index += length; 175 | data.into_py_any(self.py) 176 | } 177 | 178 | fn read_map(&mut self, length: usize) -> PyResult { 179 | if length == 0 { 180 | return Ok(PyDict::new(self.py).into_any().unbind()); 181 | } 182 | let mut key_value_pairs: Vec<(PyObject, PyObject)> = Vec::with_capacity(length); 183 | for _ in 0..length { 184 | let len = self.read_string_length()?; 185 | let key = self.read_string(len)?; 186 | let value = self.read()?; 187 | key_value_pairs.push((key, value)); 188 | } 189 | Ok(key_value_pairs.into_py_dict(self.py)?.into()) 190 | } 191 | 192 | fn read_bytes(&mut self, length: usize) -> PyResult { 193 | if length == 0 { 194 | return Ok(PyBytes::new(self.py, &[]).into_any().unbind()); 195 | } 196 | let data = with_critical_section(&self.bytes, || { 197 | // Safety: 198 | // * We're using a critical section to avoid other threads mutating the bytes while 199 | // we're reading them. 200 | // * We're not mutating the bytes ourselves. 201 | // * We're not interacting with Python while using the bytes as that might indirectly 202 | // cause the bytes to be mutated. 203 | unsafe { 204 | // We have to copy the data to uphold the safety invariant. 205 | self.bytes.as_bytes()[self.index..self.index + length].to_vec() 206 | } 207 | }); 208 | self.index += length; 209 | Ok(PyBytes::new(self.py, &data).into_any().unbind()) 210 | } 211 | 212 | fn read_struct(&mut self, length: usize) -> PyResult { 213 | let tag = self.read_byte()?; 214 | let mut fields = Vec::with_capacity(length); 215 | for _ in 0..length { 216 | fields.push(self.read()?) 217 | } 218 | let mut bolt_struct = Structure { tag, fields } 219 | .into_pyobject(self.py)? 220 | .into_any() 221 | .unbind(); 222 | let Some(hooks) = &self.hydration_hooks else { 223 | return Ok(bolt_struct); 224 | }; 225 | 226 | let attr = bolt_struct.getattr(self.py, intern!(self.py, "__class__"))?; 227 | if let Some(res) = hooks.get_item(attr)? { 228 | bolt_struct = res 229 | .call(PyTuple::new(self.py, [bolt_struct])?, None)? 230 | .into_any() 231 | .unbind(); 232 | } 233 | 234 | Ok(bolt_struct) 235 | } 236 | 237 | fn read_string_length(&mut self) -> PyResult { 238 | let marker = self.read_byte()?; 239 | let high_nibble = marker & 0xF0; 240 | match high_nibble { 241 | TINY_STRING => Ok((marker & 0x0F) as usize), 242 | STRING_8 => self.read_u8(), 243 | STRING_16 => self.read_u16(), 244 | STRING_32 => self.read_u32(), 245 | _ => Err(PyErr::new::(format!( 246 | "Invalid string length marker: {}", 247 | marker 248 | ))), 249 | } 250 | } 251 | 252 | fn read_byte(&mut self) -> PyResult { 253 | let byte = with_critical_section(&self.bytes, || { 254 | // Safety: 255 | // * We're using a critical section to avoid other threads mutating the bytes while 256 | // we're reading them. 257 | // * We're not mutating the bytes ourselves. 258 | // * We're not interacting with Python while using the bytes as that might indirectly 259 | // cause the bytes to be mutated. 260 | unsafe { self.bytes.as_bytes().get(self.index).copied() } 261 | }) 262 | .ok_or_else(|| PyErr::new::("Nothing to unpack"))?; 263 | self.index += 1; 264 | Ok(byte) 265 | } 266 | 267 | fn read_n_bytes(&mut self) -> PyResult<[u8; N]> { 268 | let to = self.index + N; 269 | with_critical_section(&self.bytes, || { 270 | // Safety: 271 | // * We're using a critical section to avoid other threads mutating the bytes while 272 | // we're reading them. 273 | // * We're not mutating the bytes ourselves. 274 | // * We're not interacting with Python while using the bytes as that might indirectly 275 | // cause the bytes to be mutated. 276 | unsafe { 277 | match self.bytes.as_bytes().get(self.index..to) { 278 | Some(b) => { 279 | self.index = to; 280 | Ok(<[u8; N]>::try_from(b).expect("we know the slice has exactly N values")) 281 | } 282 | None => Err(PyErr::new::("Nothing to unpack")), 283 | } 284 | } 285 | }) 286 | } 287 | 288 | fn read_u8(&mut self) -> PyResult { 289 | self.read_byte().map(Into::into) 290 | } 291 | 292 | fn read_u16(&mut self) -> PyResult { 293 | let data = self.read_n_bytes()?; 294 | Ok(u16::from_be_bytes(data).into()) 295 | } 296 | 297 | fn read_u32(&mut self) -> PyResult { 298 | let data = self.read_n_bytes()?; 299 | u32::from_be_bytes(data).try_into().map_err(|_| { 300 | PyErr::new::( 301 | "Server announced 32 bit sized data. Not supported by this architecture.", 302 | ) 303 | }) 304 | } 305 | 306 | fn read_i8(&mut self) -> PyResult { 307 | self.read_byte().map(|b| i8::from_be_bytes([b])) 308 | } 309 | 310 | fn read_i16(&mut self) -> PyResult { 311 | self.read_n_bytes().map(i16::from_be_bytes) 312 | } 313 | 314 | fn read_i32(&mut self) -> PyResult { 315 | self.read_n_bytes().map(i32::from_be_bytes) 316 | } 317 | 318 | fn read_i64(&mut self) -> PyResult { 319 | self.read_n_bytes().map(i64::from_be_bytes) 320 | } 321 | 322 | fn read_f64(&mut self) -> PyResult { 323 | self.read_n_bytes().map(f64::from_be_bytes) 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /testkit/.dockerignore: -------------------------------------------------------------------------------- 1 | *.py 2 | *.json 3 | -------------------------------------------------------------------------------- /testkit/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | RUN apt-get update && \ 5 | apt-get install -y locales && \ 6 | apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ 7 | localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ 8 | && rm -rf /var/lib/apt/lists/* 9 | ENV LANG=en_US.UTF-8 10 | 11 | # Using apt-get update alone in a RUN statement causes caching issues and subsequent apt-get install instructions fail. 12 | RUN apt-get --quiet update && apt-get --quiet install -y \ 13 | software-properties-common \ 14 | bash \ 15 | python3 \ 16 | python3-pip \ 17 | git \ 18 | curl \ 19 | tar \ 20 | wget \ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | # Install Build Tools 24 | RUN apt-get update && \ 25 | apt-get install -y --no-install-recommends \ 26 | make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ 27 | libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev \ 28 | libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ 29 | ca-certificates && \ 30 | apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 31 | 32 | # Install our own CAs on the image. 33 | # Assumes Linux Debian based image. 34 | COPY CAs/* /usr/local/share/ca-certificates/ 35 | # Store custom CAs somewhere where the backend can find them later. 36 | COPY CustomCAs/* /usr/local/share/custom-ca-certificates/ 37 | RUN update-ca-certificates 38 | 39 | # Install pyenv 40 | RUN git clone https://github.com/pyenv/pyenv.git .pyenv 41 | ENV PYENV_ROOT=/.pyenv 42 | ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" 43 | 44 | # Setup python version 45 | ENV PYTHON_VERSIONS="3.13 3.12 3.11 3.10 3.9 3.8 3.7" 46 | 47 | RUN for version in $PYTHON_VERSIONS; do \ 48 | pyenv install $version; \ 49 | done 50 | RUN pyenv rehash 51 | RUN pyenv global $(pyenv versions --bare --skip-aliases | sort --version-sort --reverse) 52 | 53 | # Install Latest pip and setuptools for each environment 54 | # + tox and tools for starting the tests 55 | # https://pip.pypa.io/en/stable/news/ 56 | RUN for version in $PYTHON_VERSIONS; do \ 57 | python$version -m pip install -U pip && \ 58 | python$version -m pip install -U coverage tox; \ 59 | done 60 | 61 | # Install Rust toolchain 62 | RUN curl https://sh.rustup.rs -sSf | bash -s -- -y 63 | ENV PATH="/root/.cargo/bin:${PATH}" 64 | 65 | # Installing pyarrow lib until pre-built wheel for Python 3.13 exists 66 | # https://github.com/apache/arrow/issues/43519 67 | RUN apt update && \ 68 | apt install -y -V lsb-release cmake gcc && \ 69 | distro_name=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \ 70 | code_name=$(lsb_release --codename --short) && \ 71 | wget https://apache.jfrog.io/artifactory/arrow/${distro_name}/apache-arrow-apt-source-latest-${code_name}.deb && \ 72 | apt install -y -V ./apache-arrow-apt-source-latest-${code_name}.deb && \ 73 | apt update && \ 74 | apt install -y -V libarrow-dev libarrow-dataset-dev libarrow-flight-dev libparquet-dev && \ 75 | apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 76 | ENV PYARROW_WITH_CUDA=off 77 | ENV PYARROW_WITH_GANDIVA=of 78 | -------------------------------------------------------------------------------- /testkit/_common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import os 18 | import subprocess 19 | import sys 20 | 21 | 22 | TEST_BACKEND_VERSION = os.getenv("TEST_BACKEND_VERSION", "python") 23 | 24 | 25 | def run(args, env=None, **kwargs): 26 | print(args) 27 | return subprocess.run( 28 | args, 29 | text=True, 30 | stdout=sys.stdout, 31 | stderr=sys.stderr, 32 | check=True, 33 | env=env, 34 | **kwargs, 35 | ) 36 | 37 | 38 | def run_python(args, env=None, **kwargs): 39 | cmd = [TEST_BACKEND_VERSION, "-u", *args] 40 | run(cmd, env=env, **kwargs) 41 | -------------------------------------------------------------------------------- /testkit/backend.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from pathlib import Path 18 | 19 | from _common import run_python 20 | 21 | 22 | if __name__ == "__main__": 23 | driver_path = (Path(__file__).parents[1] / "driver").absolute() 24 | run_python(["-m", "testkitbackend"], cwd=driver_path) 25 | -------------------------------------------------------------------------------- /testkit/build.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from _common import run_python 18 | 19 | 20 | if __name__ == "__main__": 21 | run_python(["-m", "pip", "install", "-U", "pip"]) 22 | run_python(["-m", "pip", "install", "-Ur", "requirements-dev.txt"]) 23 | -------------------------------------------------------------------------------- /testkit/integration.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /testkit/stress.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /testkit/testkit.json: -------------------------------------------------------------------------------- 1 | { 2 | "testkit": { 3 | "uri": "https://github.com/neo4j-drivers/testkit.git", 4 | "ref": "5.0" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /testkit/unittests.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | from _common import run_python 18 | 19 | 20 | if __name__ == "__main__": 21 | run_python(["-m", "tox", "-vv", "-f", "test"]) 22 | -------------------------------------------------------------------------------- /tests/benchmarks/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import neo4j 18 | 19 | 20 | URL = "neo4j://localhost:7687" 21 | AUTH = ("neo4j", "pass") 22 | 23 | 24 | def test_little_data(benchmark): 25 | def work(): 26 | driver.execute_query("RETURN 1 AS n") 27 | 28 | with neo4j.GraphDatabase.driver(URL, auth=AUTH) as driver: 29 | driver.verify_connectivity() 30 | benchmark.pedantic(work, rounds=5000) 31 | 32 | 33 | def test_import(benchmark): 34 | def work(): 35 | driver.execute_query("RETURN 1 AS n", param=data) 36 | 37 | data = [ 38 | *range(1000), 39 | *( 40 | { 41 | "name": f"Person {i}", 42 | "age": i, 43 | } 44 | for i in range(1000) 45 | ), 46 | f"L{'o' * 10000}ng string", 47 | ] 48 | 49 | with neo4j.GraphDatabase.driver(URL, auth=AUTH) as driver: 50 | driver.verify_connectivity() 51 | benchmark.pedantic(work, rounds=1000) 52 | 53 | 54 | def test_export_single_record(benchmark): 55 | def work(): 56 | driver.execute_query("RETURN [x IN range(0, 100000)] AS x") 57 | 58 | with neo4j.GraphDatabase.driver(URL, auth=AUTH) as driver: 59 | driver.verify_connectivity() 60 | benchmark.pedantic(work, rounds=300) 61 | 62 | 63 | def test_export_many_records(benchmark): 64 | def work(): 65 | driver.execute_query("UNWIND range(0, 1000) AS x RETURN x") 66 | 67 | with neo4j.GraphDatabase.driver(URL, auth=AUTH) as driver: 68 | driver.verify_connectivity() 69 | benchmark.pedantic(work, rounds=150) 70 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | numpy 3 | pandas 4 | pyarrow 5 | -------------------------------------------------------------------------------- /tests/v1/from_driver/test_packstream.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import struct 18 | from io import BytesIO 19 | from math import ( 20 | isnan, 21 | pi, 22 | ) 23 | from uuid import uuid4 24 | 25 | import numpy as np 26 | import pandas as pd 27 | import pytest 28 | 29 | from neo4j._codec.packstream import Structure 30 | from neo4j._codec.packstream.v1 import ( 31 | PackableBuffer, 32 | Packer, 33 | UnpackableBuffer, 34 | Unpacker, 35 | ) 36 | 37 | 38 | standard_ascii = [chr(i) for i in range(128)] 39 | not_ascii = "♥O◘♦♥O◘♦" 40 | 41 | 42 | @pytest.fixture 43 | def packer_with_buffer(): 44 | packable_buffer = Packer.new_packable_buffer() 45 | return Packer(packable_buffer), packable_buffer 46 | 47 | 48 | @pytest.fixture 49 | def unpacker_with_buffer(): 50 | unpackable_buffer = Unpacker.new_unpackable_buffer() 51 | return Unpacker(unpackable_buffer), unpackable_buffer 52 | 53 | 54 | def test_packable_buffer(packer_with_buffer): 55 | packer, packable_buffer = packer_with_buffer 56 | assert isinstance(packable_buffer, PackableBuffer) 57 | assert packable_buffer is packer.stream 58 | 59 | 60 | def test_unpackable_buffer(unpacker_with_buffer): 61 | unpacker, unpackable_buffer = unpacker_with_buffer 62 | assert isinstance(unpackable_buffer, UnpackableBuffer) 63 | assert unpackable_buffer is unpacker.unpackable 64 | 65 | 66 | @pytest.fixture 67 | def pack(packer_with_buffer): 68 | packer, packable_buffer = packer_with_buffer 69 | 70 | def _pack(*values, dehydration_hooks=None): 71 | for value in values: 72 | packer.pack(value, dehydration_hooks=dehydration_hooks) 73 | data = bytearray(packable_buffer.data) 74 | packable_buffer.clear() 75 | return data 76 | 77 | return _pack 78 | 79 | 80 | _default_out_value = object() 81 | 82 | 83 | @pytest.fixture 84 | def assert_packable(packer_with_buffer, unpacker_with_buffer): 85 | def _recursive_nan_equal(a, b): 86 | if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): 87 | return all(_recursive_nan_equal(x, y) for x, y in zip(a, b)) 88 | elif isinstance(a, dict) and isinstance(b, dict): 89 | return all(_recursive_nan_equal(a[k], b[k]) for k in a) 90 | else: 91 | return a == b or (isnan(a) and isnan(b)) 92 | 93 | def _assert(in_value, packed_value, out_value=_default_out_value): 94 | if out_value is _default_out_value: 95 | out_value = in_value 96 | nonlocal packer_with_buffer, unpacker_with_buffer 97 | packer, packable_buffer = packer_with_buffer 98 | unpacker, unpackable_buffer = unpacker_with_buffer 99 | packable_buffer.clear() 100 | unpackable_buffer.reset() 101 | 102 | packer.pack(in_value) 103 | packed_data = packable_buffer.data 104 | assert packed_data == packed_value 105 | 106 | unpackable_buffer.data = bytearray(packed_data) 107 | unpackable_buffer.used = len(packed_data) 108 | unpacked_data = unpacker.unpack() 109 | assert _recursive_nan_equal(unpacked_data, out_value) 110 | 111 | return _assert 112 | 113 | 114 | @pytest.fixture(params=(True, False)) 115 | def np_float_overflow_as_error(request): 116 | should_raise = request.param 117 | if should_raise: 118 | old_err = np.seterr(over="raise") 119 | else: 120 | old_err = np.seterr(over="ignore") 121 | yield 122 | np.seterr(**old_err) 123 | 124 | 125 | @pytest.fixture( 126 | params=( 127 | int, 128 | np.int8, 129 | np.int16, 130 | np.int32, 131 | np.int64, 132 | np.longlong, 133 | np.uint8, 134 | np.uint16, 135 | np.uint32, 136 | np.uint64, 137 | np.ulonglong, 138 | ) 139 | ) 140 | def int_type(request): 141 | if issubclass(request.param, np.number): 142 | 143 | def _int_type(value): 144 | # this avoids deprecation warning from NEP50 and forces 145 | # c-style wrapping of the value 146 | return np.array(value).astype(request.param).item() 147 | 148 | return _int_type 149 | else: 150 | return request.param 151 | 152 | 153 | @pytest.fixture( 154 | params=(float, np.float16, np.float32, np.float64, np.longdouble) 155 | ) 156 | def float_type(request, np_float_overflow_as_error): 157 | return request.param 158 | 159 | 160 | @pytest.fixture(params=(bool, np.bool_)) 161 | def bool_type(request): 162 | return request.param 163 | 164 | 165 | @pytest.fixture(params=(bytes, bytearray, np.bytes_)) 166 | def bytes_type(request): 167 | return request.param 168 | 169 | 170 | @pytest.fixture(params=(str, np.str_)) 171 | def str_type(request): 172 | return request.param 173 | 174 | 175 | @pytest.fixture( 176 | params=(list, tuple, np.array, pd.Series, pd.array, pd.arrays.SparseArray) 177 | ) 178 | def sequence_type(request): 179 | if request.param is pd.Series: 180 | 181 | def constructor(value): 182 | if not value: 183 | return pd.Series(dtype=object) 184 | return pd.Series(value) 185 | 186 | return constructor 187 | return request.param 188 | 189 | 190 | class TestPackStream: 191 | @pytest.mark.parametrize("value", (None, pd.NA)) 192 | def test_none(self, value, assert_packable): 193 | assert_packable(value, b"\xc0", None) 194 | 195 | def test_boolean(self, bool_type, assert_packable): 196 | assert_packable(bool_type(True), b"\xc3") 197 | assert_packable(bool_type(False), b"\xc2") 198 | 199 | @pytest.mark.parametrize("dtype", (bool, pd.BooleanDtype())) 200 | def test_boolean_pandas_series(self, dtype, assert_packable): 201 | value = [True, False] 202 | value_series = pd.Series(value, dtype=dtype) 203 | assert_packable(value_series, b"\x92\xc3\xc2", value) 204 | 205 | def test_negative_tiny_int(self, int_type, assert_packable): 206 | for z in range(-16, 0): 207 | z_typed = int_type(z) 208 | if z != int(z_typed): 209 | continue # not representable 210 | assert_packable(z_typed, bytes(bytearray([z + 0x100]))) 211 | 212 | @pytest.mark.parametrize( 213 | "dtype", 214 | ( 215 | int, 216 | pd.Int8Dtype(), 217 | pd.Int16Dtype(), 218 | pd.Int32Dtype(), 219 | pd.Int64Dtype(), 220 | np.int8, 221 | np.int16, 222 | np.int32, 223 | np.int64, 224 | np.longlong, 225 | ), 226 | ) 227 | def test_negative_tiny_int_pandas_series(self, dtype, assert_packable): 228 | for z in range(-16, 0): 229 | z_typed = pd.Series(z, dtype=dtype) 230 | assert_packable(z_typed, bytes(bytearray([0x91, z + 0x100])), [z]) 231 | 232 | def test_positive_tiny_int(self, int_type, assert_packable): 233 | for z in range(128): 234 | z_typed = int_type(z) 235 | if z != int(z_typed): 236 | continue # not representable 237 | assert_packable(z_typed, bytes(bytearray([z]))) 238 | 239 | def test_negative_int8(self, int_type, assert_packable): 240 | for z in range(-128, -16): 241 | z_typed = int_type(z) 242 | if z != int(z_typed): 243 | continue # not representable 244 | assert_packable(z_typed, bytes(bytearray([0xC8, z + 0x100]))) 245 | 246 | def test_positive_int16(self, int_type, assert_packable): 247 | for z in range(128, 32768): 248 | z_typed = int_type(z) 249 | if z != int(z_typed): 250 | continue # not representable 251 | expected = b"\xc9" + struct.pack(">h", z) 252 | assert_packable(z_typed, expected) 253 | 254 | def test_negative_int16(self, int_type, assert_packable): 255 | for z in range(-32768, -128): 256 | z_typed = int_type(z) 257 | if z != int(z_typed): 258 | continue # not representable 259 | expected = b"\xc9" + struct.pack(">h", z) 260 | assert_packable(z_typed, expected) 261 | 262 | def test_positive_int32(self, int_type, assert_packable): 263 | for e in range(15, 31): 264 | z = 2**e 265 | z_typed = int_type(z) 266 | if z != int(z_typed): 267 | continue # not representable 268 | expected = b"\xca" + struct.pack(">i", z) 269 | assert_packable(z_typed, expected) 270 | 271 | def test_negative_int32(self, int_type, assert_packable): 272 | for e in range(15, 31): 273 | z = -(2**e + 1) 274 | z_typed = int_type(z) 275 | if z != int(z_typed): 276 | continue # not representable 277 | expected = b"\xca" + struct.pack(">i", z) 278 | assert_packable(z_typed, expected) 279 | 280 | def test_positive_int64(self, int_type, assert_packable): 281 | for e in range(31, 63): 282 | z = 2**e 283 | z_typed = int_type(z) 284 | if z != int(z_typed): 285 | continue # not representable 286 | expected = b"\xcb" + struct.pack(">q", z) 287 | assert_packable(z_typed, expected) 288 | 289 | @pytest.mark.parametrize( 290 | "dtype", 291 | ( 292 | int, 293 | pd.Int64Dtype(), 294 | pd.UInt64Dtype(), 295 | np.int64, 296 | np.longlong, 297 | np.uint64, 298 | np.ulonglong, 299 | ), 300 | ) 301 | def test_positive_int64_pandas_series(self, dtype, assert_packable): 302 | for e in range(31, 63): 303 | z = 2**e 304 | z_typed = pd.Series(z, dtype=dtype) 305 | expected = b"\x91\xcb" + struct.pack(">q", z) 306 | assert_packable(z_typed, expected, [z]) 307 | 308 | def test_negative_int64(self, int_type, assert_packable): 309 | for e in range(31, 63): 310 | z = -(2**e + 1) 311 | z_typed = int_type(z) 312 | if z != int(z_typed): 313 | continue # not representable 314 | expected = b"\xcb" + struct.pack(">q", z) 315 | assert_packable(z_typed, expected) 316 | 317 | @pytest.mark.parametrize( 318 | "dtype", 319 | ( 320 | int, 321 | pd.Int64Dtype(), 322 | np.int64, 323 | np.longlong, 324 | ), 325 | ) 326 | def test_negative_int64_pandas_series(self, dtype, assert_packable): 327 | for e in range(31, 63): 328 | z = -(2**e + 1) 329 | z_typed = pd.Series(z, dtype=dtype) 330 | expected = b"\x91\xcb" + struct.pack(">q", z) 331 | assert_packable(z_typed, expected, [z]) 332 | 333 | def test_integer_positive_overflow(self, int_type, pack, assert_packable): 334 | with pytest.raises(OverflowError): 335 | z = 2**63 + 1 336 | z_typed = int_type(z) 337 | if z != int(z_typed): 338 | pytest.skip("not representable") 339 | pack(z_typed) 340 | 341 | def test_integer_negative_overflow(self, int_type, pack, assert_packable): 342 | with pytest.raises(OverflowError): 343 | z = -(2**63) - 1 344 | z_typed = int_type(z) 345 | if z != int(z_typed): 346 | pytest.skip("not representable") 347 | pack(z_typed) 348 | 349 | def test_float(self, float_type, assert_packable): 350 | for z in ( 351 | 0.0, 352 | -0.0, 353 | pi, 354 | 2 * pi, 355 | float("inf"), 356 | float("-inf"), 357 | float("nan"), 358 | *(float(2**e) + 0.5 for e in range(100)), 359 | *(-float(2**e) + 0.5 for e in range(100)), 360 | ): 361 | try: 362 | z_typed = float_type(z) 363 | except FloatingPointError: 364 | continue # not representable 365 | expected = b"\xc1" + struct.pack(">d", float(z_typed)) 366 | assert_packable(z_typed, expected) 367 | 368 | @pytest.mark.parametrize( 369 | "dtype", 370 | ( 371 | float, 372 | pd.Float32Dtype(), 373 | pd.Float64Dtype(), 374 | np.float16, 375 | np.float32, 376 | np.float64, 377 | np.longdouble, 378 | ), 379 | ) 380 | def test_float_pandas_series( 381 | self, dtype, np_float_overflow_as_error, assert_packable 382 | ): 383 | for z in ( 384 | 0.0, 385 | -0.0, 386 | pi, 387 | 2 * pi, 388 | float("inf"), 389 | float("-inf"), 390 | float("nan"), 391 | *(float(2**e) + 0.5 for e in range(100)), 392 | *(-float(2**e) + 0.5 for e in range(100)), 393 | ): 394 | try: 395 | z_typed = pd.Series(z, dtype=dtype) 396 | except FloatingPointError: 397 | continue # not representable 398 | if z_typed[0] is pd.NA: 399 | expected_bytes = b"\x91\xc0" # encoded as NULL 400 | expected_value = [None] 401 | else: 402 | expected_bytes = b"\x91\xc1" + struct.pack( 403 | ">d", float(z_typed[0]) 404 | ) 405 | expected_value = [float(z_typed[0])] 406 | assert_packable(z_typed, expected_bytes, expected_value) 407 | 408 | def test_empty_bytes(self, bytes_type, assert_packable): 409 | b = bytes_type(b"") 410 | assert_packable(b, b"\xcc\x00") 411 | 412 | def test_bytes_8(self, bytes_type, assert_packable): 413 | b = bytes_type(b"hello") 414 | assert_packable(b, b"\xcc\x05hello") 415 | 416 | def test_bytes_16(self, bytes_type, assert_packable): 417 | b = bytearray(40000) 418 | b_typed = bytes_type(b) 419 | assert_packable(b_typed, b"\xcd\x9c\x40" + b) 420 | 421 | def test_bytes_32(self, bytes_type, assert_packable): 422 | b = bytearray(80000) 423 | b_typed = bytes_type(b) 424 | assert_packable(b_typed, b"\xce\x00\x01\x38\x80" + b) 425 | 426 | def test_bytes_pandas_series(self, assert_packable): 427 | for b, header in ( 428 | (b"", b"\xcc\x00"), 429 | (b"hello", b"\xcc\x05"), 430 | (bytearray(40000), b"\xcd\x9c\x40"), 431 | (bytearray(80000), b"\xce\x00\x01\x38\x80"), 432 | ): 433 | b_typed = pd.Series([b]) 434 | assert_packable(b_typed, b"\x91" + header + b, [b]) 435 | 436 | def test_bytearray_size_overflow(self, bytes_type, assert_packable): 437 | stream_out = BytesIO() 438 | packer = Packer(stream_out) 439 | with pytest.raises(OverflowError): 440 | packer._pack_bytes_header(2**32) 441 | 442 | def test_empty_string(self, str_type, assert_packable): 443 | assert_packable(str_type(""), b"\x80") 444 | 445 | def test_tiny_strings(self, str_type, assert_packable): 446 | for size in range(0x10): 447 | s = str_type("A" * size) 448 | assert_packable(s, bytes(bytearray([0x80 + size]) + (b"A" * size))) 449 | 450 | def test_string_8(self, str_type, assert_packable): 451 | t = "A" * 40 452 | b = t.encode("utf-8") 453 | t_typed = str_type(t) 454 | assert_packable(t_typed, b"\xd0\x28" + b) 455 | 456 | def test_string_16(self, str_type, assert_packable): 457 | t = "A" * 40000 458 | b = t.encode("utf-8") 459 | t_typed = str_type(t) 460 | assert_packable(t_typed, b"\xd1\x9c\x40" + b) 461 | 462 | def test_string_32(self, str_type, assert_packable): 463 | t = "A" * 80000 464 | b = t.encode("utf-8") 465 | t_typed = str_type(t) 466 | assert_packable(t_typed, b"\xd2\x00\x01\x38\x80" + b) 467 | 468 | def test_unicode_string(self, str_type, assert_packable): 469 | t = "héllö" 470 | b = t.encode("utf-8") 471 | t_typed = str_type(t) 472 | assert_packable(t_typed, bytes(bytearray([0x80 + len(b)])) + b) 473 | 474 | @pytest.mark.parametrize( 475 | "dtype", 476 | ( 477 | str, 478 | np.str_, 479 | pd.StringDtype("python"), 480 | pd.StringDtype("pyarrow"), 481 | ), 482 | ) 483 | def test_string_pandas_series(self, dtype, assert_packable): 484 | values = ( 485 | ("", b"\x80"), 486 | ("A" * 40, b"\xd0\x28"), 487 | ("A" * 40000, b"\xd1\x9c\x40"), 488 | ("A" * 80000, b"\xd2\x00\x01\x38\x80"), 489 | ) 490 | for t, header in values: 491 | t_typed = pd.Series([t], dtype=dtype) 492 | assert_packable(t_typed, b"\x91" + header + t.encode("utf-8"), [t]) 493 | 494 | t_typed = pd.Series([t for t, _ in values], dtype=dtype) 495 | expected = bytes([0x90 + len(values)]) + b"".join( 496 | header + t.encode("utf-8") for t, header in values 497 | ) 498 | assert_packable(t_typed, expected, [t for t, _ in values]) 499 | 500 | def test_string_size_overflow(self): 501 | stream_out = BytesIO() 502 | packer = Packer(stream_out) 503 | with pytest.raises(OverflowError): 504 | packer._pack_string_header(2**32) 505 | 506 | def test_empty_list(self, sequence_type, assert_packable): 507 | list_ = [] 508 | list_typed = sequence_type(list_) 509 | assert_packable(list_typed, b"\x90", list_) 510 | 511 | def test_tiny_lists(self, sequence_type, assert_packable): 512 | for size in range(0x10): 513 | nums = [1] * size 514 | nums_typed = sequence_type(nums) 515 | data_out = bytearray([0x90 + size]) + bytearray([1] * size) 516 | assert_packable(nums_typed, bytes(data_out), nums) 517 | 518 | def test_list_8(self, sequence_type, assert_packable): 519 | nums = [1] * 40 520 | nums_typed = sequence_type(nums) 521 | assert_packable(nums_typed, b"\xd4\x28" + (b"\x01" * 40), nums) 522 | 523 | def test_list_16(self, sequence_type, assert_packable): 524 | nums = [1] * 40000 525 | nums_typed = sequence_type(nums) 526 | assert_packable(nums_typed, b"\xd5\x9c\x40" + (b"\x01" * 40000), nums) 527 | 528 | def test_list_32(self, sequence_type, assert_packable): 529 | nums = [1] * 80000 530 | nums_typed = sequence_type(nums) 531 | assert_packable( 532 | nums_typed, b"\xd6\x00\x01\x38\x80" + (b"\x01" * 80000), nums 533 | ) 534 | 535 | def test_nested_lists(self, sequence_type, assert_packable): 536 | list_ = [[[]]] 537 | l_typed = sequence_type([sequence_type([sequence_type([])])]) 538 | assert_packable(l_typed, b"\x91\x91\x90", list_) 539 | 540 | @pytest.mark.parametrize("as_series", (True, False)) 541 | def test_list_pandas_categorical(self, as_series, pack, assert_packable): 542 | animals = ["cat", "dog", "cat", "cat", "dog", "horse"] 543 | animals_typed = pd.Categorical(animals) 544 | if as_series: 545 | animals_typed = pd.Series(animals_typed) 546 | b = b"".join([b"\x96", *(pack(e) for e in animals)]) 547 | assert_packable(animals_typed, b, animals) 548 | 549 | def test_list_size_overflow(self): 550 | stream_out = BytesIO() 551 | packer = Packer(stream_out) 552 | with pytest.raises(OverflowError): 553 | packer._pack_list_header(2**32) 554 | 555 | def test_empty_map(self, assert_packable): 556 | assert_packable({}, b"\xa0") 557 | 558 | @pytest.mark.parametrize("size", range(0x10)) 559 | def test_tiny_maps(self, assert_packable, size): 560 | data_in = {} 561 | data_out = bytearray([0xA0 + size]) 562 | for el in range(1, size + 1): 563 | data_in[chr(64 + el)] = el 564 | data_out += bytearray([0x81, 64 + el, el]) 565 | assert_packable(data_in, bytes(data_out)) 566 | 567 | def test_map_8(self, pack, assert_packable): 568 | d = {f"A{i}": 1 for i in range(40)} 569 | b = b"".join(pack(f"A{i}", 1) for i in range(40)) 570 | assert_packable(d, b"\xd8\x28" + b) 571 | 572 | def test_map_16(self, pack, assert_packable): 573 | d = {f"A{i}": 1 for i in range(40000)} 574 | b = b"".join(pack(f"A{i}", 1) for i in range(40000)) 575 | assert_packable(d, b"\xd9\x9c\x40" + b) 576 | 577 | def test_map_32(self, pack, assert_packable): 578 | d = {f"A{i}": 1 for i in range(80000)} 579 | b = b"".join(pack(f"A{i}", 1) for i in range(80000)) 580 | assert_packable(d, b"\xda\x00\x01\x38\x80" + b) 581 | 582 | def test_empty_dataframe_maps(self, assert_packable): 583 | df = pd.DataFrame() 584 | assert_packable(df, b"\xa0", {}) 585 | 586 | @pytest.mark.parametrize("size", range(0x10)) 587 | def test_tiny_dataframes_maps(self, assert_packable, size): 588 | data_in = {} 589 | data_out = bytearray([0xA0 + size]) 590 | for el in range(1, size + 1): 591 | data_in[chr(64 + el)] = [el] 592 | data_out += bytearray([0x81, 64 + el, 0x91, el]) 593 | data_in_typed = pd.DataFrame(data_in) 594 | assert_packable(data_in_typed, bytes(data_out), data_in) 595 | 596 | def test_map_size_overflow(self): 597 | stream_out = BytesIO() 598 | packer = Packer(stream_out) 599 | with pytest.raises(OverflowError): 600 | packer._pack_map_header(2**32) 601 | 602 | @pytest.mark.parametrize( 603 | ("map_", "exc_type"), 604 | ( 605 | ({1: "1"}, TypeError), 606 | (pd.DataFrame({1: ["1"]}), TypeError), 607 | (pd.DataFrame({(1, 2): ["1"]}), TypeError), 608 | ({"x": {1: "eins", 2: "zwei", 3: "drei"}}, TypeError), 609 | ({"x": {(1, 2): "1+2i", (2, 0): "2"}}, TypeError), 610 | ), 611 | ) 612 | def test_map_key_type(self, packer_with_buffer, map_, exc_type): 613 | # maps must have string keys 614 | packer, _packable_buffer = packer_with_buffer 615 | with pytest.raises(exc_type, match="strings"): 616 | packer._pack(map_) 617 | 618 | def test_illegal_signature(self, assert_packable): 619 | with pytest.raises(ValueError): 620 | assert_packable(Structure(b"XXX"), b"\xb0XXX") 621 | 622 | def test_empty_struct(self, assert_packable): 623 | assert_packable(Structure(b"X"), b"\xb0X") 624 | 625 | def test_tiny_structs(self, assert_packable): 626 | for size in range(0x10): 627 | fields = [1] * size 628 | data_in = Structure(b"A", *fields) 629 | data_out = bytearray((0xB0 + size, 0x41, *fields)) 630 | assert_packable(data_in, bytes(data_out)) 631 | 632 | def test_struct_size_overflow(self, pack): 633 | with pytest.raises(OverflowError): 634 | fields = [1] * 16 635 | pack(Structure(b"X", *fields)) 636 | 637 | def test_illegal_uuid(self, assert_packable): 638 | with pytest.raises(ValueError): 639 | assert_packable(uuid4(), b"\xb0XXX") 640 | -------------------------------------------------------------------------------- /tests/v1/test_injection.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) "Neo4j" 2 | # Neo4j Sweden AB [https://neo4j.com] 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | import importlib 18 | import sys 19 | import traceback 20 | 21 | import pytest 22 | 23 | from neo4j._codec.hydration import DehydrationHooks 24 | from neo4j._codec.packstream import Structure 25 | from neo4j._codec.packstream.v1 import ( 26 | Packer, 27 | Unpacker, 28 | ) 29 | 30 | 31 | @pytest.fixture 32 | def packer_with_buffer(): 33 | packable_buffer = Packer.new_packable_buffer() 34 | return Packer(packable_buffer), packable_buffer 35 | 36 | 37 | @pytest.fixture 38 | def unpacker_with_buffer(): 39 | unpackable_buffer = Unpacker.new_unpackable_buffer() 40 | return Unpacker(unpackable_buffer), unpackable_buffer 41 | 42 | 43 | def test_pack_injection_works(packer_with_buffer): 44 | class TestClass: 45 | pass 46 | 47 | class TestError(Exception): 48 | pass 49 | 50 | def raise_test_exception(*args, **kwargs): 51 | raise TestError 52 | 53 | dehydration_hooks = DehydrationHooks( 54 | exact_types={TestClass: raise_test_exception}, 55 | subtypes={}, 56 | ) 57 | test_object = TestClass() 58 | packer, _ = packer_with_buffer 59 | 60 | with pytest.raises(TestError) as exc: 61 | packer.pack(test_object, dehydration_hooks=dehydration_hooks) 62 | 63 | # printing the traceback to stdout to make it easier to debug 64 | traceback.print_exception(exc.type, exc.value, exc.tb, file=sys.stdout) 65 | 66 | assert any("_rust_pack" in str(entry.statement) for entry in exc.traceback) 67 | assert not any( 68 | "_py_pack" in str(entry.statement) for entry in exc.traceback 69 | ) 70 | 71 | 72 | def test_unpack_injection_works(unpacker_with_buffer): 73 | class TestError(Exception): 74 | pass 75 | 76 | def raise_test_exception(*args, **kwargs): 77 | raise TestError 78 | 79 | hydration_hooks = {Structure: raise_test_exception} 80 | unpacker, buffer = unpacker_with_buffer 81 | 82 | buffer.reset() 83 | buffer.data = bytearray(b"\xb0\xff") 84 | 85 | with pytest.raises(TestError) as exc: 86 | unpacker.unpack(hydration_hooks) 87 | 88 | # printing the traceback to stdout to make it easier to debug 89 | traceback.print_exception(exc.type, exc.value, exc.tb, file=sys.stdout) 90 | 91 | assert any( 92 | "_rust_unpack" in str(entry.statement) for entry in exc.traceback 93 | ) 94 | assert not any( 95 | "_py_unpack" in str(entry.statement) for entry in exc.traceback 96 | ) 97 | 98 | 99 | @pytest.mark.parametrize( 100 | ("name", "package_names"), 101 | ( 102 | ("neo4j._codec.packstream._rust.v1", ()), 103 | ("neo4j._codec.packstream._rust", ("v1",)), 104 | ("neo4j._codec.packstream", ("_rust",)), 105 | ), 106 | ) 107 | def test_import_module(name, package_names): 108 | module = importlib.import_module(name) 109 | 110 | assert module.__name__ == name 111 | 112 | for package_name in package_names: 113 | package = getattr(module, package_name) 114 | assert package.__name__ == f"{name}.{package_name}" 115 | 116 | 117 | def test_rust_struct_access(): 118 | tag = b"F" 119 | fields = ["foo", False, 42, 3.14, b"bar"] 120 | struct = Structure(tag, *fields) 121 | 122 | assert struct.tag == tag 123 | assert isinstance(struct.tag, bytes) 124 | assert struct.fields == tuple(fields) 125 | 126 | 127 | def test_rust_struct_equal(): 128 | struct1 = Structure(b"F", "foo", False, 42, 3.14, b"bar") 129 | struct2 = Structure(b"F", "foo", False, 42, 3.14, b"bar") 130 | assert struct1 == struct2 131 | # [noqa] for testing correctness of equality 132 | assert not struct1 != struct2 # noqa: SIM202 133 | 134 | 135 | @pytest.mark.parametrize( 136 | "args", 137 | ( 138 | (b"F", "foo", True, 42, 3.14, b"bar"), 139 | (b"f", "foo", False, 42, 3.14, b"baz"), 140 | ), 141 | ) 142 | def test_rust_struct_not_equal(args): 143 | struct1 = Structure(b"F", "foo", False, 42, 3.14, b"bar") 144 | struct2 = Structure(*args) 145 | assert struct1 != struct2 146 | # [noqa] for testing correctness of equality 147 | assert not struct1 == struct2 # noqa: SIM201 148 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{37,38,39,310,311,312,313}-{test} 3 | # for Python 3.7 support (https://github.com/tox-dev/tox/issues/3416#issuecomment-2426989929) 4 | requires = virtualenv<20.22.0 5 | 6 | [testenv] 7 | deps = -r requirements-dev.txt 8 | commands = 9 | test: python -m pytest -v --benchmark-skip {posargs} tests 10 | --------------------------------------------------------------------------------