├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── pdm.lock ├── pymetastore ├── __init__.py ├── hive_metastore │ ├── ThriftHiveMetastore-remote │ ├── ThriftHiveMetastore.py │ ├── __init__.py │ ├── constants.py │ └── ttypes.py ├── htypes.py ├── metastore.py └── stats.py ├── pyproject.toml └── tests ├── integration └── test_metastore.py └── unit └── test_htypes.py /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | pull_request: 8 | branches: [ main ] 9 | 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | 14 | services: 15 | hive-metastore: 16 | image: ghcr.io/recap-build/hive-metastore-standalone:latest 17 | ports: 18 | - 9083:9083 19 | 20 | strategy: 21 | matrix: 22 | python-version: ['3.8'] 23 | 24 | steps: 25 | - uses: actions/checkout@v2 26 | 27 | - name: Build using Python ${{ matrix.python-version }} 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | 32 | - name: Install dependencies 33 | run: | 34 | python3.10 -m venv venv 35 | source venv/bin/activate 36 | pip install pdm 37 | pdm install 38 | 39 | - name: Run black 40 | run: | 41 | source venv/bin/activate 42 | pdm run black --check --diff pymetastore/ tests/ 43 | 44 | - name: Run isort 45 | run: | 46 | source venv/bin/activate 47 | pdm run isort pymetastore/ tests --check-only --diff 48 | 49 | - name: Run pylint 50 | run: | 51 | source venv/bin/activate 52 | pdm run pylint pymetastore/ tests/ 53 | 54 | - name: Run pyright 55 | run: | 56 | source venv/bin/activate 57 | pdm run pyright 58 | 59 | - name: Run unit tests 60 | run: | 61 | source venv/bin/activate 62 | pdm run pytest tests/unit 63 | 64 | - name: Run integration tests 65 | run: | 66 | source venv/bin/activate 67 | pdm run pytest tests/integration 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | .pdm-python 162 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/hive-thrift"] 2 | path = vendor/hive-thrift 3 | url = https://github.com/trinodb/hive-thrift.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 recap-build 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pymetastore 🐝 🐍 2 | 3 | `pymetastore` is a Python client for Hive Metastore. 4 | 5 | ## Features 6 | * Python-friendly interface for Hive Metastore 7 | * Comprehensive support for metadata operations 8 | * Fully compatible with the Hive metastore service over Thrift protocol 9 | 10 | ## Installation 11 | 12 | Install pymetastore with pip: 13 | 14 | ```bash 15 | pip install pymetastore 16 | ``` 17 | 18 | ## Quick Start 19 | 20 | Here's a taste of using `pymetastore` to connect to Hive Metastore and interact with metadata: 21 | 22 | ```python 23 | from pymetastore.metastore import HMS 24 | 25 | with HMS.create(host="localhost", port=9083) as hms: 26 | databases = hms.list_databases() 27 | database = hms.get_database(name="test_db") 28 | tables = hms.list_tables(database_name=database.name) 29 | table = hms.get_table( 30 | database_name=database.name, 31 | table_name=tables[0], 32 | ) 33 | partitions = hms.list_partitions( 34 | database_name=database.name, 35 | table_name=table.name, 36 | ) 37 | partition = hms.get_partition( 38 | database_name=database.name, 39 | table_name=table.name, 40 | partition_name="partition=1", 41 | ) 42 | ``` 43 | -------------------------------------------------------------------------------- /pdm.lock: -------------------------------------------------------------------------------- 1 | # This file is @generated by PDM. 2 | # It is not intended for manual editing. 3 | 4 | [[package]] 5 | name = "astroid" 6 | version = "2.15.5" 7 | requires_python = ">=3.7.2" 8 | summary = "An abstract syntax tree for Python with inference support." 9 | dependencies = [ 10 | "lazy-object-proxy>=1.4.0", 11 | "typing-extensions>=4.0.0; python_version < \"3.11\"", 12 | "wrapt<2,>=1.11; python_version < \"3.11\"", 13 | "wrapt<2,>=1.14; python_version >= \"3.11\"", 14 | ] 15 | 16 | [[package]] 17 | name = "black" 18 | version = "23.3.0" 19 | requires_python = ">=3.7" 20 | summary = "The uncompromising code formatter." 21 | dependencies = [ 22 | "click>=8.0.0", 23 | "mypy-extensions>=0.4.3", 24 | "packaging>=22.0", 25 | "pathspec>=0.9.0", 26 | "platformdirs>=2", 27 | "tomli>=1.1.0; python_version < \"3.11\"", 28 | "typing-extensions>=3.10.0.0; python_version < \"3.10\"", 29 | ] 30 | 31 | [[package]] 32 | name = "click" 33 | version = "8.1.3" 34 | requires_python = ">=3.7" 35 | summary = "Composable command line interface toolkit" 36 | dependencies = [ 37 | "colorama; platform_system == \"Windows\"", 38 | ] 39 | 40 | [[package]] 41 | name = "colorama" 42 | version = "0.4.6" 43 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" 44 | summary = "Cross-platform colored terminal text." 45 | 46 | [[package]] 47 | name = "dill" 48 | version = "0.3.6" 49 | requires_python = ">=3.7" 50 | summary = "serialize all of python" 51 | 52 | [[package]] 53 | name = "exceptiongroup" 54 | version = "1.1.1" 55 | requires_python = ">=3.7" 56 | summary = "Backport of PEP 654 (exception groups)" 57 | 58 | [[package]] 59 | name = "iniconfig" 60 | version = "2.0.0" 61 | requires_python = ">=3.7" 62 | summary = "brain-dead simple config-ini parsing" 63 | 64 | [[package]] 65 | name = "isort" 66 | version = "5.12.0" 67 | requires_python = ">=3.8.0" 68 | summary = "A Python utility / library to sort Python imports." 69 | 70 | [[package]] 71 | name = "lazy-object-proxy" 72 | version = "1.9.0" 73 | requires_python = ">=3.7" 74 | summary = "A fast and thorough lazy object proxy." 75 | 76 | [[package]] 77 | name = "mccabe" 78 | version = "0.7.0" 79 | requires_python = ">=3.6" 80 | summary = "McCabe checker, plugin for flake8" 81 | 82 | [[package]] 83 | name = "mypy-extensions" 84 | version = "1.0.0" 85 | requires_python = ">=3.5" 86 | summary = "Type system extensions for programs checked with the mypy type checker." 87 | 88 | [[package]] 89 | name = "nodeenv" 90 | version = "1.8.0" 91 | requires_python = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" 92 | summary = "Node.js virtual environment builder" 93 | dependencies = [ 94 | "setuptools", 95 | ] 96 | 97 | [[package]] 98 | name = "packaging" 99 | version = "23.1" 100 | requires_python = ">=3.7" 101 | summary = "Core utilities for Python packages" 102 | 103 | [[package]] 104 | name = "pathspec" 105 | version = "0.11.1" 106 | requires_python = ">=3.7" 107 | summary = "Utility library for gitignore style pattern matching of file paths." 108 | 109 | [[package]] 110 | name = "platformdirs" 111 | version = "3.7.0" 112 | requires_python = ">=3.7" 113 | summary = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." 114 | 115 | [[package]] 116 | name = "pluggy" 117 | version = "1.0.0" 118 | requires_python = ">=3.6" 119 | summary = "plugin and hook calling mechanisms for python" 120 | 121 | [[package]] 122 | name = "pylint" 123 | version = "2.17.4" 124 | requires_python = ">=3.7.2" 125 | summary = "python code static checker" 126 | dependencies = [ 127 | "astroid<=2.17.0-dev0,>=2.15.4", 128 | "colorama>=0.4.5; sys_platform == \"win32\"", 129 | "dill>=0.2; python_version < \"3.11\"", 130 | "dill>=0.3.6; python_version >= \"3.11\"", 131 | "isort<6,>=4.2.5", 132 | "mccabe<0.8,>=0.6", 133 | "platformdirs>=2.2.0", 134 | "tomli>=1.1.0; python_version < \"3.11\"", 135 | "tomlkit>=0.10.1", 136 | "typing-extensions>=3.10.0; python_version < \"3.10\"", 137 | ] 138 | 139 | [[package]] 140 | name = "pyright" 141 | version = "1.1.315" 142 | requires_python = ">=3.7" 143 | summary = "Command line wrapper for pyright" 144 | dependencies = [ 145 | "nodeenv>=1.6.0", 146 | ] 147 | 148 | [[package]] 149 | name = "pytest" 150 | version = "7.3.2" 151 | requires_python = ">=3.7" 152 | summary = "pytest: simple powerful testing with Python" 153 | dependencies = [ 154 | "colorama; sys_platform == \"win32\"", 155 | "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", 156 | "iniconfig", 157 | "packaging", 158 | "pluggy<2.0,>=0.12", 159 | "tomli>=1.0.0; python_version < \"3.11\"", 160 | ] 161 | 162 | [[package]] 163 | name = "setuptools" 164 | version = "68.0.0" 165 | requires_python = ">=3.7" 166 | summary = "Easily download, build, install, upgrade, and uninstall Python packages" 167 | 168 | [[package]] 169 | name = "six" 170 | version = "1.16.0" 171 | requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" 172 | summary = "Python 2 and 3 compatibility utilities" 173 | 174 | [[package]] 175 | name = "thrift" 176 | version = "0.16.0" 177 | summary = "Python bindings for the Apache Thrift RPC system" 178 | dependencies = [ 179 | "six>=1.7.2", 180 | ] 181 | 182 | [[package]] 183 | name = "tomli" 184 | version = "2.0.1" 185 | requires_python = ">=3.7" 186 | summary = "A lil' TOML parser" 187 | 188 | [[package]] 189 | name = "tomlkit" 190 | version = "0.11.8" 191 | requires_python = ">=3.7" 192 | summary = "Style preserving TOML library" 193 | 194 | [[package]] 195 | name = "typing-extensions" 196 | version = "4.6.3" 197 | requires_python = ">=3.7" 198 | summary = "Backported and Experimental Type Hints for Python 3.7+" 199 | 200 | [[package]] 201 | name = "wrapt" 202 | version = "1.15.0" 203 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" 204 | summary = "Module for decorators, wrappers and monkey patching." 205 | 206 | [metadata] 207 | lock_version = "4.2" 208 | cross_platform = true 209 | groups = ["default", "style", "test"] 210 | content_hash = "sha256:1258303b9afc6149cea49c7071c37a1753b3ef6d6a8fc017b14130079af56f85" 211 | 212 | [metadata.files] 213 | "astroid 2.15.5" = [ 214 | {url = "https://files.pythonhosted.org/packages/6f/51/868921f570a1ad2ddefd04594e1f95aacd6208c85f6b0ab75401acf65cfb/astroid-2.15.5-py3-none-any.whl", hash = "sha256:078e5212f9885fa85fbb0cf0101978a336190aadea6e13305409d099f71b2324"}, 215 | {url = "https://files.pythonhosted.org/packages/e9/8d/338debdfc65383c2e1a0eaf336810ced0e8bc58a3f64d8f957cfb7b19e84/astroid-2.15.5.tar.gz", hash = "sha256:1039262575027b441137ab4a62a793a9b43defb42c32d5670f38686207cd780f"}, 216 | ] 217 | "black 23.3.0" = [ 218 | {url = "https://files.pythonhosted.org/packages/06/1e/273d610249f0335afb1ddb03664a03223f4826e3d1a95170a0142cb19fb4/black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, 219 | {url = "https://files.pythonhosted.org/packages/12/4b/99c71d1cf1353edd5aff2700b8960f92e9b805c9dab72639b67dbb449d3a/black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, 220 | {url = "https://files.pythonhosted.org/packages/13/0a/ed8b66c299e896780e4528eed4018f5b084da3b9ba4ee48328550567d866/black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, 221 | {url = "https://files.pythonhosted.org/packages/13/25/cfa06788d0a936f2445af88f13604b5bcd5c9d050db618c718e6ebe66f74/black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, 222 | {url = "https://files.pythonhosted.org/packages/21/14/d5a2bec5fb15f9118baab7123d344646fac0b1c6939d51c2b05259cd2d9c/black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, 223 | {url = "https://files.pythonhosted.org/packages/24/eb/2d2d2c27cb64cfd073896f62a952a802cd83cf943a692a2f278525b57ca9/black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, 224 | {url = "https://files.pythonhosted.org/packages/27/70/07aab2623cfd3789786f17e051487a41d5657258c7b1ef8f780512ffea9c/black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, 225 | {url = "https://files.pythonhosted.org/packages/29/b1/b584fc863c155653963039664a592b3327b002405043b7e761b9b0212337/black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, 226 | {url = "https://files.pythonhosted.org/packages/3c/d7/85f3d79f9e543402de2244c4d117793f262149e404ea0168841613c33e07/black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, 227 | {url = "https://files.pythonhosted.org/packages/3f/0d/81dd4194ce7057c199d4f28e4c2a885082d9d929e7a55c514b23784f7787/black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, 228 | {url = "https://files.pythonhosted.org/packages/49/36/15d2122f90ff1cd70f06892ebda777b650218cf84b56b5916a993dc1359a/black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, 229 | {url = "https://files.pythonhosted.org/packages/49/d7/f3b7da6c772800f5375aeb050a3dcf682f0bbeb41d313c9c2820d0156e4e/black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, 230 | {url = "https://files.pythonhosted.org/packages/69/49/7e1f0cf585b0d607aad3f971f95982cc4208fc77f92363d632d23021ee57/black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, 231 | {url = "https://files.pythonhosted.org/packages/6d/b4/0f13ab7f5e364795ff82b76b0f9a4c9c50afda6f1e2feeb8b03fdd7ec57d/black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, 232 | {url = "https://files.pythonhosted.org/packages/ad/e7/4642b7f462381799393fbad894ba4b32db00870a797f0616c197b07129a9/black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, 233 | {url = "https://files.pythonhosted.org/packages/c0/53/42e312c17cfda5c8fc4b6b396a508218807a3fcbb963b318e49d3ddd11d5/black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, 234 | {url = "https://files.pythonhosted.org/packages/ca/44/eb41edd3f558a6139f09eee052dead4a7a464e563b822ddf236f5a8ee286/black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, 235 | {url = "https://files.pythonhosted.org/packages/ce/f4/2b0c6ac9e1f8584296747f66dd511898b4ebd51d6510dba118279bff53b6/black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, 236 | {url = "https://files.pythonhosted.org/packages/d1/6e/5810b6992ed70403124c67e8b3f62858a32b35405177553f1a78ed6b6e31/black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, 237 | {url = "https://files.pythonhosted.org/packages/d6/36/66370f5017b100225ec4950a60caeef60201a10080da57ddb24124453fba/black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, 238 | {url = "https://files.pythonhosted.org/packages/d7/6f/d3832960a3b646b333b7f0d80d336a3c123012e9d9d5dba4a622b2b6181d/black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, 239 | {url = "https://files.pythonhosted.org/packages/db/f4/7908f71cc71da08df1317a3619f002cbf91927fb5d3ffc7723905a2113f7/black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, 240 | {url = "https://files.pythonhosted.org/packages/de/b4/76f152c5eb0be5471c22cd18380d31d188930377a1a57969073b89d6615d/black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, 241 | {url = "https://files.pythonhosted.org/packages/eb/a5/17b40bfd9b607b69fa726b0b3a473d14b093dcd5191ea1a1dd664eccfee3/black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, 242 | {url = "https://files.pythonhosted.org/packages/fd/5b/fc2d7922c1a6bb49458d424b5be71d251f2d0dc97be9534e35d171bdc653/black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, 243 | ] 244 | "click 8.1.3" = [ 245 | {url = "https://files.pythonhosted.org/packages/59/87/84326af34517fca8c58418d148f2403df25303e02736832403587318e9e8/click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, 246 | {url = "https://files.pythonhosted.org/packages/c2/f1/df59e28c642d583f7dacffb1e0965d0e00b218e0186d7858ac5233dce840/click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, 247 | ] 248 | "colorama 0.4.6" = [ 249 | {url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, 250 | {url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, 251 | ] 252 | "dill 0.3.6" = [ 253 | {url = "https://files.pythonhosted.org/packages/7c/e7/364a09134e1062d4d5ff69b853a56cf61c223e0afcc6906b6832bcd51ea8/dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, 254 | {url = "https://files.pythonhosted.org/packages/be/e3/a84bf2e561beed15813080d693b4b27573262433fced9c1d1fea59e60553/dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, 255 | ] 256 | "exceptiongroup 1.1.1" = [ 257 | {url = "https://files.pythonhosted.org/packages/61/97/17ed81b7a8d24d8f69b62c0db37abbd8c0042d4b3fc429c73dab986e7483/exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, 258 | {url = "https://files.pythonhosted.org/packages/cc/38/57f14ddc8e8baeddd8993a36fe57ce7b4ba174c35048b9a6d270bb01e833/exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, 259 | ] 260 | "iniconfig 2.0.0" = [ 261 | {url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, 262 | {url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, 263 | ] 264 | "isort 5.12.0" = [ 265 | {url = "https://files.pythonhosted.org/packages/0a/63/4036ae70eea279c63e2304b91ee0ac182f467f24f86394ecfe726092340b/isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, 266 | {url = "https://files.pythonhosted.org/packages/a9/c4/dc00e42c158fc4dda2afebe57d2e948805c06d5169007f1724f0683010a9/isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, 267 | ] 268 | "lazy-object-proxy 1.9.0" = [ 269 | {url = "https://files.pythonhosted.org/packages/00/74/46a68f51457639c0cd79e385e2f49c0fa7324470997ac096108669c1e182/lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, 270 | {url = "https://files.pythonhosted.org/packages/11/04/fa820296cb937b378d801cdc81c19de06624cfed481c1b8a6b439255a188/lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, 271 | {url = "https://files.pythonhosted.org/packages/11/fe/be1eb76d83f1b5242c492b410ce86c59db629c0b0f0f8e75018dfd955c30/lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, 272 | {url = "https://files.pythonhosted.org/packages/16/f2/e74981dedeb1a858cd5db9bcec81c4107da374249bc6894613472e01996f/lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, 273 | {url = "https://files.pythonhosted.org/packages/18/1b/04ac4490a62ae1916f88e629e74192ada97d74afc927453d005f003e5a8f/lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, 274 | {url = "https://files.pythonhosted.org/packages/1d/5d/25b9007c65f45805e711b56beac50ba395214e9e556cc8ee57f0882f88a9/lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, 275 | {url = "https://files.pythonhosted.org/packages/20/c0/8bab72a73607d186edad50d0168ca85bd2743cfc55560c9d721a94654b20/lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, 276 | {url = "https://files.pythonhosted.org/packages/27/a1/7cc10ca831679c5875c18ae6e0a468f7787ecd31fdd53598f91ea50df58d/lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, 277 | {url = "https://files.pythonhosted.org/packages/31/ad/e8605300f51061284cc57ca0f4ef582047c7f309bda1bb1c3c19b64af5c9/lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, 278 | {url = "https://files.pythonhosted.org/packages/4c/a4/cdd6f41a675a89ab584c78019a24adc533829764bcc85b0e24ed2678020c/lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, 279 | {url = "https://files.pythonhosted.org/packages/4d/7b/a959ff734bd3d8df7b761bfeaec6428549b77267072676a337b774f3b3ef/lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, 280 | {url = "https://files.pythonhosted.org/packages/4e/cb/aca3f4d89d3efbed724fd9504a96dafbe2d903ea908355a335acb110a5cd/lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, 281 | {url = "https://files.pythonhosted.org/packages/51/28/5c6dfb51df2cbb6d771a3b0d009f1edeab01f5cb16303ce32764b01636c0/lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, 282 | {url = "https://files.pythonhosted.org/packages/5b/a6/3c0a8b2ad6ce7af133ed54321b0ead5509303be3a80f98506af198e50cb7/lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, 283 | {url = "https://files.pythonhosted.org/packages/5c/76/0b16dc53e9ee5b24c621d808f46cca11e5e86c602b6bcd6dc27f9504af5b/lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, 284 | {url = "https://files.pythonhosted.org/packages/69/1f/51657d681711476287c9ff643428be0f9663addc1d341d4be1bad89290bd/lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, 285 | {url = "https://files.pythonhosted.org/packages/69/da/58391196d8a41fa8fa69b47e8a7893f279d369939e4994b3bc8648ff0433/lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, 286 | {url = "https://files.pythonhosted.org/packages/70/e7/f3735f8e47cb29a207568c5b8d28d9f5673228789b66cb0c48d488a37f94/lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, 287 | {url = "https://files.pythonhosted.org/packages/82/ac/d079d3ad377ba72e29d16ac077f8626ad4d3f55369c93168d0b81153d9a2/lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, 288 | {url = "https://files.pythonhosted.org/packages/86/93/e921f7a795e252df7248e0d220dc69a9443ad507fe258dea51a32e5435ca/lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, 289 | {url = "https://files.pythonhosted.org/packages/8d/6d/10420823a979366bf43ca5e69433c0c588865883566b96b6e3ed5b51c1f8/lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, 290 | {url = "https://files.pythonhosted.org/packages/9d/d7/81d466f2e69784bd416797824a2b8794aaf0b864a2390062ea197f06f0fc/lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, 291 | {url = "https://files.pythonhosted.org/packages/a7/51/6626c133e966698d53d65bcd90e34ad4986c5f0968c2328b3e9de51dbcf1/lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, 292 | {url = "https://files.pythonhosted.org/packages/a8/32/c1a67f76ec6923a8a8b1bc006b7cb3d195e386e03fe56e20fe38fce0321e/lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, 293 | {url = "https://files.pythonhosted.org/packages/b0/78/78962cb6f6d31a952acbc54e7838a5a85d952144973bd6e7ad24323dd466/lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, 294 | {url = "https://files.pythonhosted.org/packages/b1/80/2d9583fa8e5ac47ef183d811d26d833477b7ed02b64c17dd2ede68a3f9cf/lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, 295 | {url = "https://files.pythonhosted.org/packages/c9/8f/c8aab72c72634de0c726a98a1e4c84a93ef20049ee0427c871214f6a58d5/lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, 296 | {url = "https://files.pythonhosted.org/packages/cd/b6/84efe6e878e94f20cf9564ac3ede5e98d37c692b07080aef50cc4341052e/lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, 297 | {url = "https://files.pythonhosted.org/packages/d0/f4/95999792ce5f9223bac10cb31b1724723ecd0ba13e081c5fb806d7f5b9c4/lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, 298 | {url = "https://files.pythonhosted.org/packages/db/92/284ab10a6d0f82da36a20d9c1464c30bb318d1a6dd0ae476de9f890e7abd/lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, 299 | {url = "https://files.pythonhosted.org/packages/e7/86/ec93d495197f1006d7c9535e168fe763b3cc21928fb35c8f9ce08944b614/lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, 300 | {url = "https://files.pythonhosted.org/packages/ed/9b/44c370c8bbba32fd0217b4f15ca99f750d669d653c7f1eefa051627710e8/lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, 301 | {url = "https://files.pythonhosted.org/packages/f5/4f/9ad496dc26a10ed9ab8f088732f08dc1f88488897d6c9ac5e3432a254c30/lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, 302 | {url = "https://files.pythonhosted.org/packages/fb/f4/c5d6d771e70ec7a9483a98054e8a5f386eda5b18b6c96544d251558c6c92/lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, 303 | {url = "https://files.pythonhosted.org/packages/fc/8d/8e0fbfeec6e51184326e0886443e44142ce22d89fa9e9c3152900e654fa0/lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, 304 | {url = "https://files.pythonhosted.org/packages/fe/bb/0fcc8585ffb7285df94382e20b54d54ca62a1bcf594f6f18d8feb3fc3b98/lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, 305 | ] 306 | "mccabe 0.7.0" = [ 307 | {url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, 308 | {url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, 309 | ] 310 | "mypy-extensions 1.0.0" = [ 311 | {url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, 312 | {url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, 313 | ] 314 | "nodeenv 1.8.0" = [ 315 | {url = "https://files.pythonhosted.org/packages/1a/e6/6d2ead760a9ddb35e65740fd5a57e46aadd7b0c49861ab24f94812797a1c/nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, 316 | {url = "https://files.pythonhosted.org/packages/48/92/8e83a37d3f4e73c157f9fcf9fb98ca39bd94701a469dc093b34dca31df65/nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, 317 | ] 318 | "packaging 23.1" = [ 319 | {url = "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, 320 | {url = "https://files.pythonhosted.org/packages/b9/6c/7c6658d258d7971c5eb0d9b69fa9265879ec9a9158031206d47800ae2213/packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, 321 | ] 322 | "pathspec 0.11.1" = [ 323 | {url = "https://files.pythonhosted.org/packages/95/60/d93628975242cc515ab2b8f5b2fc831d8be2eff32f5a1be4776d49305d13/pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, 324 | {url = "https://files.pythonhosted.org/packages/be/c8/551a803a6ebb174ec1c124e68b449b98a0961f0b737def601e3c1fbb4cfd/pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, 325 | ] 326 | "platformdirs 3.7.0" = [ 327 | {url = "https://files.pythonhosted.org/packages/14/62/d8277379c30fc6b0347aaf8ff63e5c9e013e0da95f40ba957c5e098b06c2/platformdirs-3.7.0-py3-none-any.whl", hash = "sha256:cfd065ba43133ff103ab3bd10aecb095c2a0035fcd1f07217c9376900d94ba07"}, 328 | {url = "https://files.pythonhosted.org/packages/a1/45/ff5224bbf1a9f23d95f70890659a7c2639a25d594adfe4a5ca9221f6cae5/platformdirs-3.7.0.tar.gz", hash = "sha256:87fbf6473e87c078d536980ba970a472422e94f17b752cfad17024c18876d481"}, 329 | ] 330 | "pluggy 1.0.0" = [ 331 | {url = "https://files.pythonhosted.org/packages/9e/01/f38e2ff29715251cf25532b9082a1589ab7e4f571ced434f98d0139336dc/pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, 332 | {url = "https://files.pythonhosted.org/packages/a1/16/db2d7de3474b6e37cbb9c008965ee63835bba517e22cdb8c35b5116b5ce1/pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, 333 | ] 334 | "pylint 2.17.4" = [ 335 | {url = "https://files.pythonhosted.org/packages/04/4c/3f7d42a1378c40813772bc5f25184144da09f00ffbe3f60ae985ffa7e10f/pylint-2.17.4-py3-none-any.whl", hash = "sha256:7a1145fb08c251bdb5cca11739722ce64a63db479283d10ce718b2460e54123c"}, 336 | {url = "https://files.pythonhosted.org/packages/7e/d4/aba77d10841710fea016422f419dfe501dee05fa0fc3898dc048f7bf3f60/pylint-2.17.4.tar.gz", hash = "sha256:5dcf1d9e19f41f38e4e85d10f511e5b9c35e1aa74251bf95cdd8cb23584e2db1"}, 337 | ] 338 | "pyright 1.1.315" = [ 339 | {url = "https://files.pythonhosted.org/packages/63/43/a0669b9c47ae7937a1352ec62be2a9637588715258e46f8632779eca91b7/pyright-1.1.315-py3-none-any.whl", hash = "sha256:262744daa7455f6056f6d6609b6278258e2fd7bd100d8937203516ee71d0eb93"}, 340 | {url = "https://files.pythonhosted.org/packages/a5/b6/c98d5f00a5961926e9840c1e8d44e54128885afcca6e5f9834754846f8f7/pyright-1.1.315.tar.gz", hash = "sha256:f124400a1a72ccc5fd04836e7e48ed235e8f7929bcc42b270a571fb34df17979"}, 341 | ] 342 | "pytest 7.3.2" = [ 343 | {url = "https://files.pythonhosted.org/packages/58/2a/07c65fdc40846ecb8a9dcda2c38fcb5a06a3e39d08d4a4960916431951cb/pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, 344 | {url = "https://files.pythonhosted.org/packages/7a/d0/de969198293cdea22b3a6fb99a99aeeddb7b3827f0823b33c5dc0734bbe5/pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, 345 | ] 346 | "setuptools 68.0.0" = [ 347 | {url = "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, 348 | {url = "https://files.pythonhosted.org/packages/dc/98/5f896af066c128669229ff1aa81553ac14cfb3e5e74b6b44594132b8540e/setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, 349 | ] 350 | "six 1.16.0" = [ 351 | {url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, 352 | {url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, 353 | ] 354 | "thrift 0.16.0" = [ 355 | {url = "https://files.pythonhosted.org/packages/e4/23/dd951c9883cb49a73b750bdfe91e39d78e8a3f1f7175608634f381a197d5/thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, 356 | ] 357 | "tomli 2.0.1" = [ 358 | {url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, 359 | {url = "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, 360 | ] 361 | "tomlkit 0.11.8" = [ 362 | {url = "https://files.pythonhosted.org/packages/10/37/dd53019ccb72ef7d73fff0bee9e20b16faff9658b47913a35d79e89978af/tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, 363 | {url = "https://files.pythonhosted.org/packages/ef/a8/b1c193be753c02e2a94af6e37ee45d3378a74d44fe778c2434a63af92731/tomlkit-0.11.8-py3-none-any.whl", hash = "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171"}, 364 | ] 365 | "typing-extensions 4.6.3" = [ 366 | {url = "https://files.pythonhosted.org/packages/42/56/cfaa7a5281734dadc842f3a22e50447c675a1c5a5b9f6ad8a07b467bffe7/typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, 367 | {url = "https://files.pythonhosted.org/packages/5f/86/d9b1518d8e75b346a33eb59fa31bdbbee11459a7e2cc5be502fa779e96c5/typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, 368 | ] 369 | "wrapt 1.15.0" = [ 370 | {url = "https://files.pythonhosted.org/packages/0c/6e/f80c23efc625c10460240e31dcb18dd2b34b8df417bc98521fbfd5bc2e9a/wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, 371 | {url = "https://files.pythonhosted.org/packages/0f/9a/179018bb3f20071f39597cd38fc65d6285d7b89d57f6c51f502048ed28d9/wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, 372 | {url = "https://files.pythonhosted.org/packages/12/5a/fae60a8bc9b07a3a156989b79e14c58af05ab18375749ee7c12b2f0dddbd/wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, 373 | {url = "https://files.pythonhosted.org/packages/18/f6/659d7c431a57da9c9a86945834ab2bf512f1d9ebefacea49135a0135ef1a/wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, 374 | {url = "https://files.pythonhosted.org/packages/1e/3c/cb96dbcafbf3a27413fb15e0a1997c4610283f895dc01aca955cd2fda8b9/wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, 375 | {url = "https://files.pythonhosted.org/packages/20/01/baec2650208284603961d61f53ee6ae8e3eff63489c7230dff899376a6f6/wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, 376 | {url = "https://files.pythonhosted.org/packages/21/42/36c98e9c024978f52c218f22eba1addd199a356ab16548af143d3a72ac0d/wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, 377 | {url = "https://files.pythonhosted.org/packages/23/0a/9964d7141b8c5e31c32425d3412662a7873aaf0c0964166f4b37b7db51b6/wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, 378 | {url = "https://files.pythonhosted.org/packages/29/41/f05bf85417473cf6fe4eec7396c63762e5a457a42102bd1b8af059af6586/wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, 379 | {url = "https://files.pythonhosted.org/packages/2b/fb/c31489631bb94ac225677c1090f787a4ae367614b5277f13dbfde24b2b69/wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, 380 | {url = "https://files.pythonhosted.org/packages/2d/47/16303c59a890696e1a6fd82ba055fc4e0f793fb4815b5003f1f85f7202ce/wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, 381 | {url = "https://files.pythonhosted.org/packages/2e/ce/90dcde9ff9238689f111f07b46da2db570252445a781ea147ff668f651b0/wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, 382 | {url = "https://files.pythonhosted.org/packages/31/e6/6ac59c5570a7b9aaecb10de39f70dacd0290620330277e60b29edcf8bc9a/wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, 383 | {url = "https://files.pythonhosted.org/packages/39/ee/2b8d608f2bcf86242daadf5b0b746c11d3657b09892345f10f171b5ca3ac/wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, 384 | {url = "https://files.pythonhosted.org/packages/44/a1/40379212a0b678f995fdb4f4f28aeae5724f3212cdfbf97bee8e6fba3f1b/wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, 385 | {url = "https://files.pythonhosted.org/packages/45/90/a959fa50084d7acc2e628f093c9c2679dd25085aa5085a22592e028b3e06/wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, 386 | {url = "https://files.pythonhosted.org/packages/47/dd/bee4d33058656c0b2e045530224fcddd9492c354af5d20499e5261148dcb/wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, 387 | {url = "https://files.pythonhosted.org/packages/48/65/0061e7432ca4b635e96e60e27e03a60ddaca3aeccc30e7415fed0325c3c2/wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, 388 | {url = "https://files.pythonhosted.org/packages/4a/7b/c63103817bd2f3b0145608ef642ce90d8b6d1e5780d218bce92e93045e06/wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, 389 | {url = "https://files.pythonhosted.org/packages/50/eb/af864a01300878f69b4949f8381ad57d5519c1791307e9fd0bc7f5ab50a5/wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, 390 | {url = "https://files.pythonhosted.org/packages/54/21/282abeb456f22d93533b2d373eeb393298a30b0cb0683fa8a4ed77654273/wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, 391 | {url = "https://files.pythonhosted.org/packages/55/20/90f5affc2c879db408124ce14b9443b504f961e47a517dff4f24a00df439/wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, 392 | {url = "https://files.pythonhosted.org/packages/5d/c4/3cc25541ec0404dd1d178e7697a34814d77be1e489cd6f8cb055ac688314/wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, 393 | {url = "https://files.pythonhosted.org/packages/65/be/3ae5afe9d78d97595b28914fa7e375ebc6329549d98f02768d5a08f34937/wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, 394 | {url = "https://files.pythonhosted.org/packages/6b/b0/bde5400fdf6d18cb7ef527831de0f86ac206c4da1670b67633e5a547b05f/wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, 395 | {url = "https://files.pythonhosted.org/packages/78/f2/106d90140a93690eab240fae76759d62dae639fcec1bd098eccdb83aa38f/wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, 396 | {url = "https://files.pythonhosted.org/packages/7f/b6/6dc0ddacd20337b4ce6ab0d6b0edc7da3898f85c4f97df7f30267e57509e/wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, 397 | {url = "https://files.pythonhosted.org/packages/81/1e/0bb8f01c6ac5baba66ef1ab65f4644bede856c3c7aede18c896be222151c/wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, 398 | {url = "https://files.pythonhosted.org/packages/88/f1/4dfaa1ad111d2a48429dca133e46249922ee2f279e9fdd4ab5b149cd6c71/wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, 399 | {url = "https://files.pythonhosted.org/packages/8a/1c/740c3ad1b7754dd7213f4df09ccdaf6b19e36da5ff3a269444ba9e103f1b/wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, 400 | {url = "https://files.pythonhosted.org/packages/8f/87/ba6dc86e8edb28fd1e314446301802751bd3157e9780385c9eef633994b9/wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, 401 | {url = "https://files.pythonhosted.org/packages/94/55/91dd3a7efbc1db2b07bbfc490d48e8484852c355d55e61e8b1565d7725f6/wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, 402 | {url = "https://files.pythonhosted.org/packages/96/37/a33c1220e8a298ab18eb070b6a59e4ccc3f7344b434a7ac4bd5d4bdccc97/wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, 403 | {url = "https://files.pythonhosted.org/packages/9b/50/383c155a05e3e0361d209e3f55ec823f3736c7a46b29923ea33ab85e8d70/wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, 404 | {url = "https://files.pythonhosted.org/packages/9d/40/fee1288d654c80fe1bc5ecee1c8d58f761a39bb30c73f1ce106701dd8b0a/wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, 405 | {url = "https://files.pythonhosted.org/packages/a2/3e/ee671ac60945154dfa3a406b8cb5cef2e3b4fa31c7d04edeb92716342026/wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, 406 | {url = "https://files.pythonhosted.org/packages/a4/af/8552671e4e7674fcae14bd3976dd9dc6a2b7294730e4a9a94597ac292a21/wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, 407 | {url = "https://files.pythonhosted.org/packages/a6/32/f4868adc994648fac4cfe347bcc1381c9afcb1602c8ba0910f36b96c5449/wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, 408 | {url = "https://files.pythonhosted.org/packages/a7/da/04883b14284c437eac98c7ad2959197f02acbabd57d5ea8ff4893a7c3920/wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, 409 | {url = "https://files.pythonhosted.org/packages/a9/64/886e512f438f12424b48a3ab23ae2583ec633be6e13eb97b0ccdff8e328a/wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, 410 | {url = "https://files.pythonhosted.org/packages/aa/24/bbd64ee4e1db9c75ec2a9677c538866f81800bcd2a8abd1a383369369cf5/wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, 411 | {url = "https://files.pythonhosted.org/packages/af/23/cf5dbfd676480fa8fc6eecc4c413183cd8e14369321c5111fec5c12550e9/wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, 412 | {url = "https://files.pythonhosted.org/packages/af/7f/25913aacbe0c2c68e7354222bdefe4e840489725eb835e311c581396f91f/wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, 413 | {url = "https://files.pythonhosted.org/packages/b1/8b/f4c02cf1f841dede987f93c37d42256dc4a82cd07173ad8a5458eee1c412/wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, 414 | {url = "https://files.pythonhosted.org/packages/b2/b0/a56b129822568d9946e009e8efd53439b9dd38cc1c4af085aa44b2485b40/wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, 415 | {url = "https://files.pythonhosted.org/packages/b6/0c/435198dbe6961c2343ca725be26b99c8aee615e32c45bc1cb2a960b06183/wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, 416 | {url = "https://files.pythonhosted.org/packages/b7/3d/9d3cd75f7fc283b6e627c9b0e904189c41ca144185fd8113a1a094dec8ca/wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, 417 | {url = "https://files.pythonhosted.org/packages/b9/40/975fbb1ab03fa987900bacc365645c4cbead22baddd273b4f5db7f9843d2/wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, 418 | {url = "https://files.pythonhosted.org/packages/bd/47/57ffe222af59fae1eb56bca7d458b704a9b59380c47f0921efb94dc4786a/wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, 419 | {url = "https://files.pythonhosted.org/packages/c3/12/5fabf0014a0f30eb3975b7199ac2731215a40bc8273083f6a89bd6cadec6/wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, 420 | {url = "https://files.pythonhosted.org/packages/c4/e3/01f879f8e7c1221c72dbd4bfa106624ee3d01cb8cbc82ef57fbb95880cac/wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, 421 | {url = "https://files.pythonhosted.org/packages/c7/cd/18d95465323f29e3f3fd3ff84f7acb402a6a61e6caf994dced7140d78f85/wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, 422 | {url = "https://files.pythonhosted.org/packages/ca/1c/5caf61431705b3076ca1152abfd6da6304697d7d4fe48bb3448a6decab40/wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, 423 | {url = "https://files.pythonhosted.org/packages/cd/a0/84b8fe24af8d7f7374d15e0da1cd5502fff59964bbbf34982df0ca2c9047/wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, 424 | {url = "https://files.pythonhosted.org/packages/cd/f0/060add4fcb035024f84fb3b5523fb2b119ac08608af3f61dbdda38477900/wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, 425 | {url = "https://files.pythonhosted.org/packages/cf/b1/3c24fc0f6b589ad8c99cfd1cd3e586ef144e16aaf9381ed952d047a7ee54/wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, 426 | {url = "https://files.pythonhosted.org/packages/d1/74/3c99ce16947f7af901f6203ab4a3d0908c4db06e800571dabfe8525fa925/wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, 427 | {url = "https://files.pythonhosted.org/packages/d2/60/9fe25f4cd910ae94e75a1fd4772b058545e107a82629a5ca0f2cd7cc34d5/wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, 428 | {url = "https://files.pythonhosted.org/packages/d7/4b/1bd4837362d31d402b9bc1a27cdd405baf994dbf9942696f291d2f7eeb73/wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, 429 | {url = "https://files.pythonhosted.org/packages/dd/42/9eedee19435dfc0478cdb8bdc71800aab15a297d1074f1aae0d9489adbc3/wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, 430 | {url = "https://files.pythonhosted.org/packages/dd/e9/85e780a6b70191114b13b129867cec2fab84279f6beb788e130a26e4ca58/wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, 431 | {url = "https://files.pythonhosted.org/packages/dd/eb/389f9975a6be31ddd19d29128a11f1288d07b624e464598a4b450f8d007e/wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, 432 | {url = "https://files.pythonhosted.org/packages/de/77/e2ebfa2f46c19094888a364fdb59aeab9d3336a3ad7ccdf542de572d2a35/wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, 433 | {url = "https://files.pythonhosted.org/packages/e8/86/fc38e58843159bdda745258d872b1187ad916087369ec57ef93f5e832fa8/wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, 434 | {url = "https://files.pythonhosted.org/packages/ec/f4/f84538a367105f0a7e507f0c6766d3b15b848fd753647bbf0c206399b322/wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, 435 | {url = "https://files.pythonhosted.org/packages/ee/25/83f5dcd9f96606521da2d0e7a03a18800264eafb59b569ff109c4d2fea67/wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, 436 | {url = "https://files.pythonhosted.org/packages/f6/89/bf77b063c594795aaa056cac7b467463702f346d124d46d7f06e76e8cd97/wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, 437 | {url = "https://files.pythonhosted.org/packages/f6/d3/3c6bd4db883537c40eb9d41d738d329d983d049904f708267f3828a60048/wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, 438 | {url = "https://files.pythonhosted.org/packages/f8/49/10013abe31f6892ae57c5cc260f71b7e08f1cc00f0d7b2bcfa482ea74349/wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, 439 | {url = "https://files.pythonhosted.org/packages/f8/7d/73e4e3cdb2c780e13f9d87dc10488d7566d8fd77f8d68f0e416bfbd144c7/wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, 440 | {url = "https://files.pythonhosted.org/packages/f8/f8/e068dafbb844c1447c55b23c921f3d338cddaba4ea53187a7dd0058452d9/wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, 441 | {url = "https://files.pythonhosted.org/packages/fb/2d/b6fd53b7dbf94d542866cbf1021b9a62595177fc8405fd75e0a5bf3fa3b8/wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, 442 | {url = "https://files.pythonhosted.org/packages/fb/bd/ca7fd05a45e7022f3b780a709bbdb081a6138d828ecdb5b7df113a3ad3be/wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, 443 | {url = "https://files.pythonhosted.org/packages/fd/8a/db55250ad0b536901173d737781e3b5a7cc7063c46b232c2e3a82a33c032/wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, 444 | {url = "https://files.pythonhosted.org/packages/ff/f6/c044dec6bec4ce64fbc92614c5238dd432780b06293d2efbcab1a349629c/wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, 445 | ] 446 | -------------------------------------------------------------------------------- /pymetastore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/criccomini/pymetastore/60e20f4105857ef01359d93a946e4b396521e19b/pymetastore/__init__.py -------------------------------------------------------------------------------- /pymetastore/hive_metastore/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['ttypes', 'constants', 'ThriftHiveMetastore'] 2 | -------------------------------------------------------------------------------- /pymetastore/hive_metastore/constants.py: -------------------------------------------------------------------------------- 1 | # 2 | # Autogenerated by Thrift Compiler (0.18.1) 3 | # 4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | # 6 | # options string: py 7 | # 8 | 9 | from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException 10 | from thrift.protocol.TProtocol import TProtocolException 11 | from thrift.TRecursive import fix_spec 12 | 13 | import sys 14 | from .ttypes import * 15 | DDL_TIME = "transient_lastDdlTime" 16 | HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__" 17 | HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__" 18 | HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__" 19 | IS_ARCHIVED = "is_archived" 20 | ORIGINAL_LOCATION = "original_location" 21 | IS_IMMUTABLE = "immutable" 22 | META_TABLE_COLUMNS = "columns" 23 | META_TABLE_COLUMN_TYPES = "columns.types" 24 | BUCKET_FIELD_NAME = "bucket_field_name" 25 | BUCKET_COUNT = "bucket_count" 26 | FIELD_TO_DIMENSION = "field_to_dimension" 27 | META_TABLE_NAME = "name" 28 | META_TABLE_DB = "db" 29 | META_TABLE_LOCATION = "location" 30 | META_TABLE_SERDE = "serde" 31 | META_TABLE_PARTITION_COLUMNS = "partition_columns" 32 | META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types" 33 | FILE_INPUT_FORMAT = "file.inputformat" 34 | FILE_OUTPUT_FORMAT = "file.outputformat" 35 | META_TABLE_STORAGE = "storage_handler" 36 | TABLE_IS_TRANSACTIONAL = "transactional" 37 | TABLE_NO_AUTO_COMPACT = "no_auto_compaction" 38 | TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties" 39 | TABLE_BUCKETING_VERSION = "bucketing_version" 40 | -------------------------------------------------------------------------------- /pymetastore/htypes.py: -------------------------------------------------------------------------------- 1 | """ 2 | htypes module contains the Hive metastore types, including primitive and complex types. 3 | User defined types are defined by the user, using a combination of complex 4 | and primitive types and thus are not included in this module. 5 | there are also some helper functions to convert between Hive metastore types and HType objects. 6 | """ 7 | from collections import namedtuple 8 | from enum import Enum 9 | from typing import Any, List, Optional 10 | 11 | 12 | class PrimitiveCategory(Enum): 13 | """An Enumeration holding all the primitive types supported by Hive.""" 14 | 15 | VOID = "VOID" 16 | BOOLEAN = "BOOLEAN" 17 | BYTE = "BYTE" 18 | SHORT = "SHORT" 19 | INT = "INT" 20 | LONG = "LONG" 21 | FLOAT = "FLOAT" 22 | DOUBLE = "DOUBLE" 23 | STRING = "STRING" 24 | DATE = "DATE" 25 | TIMESTAMP = "TIMESTAMP" 26 | TIMESTAMPLOCALTZ = "TIMESTAMPLOCALTZ" 27 | BINARY = "BINARY" 28 | DECIMAL = "DECIMAL" 29 | VARCHAR = "VARCHAR" 30 | CHAR = "CHAR" 31 | INTERVAL_YEAR_MONTH = "INTERVAL_YEAR_MONTH" 32 | INTERVAL_DAY_TIME = "INTERVAL_DAY_TIME" 33 | UNKNOWN = "UNKNOWN" 34 | 35 | 36 | class HTypeCategory(Enum): 37 | """ 38 | An Enumeration holding all the complex types supported by Hive plus 39 | the primitive types. 40 | """ 41 | 42 | PRIMITIVE = PrimitiveCategory 43 | STRUCT = "STRUCT" 44 | MAP = "MAP" 45 | LIST = "LIST" 46 | UNION = "UNION" 47 | 48 | 49 | class HType: 50 | """ 51 | The base class for all Hive metastore types. 52 | It contains the name of the type and the category of the type. 53 | """ 54 | 55 | def __init__(self, name: str, category: HTypeCategory): 56 | self.name = name 57 | self.category = category 58 | 59 | def __str__(self): 60 | return f"{self.__class__.__name__}(name={self.name}, category={self.category})" 61 | 62 | def __repr__(self): 63 | return f"{self.__class__.__name__}('{self.name}', {self.category})" 64 | 65 | def __eq__(self, other): 66 | if isinstance(other, HType): 67 | return (self.name == other.name) and (self.category == other.category) 68 | return False 69 | 70 | 71 | class HPrimitiveType(HType): 72 | """ 73 | A class representing a primitive type in Hive metastore. 74 | It is initialized by passing a PrimitiveCategory enum value. 75 | """ 76 | 77 | def __init__(self, primitive_type: PrimitiveCategory): 78 | super().__init__(primitive_type.value, HTypeCategory.PRIMITIVE) 79 | self.primitive_type = primitive_type 80 | 81 | def __str__(self): 82 | return f"{self.__class__.__name__}(name={self.name}, category={self.category})" 83 | 84 | def __repr__(self): 85 | return f"{self.__class__.__name__}({self.primitive_type})" 86 | 87 | 88 | class HMapType(HType): 89 | """ 90 | A class representing a map type in Hive metastore. 91 | It is initialized by passing two lists. A list of field names and a list of field types. 92 | """ 93 | 94 | def __init__(self, key_type: HType, value_type: HType): 95 | super().__init__("MAP", HTypeCategory.MAP) 96 | self.key_type = key_type 97 | self.value_type = value_type 98 | 99 | def __str__(self): 100 | return f"{super().__str__()}, key_type={str(self.key_type)}, value_type={str(self.value_type)})" 101 | 102 | def __repr__(self): 103 | return f"HMapType({repr(self.key_type)}, {repr(self.value_type)})" 104 | 105 | def __eq__(self, other): 106 | if isinstance(other, self.__class__): 107 | return ( 108 | super().__eq__(other) 109 | and (self.key_type == other.key_type) 110 | and (self.value_type == other.value_type) 111 | ) 112 | return False 113 | 114 | 115 | class HListType(HType): 116 | """ 117 | A class representing a List or array type in Hive metastore. 118 | It is initialized by passing the element type of the list. 119 | """ 120 | 121 | def __init__(self, element_type: HType): 122 | super().__init__("LIST", HTypeCategory.LIST) 123 | self.element_type = element_type 124 | 125 | def __str__(self): 126 | return f"{super().__str__()}, element_type={str(self.element_type)})" 127 | 128 | def __repr__(self): 129 | return f"HListType({repr(self.element_type)})" 130 | 131 | def __eq__(self, other): 132 | if isinstance(other, self.__class__): 133 | return super().__eq__(other) and (self.element_type == other.element_type) 134 | return False 135 | 136 | 137 | class HUnionType(HType): 138 | """ 139 | A class representing a Union type in Hive metastore. 140 | It is initialized by passing a list of types. 141 | """ 142 | 143 | def __init__(self, types: List[HType]): 144 | super().__init__("UNION", HTypeCategory.UNION) 145 | self.types = types 146 | 147 | def __str__(self): 148 | return f"{super().__str__()}, types={str(self.types)})" 149 | 150 | def __repr__(self): 151 | return f"HUnionType({repr(self.types)})" 152 | 153 | def __eq__(self, other): 154 | if isinstance(other, self.__class__): 155 | return super().__eq__(other) and (self.types == other.types) 156 | return False 157 | 158 | 159 | class HVarcharType(HType): 160 | """ 161 | A class representing a Varchar type in Hive metastore. 162 | Varchar is primitive but parameterized by length. 163 | Varchar by definition has a maximum length of 65535. 164 | """ 165 | 166 | def __init__(self, length: int): 167 | MAX_VARCHAR_LENGTH = 65535 168 | if length > MAX_VARCHAR_LENGTH: 169 | raise ValueError("Varchar length cannot exceed 65535") 170 | super().__init__("VARCHAR", HTypeCategory.PRIMITIVE) 171 | self.length = length 172 | 173 | def __str__(self): 174 | return f"HVarcharType(length={self.length})" 175 | 176 | def __repr__(self): 177 | return f"HVarcharType({self.length})" 178 | 179 | def __eq__(self, other): 180 | if isinstance(other, self.__class__): 181 | return super().__eq__(other) and (self.length == other.length) 182 | return False 183 | 184 | 185 | class HCharType(HType): 186 | """ 187 | A class representing a Char type in Hive metastore. 188 | Char is primitive but parameterized by length. 189 | Char by definition has a maximum length of 255. 190 | """ 191 | 192 | def __init__(self, length: int): 193 | MAX_CHAR_LENGTH = 255 194 | if length > MAX_CHAR_LENGTH: 195 | raise ValueError("Char length cannot exceed 255") 196 | super().__init__("CHAR", HTypeCategory.PRIMITIVE) 197 | self.length = length 198 | 199 | def __str__(self): 200 | return f"HCharType(length={self.length})" 201 | 202 | def __repr__(self): 203 | return f"HCharType({self.length})" 204 | 205 | def __eq__(self, other): 206 | if isinstance(other, self.__class__): 207 | return super().__eq__(other) and (self.length == other.length) 208 | return False 209 | 210 | 211 | class HDecimalType(HType): 212 | """ 213 | A class representing a Decimal type in Hive metastore. 214 | Decimal is primitive but parameterized by precision and scale. 215 | Decimal by definition has a maximum precision and scale of 38. 216 | """ 217 | 218 | def __init__(self, precision: int, scale: int): 219 | MAX_PRECISION = 38 220 | MAX_SCALE = 38 221 | if precision > MAX_PRECISION: 222 | raise ValueError("Decimal precision cannot exceed 38") 223 | if scale > MAX_SCALE: 224 | raise ValueError("Decimal scale cannot exceed 38") 225 | super().__init__("DECIMAL", HTypeCategory.PRIMITIVE) 226 | self.precision = precision 227 | self.scale = scale 228 | 229 | def __str__(self): 230 | return f"HDecimalType(precision={self.precision}, scale={self.scale})" 231 | 232 | def __repr__(self): 233 | return f"HDecimalType({self.precision}, {self.scale})" 234 | 235 | def __eq__(self, other): 236 | if isinstance(other, self.__class__): 237 | return ( 238 | super().__eq__(other) 239 | and (self.precision == other.precision) 240 | and (self.scale == other.scale) 241 | ) 242 | return False 243 | 244 | 245 | class HStructType(HType): 246 | """ 247 | A class representing a Struct type in Hive metastore. 248 | Struct is parameterized by a list of names and types. 249 | """ 250 | 251 | def __init__(self, names: List[str], types: List[HType]): 252 | if len(names) != len(types): 253 | raise ValueError("mismatched size of names and types.") 254 | if None in names: 255 | raise ValueError("names cannot contain None") 256 | if None in types: 257 | raise ValueError("types cannot contain None") 258 | 259 | super().__init__("STRUCT", HTypeCategory.STRUCT) 260 | self.names = names 261 | self.types = types 262 | 263 | def __str__(self): 264 | return f"{super().__str__()}, names={self.names}, types={self.types})" 265 | 266 | def __repr__(self): 267 | return f"HStructType({self.names}, {self.types})" 268 | 269 | def __eq__(self, other): 270 | if isinstance(other, self.__class__): 271 | return ( 272 | super().__eq__(other) 273 | and (self.names == other.names) 274 | and (self.types == other.types) 275 | ) 276 | return False 277 | 278 | 279 | # We need to maintain a list of the expected serialized type names. 280 | # Thrift will return the type in string format and we will have to parse it to get the type. 281 | class SerdeTypeNameConstants(Enum): 282 | """ 283 | Serde TypeNameConstants is an enum class that contains the list of expected type names 284 | as they are used by Thrift. 285 | """ 286 | 287 | # Primitive types 288 | VOID = "void" 289 | BOOLEAN = "boolean" 290 | TINYINT = "tinyint" 291 | SMALLINT = "smallint" 292 | INT = "int" 293 | BIGINT = "bigint" 294 | FLOAT = "float" 295 | DOUBLE = "double" 296 | STRING = "string" 297 | DATE = "date" 298 | CHAR = "char" 299 | VARCHAR = "varchar" 300 | TIMESTAMP = "timestamp" 301 | TIMESTAMPLOCALTZ = "timestamp with local time zone" 302 | DECIMAL = "decimal" 303 | BINARY = "binary" 304 | INTERVAL_YEAR_MONTH = "interval_year_month" 305 | INTERVAL_DAY_TIME = "interval_day_time" 306 | # Complex types 307 | LIST = "array" 308 | MAP = "map" 309 | STRUCT = "struct" 310 | UNION = "uniontype" 311 | UNKNOWN = "unknown" 312 | 313 | 314 | # We need a mapping between the two definitions of primitive types. 315 | # The first is the one used by Hive and the second is the one used by Thrift returned values. 316 | primitive_to_serde_mapping = { 317 | PrimitiveCategory.VOID: SerdeTypeNameConstants.VOID, 318 | PrimitiveCategory.BOOLEAN: SerdeTypeNameConstants.BOOLEAN, 319 | PrimitiveCategory.BYTE: SerdeTypeNameConstants.TINYINT, 320 | PrimitiveCategory.SHORT: SerdeTypeNameConstants.SMALLINT, 321 | PrimitiveCategory.INT: SerdeTypeNameConstants.INT, 322 | PrimitiveCategory.LONG: SerdeTypeNameConstants.BIGINT, 323 | PrimitiveCategory.FLOAT: SerdeTypeNameConstants.FLOAT, 324 | PrimitiveCategory.DOUBLE: SerdeTypeNameConstants.DOUBLE, 325 | PrimitiveCategory.STRING: SerdeTypeNameConstants.STRING, 326 | PrimitiveCategory.DATE: SerdeTypeNameConstants.DATE, 327 | PrimitiveCategory.TIMESTAMP: SerdeTypeNameConstants.TIMESTAMP, 328 | PrimitiveCategory.TIMESTAMPLOCALTZ: SerdeTypeNameConstants.TIMESTAMPLOCALTZ, 329 | PrimitiveCategory.BINARY: SerdeTypeNameConstants.BINARY, 330 | PrimitiveCategory.DECIMAL: SerdeTypeNameConstants.DECIMAL, 331 | PrimitiveCategory.VARCHAR: SerdeTypeNameConstants.VARCHAR, 332 | PrimitiveCategory.CHAR: SerdeTypeNameConstants.CHAR, 333 | PrimitiveCategory.INTERVAL_YEAR_MONTH: SerdeTypeNameConstants.INTERVAL_YEAR_MONTH, 334 | PrimitiveCategory.INTERVAL_DAY_TIME: SerdeTypeNameConstants.INTERVAL_DAY_TIME, 335 | PrimitiveCategory.UNKNOWN: SerdeTypeNameConstants.UNKNOWN, 336 | } 337 | 338 | """ We also need the inverse though. 339 | """ 340 | serde_to_primitive_mapping = {v.name: k for k, v in primitive_to_serde_mapping.items()} 341 | 342 | """ This function gets us the serde type name from the primitive one.""" 343 | 344 | 345 | def primitive_to_serde(primitive_category: PrimitiveCategory) -> SerdeTypeNameConstants: 346 | return primitive_to_serde_mapping[primitive_category] 347 | 348 | 349 | """ This function gets us the primitive type name from the serde one.""" 350 | 351 | 352 | def serde_to_primitive(serde_type_name: str) -> Optional[PrimitiveCategory]: 353 | return serde_to_primitive_mapping.get(serde_type_name) 354 | 355 | 356 | """This function returns the serde enum key from the string we get from Thrift""" 357 | 358 | 359 | def get_serde_type_by_value(value): 360 | for member in SerdeTypeNameConstants: 361 | if member.value == value: 362 | return member.name 363 | return None 364 | 365 | 366 | """ 367 | To be used for tokening the Thrift type string. We need to tokenize the 368 | string to get the type name and the type parameters. 369 | """ 370 | 371 | 372 | class Token(namedtuple("Token", ["position", "text", "type"])): 373 | def __new__(cls, position: int, text: str, type_: bool) -> Any: 374 | if text is None: 375 | raise ValueError("text is null") 376 | return super().__new__(cls, position, text, type_) 377 | 378 | def __str__(self): 379 | return f"{self.position}:{self.text}" 380 | 381 | 382 | class PrimitiveParts(namedtuple("PrimitiveParts", ["type_name", "type_params"])): 383 | def __new__(cls, type_name: str, type_params: List[str]) -> Any: 384 | if type_name is None: 385 | raise ValueError("type_name is null") 386 | if type_params is None: 387 | raise ValueError("type_params is null") 388 | return super().__new__(cls, type_name, type_params) 389 | 390 | def __str__(self): 391 | return f"{self.type_name}:{self.type_params}" 392 | 393 | 394 | class TypeParser: 395 | """ 396 | Util functions to parse the Thrift column types (as strings) and return the 397 | expected hive type 398 | """ 399 | 400 | def __init__(self, type_info_string: str): 401 | self.type_string = type_info_string 402 | self.type_tokens = self.tokenize(type_info_string) 403 | self.index = 0 404 | 405 | # we want the base name of the type, in general the format of a 406 | # parameterized type is (, , ...), so the base 407 | # name is the string before the first '(' 408 | @staticmethod 409 | def get_base_name(type_name: str) -> str: 410 | index = type_name.find("(") 411 | if index == -1: 412 | return type_name 413 | return type_name[:index] 414 | 415 | # Is the type named using the allowed characters? 416 | @staticmethod 417 | def is_valid_type_char(c: str) -> bool: 418 | return c.isalnum() or c in ("_", ".", " ", "$") 419 | 420 | # The concept of the tokenizer is to split the type string into tokens. The 421 | # tokens are either type names or type parameters. The type parameters are 422 | # enclosed in parentheses. 423 | def tokenize(self, type_info_string: str) -> List[Token]: 424 | tokens = [] 425 | begin = 0 426 | end = 1 427 | while end <= len(type_info_string): 428 | if ( 429 | begin > 0 430 | and type_info_string[begin - 1] == "(" 431 | and type_info_string[begin] == "'" 432 | ): 433 | begin += 1 434 | end += 1 435 | while type_info_string[end] != "'": 436 | end += 1 437 | 438 | elif type_info_string[begin] == "'" and type_info_string[begin + 1] == ")": 439 | begin += 1 440 | end += 1 441 | 442 | if ( 443 | end == len(type_info_string) 444 | or not self.is_valid_type_char(type_info_string[end - 1]) 445 | or not self.is_valid_type_char(type_info_string[end]) 446 | ): 447 | token = Token( 448 | begin, 449 | type_info_string[begin:end], 450 | self.is_valid_type_char(type_info_string[begin]), 451 | ) 452 | tokens.append(token) 453 | begin = end 454 | end += 1 455 | 456 | return tokens 457 | 458 | def peek(self) -> Optional[Token]: 459 | if self.index >= len(self.type_tokens): 460 | return None 461 | return self.type_tokens[self.index] 462 | 463 | def expect(self, item: str, alternative: Optional[str] = None) -> Token: 464 | if self.index >= len(self.type_tokens): 465 | raise ValueError(f"Error: {item} expected at the end of 'typeInfoString'") 466 | 467 | token = self.type_tokens[self.index] 468 | 469 | if item == "type": 470 | if ( 471 | token.text 472 | not in [ 473 | HTypeCategory.LIST.value, 474 | HTypeCategory.MAP.value, 475 | HTypeCategory.STRUCT.value, 476 | HTypeCategory.UNION.value, 477 | ] 478 | and (self.get_base_name(token.text) is None) 479 | and (token.text != alternative) 480 | ): 481 | raise ValueError( 482 | f"Error: {item} expected at the position {token.position} " 483 | f"of 'typeInfoString' but '{token.text}' is found." 484 | ) 485 | elif item == "name": 486 | if not token.type and token.text != alternative: 487 | raise ValueError( 488 | f"Error: {item} expected at the position {token.position} " 489 | f"of 'typeInfoString' but '{token.text}' is found." 490 | ) 491 | elif item != token.text and token.text != alternative: 492 | raise ValueError( 493 | f"Error: {item} expected at the position {token.position} of " 494 | f"'typeInfoString' but '{token.text}' is found." 495 | ) 496 | 497 | self.index += 1 # increment the index 498 | return token 499 | 500 | def parse_params(self) -> List[str]: 501 | params = [] 502 | 503 | token: Optional[Token] = self.peek() 504 | if token is not None and token.text == "(": 505 | token = self.expect("(", None) 506 | token = self.peek() 507 | while token is None or token.text != ")": 508 | token = self.expect("name", None) 509 | params.append(token.text) 510 | token = self.expect(",", ")") 511 | 512 | if not params: 513 | raise ValueError( 514 | "type parameters expected for type string 'typeInfoString'" 515 | ) 516 | 517 | return params 518 | 519 | def parse_type(self) -> HType: 520 | token = self.expect("type") 521 | 522 | # first we take care of primitive types. 523 | serde_val = get_serde_type_by_value(token.text) 524 | if serde_val and serde_val not in ( 525 | HTypeCategory.LIST.value, 526 | HTypeCategory.MAP.value, 527 | HTypeCategory.STRUCT.value, 528 | HTypeCategory.UNION.value, 529 | ): 530 | primitive_type = serde_to_primitive(serde_val) 531 | if primitive_type and primitive_type is not PrimitiveCategory.UNKNOWN: 532 | params = self.parse_params() 533 | if primitive_type in ( 534 | PrimitiveCategory.CHAR, 535 | PrimitiveCategory.VARCHAR, 536 | ): 537 | if len(params) == 0: 538 | raise ValueError( 539 | "char/varchar type must have a length specified" 540 | ) 541 | if len(params) == 1: 542 | length = int(params[0]) 543 | if primitive_type is PrimitiveCategory.CHAR: 544 | return HCharType(length) 545 | elif primitive_type is PrimitiveCategory.VARCHAR: 546 | return HVarcharType(length) 547 | else: 548 | raise ValueError( 549 | f"Error: {token.text} type takes only one parameter, but instead " 550 | f"{len(params)} parameters are found." 551 | ) 552 | elif primitive_type is PrimitiveCategory.DECIMAL: 553 | if len(params) == 0: 554 | return HDecimalType(10, 0) 555 | elif len(params) == 1: 556 | precision = int(params[0]) 557 | return HDecimalType(precision, 0) 558 | elif len(params) == 2: 559 | precision = int(params[0]) 560 | scale = int(params[1]) 561 | return HDecimalType(precision, scale) 562 | else: 563 | raise ValueError( 564 | f"Error: {token.text} type takes only two parameters, but instead " 565 | f"{len(params)} parameters are found." 566 | ) 567 | else: 568 | return HPrimitiveType(primitive_type) 569 | 570 | # next we take care of complex types. 571 | if HTypeCategory.LIST.value == serde_val: 572 | self.expect("<") 573 | element_type = self.parse_type() 574 | self.expect(">") 575 | return HListType(element_type) 576 | 577 | if HTypeCategory.MAP.value == serde_val: 578 | self.expect("<") 579 | key_type = self.parse_type() 580 | self.expect(",") 581 | value_type = self.parse_type() 582 | self.expect(">") 583 | return HMapType(key_type, value_type) 584 | 585 | if HTypeCategory.STRUCT.value == serde_val: 586 | self.expect("<") 587 | names = [] 588 | types = [] 589 | token: Optional[Token] = self.peek() 590 | while token is not None and token.text != ">": 591 | field_name = self.expect("name") 592 | self.expect(":") 593 | field_type = self.parse_type() 594 | names.append(field_name.text) 595 | types.append(field_type) 596 | token: Optional[Token] = self.peek() 597 | if token is not None and token.text == ",": 598 | self.expect(",") 599 | token = self.peek() 600 | self.expect(">") 601 | return HStructType(names, types) 602 | 603 | if HTypeCategory.UNION.value == serde_val: 604 | self.expect("<") 605 | types = [] 606 | token = self.peek() 607 | while token is not None and token.text != ">": 608 | field_type = self.parse_type() 609 | types.append(field_type) 610 | token = self.peek() 611 | if token is not None and token.text == ",": 612 | self.expect(",") 613 | token = self.peek() 614 | self.expect(">") 615 | return HUnionType(types) 616 | 617 | # if we reach this point and we haven't figure out what to do, then we 618 | # better raise an error. 619 | raise ValueError(f"Error: {token.text} is not a valid type.") 620 | -------------------------------------------------------------------------------- /pymetastore/metastore.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from enum import Enum 3 | from typing import Dict, List, Optional 4 | 5 | from thrift.protocol.TBinaryProtocol import TBinaryProtocol 6 | from thrift.transport import TSocket, TTransport 7 | 8 | from .hive_metastore import ThriftHiveMetastore as hms 9 | from .hive_metastore.ttypes import ( 10 | Database, 11 | FieldSchema, 12 | Partition, 13 | PrincipalType, 14 | SerDeInfo, 15 | StorageDescriptor, 16 | Table, 17 | ) 18 | from .htypes import HType, TypeParser 19 | from .stats import ( 20 | BinaryTypeStats, 21 | BooleanTypeStats, 22 | ColumnStats, 23 | DateTypeStats, 24 | DecimalTypeStats, 25 | DoubleTypeStats, 26 | LongTypeStats, 27 | StringTypeStats, 28 | ) 29 | 30 | 31 | class HPrincipalType(Enum): 32 | ROLE = "ROLE" 33 | USER = "USER" 34 | GROUP = "GROUP" 35 | 36 | 37 | @dataclass 38 | class HPrivilegeGrantInfo: 39 | privilege: str 40 | grantor: str 41 | grantor_type: HPrincipalType 42 | create_time: int 43 | grant_option: bool 44 | 45 | 46 | @dataclass 47 | class HPrincipalPrivilegeSet: 48 | user_privileges: List[HPrivilegeGrantInfo] 49 | group_privileges: List[HPrivilegeGrantInfo] 50 | role_privileges: List[HPrivilegeGrantInfo] 51 | 52 | 53 | @dataclass 54 | class HDatabase: 55 | name: str 56 | location: Optional[str] = None 57 | owner_name: Optional[str] = None 58 | owner_type: Optional[HPrincipalType] = None 59 | comment: Optional[str] = None 60 | parameters: Optional[Dict[str, str]] = None 61 | 62 | 63 | @dataclass 64 | class HColumn: 65 | name: str 66 | type: HType 67 | comment: Optional[str] = None 68 | 69 | 70 | class HSortingOrder(Enum): 71 | ASC = 1 72 | DESC = 0 73 | 74 | 75 | @dataclass 76 | class HSortingColumn: 77 | column: str 78 | order: HSortingOrder 79 | 80 | 81 | class BucketingVersion(Enum): 82 | V1 = 1 83 | V2 = 2 84 | 85 | 86 | @dataclass 87 | class HiveBucketProperty: 88 | bucketed_by: List[str] 89 | bucket_count: int 90 | version: BucketingVersion = BucketingVersion.V1 91 | sorting_columns: List[HSortingColumn] = field(default_factory=list) 92 | 93 | 94 | @dataclass 95 | class StorageFormat: 96 | serde: str 97 | input_format: str 98 | output_format: str 99 | 100 | 101 | @dataclass 102 | class HStorage: 103 | storage_format: StorageFormat 104 | skewed: bool = False 105 | location: Optional[str] = None 106 | bucket_property: Optional[HiveBucketProperty] = None 107 | serde_parameters: Optional[Dict[str, str]] = None 108 | 109 | 110 | @dataclass 111 | class HTable: 112 | database_name: str 113 | name: str 114 | table_type: str 115 | columns: List[HColumn] 116 | partition_columns: List[HColumn] 117 | storage: HStorage 118 | parameters: Dict[str, str] 119 | view_original_text: Optional[str] = None 120 | view_expanded_text: Optional[str] = None 121 | write_id: Optional[int] = None 122 | owner: Optional[str] = None 123 | 124 | 125 | @dataclass 126 | class HSkewedInfo: 127 | skewed_col_names: List[str] 128 | skewed_col_values: List[List[str]] 129 | skewed_col_value_location_maps: Dict[List[str], str] 130 | 131 | 132 | @dataclass 133 | class HPartition: 134 | database_name: str 135 | table_name: str 136 | values: List[str] 137 | parameters: Dict[str, str] 138 | create_time: int 139 | last_access_time: int 140 | sd: HStorage 141 | cat_name: str 142 | write_id: int 143 | 144 | 145 | class HMS: 146 | def __init__(self, client: hms.Client): 147 | self.client = client 148 | 149 | @staticmethod 150 | def create(host: str = "localhost", port: int = 9083) -> "_HMSConnection": 151 | return _HMSConnection(host, port) 152 | 153 | def list_databases(self) -> List[str]: 154 | databases = self.client.get_all_databases() 155 | db_names = [] 156 | for database in databases: 157 | db_names.append(database) 158 | return db_names 159 | 160 | def get_database(self, name: str) -> HDatabase: 161 | db: Database = self.client.get_database(name) 162 | 163 | if db.ownerType is PrincipalType.USER: 164 | owner_type = HPrincipalType.USER 165 | elif db.ownerType is PrincipalType.ROLE: 166 | owner_type = HPrincipalType.ROLE 167 | else: 168 | owner_type = None 169 | 170 | return HDatabase( 171 | db.name, # pyright: ignore[reportGeneralTypeIssues] 172 | db.locationUri, 173 | db.ownerName, 174 | owner_type, 175 | db.description, 176 | db.parameters, 177 | ) 178 | 179 | def list_tables(self, database_name: str) -> List[str]: 180 | return self.client.get_all_tables(database_name) 181 | 182 | def list_columns(self, database_name: str, table_name: str) -> List[str]: 183 | # TODO: Rather than ignore these pyright errors, do appropriate None handling 184 | columns = self.client.get_table( 185 | database_name, 186 | table_name, 187 | ).sd.cols # pyright: ignore[reportOptionalMemberAccess] 188 | self.client.get_schema(database_name, table_name) 189 | column_names = [] 190 | for column in columns: # pyright: ignore[reportOptionalIterable] 191 | column_names.append(column.name) 192 | return column_names 193 | 194 | def list_partitions( 195 | self, 196 | database_name: str, 197 | table_name: str, 198 | max_parts: int = -1, 199 | ) -> List[str]: 200 | partitions = self.client.get_partition_names( 201 | database_name, 202 | table_name, 203 | max_parts, 204 | ) 205 | return partitions 206 | 207 | def get_partitions( 208 | self, 209 | database_name: str, 210 | table_name: str, 211 | max_parts: int = -1, 212 | ) -> List[HPartition]: 213 | result_partitions = [] 214 | partitions: List[Partition] = self.client.get_partitions( 215 | database_name, 216 | table_name, 217 | max_parts, 218 | ) 219 | 220 | assert isinstance(partitions, list) 221 | 222 | for partition in partitions: 223 | assert isinstance(partition, Partition) 224 | assert isinstance(partition.sd, StorageDescriptor) 225 | assert isinstance(partition.sd.serdeInfo, SerDeInfo) 226 | assert isinstance(partition.dbName, str) 227 | assert isinstance(partition.tableName, str) 228 | assert isinstance(partition.values, list) 229 | assert isinstance(partition.parameters, dict) 230 | assert isinstance(partition.createTime, int) 231 | assert isinstance(partition.lastAccessTime, int) 232 | assert isinstance(partition.catName, str) 233 | 234 | serialization_lib = ( 235 | "" 236 | if partition.sd.serdeInfo.serializationLib is None 237 | else partition.sd.serdeInfo.serializationLib 238 | ) 239 | input_format = ( 240 | "" if partition.sd.inputFormat is None else partition.sd.inputFormat 241 | ) 242 | output_format = ( 243 | "" if partition.sd.outputFormat is None else partition.sd.outputFormat 244 | ) 245 | bucket_cols = ( 246 | [] if partition.sd.bucketCols is None else partition.sd.bucketCols 247 | ) 248 | sort_cols = [] if partition.sd.sortCols is None else partition.sd.sortCols 249 | num_buckets = ( 250 | 0 if partition.sd.numBuckets is None else partition.sd.numBuckets 251 | ) 252 | is_skewed = partition.sd.skewedInfo is not None 253 | 254 | assert isinstance(serialization_lib, str) 255 | assert isinstance(input_format, str) 256 | assert isinstance(output_format, str) 257 | assert isinstance(bucket_cols, list) 258 | assert isinstance(sort_cols, list) 259 | assert isinstance(num_buckets, int) 260 | 261 | storage_format = StorageFormat( 262 | serialization_lib, 263 | input_format, 264 | output_format, 265 | ) 266 | 267 | bucket_property = HiveBucketProperty( 268 | bucket_cols, 269 | num_buckets, 270 | BucketingVersion.V1, 271 | sort_cols, 272 | ) 273 | 274 | sd = HStorage( 275 | storage_format, 276 | is_skewed, 277 | partition.sd.location, 278 | bucket_property, 279 | partition.sd.serdeInfo.parameters, 280 | ) 281 | 282 | result_partition = HPartition( 283 | partition.dbName, 284 | partition.tableName, 285 | partition.values, 286 | partition.parameters, 287 | partition.createTime, 288 | partition.lastAccessTime, 289 | sd, 290 | partition.catName, 291 | partition.writeId, 292 | ) 293 | 294 | result_partitions.append(result_partition) 295 | 296 | return result_partitions 297 | 298 | def get_partition( 299 | self, 300 | database_name: str, 301 | table_name: str, 302 | partition_name: str, 303 | ) -> HPartition: 304 | partition: Partition = self.client.get_partition_by_name( 305 | database_name, 306 | table_name, 307 | partition_name, 308 | ) 309 | assert partition is not None 310 | assert isinstance(partition, Partition) 311 | assert isinstance(partition.sd, StorageDescriptor) 312 | assert isinstance(partition.sd.serdeInfo, SerDeInfo) 313 | assert isinstance(partition.sd.serdeInfo.serializationLib, str) 314 | assert partition.sd.inputFormat is not None 315 | assert partition.sd.outputFormat is not None 316 | 317 | serialization_lib = partition.sd.serdeInfo.serializationLib 318 | input_format = partition.sd.inputFormat 319 | output_format = partition.sd.outputFormat 320 | storage_format = StorageFormat( 321 | serialization_lib, 322 | input_format, 323 | output_format, 324 | ) 325 | sort_cols = [] if partition.sd.sortCols is None else partition.sd.sortCols 326 | bucket_cols = [] if partition.sd.bucketCols is None else partition.sd.bucketCols 327 | num_buckets = partition.sd.numBuckets or 0 328 | is_skewed = partition.sd.skewedInfo is not None 329 | location = "" if partition.sd.location is None else partition.sd.location 330 | serde_parameters = ( 331 | {} 332 | if partition.sd.serdeInfo.parameters is None 333 | else partition.sd.serdeInfo.parameters 334 | ) 335 | cat_name = "" if partition.catName is None else partition.catName 336 | write_id = -1 if partition.writeId is None else partition.writeId 337 | last_access_time = ( 338 | -1 if partition.lastAccessTime is None else partition.lastAccessTime 339 | ) 340 | partition_parameters = ( 341 | {} if partition.parameters is None else partition.parameters 342 | ) 343 | create_time = -1 if partition.createTime is None else partition.createTime 344 | values = [] if partition.values is None else partition.values 345 | db_name = partition.dbName 346 | partition_table_name = partition.tableName 347 | 348 | assert isinstance(sort_cols, list) 349 | assert isinstance(bucket_cols, list) 350 | assert isinstance(num_buckets, int) 351 | assert isinstance(location, str) 352 | assert isinstance(serde_parameters, dict) 353 | assert isinstance(cat_name, str) 354 | assert isinstance(write_id, int) 355 | assert isinstance(last_access_time, int) 356 | assert isinstance(partition_parameters, dict) 357 | assert isinstance(create_time, int) 358 | assert isinstance(values, list) 359 | assert isinstance(db_name, str) 360 | assert isinstance(partition_table_name, str) 361 | 362 | bucket_property = HiveBucketProperty( 363 | bucket_cols, 364 | num_buckets, 365 | BucketingVersion.V1, 366 | sort_cols, 367 | ) 368 | sd = HStorage( 369 | storage_format, 370 | is_skewed, 371 | location, 372 | bucket_property, 373 | serde_parameters, 374 | ) 375 | result_partition = HPartition( 376 | db_name, 377 | partition_table_name, 378 | values, 379 | partition_parameters, 380 | create_time, 381 | last_access_time, 382 | sd, 383 | cat_name, 384 | write_id, 385 | ) 386 | 387 | return result_partition 388 | 389 | def get_table(self, database_name: str, table_name: str) -> HTable: 390 | table: Table = self.client.get_table(database_name, table_name) 391 | 392 | columns = [] 393 | 394 | partition_columns = [] 395 | if table.partitionKeys is not None: 396 | if isinstance(table.partitionKeys, list): 397 | t_part_columns: List[FieldSchema] = table.partitionKeys 398 | for column in t_part_columns: 399 | if column is not None: 400 | if isinstance(column, FieldSchema): 401 | if column.type is not None: 402 | type_parser = TypeParser(column.type) 403 | else: 404 | raise TypeError("Expected type to be str, got None") 405 | if column.comment is not None: 406 | comment = column.comment 407 | else: 408 | comment = "" 409 | if column.name is not None: 410 | name = column.name 411 | else: 412 | raise TypeError("Expected name to be str, got None") 413 | partition_columns.append( 414 | HColumn(name, type_parser.parse_type(), comment) 415 | ) 416 | 417 | if table.sd is not None: 418 | if table.sd.cols is not None: 419 | if isinstance(table.sd.cols, list): 420 | t_columns: List[FieldSchema] = table.sd.cols 421 | for column in t_columns: 422 | if column is not None: 423 | if isinstance(column, FieldSchema): 424 | if column.type is not None: 425 | type_parser = TypeParser(column.type) 426 | else: 427 | raise TypeError("Expected type to be str, got None") 428 | if column.comment is not None: 429 | comment = column.comment 430 | else: 431 | comment = "" 432 | if column.name is not None: 433 | name = column.name 434 | else: 435 | raise TypeError("Expected name to be str, got None") 436 | columns.append( 437 | HColumn(name, type_parser.parse_type(), comment) 438 | ) 439 | 440 | if table.sd.serdeInfo is not None: 441 | if isinstance(table.sd.serdeInfo, SerDeInfo): 442 | if table.sd.serdeInfo.serializationLib is not None: 443 | if isinstance(table.sd.serdeInfo.serializationLib, str): 444 | serde = table.sd.serdeInfo.serializationLib 445 | else: 446 | raise TypeError( 447 | f"Expected serializationLib to be str, got {type(table.sd.serdeInfo.serializationLib)}" 448 | ) 449 | else: 450 | raise TypeError( 451 | f"Expected serdeInfo to be str, got {type(table.sd.serdeInfo)}" 452 | ) 453 | else: 454 | raise TypeError( 455 | f"Expected serdeInfo to be SerDeInfo, got {type(table.sd.serdeInfo)}" 456 | ) 457 | else: 458 | raise TypeError("Expected serdeInfo to be SerDeInfo, got None") 459 | 460 | if table.sd.inputFormat is not None: 461 | if isinstance(table.sd.inputFormat, str): 462 | input_format = table.sd.inputFormat 463 | else: 464 | raise TypeError( 465 | f"Expected inputFormat to be str, got {type(table.sd.inputFormat)}" 466 | ) 467 | else: 468 | raise TypeError("Expected inputFormat to be str, got None") 469 | 470 | if table.sd.outputFormat is not None: 471 | if isinstance(table.sd.outputFormat, str): 472 | output_format = table.sd.outputFormat 473 | else: 474 | raise TypeError( 475 | f"Expected outputFormat to be str, got {type(table.sd.outputFormat)}" 476 | ) 477 | else: 478 | raise TypeError("Expected outputFormat to be str, got None") 479 | 480 | storage_format = StorageFormat(serde, input_format, output_format) 481 | 482 | bucket_property = None 483 | if table.sd.bucketCols is not None: 484 | sort_cols = [] 485 | if table.sd.sortCols is not None: 486 | if isinstance(table.sd.sortCols, list): 487 | for order in table.sd.sortCols: 488 | sort_cols.append(HSortingColumn(order.col, order.order)) 489 | else: 490 | raise TypeError( 491 | f"Expected bucketCols to be list, got {type(table.sd.sortCols)}" 492 | ) 493 | 494 | version = BucketingVersion.V1 495 | if table.parameters is not None: 496 | if isinstance(table.parameters, dict): 497 | if ( 498 | table.parameters.get( 499 | "TABLE_BUCKETING_VERSION", BucketingVersion.V1 500 | ) 501 | == BucketingVersion.V2 502 | ): 503 | version = BucketingVersion.V2 504 | else: 505 | raise TypeError( 506 | f"Expected parameters to be dict, got {type(table.parameters)}" 507 | ) 508 | else: 509 | raise TypeError( 510 | f"Expected parameters to be dict, got {type(table.parameters)}" 511 | ) 512 | 513 | if table.sd.numBuckets is not None: 514 | if isinstance(table.sd.numBuckets, int): 515 | num_buckets = table.sd.numBuckets 516 | else: 517 | raise TypeError( 518 | f"Expected numBuckets to be int, got {type(table.sd.numBuckets)}" 519 | ) 520 | else: 521 | raise TypeError( 522 | f"Expected numBuckets to be int, got {type(table.sd.numBuckets)}" 523 | ) 524 | 525 | bucket_property = HiveBucketProperty( 526 | table.sd.bucketCols, num_buckets, version, sort_cols 527 | ) 528 | 529 | if table.sd.skewedInfo is None: 530 | is_skewed = False 531 | else: 532 | is_skewed = True 533 | 534 | if table.sd.location is not None: 535 | if isinstance(table.sd.location, str): 536 | location = table.sd.location 537 | else: 538 | raise TypeError( 539 | f"Expected location to be str, got {type(table.sd.location)}" 540 | ) 541 | else: 542 | location = None 543 | 544 | if table.sd.serdeInfo is not None: 545 | if isinstance(table.sd.serdeInfo, SerDeInfo): 546 | serde_info = table.sd.serdeInfo 547 | else: 548 | raise TypeError( 549 | f"Expected serdeInfo to be SerDeInfo, got {type(table.sd.serdeInfo)}" 550 | ) 551 | else: 552 | raise TypeError( 553 | f"Expected serdeInfo to be SerDeInfo, got {type(table.sd.serdeInfo)}" 554 | ) 555 | if serde_info.parameters is not None: 556 | if isinstance(serde_info.parameters, dict): 557 | serde_parameters = serde_info.parameters 558 | else: 559 | raise TypeError( 560 | f"Expected serdeInfo.parameters to be dict, got {type(serde_info.parameters)}" 561 | ) 562 | else: 563 | raise TypeError( 564 | f"Expected serdeInfo.parameters to be dict, got {type(serde_info.parameters)}" 565 | ) 566 | else: 567 | raise TypeError( 568 | f"Expected sd to be StorageDescriptor, got {type(table.sd)}" 569 | ) 570 | 571 | storage = HStorage( 572 | storage_format, 573 | is_skewed, 574 | location, 575 | bucket_property, 576 | serde_parameters, 577 | ) 578 | 579 | if table.parameters is not None: 580 | if isinstance(table.parameters, dict): 581 | params = table.parameters 582 | else: 583 | raise TypeError( 584 | f"Expected parameters to be dict, got {type(table.parameters)}" 585 | ) 586 | else: 587 | raise TypeError( 588 | f"Expected parameters to be dict, got {type(table.parameters)}" 589 | ) 590 | 591 | if table.tableType is not None: 592 | if isinstance(table.tableType, str): 593 | table_type = table.tableType 594 | else: 595 | raise TypeError( 596 | f"Expected tableType to be str, got {type(table.tableType)}" 597 | ) 598 | else: 599 | raise TypeError( 600 | f"Expected tableType to be str, got {type(table.tableType)}" 601 | ) 602 | 603 | if table.tableName is not None: 604 | table_name = table.tableName 605 | else: 606 | raise TypeError( 607 | f"Expected tableName to be str, got {type(table.tableName)}" 608 | ) 609 | 610 | if table.dbName is not None: 611 | db_name = table.dbName 612 | else: 613 | raise TypeError(f"Expected dbName to be str, got {type(table.dbName)}") 614 | 615 | return HTable( 616 | db_name, 617 | table_name, 618 | table_type, 619 | columns, 620 | partition_columns, 621 | storage, 622 | params, 623 | table.viewOriginalText, 624 | table.viewExpandedText, 625 | table.writeId, 626 | table.owner, 627 | ) 628 | 629 | def get_table_stats( 630 | self, 631 | table: HTable, 632 | columns: List[HColumn] | None = None, 633 | ) -> List[ColumnStats]: 634 | columns = columns or [] 635 | 636 | if len(columns) > 0: 637 | assert all(isinstance(column, HColumn) for column in columns) 638 | 639 | assert isinstance(table, HTable) 640 | assert table.name is not None 641 | assert table.database_name is not None 642 | assert table.columns is not None 643 | assert len(table.columns) > 0 644 | 645 | if len(columns) == 0: 646 | columns = table.columns 647 | 648 | column_names = [column.name for column in columns] 649 | results = [] 650 | 651 | for column in column_names: 652 | res = self.client.get_table_column_statistics( 653 | table.database_name, 654 | table.name, 655 | column, 656 | ) 657 | results.append(res) 658 | 659 | result_columns = [] 660 | for col in results: 661 | for obj in col.statsObj: 662 | name = obj.colName 663 | col_type = obj.colType 664 | stats = obj.statsData 665 | 666 | if stats.booleanStats is not None: 667 | c_stats = BooleanTypeStats( 668 | numTrues=stats.booleanStats.numTrues, 669 | numFalses=stats.booleanStats.numFalses, 670 | numNulls=stats.booleanStats.numNulls, 671 | ) 672 | elif stats.longStats is not None: 673 | c_stats = LongTypeStats( 674 | lowValue=stats.longStats.lowValue, 675 | highValue=stats.longStats.highValue, 676 | numNulls=stats.longStats.numNulls, 677 | cardinality=stats.longStats.numDVs, 678 | ) 679 | elif stats.doubleStats is not None: 680 | c_stats = DoubleTypeStats( 681 | cardinality=stats.doubleStats.numDVs, 682 | lowValue=stats.doubleStats.lowValue, 683 | highValue=stats.doubleStats.highValue, 684 | numNulls=stats.doubleStats.numNulls, 685 | ) 686 | elif stats.stringStats is not None: 687 | c_stats = StringTypeStats( 688 | avgColLen=stats.stringStats.avgColLen, 689 | maxColLen=stats.stringStats.maxColLen, 690 | cardinality=stats.stringStats.numDVs, 691 | numNulls=stats.stringStats.numNulls, 692 | ) 693 | elif stats.binaryStats is not None: 694 | c_stats = BinaryTypeStats( 695 | avgColLen=stats.binaryStats.avgColLen, 696 | maxColLen=stats.binaryStats.maxColLen, 697 | numNulls=stats.binaryStats.numNulls, 698 | ) 699 | elif stats.decimalStats is not None: 700 | c_stats = DecimalTypeStats( 701 | lowValue=stats.decimalStats.lowValue, 702 | highValue=stats.decimalStats.highValue, 703 | numNulls=stats.decimalStats.numNulls, 704 | cardinality=stats.decimalStats.numDVs, 705 | ) 706 | elif stats.dateStats is not None: 707 | c_stats = DateTypeStats( 708 | cardinality=stats.dateStats.numDVs, 709 | lowValue=stats.dateStats.lowValue, 710 | highValue=stats.dateStats.highValue, 711 | numNulls=stats.dateStats.numNulls, 712 | ) 713 | else: 714 | c_stats = None 715 | 716 | result = ColumnStats( 717 | catName=col.statsDesc.catName, 718 | dbName=col.statsDesc.dbName, 719 | tableName=col.statsDesc.tableName, 720 | partName=col.statsDesc.partName, 721 | isTblLevel=col.statsDesc.isTblLevel, 722 | lastAnalyzed=col.statsDesc.lastAnalyzed, 723 | columnName=name, 724 | columnType=col_type, 725 | stats=c_stats, 726 | ) 727 | 728 | result_columns.append(result) 729 | 730 | return result_columns 731 | 732 | 733 | class _HMSConnection: 734 | def __init__(self, host, port): 735 | self.host = host 736 | self.port = port 737 | self.transport = None 738 | 739 | def __enter__(self): 740 | socket = TSocket.TSocket(self.host, self.port) 741 | self.transport = TTransport.TBufferedTransport(socket) 742 | protocol = TBinaryProtocol(self.transport) 743 | self.transport.open() 744 | return HMS(hms.Client(protocol)) 745 | 746 | def __exit__(self, type, value, traceback): 747 | if self.transport is not None: 748 | self.transport.close() 749 | -------------------------------------------------------------------------------- /pymetastore/stats.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from pymetastore.hive_metastore import ttypes 5 | 6 | 7 | @dataclass 8 | class TypeStats: 9 | numNulls: int 10 | typeName: str 11 | 12 | 13 | @dataclass 14 | class BooleanTypeStats(TypeStats): 15 | numTrues: int 16 | numFalses: int 17 | 18 | def __init__(self, numTrues: int, numFalses: int, numNulls: int): 19 | super().__init__(numNulls, typeName="boolean") 20 | self.numTrues = numTrues 21 | self.numFalses = numFalses 22 | 23 | 24 | @dataclass 25 | class DoubleTypeStats(TypeStats): 26 | cardinality: int 27 | lowValue: Optional[float] 28 | highValue: Optional[float] 29 | 30 | def __init__( 31 | self, 32 | cardinality: int, 33 | numNulls: int, 34 | lowValue: Optional[float] = None, 35 | highValue: Optional[float] = None, 36 | ): 37 | super().__init__(typeName="double", numNulls=numNulls) 38 | self.cardinality = cardinality 39 | self.lowValue = lowValue 40 | self.highValue = highValue 41 | 42 | 43 | @dataclass 44 | class LongTypeStats(TypeStats): 45 | cardinality: int 46 | lowValue: Optional[int] 47 | highValue: Optional[int] 48 | 49 | def __init__( 50 | self, 51 | cardinality: int, 52 | numNulls: int, 53 | lowValue: Optional[int] = None, 54 | highValue: Optional[int] = None, 55 | ): 56 | super().__init__(typeName="long", numNulls=numNulls) 57 | self.cardinality = cardinality 58 | self.lowValue = lowValue 59 | self.highValue = highValue 60 | 61 | 62 | @dataclass 63 | class StringTypeStats(TypeStats): 64 | maxColLen: int 65 | avgColLen: float 66 | cardinality: int 67 | 68 | def __init__( 69 | self, maxColLen: int, avgColLen: float, cardinality: int, numNulls: int 70 | ): 71 | super().__init__(typeName="string", numNulls=numNulls) 72 | self.maxColLen = maxColLen 73 | self.avgColLen = avgColLen 74 | self.cardinality = cardinality 75 | 76 | 77 | @dataclass 78 | class BinaryTypeStats(TypeStats): 79 | maxColLen: int 80 | avgColLen: float 81 | 82 | def __init__(self, maxColLen: int, avgColLen: float, numNulls: int): 83 | super().__init__(typeName="binary", numNulls=numNulls) 84 | self.maxColLen = maxColLen 85 | self.avgColLen = avgColLen 86 | 87 | 88 | @dataclass 89 | class DecimalTypeStats(TypeStats): 90 | cardinality: int 91 | lowValue: Optional[ttypes.Decimal] 92 | highValue: Optional[ttypes.Decimal] 93 | 94 | def __init__( 95 | self, 96 | cardinality: int, 97 | numNulls: int, 98 | lowValue: Optional[ttypes.Decimal] = None, 99 | highValue: Optional[ttypes.Decimal] = None, 100 | ): 101 | super().__init__(typeName="decimal", numNulls=numNulls) 102 | self.cardinality = cardinality 103 | self.lowValue = lowValue 104 | self.highValue = highValue 105 | 106 | 107 | @dataclass 108 | class DateTypeStats(TypeStats): 109 | cardinality: int 110 | lowValue: Optional[ttypes.Date] 111 | highValue: Optional[ttypes.Date] 112 | 113 | def __init__( 114 | self, 115 | cardinality: int, 116 | numNulls: int, 117 | lowValue: Optional[ttypes.Date] = None, 118 | highValue: Optional[ttypes.Date] = None, 119 | ): 120 | super().__init__(typeName="date", numNulls=numNulls) 121 | self.cardinality = cardinality 122 | self.lowValue = lowValue 123 | self.highValue = highValue 124 | 125 | 126 | @dataclass 127 | class ColumnStats: 128 | isTblLevel: bool 129 | dbName: str 130 | tableName: str 131 | columnName: str 132 | columnType: str 133 | stats: Optional[TypeStats] = None # not all column types have stats 134 | partName: Optional[str] = None 135 | lastAnalyzed: Optional[int] = None 136 | catName: Optional[str] = None 137 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | 2 | [project] 3 | name = "pymetastore" 4 | version = "0.4.2" 5 | description = "A Python client for the Thrift interface to Hive Metastore" 6 | authors = [ 7 | {name = "Chris Riccomini", email = "criccomini@apache.org"}, 8 | {name = "Kostas Pardalis"}, 9 | ] 10 | dependencies = [ 11 | "thrift>=0.16.0", 12 | ] 13 | requires-python = ">=3.8" 14 | readme = "README.md" 15 | license = {text = "MIT"} 16 | keywords = [ 17 | "data", 18 | "data catalog", 19 | "data discovery", 20 | "data engineering", 21 | "data governance", 22 | "data infrastructure", 23 | "data integration", 24 | "data pipelines", 25 | "hcatalog", 26 | "hive", 27 | "hive metastore", 28 | "metadata", 29 | "metastore", 30 | "thrift", 31 | "python", 32 | "recap", 33 | ] 34 | 35 | [project.urls] 36 | repository = "https://github.com/recap-build/pymetastore" 37 | 38 | [build-system] 39 | requires = ["pdm-backend"] 40 | build-backend = "pdm.backend" 41 | 42 | [tool.pdm.dev-dependencies] 43 | test = [ 44 | "pytest>=7.3.2", 45 | ] 46 | style = [ 47 | "black>=23.3.0", 48 | "isort>=5.11.5", 49 | "pylint>=2.13.9", 50 | "pyright>=1.1.315", 51 | ] 52 | 53 | [tool.pdm.scripts] 54 | thrift = "thrift -r --gen py --out pymetastore vendor/hive-thrift/src/main/thrift/hive_metastore.thrift" 55 | black = "black pymetastore/ tests/" 56 | isort = "isort pymetastore/ tests/" 57 | style = {composite = ["black", "isort"]} 58 | 59 | [tool.black] 60 | exclude = "pymetastore/hive_metastore/" 61 | 62 | [tool.isort] 63 | profile = "black" 64 | case_sensitive = true 65 | skip_glob = ["pymetastore/hive_metastore/*"] 66 | 67 | [tool.pylint.master] 68 | ignore = "pymetastore/hive_metastore" 69 | fail-under = 7 70 | 71 | [tool.pylint.messages_control] 72 | max-line-length = 110 73 | 74 | [tool.pyright] 75 | include = ["pymetastore/"] 76 | pythonPlatform = "All" 77 | exclude = [ 78 | "**/__pycache__", 79 | "pymetastore/hive_metastore" 80 | ] 81 | 82 | [tool.pytest.ini_options] 83 | markers = [ 84 | "integration: marks tests as an integration test", 85 | ] 86 | -------------------------------------------------------------------------------- /tests/integration/test_metastore.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration tests for the metastore module. 3 | """ 4 | import os 5 | 6 | import pytest 7 | from thrift.protocol import TBinaryProtocol 8 | from thrift.transport import TSocket, TTransport 9 | 10 | from pymetastore.hive_metastore import ttypes 11 | from pymetastore.hive_metastore.ThriftHiveMetastore import Client 12 | from pymetastore.htypes import ( 13 | HCharType, 14 | HDecimalType, 15 | HListType, 16 | HMapType, 17 | HPrimitiveType, 18 | HStructType, 19 | HType, 20 | HTypeCategory, 21 | HUnionType, 22 | HVarcharType, 23 | PrimitiveCategory, 24 | ) 25 | from pymetastore.metastore import ( 26 | HMS, 27 | BucketingVersion, 28 | HColumn, 29 | HDatabase, 30 | HPartition, 31 | HStorage, 32 | HTable, 33 | HiveBucketProperty, 34 | StorageFormat, 35 | ) 36 | from pymetastore.stats import ( 37 | BinaryTypeStats, 38 | BooleanTypeStats, 39 | DateTypeStats, 40 | DecimalTypeStats, 41 | DoubleTypeStats, 42 | LongTypeStats, 43 | StringTypeStats, 44 | ) 45 | 46 | 47 | @pytest.fixture(scope="module") 48 | def hive_client(): 49 | """ 50 | Create a connection to the Hive Metastore. 51 | 52 | This function opens a connection to the Hive Metastore on a specified host and port. 53 | The host and port can be defined in environment variables HMS_HOST and HMS_PORT. 54 | If not defined, it defaults to "localhost" and 9083, respectively. 55 | 56 | The function uses a context manager (via the `yield` keyword) to ensure that the 57 | transport connection is properly closed after use. 58 | 59 | Yields: 60 | client: A handle to the Hive Metastore client. 61 | 62 | Raises: 63 | TTransportException: If a connection to the Hive Metastore cannot be established. 64 | """ 65 | host = os.environ.get("HMS_HOST", "localhost") 66 | port = int(os.environ.get("HMS_PORT", 9083)) 67 | transport = TSocket.TSocket(host, port) 68 | transport = TTransport.TBufferedTransport(transport) 69 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 70 | client = Client(protocol) 71 | transport.open() 72 | 73 | yield client 74 | 75 | transport.close() 76 | 77 | 78 | @pytest.fixture(scope="module", autouse=True) 79 | # pylint: disable=redefined-outer-name 80 | # pylint: disable=too-many-locals 81 | # pylint: disable=invalid-name 82 | def setup_data(hive_client): 83 | """ 84 | Set up initial data for testing the core functionality of pymetastore. 85 | 86 | This function creates a test database, several test tables, and partitions for 87 | these tables in the Hive metastore. It tests various data types, including 88 | primitive types, parameterized types, and table statistics. 89 | 90 | Args: 91 | hive_client: A handle to the Hive metastore client. 92 | """ 93 | db = ttypes.Database( 94 | name="test_db", 95 | description="This is a test database", 96 | locationUri="/tmp/test_db.db", 97 | parameters={}, 98 | ownerName="owner", 99 | ) 100 | 101 | if "test_db" in hive_client.get_all_databases(): 102 | hive_client.drop_database("test_db", True, True) 103 | hive_client.create_database(db) 104 | 105 | cols = [ 106 | ttypes.FieldSchema(name="col1", type="int", comment="c1"), 107 | ttypes.FieldSchema(name="col2", type="string", comment="c2"), 108 | ] 109 | 110 | partition_keys = [ttypes.FieldSchema(name="partition", type="string", comment="")] 111 | 112 | serde_info = ttypes.SerDeInfo( 113 | name="test_serde", 114 | serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", 115 | parameters={"field.delim": ","}, 116 | ) 117 | 118 | storage_desc = ttypes.StorageDescriptor( 119 | location="/tmp/test_db/test_table", 120 | cols=cols, 121 | inputFormat="org.apache.hadoop.mapred.TextInputFormat", 122 | outputFormat="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 123 | compressed=False, 124 | numBuckets=-1, 125 | serdeInfo=serde_info, 126 | bucketCols=[], 127 | ) 128 | 129 | table = ttypes.Table( 130 | tableName="test_table", 131 | dbName="test_db", 132 | owner="owner", 133 | createTime=0, 134 | lastAccessTime=0, 135 | retention=0, 136 | sd=storage_desc, 137 | partitionKeys=partition_keys, 138 | # We have to manually set the EXTERNAL parameter to TRUE, otherwise the 139 | # metastore returns tableType=None. See https://github.com/recap-build/pymetastore/issues/37. 140 | parameters={"EXTERNAL": "TRUE"}, 141 | tableType="EXTERNAL_TABLE", 142 | ) 143 | 144 | if "test_table" in hive_client.get_all_tables("test_db"): 145 | hive_client.drop_table("test_db", "test_table", True) 146 | hive_client.create_table(table) 147 | 148 | for i in range(1, 6): 149 | partition = ttypes.Partition( 150 | dbName="test_db", 151 | tableName="test_table", 152 | values=[str(i)], 153 | sd=storage_desc, 154 | parameters={}, 155 | ) 156 | hive_client.add_partition(partition) 157 | 158 | # testing primitive Types 159 | cols2 = [ 160 | ttypes.FieldSchema(name="col3", type="void", comment="c3"), 161 | ttypes.FieldSchema(name="col4", type="boolean", comment="c4"), 162 | ttypes.FieldSchema(name="col5", type="tinyint", comment="c5"), 163 | ttypes.FieldSchema(name="col6", type="smallint", comment="c6"), 164 | ttypes.FieldSchema(name="col7", type="bigint", comment="c7"), 165 | ttypes.FieldSchema(name="col8", type="float", comment="c8"), 166 | ttypes.FieldSchema(name="col9", type="double", comment="c9"), 167 | ttypes.FieldSchema(name="col10", type="date", comment="c10"), 168 | ttypes.FieldSchema(name="col11", type="timestamp", comment="c11"), 169 | ttypes.FieldSchema( 170 | name="col12", type="timestamp with local time zone", comment="c12" 171 | ), 172 | ttypes.FieldSchema(name="col13", type="interval_year_month", comment="c13"), 173 | ttypes.FieldSchema(name="col14", type="interval_day_time", comment="c14"), 174 | ttypes.FieldSchema(name="col15", type="binary", comment="c15"), 175 | ] 176 | 177 | storage_desc = ttypes.StorageDescriptor( 178 | location="/tmp/test_db/test_table2", 179 | cols=cols2, 180 | inputFormat="org.apache.hadoop.mapred.TextInputFormat", 181 | outputFormat="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 182 | compressed=False, 183 | numBuckets=-1, 184 | serdeInfo=serde_info, 185 | bucketCols=[], 186 | ) 187 | 188 | table = ttypes.Table( 189 | tableName="test_table2", 190 | dbName="test_db", 191 | owner="owner", 192 | createTime=0, 193 | lastAccessTime=0, 194 | retention=0, 195 | sd=storage_desc, 196 | partitionKeys=partition_keys, 197 | # We have to manually set the EXTERNAL parameter to TRUE, otherwise the 198 | # metastore returns tableType=None. See https://github.com/recap-build/pymetastore/issues/37. 199 | parameters={"EXTERNAL": "TRUE"}, 200 | tableType="EXTERNAL_TABLE", 201 | ) 202 | 203 | if "test_table2" in hive_client.get_all_tables("test_db"): 204 | hive_client.drop_table("test_db", "test_table2", True) 205 | hive_client.create_table(table) 206 | 207 | # testing Parameterized Types 208 | cols3 = [ 209 | ttypes.FieldSchema(name="col16", type="decimal(10,2)", comment="c16"), 210 | ttypes.FieldSchema(name="col17", type="varchar(10)", comment="c17"), 211 | ttypes.FieldSchema(name="col18", type="char(10)", comment="c18"), 212 | ttypes.FieldSchema(name="col19", type="array", comment="c19"), 213 | ttypes.FieldSchema(name="col20", type="map", comment="c20"), 214 | ttypes.FieldSchema(name="col21", type="struct", comment="c21"), 215 | ttypes.FieldSchema(name="col22", type="uniontype", comment="c22"), 216 | ] 217 | 218 | storage_desc = ttypes.StorageDescriptor( 219 | location="/tmp/test_db/test_table3", 220 | cols=cols3, 221 | inputFormat="org.apache.hadoop.mapred.TextInputFormat", 222 | outputFormat="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 223 | compressed=False, 224 | numBuckets=-1, 225 | serdeInfo=serde_info, 226 | bucketCols=[], 227 | ) 228 | 229 | table = ttypes.Table( 230 | tableName="test_table3", 231 | dbName="test_db", 232 | owner="owner", 233 | createTime=0, 234 | lastAccessTime=0, 235 | retention=0, 236 | sd=storage_desc, 237 | partitionKeys=partition_keys, 238 | # We have to manually set the EXTERNAL parameter to TRUE, otherwise the 239 | # metastore returns tableType=None. See https://github.com/recap-build/pymetastore/issues/37. 240 | parameters={"EXTERNAL": "TRUE"}, 241 | tableType="EXTERNAL_TABLE", 242 | ) 243 | 244 | if "test_table3" in hive_client.get_all_tables("test_db"): 245 | hive_client.drop_table("test_db", "test_table3", True) 246 | hive_client.create_table(table) 247 | 248 | cols4 = [ 249 | ttypes.FieldSchema(name="col4", type="boolean", comment="c4"), 250 | ttypes.FieldSchema(name="col5", type="double", comment="c5"), 251 | ttypes.FieldSchema(name="col6", type="bigint", comment="c6"), 252 | ttypes.FieldSchema(name="col7", type="string", comment="c7"), 253 | ttypes.FieldSchema(name="col8", type="binary", comment="c8"), 254 | ttypes.FieldSchema(name="col9", type="decimal(10,2)", comment="c9"), 255 | ttypes.FieldSchema(name="col10", type="date", comment="c10"), 256 | ] 257 | 258 | storage_desc = ttypes.StorageDescriptor( 259 | location="/tmp/test_db/test_table4", 260 | cols=cols4, 261 | inputFormat="org.apache.hadoop.mapred.TextInputFormat", 262 | outputFormat="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 263 | compressed=False, 264 | numBuckets=-1, 265 | serdeInfo=serde_info, 266 | bucketCols=[], 267 | ) 268 | 269 | table = ttypes.Table( 270 | tableName="test_table4", 271 | dbName="test_db", 272 | owner="owner", 273 | createTime=0, 274 | lastAccessTime=0, 275 | retention=0, 276 | sd=storage_desc, 277 | partitionKeys=partition_keys, 278 | # We have to manually set the EXTERNAL parameter to TRUE, otherwise the 279 | # metastore returns tableType=None. See https://github.com/recap-build/pymetastore/issues/37. 280 | parameters={"EXTERNAL": "TRUE"}, 281 | tableType="EXTERNAL_TABLE", 282 | ) 283 | 284 | if "test_table4" in hive_client.get_all_tables("test_db"): 285 | hive_client.drop_table("test_db", "test_table4", True) 286 | hive_client.create_table(table) 287 | 288 | stats_desc = ttypes.ColumnStatisticsDesc(True, "test_db", "test_table4") 289 | 290 | stats_bool = ttypes.BooleanColumnStatsData(numTrues=10, numFalses=10, numNulls=0) 291 | stats_obj_bool = ttypes.ColumnStatisticsObj( 292 | "col4", "boolean", ttypes.ColumnStatisticsData(booleanStats=stats_bool) 293 | ) 294 | 295 | stats_double = ttypes.DoubleColumnStatsData( 296 | lowValue=0, highValue=1, numNulls=0, numDVs=100 297 | ) 298 | stats_obj_double = ttypes.ColumnStatisticsObj( 299 | "col5", "double", ttypes.ColumnStatisticsData(doubleStats=stats_double) 300 | ) 301 | stats_long = ttypes.LongColumnStatsData( 302 | lowValue=0, highValue=100, numNulls=0, numDVs=100 303 | ) 304 | stats_obj_long = ttypes.ColumnStatisticsObj( 305 | "col6", "bigint", ttypes.ColumnStatisticsData(longStats=stats_long) 306 | ) 307 | stats_string = ttypes.StringColumnStatsData( 308 | avgColLen=10, maxColLen=10, numNulls=5, numDVs=10 309 | ) 310 | stats_obj_string = ttypes.ColumnStatisticsObj( 311 | "col7", "string", ttypes.ColumnStatisticsData(stringStats=stats_string) 312 | ) 313 | stats_binary = ttypes.BinaryColumnStatsData(avgColLen=10, maxColLen=10, numNulls=5) 314 | stats_obj_binary = ttypes.ColumnStatisticsObj( 315 | "col8", "binary", ttypes.ColumnStatisticsData(binaryStats=stats_binary) 316 | ) 317 | stats_decimal = ttypes.DecimalColumnStatsData( 318 | lowValue=ttypes.Decimal(1, b"123445.3"), 319 | highValue=ttypes.Decimal(1, b"1232324124"), 320 | numNulls=0, 321 | numDVs=100, 322 | ) 323 | stats_obj_decimal = ttypes.ColumnStatisticsObj( 324 | "col9", "decimal(10,2)", ttypes.ColumnStatisticsData(decimalStats=stats_decimal) 325 | ) 326 | stats_date = ttypes.DateColumnStatsData( 327 | lowValue=ttypes.Date(0), highValue=ttypes.Date(1), numNulls=0, numDVs=100 328 | ) 329 | stats_obj_date = ttypes.ColumnStatisticsObj( 330 | "col10", "date", ttypes.ColumnStatisticsData(dateStats=stats_date) 331 | ) 332 | col_stats = ttypes.ColumnStatistics( 333 | stats_desc, 334 | [ 335 | stats_obj_bool, 336 | stats_obj_double, 337 | stats_obj_long, 338 | stats_obj_string, 339 | stats_obj_binary, 340 | stats_obj_decimal, 341 | stats_obj_date, 342 | ], 343 | ) 344 | hive_client.update_table_column_statistics(col_stats) 345 | 346 | 347 | # pylint: disable=redefined-outer-name 348 | def test_list_databases(hive_client): 349 | """ 350 | Test the functionality of listing all databases. 351 | 352 | This function retrieves all databases and checks that the database "test_db" is present. 353 | 354 | Args: 355 | hive_client: A handle to the Hive metastore client. 356 | 357 | Raises: 358 | AssertionError: If "test_db" is not found in the list of databases. 359 | """ 360 | assert "test_db" in HMS(hive_client).list_databases() 361 | 362 | 363 | # pylint: disable=redefined-outer-name 364 | def test_get_database(hive_client): 365 | """ 366 | Test the functionality of retrieving database metadata. 367 | 368 | This function retrieves the database "test_db" and checks that the properties 369 | of the database match the expected values. 370 | 371 | Args: 372 | hive_client: A handle to the Hive metastore client. 373 | 374 | Raises: 375 | AssertionError: If the properties of the fetched database do not match the expected values. 376 | """ 377 | hms = HMS(hive_client) 378 | database = hms.get_database("test_db") 379 | assert isinstance(database, HDatabase) 380 | assert database.name == "test_db" 381 | assert database.location == "file:/tmp/test_db.db" 382 | assert database.owner_name == "owner" 383 | 384 | 385 | # pylint: disable=redefined-outer-name 386 | def test_list_tables(hive_client): 387 | """ 388 | Test the functionality of listing all tables in a database. 389 | 390 | This function retrieves all tables from the database "test_db" and checks 391 | that the table "test_table" is present. 392 | 393 | Args: 394 | hive_client: A handle to the Hive metastore client. 395 | 396 | Raises: 397 | AssertionError: If "test_table" is not found in the list of tables in "test_db". 398 | """ 399 | hms = HMS(hive_client) 400 | tables = hms.list_tables("test_db") 401 | assert isinstance(tables, list) 402 | assert "test_table" in tables 403 | 404 | 405 | # pylint: disable=redefined-outer-name 406 | def test_get_table(hive_client): 407 | """ 408 | Test the functionality of retrieving table metadata. 409 | 410 | This function retrieves the table "test_table" from the database "test_db" and 411 | checks that the properties of the table match the expected values. This includes 412 | checks on the table name, owner, location, columns, and table type. 413 | 414 | Args: 415 | hive_client: A handle to the Hive metastore client. 416 | 417 | Raises: 418 | AssertionError: If the properties of the fetched table do not match the expected values. 419 | """ 420 | hms = HMS(hive_client) 421 | table = hms.get_table("test_db", "test_table") 422 | assert isinstance(table, HTable) 423 | assert table.database_name == "test_db" 424 | assert table.name == "test_table" 425 | assert table.owner == "owner" 426 | assert table.storage.location == "file:/tmp/test_db/test_table" 427 | assert len(table.columns) == 2 428 | assert isinstance(table.columns[0], HColumn) 429 | assert isinstance(table.columns[1], HColumn) 430 | assert len(table.partition_columns) == 1 431 | assert isinstance(table.partition_columns[0], HColumn) 432 | assert isinstance(table.parameters, dict) 433 | assert len(table.parameters) == 2 434 | assert table.parameters.get("EXTERNAL") == "TRUE" 435 | # this is not a parameter of the table we created, but the metastore adds it 436 | assert table.parameters.get("transient_lastDdlTime") is not None 437 | assert table.table_type == "EXTERNAL_TABLE" 438 | 439 | 440 | # pylint: disable=redefined-outer-name 441 | def test_get_table_columns(hive_client): 442 | """ 443 | Test the functionality of retrieving table columns, including those defined for partitioning. 444 | 445 | This function retrieves the table "test_table" from the database "test_db" and 446 | fetches the column and partition column definitions. It then checks that the types 447 | and properties of the columns are correctly recognized and are of the expected types. 448 | 449 | Args: 450 | hive_client: A handle to the Hive metastore client. 451 | 452 | Raises: 453 | AssertionError: If the column types or properties do not match the expected values. 454 | """ 455 | hms = HMS(hive_client) 456 | table = hms.get_table("test_db", "test_table") 457 | columns = table.columns 458 | partition_columns = table.partition_columns 459 | 460 | assert columns[0].name == "col1" 461 | assert isinstance(columns[0].type, HType) 462 | assert isinstance(columns[0].type, HPrimitiveType) 463 | assert columns[0].type.name == "INT" 464 | assert columns[0].type.category == HTypeCategory.PRIMITIVE 465 | assert columns[0].comment == "c1" 466 | 467 | assert columns[1].name == "col2" 468 | assert isinstance(columns[1].type, HType) 469 | assert isinstance(columns[1].type, HPrimitiveType) 470 | assert columns[1].type.name == "STRING" 471 | assert columns[1].type.category == HTypeCategory.PRIMITIVE 472 | assert columns[1].comment == "c2" 473 | 474 | assert partition_columns[0].name == "partition" 475 | assert isinstance(partition_columns[0].type, HType) 476 | assert isinstance(partition_columns[0].type, HPrimitiveType) 477 | assert partition_columns[0].type.name == "STRING" 478 | assert partition_columns[0].type.category == HTypeCategory.PRIMITIVE 479 | assert partition_columns[0].comment == "" 480 | 481 | 482 | # pylint: disable=redefined-outer-name 483 | def test_list_columns(hive_client): 484 | """ 485 | Test the functionality of listing all column names of a table. 486 | 487 | This function retrieves all column names from the table "test_table" in the database 488 | "test_db". It then checks that the names of the returned columns are as expected. 489 | 490 | Args: 491 | hive_client: A handle to the Hive metastore client. 492 | 493 | Raises: 494 | AssertionError: If the names of the fetched columns do not match the expected values. 495 | """ 496 | hms = HMS(hive_client) 497 | columns = hms.list_columns("test_db", "test_table") 498 | 499 | assert isinstance(columns, list) 500 | assert len(columns) == 2 501 | assert columns[0] == "col1" 502 | assert columns[1] == "col2" 503 | 504 | 505 | # pylint: disable=redefined-outer-name 506 | def test_list_partitions(hive_client): 507 | """ 508 | Test the functionality of listing all partitions of a table. 509 | 510 | This function retrieves all partition names from the table "test_table" in the database 511 | "test_db". It then checks that all expected partitions are found. 512 | 513 | Args: 514 | hive_client: A handle to the Hive metastore client. 515 | 516 | Raises: 517 | AssertionError: If not all expected partitions are found. 518 | """ 519 | hms = HMS(hive_client) 520 | partitions = hms.list_partitions("test_db", "test_table") 521 | 522 | assert isinstance(partitions, list) 523 | assert len(partitions) == 5 524 | for i in range(1, 6): 525 | assert f"partition={i}" in partitions 526 | 527 | 528 | # pylint: disable=redefined-outer-name 529 | def test_get_partitions(hive_client): 530 | """ 531 | Test the functionality of retrieving all partitions of a table. 532 | 533 | This function retrieves all partitions from the table "test_table" in the database 534 | "test_db". It then checks that the properties of each returned partition are as 535 | expected and that all expected partitions are found. 536 | 537 | Args: 538 | hive_client: A handle to the Hive metastore client. 539 | 540 | Raises: 541 | AssertionError: If the properties of the fetched partitions do not match the 542 | expected values, or if not all expected partitions are found. 543 | """ 544 | hms = HMS(hive_client) 545 | partitions = hms.get_partitions("test_db", "test_table") 546 | expected_storage = HStorage( 547 | storage_format=StorageFormat( 548 | serde="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", 549 | input_format="org.apache.hadoop.mapred.TextInputFormat", 550 | output_format="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 551 | ), 552 | skewed=True, 553 | location="file:/tmp/test_db/test_table", 554 | bucket_property=HiveBucketProperty( 555 | bucketed_by=[], 556 | bucket_count=-1, 557 | version=BucketingVersion.V1, 558 | sorting_columns=[], 559 | ), 560 | serde_parameters={"field.delim": ","}, 561 | ) 562 | 563 | assert isinstance(partitions, list) 564 | assert len(partitions) == 5 # we created 5 partitions in the setup_data fixture 565 | 566 | missing_partitions = {1, 2, 3, 4, 5} 567 | 568 | for partition in partitions: 569 | assert isinstance(partition, HPartition) 570 | missing_partitions.discard(int(partition.values[0])) 571 | assert partition.database_name == "test_db" 572 | assert partition.table_name == "test_table" 573 | assert partition.sd == expected_storage 574 | assert partition.create_time != 0 575 | assert partition.last_access_time == 0 576 | assert partition.cat_name == "hive" 577 | assert isinstance(partition.parameters, dict) 578 | assert partition.write_id == -1 579 | 580 | assert missing_partitions == set() 581 | 582 | 583 | # pylint: disable=redefined-outer-name 584 | def test_get_partition(hive_client): 585 | """ 586 | Test the functionality of retrieving a single partition. 587 | 588 | This function retrieves a specific partition "partition=1" from the table 589 | "test_table" in the database "test_db". It then checks that the properties 590 | of the returned partition are as expected. 591 | 592 | Args: 593 | hive_client: A handle to the Hive metastore client. 594 | 595 | Raises: 596 | AssertionError: If the properties of the fetched partition do not match the expected values. 597 | """ 598 | hms = HMS(hive_client) 599 | partition = hms.get_partition("test_db", "test_table", "partition=1") 600 | expected_storage = HStorage( 601 | storage_format=StorageFormat( 602 | serde="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", 603 | input_format="org.apache.hadoop.mapred.TextInputFormat", 604 | output_format="org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", 605 | ), 606 | skewed=True, 607 | location="file:/tmp/test_db/test_table", 608 | bucket_property=HiveBucketProperty( 609 | bucketed_by=[], 610 | bucket_count=-1, 611 | version=BucketingVersion.V1, 612 | sorting_columns=[], 613 | ), 614 | serde_parameters={"field.delim": ","}, 615 | ) 616 | 617 | assert isinstance(partition, HPartition) 618 | assert partition.database_name == "test_db" 619 | assert partition.table_name == "test_table" 620 | assert partition.values == ["1"] 621 | assert partition.sd == expected_storage 622 | assert partition.create_time != 0 623 | assert partition.last_access_time == 0 624 | assert partition.cat_name == "hive" 625 | assert isinstance(partition.parameters, dict) 626 | assert partition.write_id == -1 627 | 628 | 629 | # pylint: disable=redefined-outer-name 630 | # pylint: disable=too-many-statements 631 | def test_primitive_types(hive_client): 632 | """ 633 | Test the functionality of handling primitive types. 634 | 635 | This function retrieves the table "test_table2" from the database "test_db" and 636 | fetches the column definitions. It then checks that the types and properties 637 | of the columns are correctly recognized and are of the expected primitive types. 638 | 639 | Args: 640 | hive_client: A handle to the Hive metastore client. 641 | 642 | Raises: 643 | AssertionError: If the column types or properties do not match the expected values. 644 | """ 645 | hms = HMS(hive_client) 646 | 647 | table = hms.get_table("test_db", "test_table2") 648 | columns = table.columns 649 | 650 | assert len(columns) == 13 651 | 652 | assert columns[0].name == "col3" 653 | assert isinstance(columns[0].type, HType) 654 | assert isinstance(columns[0].type, HPrimitiveType) 655 | assert columns[0].type.name == "VOID" 656 | assert columns[0].type.category == HTypeCategory.PRIMITIVE 657 | assert columns[0].comment == "c3" 658 | 659 | assert columns[1].name == "col4" 660 | assert isinstance(columns[1].type, HType) 661 | assert isinstance(columns[1].type, HPrimitiveType) 662 | assert columns[1].type.name == "BOOLEAN" 663 | assert columns[1].type.category == HTypeCategory.PRIMITIVE 664 | assert columns[1].comment == "c4" 665 | 666 | assert columns[2].name == "col5" 667 | assert isinstance(columns[2].type, HType) 668 | assert isinstance(columns[2].type, HPrimitiveType) 669 | assert columns[2].type.name == "BYTE" 670 | assert columns[2].type.category == HTypeCategory.PRIMITIVE 671 | assert columns[2].comment == "c5" 672 | 673 | assert columns[3].name == "col6" 674 | assert isinstance(columns[3].type, HType) 675 | assert isinstance(columns[3].type, HPrimitiveType) 676 | assert columns[3].type.name == "SHORT" 677 | assert columns[3].type.category == HTypeCategory.PRIMITIVE 678 | assert columns[3].comment == "c6" 679 | 680 | assert columns[4].name == "col7" 681 | assert isinstance(columns[4].type, HType) 682 | assert isinstance(columns[4].type, HPrimitiveType) 683 | assert columns[4].type.name == "LONG" 684 | assert columns[4].type.category == HTypeCategory.PRIMITIVE 685 | assert columns[4].comment == "c7" 686 | 687 | assert columns[5].name == "col8" 688 | assert isinstance(columns[5].type, HType) 689 | assert isinstance(columns[5].type, HPrimitiveType) 690 | assert columns[5].type.name == "FLOAT" 691 | assert columns[5].type.category == HTypeCategory.PRIMITIVE 692 | assert columns[5].comment == "c8" 693 | 694 | assert columns[6].name == "col9" 695 | assert isinstance(columns[6].type, HType) 696 | assert isinstance(columns[6].type, HPrimitiveType) 697 | assert columns[6].type.name == "DOUBLE" 698 | assert columns[6].type.category == HTypeCategory.PRIMITIVE 699 | assert columns[6].comment == "c9" 700 | 701 | assert columns[7].name == "col10" 702 | assert isinstance(columns[7].type, HType) 703 | assert isinstance(columns[7].type, HPrimitiveType) 704 | assert columns[7].type.name == "DATE" 705 | assert columns[7].type.category == HTypeCategory.PRIMITIVE 706 | assert columns[7].comment == "c10" 707 | 708 | assert columns[8].name == "col11" 709 | assert isinstance(columns[8].type, HType) 710 | assert isinstance(columns[8].type, HPrimitiveType) 711 | assert columns[8].type.name == "TIMESTAMP" 712 | assert columns[8].type.category == HTypeCategory.PRIMITIVE 713 | assert columns[8].comment == "c11" 714 | 715 | assert columns[9].name == "col12" 716 | assert isinstance(columns[9].type, HType) 717 | assert isinstance(columns[9].type, HPrimitiveType) 718 | assert columns[9].type.name == "TIMESTAMPLOCALTZ" 719 | assert columns[9].type.category == HTypeCategory.PRIMITIVE 720 | assert columns[9].comment == "c12" 721 | 722 | assert columns[10].name == "col13" 723 | assert isinstance(columns[10].type, HType) 724 | assert isinstance(columns[10].type, HPrimitiveType) 725 | assert columns[10].type.name == "INTERVAL_YEAR_MONTH" 726 | assert columns[10].type.category == HTypeCategory.PRIMITIVE 727 | assert columns[10].comment == "c13" 728 | 729 | assert columns[11].name == "col14" 730 | assert isinstance(columns[11].type, HType) 731 | assert isinstance(columns[11].type, HPrimitiveType) 732 | assert columns[11].type.name == "INTERVAL_DAY_TIME" 733 | assert columns[11].type.category == HTypeCategory.PRIMITIVE 734 | assert columns[11].comment == "c14" 735 | 736 | assert columns[12].name == "col15" 737 | assert isinstance(columns[12].type, HType) 738 | assert isinstance(columns[12].type, HPrimitiveType) 739 | assert columns[12].type.name == "BINARY" 740 | assert columns[12].type.category == HTypeCategory.PRIMITIVE 741 | assert columns[12].comment == "c15" 742 | 743 | 744 | # pylint: disable=redefined-outer-name 745 | # pylint: disable=too-many-statements 746 | def test_parameterized_types(hive_client): 747 | """ 748 | Test the functionality of handling parameterized types. 749 | 750 | This function retrieves the table "test_table3" from the database "test_db" and 751 | fetches the column definitions. It then checks that the types and properties 752 | of the columns are correctly recognized and are of the expected types. 753 | 754 | Args: 755 | hive_client: A handle to the Hive metastore client. 756 | 757 | Raises: 758 | AssertionError: If the column types or properties do not match the expected values. 759 | """ 760 | hms = HMS(hive_client) 761 | table = hms.get_table("test_db", "test_table3") 762 | columns = table.columns 763 | 764 | assert len(columns) == 7 765 | 766 | assert columns[0].name == "col16" 767 | assert isinstance(columns[0].type, HType) 768 | assert isinstance(columns[0].type, HDecimalType) 769 | assert columns[0].type.name == "DECIMAL" 770 | assert columns[0].type.category == HTypeCategory.PRIMITIVE 771 | assert columns[0].comment == "c16" 772 | assert columns[0].type.precision == 10 773 | assert columns[0].type.scale == 2 774 | 775 | assert columns[1].name == "col17" 776 | assert isinstance(columns[1].type, HType) 777 | assert isinstance(columns[1].type, HVarcharType) 778 | assert columns[1].type.name == "VARCHAR" 779 | assert columns[1].type.category == HTypeCategory.PRIMITIVE 780 | assert columns[1].comment == "c17" 781 | assert columns[1].type.length == 10 782 | 783 | assert columns[2].name == "col18" 784 | assert isinstance(columns[2].type, HType) 785 | assert isinstance(columns[2].type, HCharType) 786 | assert columns[2].type.name == "CHAR" 787 | assert columns[2].type.category == HTypeCategory.PRIMITIVE 788 | assert columns[2].comment == "c18" 789 | assert columns[2].type.length == 10 790 | 791 | assert columns[3].name == "col19" 792 | assert isinstance(columns[3].type, HType) 793 | assert isinstance(columns[3].type, HListType) 794 | assert columns[3].type.name == "LIST" 795 | assert columns[3].type.category == HTypeCategory.LIST 796 | assert columns[3].comment == "c19" 797 | assert isinstance(columns[3].type.element_type, HType) 798 | assert isinstance(columns[3].type.element_type, HPrimitiveType) 799 | assert columns[3].type.element_type.name == "INT" 800 | 801 | assert columns[4].name == "col20" 802 | assert isinstance(columns[4].type, HType) 803 | assert isinstance(columns[4].type, HMapType) 804 | assert columns[4].type.name == "MAP" 805 | assert columns[4].type.category == HTypeCategory.MAP 806 | assert columns[4].comment == "c20" 807 | assert isinstance(columns[4].type.key_type, HType) 808 | assert isinstance(columns[4].type.key_type, HPrimitiveType) 809 | assert columns[4].type.key_type.name == "INT" 810 | assert isinstance(columns[4].type.value_type, HType) 811 | assert isinstance(columns[4].type.value_type, HPrimitiveType) 812 | assert columns[4].type.value_type.name == "STRING" 813 | 814 | assert columns[5].name == "col21" 815 | assert isinstance(columns[5].type, HType) 816 | assert isinstance(columns[5].type, HStructType) 817 | assert columns[5].type.name == "STRUCT" 818 | assert columns[5].type.category == HTypeCategory.STRUCT 819 | assert columns[5].comment == "c21" 820 | assert len(columns[5].type.names) == 2 821 | assert len(columns[5].type.types) == 2 822 | assert columns[5].type.names[0] == "a" 823 | assert columns[5].type.names[1] == "b" 824 | assert isinstance(columns[5].type.types[0], HType) 825 | assert isinstance(columns[5].type.types[0], HPrimitiveType) 826 | assert columns[5].type.types[0].name == "INT" 827 | assert isinstance(columns[5].type.types[1], HType) 828 | assert isinstance(columns[5].type.types[1], HPrimitiveType) 829 | assert columns[5].type.types[1].name == "STRING" 830 | 831 | assert columns[6].name == "col22" 832 | assert isinstance(columns[6].type, HType) 833 | assert isinstance(columns[6].type, HUnionType) 834 | assert columns[6].type.name == "UNION" 835 | assert columns[6].type.category == HTypeCategory.UNION 836 | assert columns[6].comment == "c22" 837 | assert len(columns[6].type.types) == 2 838 | assert isinstance(columns[6].type.types[0], HType) 839 | assert isinstance(columns[6].type.types[0], HPrimitiveType) 840 | assert columns[6].type.types[0].name == "INT" 841 | assert isinstance(columns[6].type.types[1], HType) 842 | assert isinstance(columns[6].type.types[1], HPrimitiveType) 843 | assert columns[6].type.types[1].name == "STRING" 844 | 845 | 846 | # pylint: disable=redefined-outer-name 847 | def test_table_stats(hive_client): 848 | """ 849 | Test the table statistics fetching functionality of the HMS class. 850 | 851 | This function retrieves the table "test_table4" from the database "test_db" and 852 | then fetches the statistics for seven columns of different types. The function 853 | checks that the returned statistics are correct and of the expected types. 854 | 855 | Args: 856 | hive_client: A handle to the Hive metastore client. 857 | 858 | Raises: 859 | AssertionError: If the fetched statistics are not as expected. 860 | """ 861 | hms = HMS(hive_client) 862 | table = hms.get_table("test_db", "test_table4") 863 | 864 | statistics = hms.get_table_stats( 865 | table, 866 | [ 867 | HColumn("col4", HPrimitiveType(PrimitiveCategory.BOOLEAN)), 868 | HColumn("col5", HPrimitiveType(PrimitiveCategory.DOUBLE)), 869 | HColumn("col7", HPrimitiveType(PrimitiveCategory.STRING)), 870 | HColumn("col8", HPrimitiveType(PrimitiveCategory.BINARY)), 871 | HColumn("col9", HDecimalType(1, 1)), 872 | HColumn("col10", HPrimitiveType(PrimitiveCategory.DATE)), 873 | HColumn("col6", HPrimitiveType(PrimitiveCategory.LONG)), 874 | ], 875 | ) 876 | 877 | assert len(statistics) == 7 878 | assert statistics[0].tableName == "test_table4" 879 | assert statistics[0].dbName == "test_db" 880 | assert statistics[0].stats is not None 881 | 882 | assert isinstance(statistics[0].stats, BooleanTypeStats) 883 | assert statistics[0].stats.numTrues == 10 884 | assert statistics[0].stats.numFalses == 10 885 | assert statistics[0].stats.numNulls == 0 886 | 887 | assert isinstance(statistics[1].stats, DoubleTypeStats) 888 | assert statistics[1].stats.lowValue == 0 889 | assert statistics[1].stats.highValue == 1 890 | assert statistics[1].stats.numNulls == 0 891 | assert statistics[1].stats.cardinality == 100 892 | 893 | assert isinstance(statistics[2].stats, StringTypeStats) 894 | assert statistics[2].stats.avgColLen == 10 895 | assert statistics[2].stats.maxColLen == 10 896 | assert statistics[2].stats.numNulls == 5 897 | assert statistics[2].stats.cardinality == 10 898 | 899 | assert isinstance(statistics[3].stats, BinaryTypeStats) 900 | assert statistics[3].stats.avgColLen == 10 901 | assert statistics[3].stats.maxColLen == 10 902 | assert statistics[3].stats.numNulls == 5 903 | 904 | assert isinstance(statistics[4].stats, DecimalTypeStats) 905 | assert statistics[4].stats.lowValue == ttypes.Decimal(scale=1, unscaled=b"123445.3") 906 | assert statistics[4].stats.highValue == ttypes.Decimal(1, b"1232324124") 907 | assert statistics[4].stats.numNulls == 0 908 | assert statistics[4].stats.cardinality == 100 909 | 910 | assert isinstance(statistics[5].stats, DateTypeStats) 911 | assert statistics[5].stats.lowValue == ttypes.Date(0) 912 | assert statistics[5].stats.highValue == ttypes.Date(1) 913 | assert statistics[5].stats.numNulls == 0 914 | assert statistics[5].stats.cardinality == 100 915 | 916 | assert isinstance(statistics[6].stats, LongTypeStats) 917 | assert statistics[6].stats.lowValue == 0 918 | assert statistics[6].stats.highValue == 100 919 | assert statistics[6].stats.numNulls == 0 920 | assert statistics[6].stats.cardinality == 100 921 | -------------------------------------------------------------------------------- /tests/unit/test_htypes.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pymetastore.htypes import ( 4 | HCharType, 5 | HDecimalType, 6 | HListType, 7 | HMapType, 8 | HPrimitiveType, 9 | HStructType, 10 | HType, 11 | HTypeCategory, 12 | HUnionType, 13 | HVarcharType, 14 | PrimitiveCategory, 15 | SerdeTypeNameConstants, 16 | Token, 17 | TypeParser, 18 | ) 19 | 20 | # Although type systems change, it is a rare event and even if it happens, we 21 | # should be very careful as way too many things can break. For this reason 22 | # we'll be testing values and members for the types we define to catch any 23 | # changes. For this reason we'll be primarily be testing the values and not the 24 | # utility functions for the conversion between types. These functions are 25 | # trivial and they primarily expose a thin API over the mappings we define. 26 | 27 | 28 | # tests for primitive types. 29 | def test_primitivecategory_members(): 30 | expected_members = { 31 | "VOID", 32 | "BOOLEAN", 33 | "BYTE", 34 | "SHORT", 35 | "INT", 36 | "LONG", 37 | "FLOAT", 38 | "DOUBLE", 39 | "STRING", 40 | "DATE", 41 | "TIMESTAMP", 42 | "TIMESTAMPLOCALTZ", 43 | "BINARY", 44 | "DECIMAL", 45 | "VARCHAR", 46 | "CHAR", 47 | "INTERVAL_YEAR_MONTH", 48 | "INTERVAL_DAY_TIME", 49 | "UNKNOWN", 50 | } 51 | 52 | actual_members = {member.name for member in PrimitiveCategory} 53 | assert actual_members == expected_members 54 | 55 | 56 | def test_primitivecategory_values(): 57 | expected_values = { 58 | "VOID", 59 | "BOOLEAN", 60 | "BYTE", 61 | "SHORT", 62 | "INT", 63 | "LONG", 64 | "FLOAT", 65 | "DOUBLE", 66 | "STRING", 67 | "DATE", 68 | "TIMESTAMP", 69 | "TIMESTAMPLOCALTZ", 70 | "BINARY", 71 | "DECIMAL", 72 | "VARCHAR", 73 | "CHAR", 74 | "INTERVAL_YEAR_MONTH", 75 | "INTERVAL_DAY_TIME", 76 | "UNKNOWN", 77 | } 78 | 79 | actual_values = {member.value for member in PrimitiveCategory} 80 | assert actual_values == expected_values 81 | 82 | 83 | # tests for HType definitions. 84 | def test_htype_members(): 85 | expected_members = {"PRIMITIVE", "STRUCT", "MAP", "LIST", "UNION"} 86 | 87 | actual_members = {member.name for member in HTypeCategory} 88 | assert actual_members == expected_members 89 | 90 | 91 | def test_htype_values(): 92 | expected_values = {PrimitiveCategory, "STRUCT", "MAP", "LIST", "UNION"} 93 | 94 | actual_values = {member.value for member in HTypeCategory} 95 | assert actual_values == expected_values 96 | 97 | 98 | # Tests for the Serde definitions. Remember that we are converting between 99 | # Thrift and Hive types so we need to account for that. It's important to test 100 | # that stuff here because the thrift types are auto generated and we should 101 | # catch any changes that might happen in the .thrift files as soon as possible. 102 | def test_serdetypes_members(): 103 | expected_members = { 104 | "VOID", 105 | "BOOLEAN", 106 | "TINYINT", 107 | "SMALLINT", 108 | "INT", 109 | "BIGINT", 110 | "FLOAT", 111 | "DOUBLE", 112 | "STRING", 113 | "DATE", 114 | "CHAR", 115 | "VARCHAR", 116 | "TIMESTAMP", 117 | "TIMESTAMPLOCALTZ", 118 | "DECIMAL", 119 | "BINARY", 120 | "INTERVAL_YEAR_MONTH", 121 | "INTERVAL_DAY_TIME", 122 | "LIST", 123 | "MAP", 124 | "STRUCT", 125 | "UNION", 126 | "UNKNOWN", 127 | } 128 | 129 | actual_members = {member.name for member in SerdeTypeNameConstants} 130 | assert actual_members == expected_members 131 | 132 | 133 | def test_serdetypes_values(): 134 | expected_values = { 135 | "void", 136 | "boolean", 137 | "tinyint", 138 | "smallint", 139 | "int", 140 | "bigint", 141 | "float", 142 | "double", 143 | "string", 144 | "date", 145 | "char", 146 | "varchar", 147 | "timestamp", 148 | "timestamp with local time zone", 149 | "decimal", 150 | "binary", 151 | "interval_year_month", 152 | "interval_day_time", 153 | "array", 154 | "map", 155 | "struct", 156 | "uniontype", 157 | "unknown", 158 | } 159 | 160 | actual_values = {member.value for member in SerdeTypeNameConstants} 161 | assert actual_values == expected_values 162 | 163 | 164 | def test_token_creation(): 165 | # Test successful creation of Token instance 166 | token = Token(position=0, text="double", type_=True) 167 | assert token.position == 0 168 | assert token.text == "double" 169 | assert token.type is True 170 | 171 | # Test the string representation of the Token instance 172 | assert str(token) == "0:double" 173 | 174 | # Test failure when providing None as text 175 | with pytest.raises(ValueError): 176 | Token(position=0, text=None, type_=True) 177 | 178 | 179 | def test_token_properties(): 180 | token = Token(position=1, text="<", type_=False) 181 | 182 | # Test the properties (position, text, type_) of the Token instance 183 | assert token.position == 1 184 | assert token.text == "<" 185 | assert token.type is False 186 | 187 | # Test the string representation of the Token instance 188 | assert str(token) == "1:<" 189 | 190 | 191 | # Tests for the TypeParser class 192 | def test_typeparser_tokenize(): 193 | parser = TypeParser("STRING") 194 | assert parser.index == 0 195 | assert parser.type_tokens == [Token(position=0, text="STRING", type_=True)] 196 | parser = TypeParser("STRUCT") 197 | assert parser.index == 0 198 | 199 | assert parser.type_tokens == [ 200 | Token(position=0, text="STRUCT", type_=True), 201 | Token(position=6, text="<", type_=False), 202 | Token(position=7, text="a", type_=True), 203 | Token(position=8, text=":", type_=False), 204 | Token(position=9, text="INT", type_=True), 205 | Token(position=12, text=",", type_=False), 206 | Token(position=13, text="b", type_=True), 207 | Token(position=14, text=":", type_=False), 208 | Token(position=15, text="STRING", type_=True), 209 | Token(position=21, text=">", type_=False), 210 | ] 211 | 212 | parser = TypeParser("MAP>") 213 | assert parser.index == 0 214 | assert parser.type_tokens == [ 215 | Token(position=0, text="MAP", type_=True), 216 | Token(position=3, text="<", type_=False), 217 | Token(position=4, text="STRING", type_=True), 218 | Token(position=10, text=",", type_=False), 219 | Token(position=11, text="STRUCT", type_=True), 220 | Token(position=17, text="<", type_=False), 221 | Token(position=18, text="a", type_=True), 222 | Token(position=19, text=":", type_=False), 223 | Token(position=20, text="INT", type_=True), 224 | Token(position=23, text=",", type_=False), 225 | Token(position=24, text="b", type_=True), 226 | Token(position=25, text=":", type_=False), 227 | Token(position=26, text="STRING", type_=True), 228 | Token(position=32, text=">", type_=False), 229 | Token(position=33, text=">", type_=False), 230 | ] 231 | 232 | parser = TypeParser("UNIONTYPE,struct>") 233 | assert parser.index == 0 234 | 235 | assert parser.type_tokens == [ 236 | Token(position=0, text="UNIONTYPE", type_=True), 237 | Token(position=9, text="<", type_=False), 238 | Token(position=10, text="int", type_=True), 239 | Token(position=13, text=",", type_=False), 240 | Token(position=14, text="double", type_=True), 241 | Token(position=20, text=",", type_=False), 242 | Token(position=21, text="array", type_=True), 243 | Token(position=26, text="<", type_=False), 244 | Token(position=27, text="string", type_=True), 245 | Token(position=33, text=">", type_=False), 246 | Token(position=34, text=",", type_=False), 247 | Token(position=35, text="struct", type_=True), 248 | Token(position=41, text="<", type_=False), 249 | Token(position=42, text="a", type_=True), 250 | Token(position=43, text=":", type_=False), 251 | Token(position=44, text="int", type_=True), 252 | Token(position=47, text=",", type_=False), 253 | Token(position=48, text="b", type_=True), 254 | Token(position=49, text=":", type_=False), 255 | Token(position=50, text="string", type_=True), 256 | Token(position=56, text=">", type_=False), 257 | Token(position=57, text=">", type_=False), 258 | ] 259 | parser = TypeParser("uniontype") 260 | assert parser.index == 0 261 | assert parser.type_tokens == [ 262 | Token(position=0, text="uniontype", type_=True), 263 | Token(position=9, text="<", type_=False), 264 | Token(position=10, text="integer", type_=True), 265 | Token(position=17, text=",", type_=False), 266 | Token(position=18, text="string", type_=True), 267 | Token(position=24, text=">", type_=False), 268 | ] 269 | 270 | parser = TypeParser( 271 | "STRUCT" 272 | ) 273 | assert parser.index == 0 274 | assert parser.type_tokens == [ 275 | Token(position=0, text="STRUCT", type_=True), 276 | Token(position=6, text="<", type_=False), 277 | Token(position=7, text="houseno", type_=True), 278 | Token(position=14, text=":", type_=False), 279 | Token(position=15, text="STRING", type_=True), 280 | Token(position=21, text=",", type_=False), 281 | Token(position=22, text="streetname", type_=True), 282 | Token(position=32, text=":", type_=False), 283 | Token(position=33, text="STRING", type_=True), 284 | Token(position=39, text=",", type_=False), 285 | Token(position=40, text="town", type_=True), 286 | Token(position=44, text=":", type_=False), 287 | Token(position=45, text="STRING", type_=True), 288 | Token(position=51, text=",", type_=False), 289 | Token(position=52, text="postcode", type_=True), 290 | Token(position=60, text=":", type_=False), 291 | Token(position=61, text="STRING", type_=True), 292 | Token(position=67, text=">", type_=False), 293 | ] 294 | 295 | 296 | # Basic Types 297 | def test_basic_type_parser(): 298 | # String Type 299 | parser = TypeParser("string") 300 | ptype = parser.parse_type() 301 | assert ptype.name == HPrimitiveType(PrimitiveCategory.STRING).name 302 | assert ptype.category == HPrimitiveType(PrimitiveCategory.STRING).category 303 | 304 | # Int Type 305 | parser = TypeParser("int") 306 | ptype = parser.parse_type() 307 | assert ptype.name == HPrimitiveType(PrimitiveCategory.INT).name 308 | assert ptype.category == HPrimitiveType(PrimitiveCategory.INT).category 309 | 310 | # Long Type 311 | parser = TypeParser("double") 312 | ptype = parser.parse_type() 313 | assert ptype.name == HPrimitiveType(PrimitiveCategory.DOUBLE).name 314 | assert ptype.category == HPrimitiveType(PrimitiveCategory.DOUBLE).category 315 | 316 | # Float Type 317 | parser = TypeParser("float") 318 | ptype = parser.parse_type() 319 | assert ptype.name == HPrimitiveType(PrimitiveCategory.FLOAT).name 320 | assert ptype.category == HPrimitiveType(PrimitiveCategory.FLOAT).category 321 | 322 | # Boolean Type 323 | parser = TypeParser("boolean") 324 | ptype = parser.parse_type() 325 | assert ptype.name == HPrimitiveType(PrimitiveCategory.BOOLEAN).name 326 | assert ptype.category == HPrimitiveType(PrimitiveCategory.BOOLEAN).category 327 | 328 | # Binary Type 329 | parser = TypeParser("binary") 330 | ptype = parser.parse_type() 331 | assert ptype.name == HPrimitiveType(PrimitiveCategory.BINARY).name 332 | assert ptype.category == HPrimitiveType(PrimitiveCategory.BINARY).category 333 | 334 | # Timestamp Type 335 | parser = TypeParser("timestamp") 336 | ptype = parser.parse_type() 337 | assert ptype.name == HPrimitiveType(PrimitiveCategory.TIMESTAMP).name 338 | assert ptype.category == HPrimitiveType(PrimitiveCategory.TIMESTAMP).category 339 | 340 | # Date Type 341 | parser = TypeParser("date") 342 | ptype = parser.parse_type() 343 | assert ptype.name == HPrimitiveType(PrimitiveCategory.DATE).name 344 | assert ptype.category == HPrimitiveType(PrimitiveCategory.DATE).category 345 | 346 | 347 | # Decimal Type 348 | def test_decimal_type_parser(): 349 | parser = TypeParser("decimal") 350 | ptype = parser.parse_type() 351 | 352 | # default precision and scale decimal 353 | assertd = HDecimalType(10, 0) 354 | assert isinstance(ptype, HDecimalType) 355 | assert ptype.precision == assertd.precision 356 | assert ptype.scale == assertd.scale 357 | 358 | # with precision defined only 359 | parser = TypeParser("decimal(20)") 360 | ptype = parser.parse_type() 361 | assertd = HDecimalType(20, 0) 362 | assert isinstance(ptype, HDecimalType) 363 | assert ptype.precision == assertd.precision 364 | assert ptype.scale == assertd.scale 365 | 366 | # with precision and scale defined 367 | parser = TypeParser("decimal(20,10)") 368 | ptype = parser.parse_type() 369 | assertd = HDecimalType(20, 10) 370 | assert isinstance(ptype, HDecimalType) 371 | assert ptype.precision == assertd.precision 372 | assert ptype.scale == assertd.scale 373 | 374 | # Test accepted boundaries for parameters 375 | parser = TypeParser("decimal(39,0)") 376 | with pytest.raises(ValueError, match="Decimal precision cannot exceed 38"): 377 | parser.parse_type() 378 | 379 | # Test accepted boundaries for parameters 380 | parser = TypeParser("decimal(37,40)") 381 | with pytest.raises(ValueError, match="Decimal scale cannot exceed 38"): 382 | parser.parse_type() 383 | 384 | # Test accepted boundaries for parameters 385 | parser = TypeParser("decimal(39,40)") 386 | with pytest.raises(ValueError, match="Decimal precision cannot exceed 38"): 387 | parser.parse_type() 388 | 389 | 390 | # CHAR Type 391 | def test_char_type_parser(): 392 | parser = TypeParser("char(10)") 393 | ptype = parser.parse_type() 394 | assert isinstance(ptype, HCharType) 395 | assert ptype.length == 10 396 | 397 | # Testing char without length 398 | parser = TypeParser("char") 399 | with pytest.raises( 400 | ValueError, match="char/varchar type must have a length specified" 401 | ): 402 | parser.parse_type() 403 | 404 | # Testing char with more than 255 length 405 | parser = TypeParser("char(65536)") 406 | with pytest.raises(ValueError, match="Char length cannot exceed 255"): 407 | parser.parse_type() 408 | 409 | # Testing char with more than one parameter 410 | parser = TypeParser("char(10,20)") 411 | with pytest.raises( 412 | ValueError, 413 | match="Error: char type takes only one parameter, but instead 2 parameters are found.", 414 | ): 415 | parser.parse_type() 416 | 417 | # VARCHAR Type 418 | 419 | 420 | def test_var_char_type_parser(): 421 | parser = TypeParser("varchar(10)") 422 | ptype = parser.parse_type() 423 | assert isinstance(ptype, HVarcharType) 424 | assert ptype.length == 10 425 | 426 | # Testing varchar without length 427 | parser = TypeParser("varchar") 428 | with pytest.raises( 429 | ValueError, match="char/varchar type must have a length specified" 430 | ): 431 | parser.parse_type() 432 | 433 | # Testing varchar with more than 65535 length 434 | parser = TypeParser("varchar(65536)") 435 | with pytest.raises(ValueError, match="Varchar length cannot exceed 65535"): 436 | parser.parse_type() 437 | 438 | # Testing varchar with more than one parameter 439 | parser = TypeParser("varchar(10,20)") 440 | with pytest.raises( 441 | ValueError, 442 | match="Error: varchar type takes only one parameter, but instead 2 parameters are found.", 443 | ): 444 | parser.parse_type() 445 | 446 | 447 | # ARRAY Type 448 | def test_array_type_parser(): 449 | parser = TypeParser("array") 450 | ptype = parser.parse_type() 451 | assert isinstance(ptype, HListType) 452 | assert ptype.element_type.name == "INT" 453 | assert ( 454 | ptype.element_type.category.PRIMITIVE 455 | == HPrimitiveType(PrimitiveCategory.INT).category 456 | ) 457 | 458 | 459 | def test_struct_type_parser(): 460 | parser = TypeParser("struct") 461 | ptype: HStructType = parser.parse_type() 462 | assert isinstance(ptype, HStructType) 463 | assert ptype.names == ["name", "age"] 464 | assert ptype.types[0].category == HPrimitiveType(PrimitiveCategory.STRING).category 465 | assert ptype.types[0].name == HPrimitiveType(PrimitiveCategory.STRING).name 466 | assert ptype.types[1].category == HPrimitiveType(PrimitiveCategory.INT).category 467 | assert ptype.types[1].name == HPrimitiveType(PrimitiveCategory.INT).name 468 | 469 | 470 | def test_map_type_parser(): 471 | parser = TypeParser("map") 472 | ptype = parser.parse_type() 473 | assert isinstance(ptype, HMapType) 474 | assert ptype.key_type.name == "STRING" 475 | assert ptype.key_type.category == HPrimitiveType(PrimitiveCategory.STRING).category 476 | assert ptype.value_type.name == "INT" 477 | assert ptype.value_type.category == HPrimitiveType(PrimitiveCategory.INT).category 478 | 479 | 480 | def test_union_type_parser(): 481 | parser = TypeParser("uniontype") 482 | ptype = parser.parse_type() 483 | assert isinstance(ptype, HUnionType) 484 | assert ptype.types[0].name == "INT" 485 | assert ptype.types[0].category == HPrimitiveType(PrimitiveCategory.INT).category 486 | assert ptype.types[1].name == "STRING" 487 | assert ptype.types[1].category == HPrimitiveType(PrimitiveCategory.STRING).category 488 | 489 | 490 | def test_htype_str_repr(): 491 | ex = HType("int", PrimitiveCategory.INT) 492 | assert str(ex) == "HType(name=int, category=PrimitiveCategory.INT)" 493 | assert repr(ex) == "HType('int', PrimitiveCategory.INT)" 494 | clone = eval(repr(ex)) 495 | assert clone == ex 496 | 497 | 498 | # pylint: disable=line-too-long 499 | def test_hmap_type_str_repr_and_eq(): 500 | typ = HMapType( 501 | HPrimitiveType(PrimitiveCategory.STRING), HPrimitiveType(PrimitiveCategory.INT) 502 | ) 503 | assert isinstance(typ, HMapType) 504 | assert ( 505 | str(typ) 506 | == "HMapType(name=MAP, category=HTypeCategory.MAP), key_type=HPrimitiveType(name=STRING, category=HTypeCategory.PRIMITIVE), value_type=HPrimitiveType(name=INT, category=HTypeCategory.PRIMITIVE))" 507 | ) 508 | assert ( 509 | repr(typ) 510 | == "HMapType(HPrimitiveType(PrimitiveCategory.STRING), HPrimitiveType(PrimitiveCategory.INT))" 511 | ) 512 | 513 | clone = eval(repr(typ)) 514 | assert clone == typ 515 | 516 | 517 | # pylint: disable=line-too-long 518 | def test_hlist_type_str_repr_and_eq(): 519 | _type = HListType(HPrimitiveType(PrimitiveCategory.INT)) 520 | assert isinstance(_type, HListType) 521 | assert ( 522 | str(_type) 523 | == "HListType(name=LIST, category=HTypeCategory.LIST), element_type=HPrimitiveType(name=INT, category=HTypeCategory.PRIMITIVE))" 524 | ) 525 | assert repr(_type) == "HListType(HPrimitiveType(PrimitiveCategory.INT))" 526 | 527 | clone = eval(repr(_type)) 528 | assert clone == _type 529 | 530 | 531 | # pylint: disable=line-too-long 532 | def test_hunion_type_str_repr_and_eq(): 533 | _type = HUnionType( 534 | [ 535 | HPrimitiveType(PrimitiveCategory.INT), 536 | HPrimitiveType(PrimitiveCategory.STRING), 537 | ] 538 | ) 539 | assert isinstance(_type, HUnionType) 540 | assert ( 541 | str(_type) 542 | == "HUnionType(name=UNION, category=HTypeCategory.UNION), types=[HPrimitiveType(PrimitiveCategory.INT), HPrimitiveType(PrimitiveCategory.STRING)])" 543 | ) 544 | assert ( 545 | repr(_type) 546 | == "HUnionType([HPrimitiveType(PrimitiveCategory.INT), HPrimitiveType(PrimitiveCategory.STRING)])" 547 | ) 548 | 549 | clone = eval(repr(_type)) 550 | assert clone == _type 551 | 552 | 553 | def test_hvarchar_type_str_repr_and_eq(): 554 | _type = HVarcharType(10) 555 | assert isinstance(_type, HVarcharType) 556 | assert str(_type) == "HVarcharType(length=10)" 557 | assert repr(_type) == "HVarcharType(10)" 558 | 559 | clone = eval(repr(_type)) 560 | assert clone == _type 561 | 562 | 563 | def test_hchar_type_str_repr_and_eq(): 564 | _type = HCharType(10) 565 | assert isinstance(_type, HCharType) 566 | assert str(_type) == "HCharType(length=10)" 567 | assert repr(_type) == "HCharType(10)" 568 | 569 | clone = eval(repr(_type)) 570 | assert clone == _type 571 | 572 | 573 | def test_hdecimal_type_str_repr_and_eq(): 574 | _type = HDecimalType(10, 2) 575 | assert isinstance(_type, HDecimalType) 576 | assert str(_type) == "HDecimalType(precision=10, scale=2)" 577 | assert repr(_type) == "HDecimalType(10, 2)" 578 | 579 | clone = eval(repr(_type)) 580 | assert clone == _type 581 | 582 | 583 | # pylint: disable=line-too-long 584 | def test_hstruct_type_str_repr_and_eq(): 585 | _type = HStructType( 586 | ["name", "age"], 587 | [ 588 | HPrimitiveType(PrimitiveCategory.STRING), 589 | HPrimitiveType(PrimitiveCategory.INT), 590 | ], 591 | ) 592 | assert isinstance(_type, HStructType) 593 | 594 | assert ( 595 | str(_type) 596 | == "HStructType(name=STRUCT, category=HTypeCategory.STRUCT), names=['name', 'age'], types=[HPrimitiveType(PrimitiveCategory.STRING), HPrimitiveType(PrimitiveCategory.INT)])" 597 | ) 598 | assert ( 599 | repr(_type) 600 | == "HStructType(['name', 'age'], [HPrimitiveType(PrimitiveCategory.STRING), HPrimitiveType(PrimitiveCategory.INT)])" 601 | ) 602 | 603 | clone = eval(repr(_type)) 604 | assert clone == _type 605 | --------------------------------------------------------------------------------