├── .github └── workflows │ ├── incremental_test.yaml │ └── test.yaml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── dbt ├── adapters │ └── databend │ │ ├── __init__.py │ │ ├── __version__.py │ │ ├── column.py │ │ ├── connections.py │ │ ├── impl.py │ │ └── relation.py └── include │ └── databend │ ├── __init__.py │ ├── dbt_project.yml │ ├── macros │ ├── adapters.sql │ ├── adapters │ │ ├── apply_grants.sql │ │ └── relation.sql │ ├── catalog.sql │ ├── materializations │ │ ├── incremental.sql │ │ ├── seed.sql │ │ ├── snapshot.sql │ │ └── table.sql │ └── utils.sql │ └── profile_template.yml ├── dev-requirements.txt ├── setup.py ├── tests ├── README.md ├── __init__.py ├── conftest.py └── functional │ └── adapter │ ├── empty │ └── test_empty.py │ ├── incremental │ └── test_incremental.py │ ├── statement_test │ ├── seeds.py │ └── test_statements.py │ ├── test_basic.py │ ├── test_changing_relation_type.py │ ├── test_list_relations_without_caching.py │ ├── test_simple_seed.py │ ├── test_timestamps.py │ └── unit_testing │ ├── test_renamed_relations.py │ └── test_unit_testing.py └── tox.ini /.github/workflows/incremental_test.yaml: -------------------------------------------------------------------------------- 1 | name: Incremental Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | services: 13 | databend: 14 | image: datafuselabs/databend 15 | env: 16 | QUERY_DEFAULT_USER: databend 17 | QUERY_DEFAULT_PASSWORD: databend 18 | MINIO_ENABLED: true 19 | ports: 20 | - 8000:8000 21 | - 9000:9000 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Setup Python 27 | uses: actions/setup-python@v5 28 | with: 29 | python-version: '3.11' 30 | 31 | - name: Pip Install 32 | run: | 33 | pip install pipenv 34 | pip install pytest 35 | pip install -r dev-requirements.txt 36 | pipenv install --dev --skip-lock 37 | 38 | - name: Verify Service Running 39 | run: | 40 | cid=$(docker ps -a | grep databend | cut -d' ' -f1) 41 | docker logs ${cid} 42 | curl -v http://localhost:8000/v1/health 43 | 44 | - name: dbt databend Incremental Test Suite 45 | run: | 46 | python -m pytest -s tests/functional/adapter/incremental/test_incremental.py 47 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: Ci Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | services: 13 | databend: 14 | image: datafuselabs/databend 15 | env: 16 | QUERY_DEFAULT_USER: databend 17 | QUERY_DEFAULT_PASSWORD: databend 18 | MINIO_ENABLED: true 19 | ports: 20 | - 8000:8000 21 | - 9000:9000 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | 26 | - name: Setup Python 27 | uses: actions/setup-python@v5 28 | with: 29 | python-version: '3.11' 30 | 31 | - name: Pip Install 32 | run: | 33 | pip install pipenv 34 | pip install pytest 35 | pip install -r dev-requirements.txt 36 | pipenv install --dev --skip-lock 37 | 38 | - name: Verify Service Running 39 | run: | 40 | cid=$(docker ps -a | grep databend | cut -d' ' -f1) 41 | docker logs ${cid} 42 | curl -v http://localhost:8000/v1/health 43 | 44 | - name: dbt databend Unit Test Suite 45 | run: | 46 | python -m pytest -s tests/functional/adapter/unit_testing/test_unit_testing.py 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .idea 6 | venv/ 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | pip-wheel-metadata/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | test.env 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | logs/ 134 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # dbt-databend Changelog 2 | 3 | - This file provides a full account of all changes to `dbt-databend`. 4 | - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases. 5 | - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. 6 | 7 | ## Previous Releases 8 | For information on prior major and minor releases, see their changelogs: 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | Welcome to contribute to dbt-databend-cloud. Here are some guides for you. 4 | 5 | ## Ways to contribute 6 | 7 | We love to accept any kinds of contributions, such as: 8 | 9 | - **Report bugs:** if you find any bug in your use, you can make an issue for it. 10 | - **Answer questions:** check out the open issues to see if you can help answer questions. 11 | - **Suggest changes:** any suggestions which you can make this project better. 12 | - **Tackle an issue:** if you see an issue you would like to resolve, please feel free to fork the course and submit 13 | a PR (check out the instructions below). 14 | 15 | ## How to make a PR 16 | 17 | 1. Check out which [contributions](#ways-to-contribute) above you are willing to make 18 | 2. Fork this repository. 19 | 3. Code and pass [integration tests](test/README.md) 20 | 4. Commit your changes to your branch 21 | 5. Open a pull request to upstream 22 | 6. Request a review from the reviewers 23 | 24 | ## How to formate your code 25 | 26 | dbt-databend-cloud use black to fmt Python. To fmt your code, you need to: 27 | 28 | First, install black 29 | ``` 30 | pip3 install black 31 | ``` 32 | Then fmt 33 | ``` 34 | black . 35 | ``` 36 | 37 | You can also integrate with IntelliJ IDEA or other editors. See [here](https://black.readthedocs.io/en/stable/integrations/editors.html) for more details. 38 | 39 | 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include dbt/include *.sql *.yml *.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Databend logo 3 | dbt logo 4 |

5 | 6 | # dbt-databend-cloud 7 | 8 | ![PyPI](https://img.shields.io/pypi/v/dbt-databend-cloud) 9 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dbt-databend-cloud) 10 | 11 | The `dbt-databend-cloud` package contains all of the code enabling [dbt](https://getdbt.com) to work with 12 | [Databend Cloud](https://databend.rs/doc/cloud/). 13 | 14 | ## Table of Contents 15 | * [Installation](#installation) 16 | * [Supported features](#supported-features) 17 | * [Profile Configuration](#profile-configuration) 18 | * [Database User Privileges](#database-user-privileges) 19 | * [Running Tests](#running-tests) 20 | * [Example](#example) 21 | * [Contributing](#contributing) 22 | 23 | ## Installation 24 | Compile by source code. 25 | 26 | ```bash 27 | $ git clone https://github.com/databendcloud/dbt-databend.git 28 | $ cd dbt-databend 29 | $ pip install . 30 | ``` 31 | Also, you can get it from pypi. 32 | 33 | ```bash 34 | $ pip install dbt-databend-cloud 35 | ``` 36 | ## Supported features 37 | 38 | | ok | Feature | 39 | |:--:|:---------------------------:| 40 | | ✅ | Table materialization | 41 | | ✅ | View materialization | 42 | | ✅ | Incremental materialization | 43 | | ❌ | Ephemeral materialization | 44 | | ✅ | Seeds | 45 | | ✅ | Sources | 46 | | ✅ | Custom data tests | 47 | | ✅ | Docs generate | 48 | | ✅ | Snapshots | 49 | | ✅ | Connection retry | 50 | 51 | Note: 52 | 53 | * Databend does not support `Ephemeral` and `SnapShot`. You can find more detail [here](https://github.com/datafuselabs/databend/issues/8685) 54 | 55 | ## Profile Configuration 56 | 57 | Databend Cloud targets should be set up using the following configuration in your `profiles.yml` file. 58 | 59 | **Example entry for profiles.yml:** 60 | 61 | ``` 62 | Your_Profile_Name: 63 | target: dev 64 | outputs: 65 | dev: 66 | type: databend 67 | host: [host] 68 | port: [port] 69 | schema: [schema(Your database)] 70 | user: [username] 71 | pass: [password] 72 | secure: [SSL] 73 | ``` 74 | 75 | | Option | Description | Required? | Example | 76 | |--------|------------------------------------------------------|-----------|--------------------------------| 77 | | type | The specific adapter to use | Required | `databend` | 78 | | host | The server (hostname) to connect to | Required | `yourorg.databend.com` | 79 | | port | The port to use | Required | `443` | 80 | | schema | Specify the schema (database) to build models into | Required | `analytics` | 81 | | user | The username to use to connect to the server | Required | `dbt_admin` | 82 | | pass | The password to use for authenticating to the server | Required | `correct-horse-battery-staple` | 83 | | secure | The SSL of host (default as True) | Optional | `True` | 84 | 85 | 86 | Note: 87 | 88 | * You can find your host, user, pass information in this [docs](https://docs.databend.com/using-databend-cloud/warehouses/connecting-a-warehouse) 89 | 90 | ## Running Tests 91 | 92 | See [tests/README.md](tests/README.md) for details on running the integration tests. 93 | 94 | ## Example 95 | 96 | Click [here](https://github.com/databendcloud/dbt-databend/wiki/How-to-use-dbt-with-Databend-Cloud) to see a simple example about using dbt with dbt-databend-cloud. 97 | 98 | ## Contributing 99 | 100 | Welcome to contribute for dbt-databend-cloud. See [Contributing Guide](CONTRIBUTING.md) for more information. 101 | -------------------------------------------------------------------------------- /dbt/adapters/databend/__init__.py: -------------------------------------------------------------------------------- 1 | from dbt.adapters.databend.connections import DatabendConnectionManager # noqa 2 | from dbt.adapters.databend.connections import DatabendCredentials 3 | from dbt.adapters.databend.impl import DatabendAdapter 4 | from dbt.adapters.databend.column import DatabendColumn # noqa 5 | from dbt.adapters.databend.relation import DatabendRelation # noqa 6 | 7 | from dbt.adapters.base import AdapterPlugin 8 | from dbt.include import databend 9 | 10 | 11 | Plugin = AdapterPlugin( 12 | adapter=DatabendAdapter, 13 | credentials=DatabendCredentials, 14 | include_path=databend.PACKAGE_PATH, 15 | ) 16 | -------------------------------------------------------------------------------- /dbt/adapters/databend/__version__.py: -------------------------------------------------------------------------------- 1 | version = "1.8.1" 2 | -------------------------------------------------------------------------------- /dbt/adapters/databend/column.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import TypeVar, Optional, Dict, Any 3 | 4 | from dbt.adapters.base.column import Column 5 | from dbt_common.exceptions import DbtRuntimeError 6 | 7 | Self = TypeVar("Self", bound="DatabendColumn") 8 | 9 | 10 | @dataclass 11 | class DatabendColumn(Column): 12 | @property 13 | def quoted(self) -> str: 14 | return '"{}"'.format(self.column) 15 | 16 | def is_string(self) -> bool: 17 | if self.dtype is None: 18 | return False 19 | return self.dtype.lower() in [ 20 | "string", 21 | "varchar", 22 | ] 23 | 24 | def is_integer(self) -> bool: 25 | if self.dtype is None: 26 | return False 27 | return self.dtype.lower().startswith("int") or self.dtype.lower() in ( 28 | "tinyint", 29 | "smallint", 30 | "bigint", 31 | ) 32 | 33 | def is_numeric(self) -> bool: 34 | return False 35 | 36 | def is_float(self) -> bool: 37 | if self.dtype is None: 38 | return False 39 | return self.dtype.lower() in ("float", "double") 40 | 41 | def string_size(self) -> int: 42 | if not self.is_string(): 43 | raise DbtRuntimeError( 44 | "Called string_size() on non-string field!" 45 | ) 46 | 47 | if self.char_size is None: 48 | return 256 49 | else: 50 | return int(self.char_size) 51 | 52 | @classmethod 53 | def string_type(cls, size: int) -> str: 54 | return "VARCHAR" 55 | 56 | @classmethod 57 | def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: 58 | return dtype 59 | 60 | def literal(self, value): 61 | return f"CAST({value} AS {self.dtype})" 62 | 63 | def can_expand_to(self, other_column: "Column") -> bool: 64 | return self.is_string() and other_column.is_string() 65 | 66 | def __repr__(self) -> str: 67 | return "".format(self.name, self.data_type) 68 | -------------------------------------------------------------------------------- /dbt/adapters/databend/connections.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from dataclasses import dataclass 3 | 4 | import agate 5 | import dbt_common.exceptions # noqa 6 | 7 | from dbt.adapters.exceptions.connection import FailedToConnectError 8 | from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials 9 | from dbt_common.clients.agate_helper import empty_table 10 | from dbt.adapters.sql import SQLConnectionManager as connection_cls 11 | from dbt.adapters.events.logging import AdapterLogger # type: ignore 12 | from dbt_common.events.functions import warn_or_error 13 | from dbt.adapters.events.types import AdapterEventWarning 14 | from dbt_common.ui import line_wrap_message, warning_tag 15 | from dbt_common.clients.agate_helper import empty_table 16 | from typing import Optional, Tuple, List, Any 17 | from databend_sqlalchemy import connector 18 | 19 | from dbt_common.exceptions import ( 20 | DbtInternalError, 21 | DbtRuntimeError, 22 | DbtConfigError, 23 | ) 24 | 25 | logger = AdapterLogger("databend") 26 | 27 | 28 | @dataclass 29 | class DatabendAdapterResponse(AdapterResponse): 30 | pass 31 | 32 | 33 | @dataclass 34 | class DatabendCredentials(Credentials): 35 | """ 36 | Defines database specific credentials that get added to 37 | profiles.yml to connect to new adapter 38 | """ 39 | 40 | host: Optional[str] = None 41 | port: Optional[int] = None 42 | database: Optional[str] = None 43 | username: Optional[str] = None 44 | password: Optional[str] = None 45 | schema: Optional[str] = None 46 | secure: Optional[bool] = None 47 | 48 | # Add credentials members here, like: 49 | # host: str 50 | # port: int 51 | # username: str 52 | # password: str 53 | 54 | _ALIASES = {"dbname": "database", "pass": "password", "user": "username"} 55 | 56 | def __init__(self, **kwargs): 57 | for k, v in kwargs.items(): 58 | setattr(self, k, v) 59 | self.database = None 60 | 61 | @classmethod 62 | def __pre_deserialize__(cls, data): 63 | data = super().__pre_deserialize__(data) 64 | if "database" not in data: 65 | data["database"] = None 66 | return data 67 | 68 | def __post_init__(self): 69 | # databend classifies database and schema as the same thing 70 | self.database = None 71 | if self.database is not None and self.database != self.schema: 72 | raise DbtRuntimeError( 73 | f" schema: {self.schema} \n" 74 | f" database: {self.database} \n" 75 | f"On Databend, database must be omitted or have the same value as" 76 | f" schema." 77 | ) 78 | 79 | @property 80 | def type(self): 81 | """Return name of adapter.""" 82 | return "databend" 83 | 84 | @property 85 | def unique_field(self): 86 | """ 87 | Hashed and included in anonymous telemetry to track adapter adoption. 88 | Pick a field that can uniquely identify one team/organization building with this adapter 89 | """ 90 | return self.schema 91 | 92 | def _connection_keys(self): 93 | """ 94 | List of keys to display in the `dbt debug` output. 95 | """ 96 | return ("host", "port", "database", "schema", "user") 97 | 98 | 99 | @dataclass 100 | class DatabendAdapterResponse(AdapterResponse): 101 | query_id: str = "" 102 | 103 | 104 | class DatabendConnectionManager(connection_cls): 105 | TYPE = "databend" 106 | 107 | @contextmanager 108 | def exception_handler(self, sql: str): 109 | """ 110 | Returns a context manager, that will handle exceptions raised 111 | from queries, catch, log, and raise dbt exceptions it knows how to handle. 112 | """ 113 | try: 114 | yield 115 | 116 | except Exception as e: 117 | logger.debug("Error running SQL: {}".format(sql)) 118 | logger.debug("Rolling back transaction.") 119 | self.rollback_if_open() 120 | raise DbtRuntimeError(str(e)) 121 | 122 | # except for DML statements where explicitly defined 123 | def add_begin_query(self, *args, **kwargs): 124 | pass 125 | 126 | def add_commit_query(self, *args, **kwargs): 127 | pass 128 | 129 | def begin(self): 130 | pass 131 | 132 | def commit(self): 133 | pass 134 | 135 | def clear_transaction(self): 136 | pass 137 | 138 | @classmethod 139 | def open(cls, connection): 140 | """ 141 | Receives a connection object and a Credentials object 142 | and moves it to the "open" state. 143 | """ 144 | if connection.state == "open": 145 | logger.debug("Connection is already open, skipping open.") 146 | return connection 147 | 148 | credentials = connection.credentials 149 | 150 | try: 151 | if credentials.secure is None: 152 | credentials.secure = True 153 | 154 | if credentials.secure: 155 | handle = connector.connect( 156 | f"https://{credentials.username}:{credentials.password}@{credentials.host}:{credentials.port}/{credentials.schema}?secure=true " 157 | ) 158 | else: 159 | handle = connector.connect( 160 | f"http://{credentials.username}:{credentials.password}@{credentials.host}:{credentials.port}/{credentials.schema}?secure=false " 161 | ) 162 | 163 | except Exception as e: 164 | logger.debug("Error opening connection: {}".format(e)) 165 | connection.handle = None 166 | connection.state = "fail" 167 | raise FailedToConnectError(str(e)) 168 | connection.state = "open" 169 | connection.handle = handle 170 | return connection 171 | 172 | @classmethod 173 | def get_response(cls, cursor): 174 | return DatabendAdapterResponse( 175 | _message="{} {}".format("adapter response", cursor.rowcount), 176 | rows_affected=cursor.rowcount, 177 | ) 178 | 179 | def execute( 180 | self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None 181 | ) -> Tuple[AdapterResponse, agate.Table]: 182 | # don't apply the query comment here 183 | # it will be applied after ';' queries are split 184 | _, cursor = self.add_query(sql, auto_begin) 185 | response = self.get_response(cursor) 186 | # table: rows, column_names=None, column_types=None, row_names=None 187 | if fetch: 188 | table = self.get_result_from_cursor(cursor, limit) 189 | else: 190 | table = dbt_common.clients.agate_helper.empty_table() 191 | return response, table 192 | 193 | def add_query(self, sql, auto_begin=False, bindings=None, abridge_sql_log=False): 194 | connection, cursor = super().add_query( 195 | sql, auto_begin, bindings=bindings, abridge_sql_log=abridge_sql_log 196 | ) 197 | 198 | if cursor is None: 199 | conn = self.get_thread_connection() 200 | if conn is None or conn.name is None: 201 | conn_name = "" 202 | else: 203 | conn_name = conn.name 204 | 205 | raise Exception( 206 | "Tried to run an empty query on model '{}'. If you are " 207 | "conditionally running\nsql, eg. in a model hook, make " 208 | "sure your `else` clause contains valid sql!\n\n" 209 | "Provided SQL:\n{}".format(conn_name, sql) 210 | ) 211 | 212 | return connection, cursor 213 | 214 | @classmethod 215 | def get_status(cls, _): 216 | """ 217 | Returns connection status 218 | """ 219 | return "OK" 220 | 221 | @classmethod 222 | def get_credentials(cls, credentials): 223 | """ 224 | Returns Databend credentials 225 | """ 226 | return credentials 227 | 228 | def cancel(self, connection): 229 | """ 230 | Gets a connection object and attempts to cancel any ongoing queries. 231 | """ 232 | connection_name = connection.name 233 | logger.debug("Cancelling query '{}'", connection_name) 234 | connection.handle.close() 235 | logger.debug("Cancel query '{}'", connection_name) 236 | 237 | @classmethod 238 | def process_results(cls, column_names, rows): 239 | 240 | return [dict(zip(column_names, row)) for row in rows] 241 | 242 | @classmethod 243 | def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Table: 244 | data: List[Any] = [] 245 | column_names: List[str] = [] 246 | 247 | if cursor.description is not None: 248 | column_names = [col[0] for col in cursor.description] 249 | if limit: 250 | rows = cursor.fetchmany(limit) 251 | else: 252 | rows = cursor.fetchall() 253 | data = cls.process_results(column_names, rows) 254 | 255 | return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names) 256 | -------------------------------------------------------------------------------- /dbt/adapters/databend/impl.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import Future 2 | from dataclasses import dataclass 3 | from typing import Callable, List, Optional, Set, Union, FrozenSet, Tuple 4 | 5 | import agate 6 | from dbt.adapters.base import AdapterConfig, available 7 | from dbt.adapters.base.impl import catch_as_completed 8 | from dbt.adapters.base.relation import InformationSchema 9 | from dbt.adapters.sql import SQLAdapter 10 | 11 | from dbt_common.clients.agate_helper import table_from_rows 12 | from dbt.adapters.events.logging import AdapterLogger 13 | from dbt.adapters.contracts.relation import RelationType 14 | from dbt_common.contracts.constraints import ConstraintType 15 | from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError 16 | from dbt_common.utils import filter_null_values 17 | from dbt_common.utils import executor 18 | 19 | import csv 20 | import io 21 | from dbt.adapters.databend.column import DatabendColumn 22 | from dbt.adapters.databend.connections import DatabendConnectionManager 23 | from dbt.adapters.databend.relation import DatabendRelation 24 | 25 | GET_CATALOG_MACRO_NAME = "get_catalog" 26 | LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" 27 | LIST_SCHEMAS_MACRO_NAME = "list_schemas" 28 | 29 | logger = AdapterLogger("databend") 30 | 31 | 32 | def _expect_row_value(key: str, row: agate.Row): 33 | if key not in row.keys(): 34 | raise DbtInternalError( 35 | f"Got a row without '{key}' column, columns: {row.keys()}" 36 | ) 37 | 38 | return row[key] 39 | 40 | 41 | @dataclass 42 | class DatabendConfig(AdapterConfig): 43 | cluster_by: Optional[Union[List[str], str]] = None 44 | 45 | 46 | class DatabendAdapter(SQLAdapter): 47 | Relation = DatabendRelation 48 | Column = DatabendColumn 49 | ConnectionManager = DatabendConnectionManager 50 | AdapterSpecificConfigs = DatabendConfig 51 | 52 | @classmethod 53 | def date_function(cls): 54 | return "NOW()" 55 | 56 | @classmethod 57 | def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: 58 | return "string" 59 | 60 | @classmethod 61 | def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: 62 | decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) 63 | return "float" if decimals else "int" 64 | 65 | @classmethod 66 | def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: 67 | return "bool" 68 | 69 | @available 70 | def get_csv_data(self, table): 71 | csv_funcs = [c.csvify for c in table._column_types] 72 | 73 | buf = io.StringIO() 74 | writer = csv.writer(buf, lineterminator="\n") 75 | 76 | for row in table.rows: 77 | writer.writerow(tuple(csv_funcs[i](d) for i, d in enumerate(row))) 78 | 79 | return buf.getvalue() 80 | 81 | @classmethod 82 | def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: 83 | return "timestamp" 84 | 85 | @classmethod 86 | def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: 87 | return "date" 88 | 89 | @classmethod 90 | def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: 91 | raise DbtRuntimeError( 92 | "`convert_time_type` is not implemented for this adapter!" 93 | ) 94 | 95 | def quote(self, identifier): 96 | return "{}".format(identifier) 97 | 98 | def check_schema_exists(self, database, schema): 99 | results = self.execute_macro( 100 | LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database} 101 | ) 102 | 103 | exists = True if schema in [row[0] for row in results] else False 104 | return exists 105 | 106 | def list_relations_without_caching( 107 | self, schema_relation: DatabendRelation 108 | ) -> List[DatabendRelation]: 109 | kwargs = {"schema_relation": schema_relation} 110 | results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) 111 | 112 | relations = [] 113 | for row in results: 114 | if len(row) != 4: 115 | raise DbtRuntimeError( 116 | f"Invalid value from 'show table extended ...', " 117 | f"got {len(row)} values, expected 4" 118 | ) 119 | _database, name, schema, type_info = row 120 | rel_type = RelationType.View if "view" in type_info else RelationType.Table 121 | relation = self.Relation.create(database=None, schema=schema, identifier=name, rt=rel_type) 122 | relations.append(relation) 123 | 124 | return relations 125 | 126 | @classmethod 127 | def _catalog_filter_table( 128 | cls, table: agate.Table, used_schemas: FrozenSet[Tuple[str, str]] 129 | ) -> agate.Table: 130 | table = table_from_rows( 131 | table.rows, 132 | table.column_names, 133 | text_only_columns=["table_schema", "table_name"], 134 | ) 135 | return super()._catalog_filter_table(table, used_schemas) 136 | 137 | def get_relation(self, database: Optional[str], schema: str, identifier: str): 138 | # if not self.Relation.include_policy.database: 139 | # database = None 140 | 141 | return super().get_relation(database, schema, identifier) 142 | 143 | def parse_show_columns( 144 | self, _relation: DatabendRelation, raw_rows: List[agate.Row] 145 | ) -> List[DatabendColumn]: 146 | rows = [ 147 | dict(zip(row._keys, row._values)) # pylint: disable=protected-access 148 | for row in raw_rows 149 | ] 150 | 151 | return [ 152 | DatabendColumn( 153 | column=column["name"], 154 | dtype=column["type"], 155 | ) 156 | for column in rows 157 | ] 158 | 159 | def get_columns_in_relation( 160 | self, relation: DatabendRelation 161 | ) -> List[DatabendColumn]: 162 | rows: List[agate.Row] = super().get_columns_in_relation(relation) 163 | 164 | return self.parse_show_columns(relation, rows) 165 | 166 | def _get_one_catalog( 167 | self, 168 | information_schema: InformationSchema, 169 | schemas: Set[str], 170 | used_schemas: FrozenSet[Tuple[str, str]], 171 | ) -> agate.Table: 172 | if len(schemas) != 1: 173 | DbtRuntimeError( 174 | f"Expected only one schema in databend _get_one_catalog, found {schemas}" 175 | ) 176 | 177 | return super()._get_one_catalog(information_schema, schemas, used_schemas) 178 | 179 | def update_column_sql( 180 | self, 181 | dst_name: str, 182 | dst_column: str, 183 | clause: str, 184 | where_clause: Optional[str] = None, 185 | ) -> str: 186 | raise DbtInternalError( 187 | "`update_column_sql` is not implemented for this adapter!" 188 | ) 189 | 190 | def run_sql_for_tests(self, sql, fetch, conn): 191 | cursor = conn.handle.cursor() 192 | try: 193 | cursor.execute(sql) 194 | if fetch == "one": 195 | if hasattr(cursor, "fetchone"): 196 | return cursor.fetchone() 197 | else: 198 | return cursor.fetchall()[0] 199 | elif fetch == "all": 200 | return cursor.fetchall() 201 | else: 202 | return 203 | except BaseException as exc: 204 | logger.error(sql) 205 | logger.error(exc) 206 | raise exc 207 | finally: 208 | conn.transaction_open = False 209 | 210 | def get_rows_different_sql( 211 | self, 212 | relation_a: DatabendRelation, 213 | relation_b: DatabendRelation, 214 | column_names: Optional[List[str]] = None, 215 | except_operator: str = "EXCEPT", 216 | ) -> str: 217 | names: List[str] 218 | if column_names is None: 219 | columns = self.get_columns_in_relation(relation_a) 220 | names = sorted((self.quote(c.name) for c in columns)) 221 | else: 222 | names = sorted((self.quote(n) for n in column_names)) 223 | 224 | alias_a = "A" 225 | alias_b = "B" 226 | columns_csv_a = ", ".join([f"{alias_a}.{name}" for name in names]) 227 | columns_csv_b = ", ".join([f"{alias_b}.{name}" for name in names]) 228 | join_condition = " AND ".join( 229 | [f"{alias_a}.{name} = {alias_b}.{name}" for name in names] 230 | ) 231 | if len(names) == 0: 232 | return f"select 0, 0" 233 | first_column = names[0] 234 | 235 | # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it 236 | COLUMNS_EQUAL_SQL = """ 237 | WITH 238 | a_except_b as ( 239 | SELECT 240 | {columns_a} 241 | FROM {relation_a} as {alias_a} 242 | LEFT OUTER JOIN {relation_b} as {alias_b} 243 | ON {join_condition} 244 | WHERE {alias_b}.{first_column} is null 245 | ), 246 | b_except_a as ( 247 | SELECT 248 | {columns_b} 249 | FROM {relation_b} as {alias_b} 250 | LEFT OUTER JOIN {relation_a} as {alias_a} 251 | ON {join_condition} 252 | WHERE {alias_a}.{first_column} is null 253 | ), 254 | diff_count as ( 255 | SELECT 256 | 1 as id, 257 | COUNT(*) as num_missing FROM ( 258 | SELECT * FROM a_except_b 259 | UNION ALL 260 | SELECT * FROM b_except_a 261 | ) as missing 262 | ), 263 | table_a as ( 264 | SELECT COUNT(*) as num_rows FROM {relation_a} 265 | ), 266 | table_b as ( 267 | SELECT COUNT(*) as num_rows FROM {relation_b} 268 | ), 269 | row_count_diff as ( 270 | SELECT 271 | 1 as id, 272 | table_a.num_rows - table_b.num_rows as difference 273 | FROM table_a, table_b 274 | ) 275 | SELECT 276 | row_count_diff.difference as row_count_difference, 277 | diff_count.num_missing as num_mismatched 278 | FROM row_count_diff 279 | INNER JOIN diff_count ON row_count_diff.id = diff_count.id 280 | """.strip() 281 | 282 | sql = COLUMNS_EQUAL_SQL.format( 283 | alias_a=alias_a, 284 | alias_b=alias_b, 285 | first_column=first_column, 286 | columns_a=columns_csv_a, 287 | columns_b=columns_csv_b, 288 | join_condition=join_condition, 289 | relation_a=str(relation_a), 290 | relation_b=str(relation_b), 291 | ) 292 | 293 | return sql 294 | 295 | @property 296 | def default_python_submission_method(self): 297 | raise NotImplementedError("default_python_submission_method is not specified") 298 | 299 | @property 300 | def python_submission_helpers(self): 301 | raise NotImplementedError("python_submission_helpers is not specified") 302 | -------------------------------------------------------------------------------- /dbt/adapters/databend/relation.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from typing import Optional, TypeVar, Any, Type, Dict, Union, Iterator, Tuple, Set 3 | 4 | from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError 5 | from dbt.adapters.base.relation import BaseRelation, Policy 6 | from dbt.adapters.contracts.relation import ( 7 | Path, 8 | RelationType, 9 | ) 10 | 11 | 12 | @dataclass 13 | class DatabendQuotePolicy(Policy): 14 | database: bool = False 15 | schema: bool = False 16 | identifier: bool = False 17 | 18 | 19 | @dataclass 20 | class DatabendIncludePolicy(Policy): 21 | database: bool = False 22 | schema: bool = True 23 | identifier: bool = True 24 | 25 | 26 | Self = TypeVar("Self", bound="DatabendRelation") 27 | 28 | 29 | @dataclass(frozen=True, eq=False, repr=False) 30 | class DatabendRelation(BaseRelation): 31 | quote_policy: Policy = field(default_factory=lambda: DatabendQuotePolicy()) 32 | include_policy: DatabendIncludePolicy = field( 33 | default_factory=lambda: DatabendIncludePolicy() 34 | ) 35 | quote_character: str = "" 36 | 37 | def __post_init__(self): 38 | if self.database != self.schema and self.database: 39 | raise DbtDatabaseError( 40 | f" schema: {self.schema} \n" 41 | f" database: {self.database} \n" 42 | f"On Databend, database must be omitted or have the same value as" 43 | f" schema." 44 | ) 45 | 46 | @classmethod 47 | def create( 48 | cls: Type[Self], 49 | database: Optional[str] = None, 50 | schema: Optional[str] = None, 51 | identifier: Optional[str] = None, 52 | rt: Optional[RelationType] = None, 53 | **kwargs, 54 | ) -> Self: 55 | database = None 56 | kwargs.update( 57 | { 58 | "path": { 59 | "database": database, 60 | "schema": schema, 61 | "identifier": identifier, 62 | }, 63 | "type": rt, 64 | } 65 | ) 66 | return cls.from_dict(kwargs) 67 | 68 | def render(self): 69 | if self.include_policy.database and self.include_policy.schema: 70 | raise DbtRuntimeError( 71 | "Got a databend relation with schema and database set to " 72 | "include, but only one can be set" 73 | ) 74 | return super().render() 75 | 76 | @classmethod 77 | def get_path( 78 | cls, relation: BaseRelation, information_schema_view: Optional[str] 79 | ) -> Path: 80 | Path.database = None 81 | return Path( 82 | database=None, 83 | schema=relation.schema, 84 | identifier="INFORMATION_SCHEMA", 85 | ) 86 | 87 | def matches( 88 | self, 89 | database: Optional[str] = None, 90 | schema: Optional[str] = None, 91 | identifier: Optional[str] = None, 92 | ): 93 | if database: 94 | raise DbtRuntimeError( 95 | f"Passed unexpected schema value {schema} to Relation.matches" 96 | ) 97 | return self.schema == schema and self.identifier == identifier 98 | -------------------------------------------------------------------------------- /dbt/include/databend/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | PACKAGE_PATH = os.path.dirname(__file__) 4 | -------------------------------------------------------------------------------- /dbt/include/databend/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: dbt_databend 2 | version: 1.5.0 3 | config-version: 2 4 | 5 | macro-paths: ["macros"] 6 | -------------------------------------------------------------------------------- /dbt/include/databend/macros/adapters.sql: -------------------------------------------------------------------------------- 1 | /* For examples of how to fill out the macros please refer to the postgres adapter and docs 2 | postgres adapter macros: https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/adapters.sql 3 | dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter 4 | */ 5 | 6 | {% macro databend__create_schema(relation) -%} 7 | '''Creates a new schema in the target database, if schema already exists, method is a no-op. ''' 8 | {%- call statement('create_schema') -%} 9 | create database if not exists {{ relation.without_identifier().include(database=False) }} 10 | {% endcall %} 11 | {% endmacro %} 12 | 13 | {% macro databend__drop_relation(relation) -%} 14 | '''Deletes relatonship identifer between tables.''' 15 | /* 16 | 1. If database exists 17 | 2. Create a new schema if passed schema does not exist already 18 | */ 19 | {% call statement('drop_relation', auto_begin=False) -%} 20 | drop {{ relation.type }} if exists {{ relation }} 21 | {%- endcall %} 22 | {% endmacro %} 23 | 24 | {% macro databend__drop_schema(relation) -%} 25 | '''drops a schema in a target database.''' 26 | /* 27 | 1. If database exists 28 | 2. search all calls of schema, and change include value to False, cascade it to backtrack 29 | */ 30 | {%- call statement('drop_schema') -%} 31 | drop database if exists {{ relation.without_identifier().include(database=False) }} 32 | {%- endcall -%} 33 | {% endmacro %} 34 | 35 | {% macro databend__get_columns_in_relation(relation) -%} 36 | '''Returns a list of Columns in a table.''' 37 | /* 38 | 1. select as many values from column as needed 39 | 2. search relations to columns 40 | 3. where table name is equal to the relation identifier 41 | 4. if a relation schema exists and table schema is equal to the relation schema 42 | 5. order in whatever way you want to call. 43 | 6. create a table by loading result from call 44 | 7. return new table 45 | */ 46 | {% call statement('get_columns_in_relation', fetch_result=True) %} 47 | select 48 | name, 49 | type, 50 | 0 as position 51 | from system.columns 52 | where 53 | table = '{{ relation.identifier }}' 54 | {% if relation.schema %} 55 | and database = '{{ relation.schema }}' 56 | {% endif %} 57 | {% endcall %} 58 | {% do return(load_result('get_columns_in_relation').table) %} 59 | {% endmacro %} 60 | 61 | -- Example of 2 of 3 required macros that do not come with a default implementation 62 | 63 | {% macro databend__list_relations_without_caching(schema_relation) -%} 64 | '''creates a table of relations withough using local caching.''' 65 | {% call statement('list_relations_without_caching', fetch_result=True) -%} 66 | select 67 | null as db, 68 | name as name, 69 | database as schema, 70 | if(engine = 'VIEW', 'view', 'table') as type 71 | from system.tables 72 | where database = '{{ schema_relation.schema }}' 73 | {% endcall %} 74 | {{ return(load_result('list_relations_without_caching').table) }} 75 | {% endmacro %} 76 | 77 | {% macro databend__list_schemas(database) -%} 78 | '''Returns a table of unique schemas.''' 79 | /* 80 | 1. search schemea by specific name 81 | 2. create a table with names 82 | */ 83 | {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} 84 | select name from system.databases 85 | {% endcall %} 86 | {{ return(load_result('list_schemas').table) }} 87 | {% endmacro %} 88 | 89 | {% macro databend__rename_relation(from_relation, to_relation) -%} 90 | '''Renames a relation in the database.''' 91 | /* 92 | 1. Search for a specific relation name 93 | 2. alter table by targeting specific name and passing in new name 94 | */ 95 | {% call statement('drop_relation') %} 96 | drop {{ to_relation.type }} if exists {{ to_relation }} 97 | {% endcall %} 98 | {% call statement('rename_relation') %} 99 | rename table {{ from_relation }} to {{ to_relation }} 100 | {% endcall %} 101 | {% endmacro %} 102 | 103 | {% macro databend__truncate_relation(relation) -%} 104 | '''Removes all rows from a targeted set of tables.''' 105 | /* 106 | 1. grab all tables tied to the relation 107 | 2. remove rows from relations 108 | */ 109 | {% call statement('truncate_relation') -%} 110 | truncate table {{ relation }} 111 | {%- endcall %} 112 | {% endmacro %} 113 | 114 | 115 | {% macro databend__make_temp_relation(base_relation, suffix) %} 116 | {% set tmp_identifier = base_relation.identifier ~ suffix %} 117 | {% set tmp_relation = base_relation.incorporate( 118 | path={"identifier": tmp_identifier, "schema": None}) -%} 119 | {% do return(tmp_relation) %} 120 | {% endmacro %} 121 | 122 | {% macro databend__generate_database_name(custom_database_name=none, node=none) -%} 123 | {% do return(None) %} 124 | {%- endmacro %} 125 | 126 | {% macro databend__current_timestamp() -%} 127 | NOW() 128 | {%- endmacro %} 129 | 130 | {% macro databend__create_table_as(temporary, relation, sql) -%} 131 | {%- set sql_header = config.get('sql_header', none) -%} 132 | 133 | {{ sql_header if sql_header is not none }} 134 | 135 | {% if temporary -%} 136 | create or replace transient table {{ relation.name }} 137 | {%- else %} 138 | create or replace table {{ relation.include(database=False) }} 139 | {{ cluster_by_clause(label="cluster by") }} 140 | {%- endif %} 141 | as {{ sql }} 142 | {%- endmacro %} 143 | 144 | {% macro databend__create_view_as(relation, sql) -%} 145 | {%- set sql_header = config.get('sql_header', none) -%} 146 | 147 | {{ sql_header if sql_header is not none }} 148 | 149 | create or replace view {{ relation.include(database=False) }} 150 | as {{ sql }} 151 | {%- endmacro %} 152 | 153 | {% macro databend__get_columns_in_query(select_sql) %} 154 | {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} 155 | select * from ( 156 | {{ select_sql }} 157 | ) as __dbt_sbq 158 | limit 0 159 | {% endcall %} 160 | 161 | {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} 162 | {% endmacro %} 163 | 164 | {% macro cluster_by_clause(label) %} 165 | {%- set cols = config.get('cluster_by', validator=validation.any[list, basestring]) -%} 166 | {%- if cols is not none %} 167 | {%- if cols is string -%} 168 | {%- set cols = [cols] -%} 169 | {%- endif -%} 170 | {{ label }} 171 | {%- for item in cols -%} 172 | {{ item }} 173 | {%- if not loop.last -%},{%- endif -%} 174 | {%- endfor -%} 175 | {%- endif %} 176 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt/include/databend/macros/adapters/apply_grants.sql: -------------------------------------------------------------------------------- 1 | {% macro databend__get_show_grant_sql(relation) %} 2 | {# Usually called from apply_grants. Should not be called directly. #} 3 | {{ adapter.raise_grant_error() }} 4 | {% endmacro %} 5 | 6 | 7 | {%- macro databend__get_grant_sql(relation, privilege, grantee) -%} 8 | {# Usually called from apply_grants. Should not be called directly. #} 9 | {{ adapter.raise_grant_error() }} 10 | {%- endmacro -%} 11 | 12 | 13 | {%- macro databend__get_revoke_sql(relation, privilege, grantee) -%} 14 | {# Usually called from apply_grants. Should not be called directly. #} 15 | {{ adapter.raise_grant_error() }} 16 | {%- endmacro -%} 17 | 18 | 19 | {% macro databend__copy_grants() %} 20 | {{ return(True) }} 21 | {% endmacro %} 22 | 23 | 24 | {% macro databend__apply_grants(relation, grant_config, should_revoke) %} 25 | {% if grant_config %} 26 | {{ adapter.raise_grant_error() }} 27 | {% endif %} 28 | {% endmacro %} 29 | 30 | 31 | {%- macro databend__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%} 32 | {# Databend does not support DCL statements yet #} 33 | {% if grant_config %} 34 | {{ adapter.raise_grant_error() }} 35 | {% endif %} 36 | {{ return([]) }} 37 | {%- endmacro %} 38 | -------------------------------------------------------------------------------- /dbt/include/databend/macros/adapters/relation.sql: -------------------------------------------------------------------------------- 1 | {% macro databend__get_or_create_relation(database, schema, identifier, type) %} 2 | {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} 3 | {% if target_relation %} 4 | {% do return([true, target_relation]) %} 5 | {% endif %} 6 | 7 | {%- set new_relation = api.Relation.create( 8 | database=database, 9 | schema=schema, 10 | identifier=identifier, 11 | type=type 12 | ) -%} 13 | {% do return([false, new_relation]) %} 14 | {% endmacro %} 15 | 16 | {% macro databend__get_database(database) %} 17 | {% call statement('get_database', fetch_result=True) %} 18 | select name 19 | from system.databases 20 | where name = '{{ database }}' 21 | {% endcall %} 22 | {% do return(load_result('get_database').table) %} 23 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/databend/macros/catalog.sql: -------------------------------------------------------------------------------- 1 | {% macro databend__get_catalog(information_schema, schemas) -%} 2 | {%- call statement('catalog', fetch_result=True) -%} 3 | select 4 | null as table_database, 5 | tables.database as table_schema, 6 | tables.name as table_name, 7 | if(tables.engine = 'VIEW', 'view', 'table') as table_type, 8 | null as table_comment, 9 | null as column_name, 10 | 0 as column_index, 11 | null as column_type, 12 | null as column_comment, 13 | null as table_owner 14 | from system.tables 15 | where tables.database != 'system' and 16 | ( 17 | {%- for schema in schemas -%} 18 | tables.database = '{{ schema }}' 19 | {%- if not loop.last %} or {% endif -%} 20 | {%- endfor -%} 21 | ) 22 | order by tables.database, tables.name 23 | {%- endcall -%} 24 | {{ return(load_result('catalog').table) }} 25 | {%- endmacro %} -------------------------------------------------------------------------------- /dbt/include/databend/macros/materializations/incremental.sql: -------------------------------------------------------------------------------- 1 | {% materialization incremental, adapter='databend' %} 2 | 3 | {%- set existing_relation = load_cached_relation(this) -%} 4 | {%- set target_relation = this.incorporate(type='table') -%} 5 | 6 | {%- set unique_key = config.get('unique_key') -%} 7 | {% if unique_key is not none and unique_key|length == 0 %} 8 | {% set unique_key = none %} 9 | {% endif %} 10 | {% if unique_key is iterable and (unique_key is not string and unique_key is not mapping) %} 11 | {% set unique_key = unique_key|join(', ') %} 12 | {% endif %} 13 | {%- set grant_config = config.get('grants') -%} 14 | {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%} 15 | {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} 16 | 17 | {%- set intermediate_relation = make_intermediate_relation(target_relation)-%} 18 | {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} 19 | {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} 20 | {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} 21 | {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} 22 | 23 | {{ drop_relation_if_exists(preexisting_intermediate_relation) }} 24 | {{ drop_relation_if_exists(preexisting_backup_relation) }} 25 | 26 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 27 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 28 | {% set to_drop = [] %} 29 | 30 | {% if existing_relation is none %} 31 | -- No existing table, simply create a new one 32 | {% call statement('main') %} 33 | {{ get_create_table_as_sql(False, target_relation, sql) }} 34 | {% endcall %} 35 | 36 | {% elif full_refresh_mode %} 37 | -- Completely replacing the old table, so create a temporary table and then swap it 38 | {% call statement('main') %} 39 | {{ get_create_table_as_sql(False, intermediate_relation, sql) }} 40 | {% endcall %} 41 | {% set need_swap = true %} 42 | 43 | {% elif unique_key is none -%} 44 | {% call statement('main') %} 45 | {{ databend__insert_into(target_relation, sql) }} 46 | {% endcall %} 47 | 48 | {% else %} 49 | {% set schema_changes = 'ignore' %} 50 | {% set incremental_strategy = config.get('incremental_strategy') %} 51 | {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} 52 | {% if incremental_strategy != 'delete_insert' and incremental_predicates %} 53 | {% do exceptions.raise_compiler_error('Cannot apply incremental predicates with ' + incremental_strategy + ' strategy.') %} 54 | {% endif %} 55 | {% if incremental_strategy == 'delete_insert' %} 56 | {% do databend__incremental_delete_insert(existing_relation, unique_key, incremental_predicates) %} 57 | {% elif incremental_strategy == 'append' %} 58 | {% call statement('main') %} 59 | {{ databend__insert_into(target_relation, sql) }} 60 | {% endcall %} 61 | {% endif %} 62 | {% endif %} 63 | 64 | -- {% if need_swap %} 65 | -- {% if existing_relation.can_exchange %} 66 | -- {% do adapter.rename_relation(intermediate_relation, backup_relation) %} 67 | -- {% do exchange_tables_atomic(backup_relation, target_relation) %} 68 | -- {% else %} 69 | -- {% do adapter.rename_relation(target_relation, backup_relation) %} 70 | -- {% do adapter.rename_relation(intermediate_relation, target_relation) %} 71 | -- {% endif %} 72 | -- {% do to_drop.append(backup_relation) %} 73 | -- {% endif %} 74 | 75 | {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} 76 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} 77 | 78 | {% do persist_docs(target_relation, model) %} 79 | 80 | {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %} 81 | {% do create_indexes(target_relation) %} 82 | {% endif %} 83 | 84 | {{ run_hooks(post_hooks, inside_transaction=True) }} 85 | 86 | {% do adapter.commit() %} 87 | 88 | {% for rel in to_drop %} 89 | {% do adapter.drop_relation(rel) %} 90 | {% endfor %} 91 | 92 | {{ run_hooks(post_hooks, inside_transaction=False) }} 93 | 94 | {{ return({'relations': [target_relation]}) }} 95 | 96 | {%- endmaterialization %} 97 | 98 | 99 | {% macro process_schema_changes(on_schema_change, source_relation, target_relation) %} 100 | 101 | {%- set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) -%} 102 | {% if not schema_changes_dict['schema_changed'] %} 103 | {{ return }} 104 | {% endif %} 105 | 106 | {% if on_schema_change == 'fail' %} 107 | {% set fail_msg %} 108 | The source and target schemas on this incremental model are out of sync! 109 | They can be reconciled in several ways: 110 | - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation. 111 | - Re-run the incremental model with `full_refresh: True` to update the target schema. 112 | - update the schema manually and re-run the process. 113 | {% endset %} 114 | {% do exceptions.raise_compiler_error(fail_msg) %} 115 | {{ return }} 116 | {% endif %} 117 | 118 | {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %} 119 | 120 | {% endmacro %} 121 | 122 | 123 | 124 | {% macro databend__incremental_delete_insert(existing_relation, unique_key, incremental_predicates) %} 125 | {% set new_data_relation = existing_relation.incorporate(path={"identifier": model['name'] 126 | + '__dbt_new_data_' + invocation_id.replace('-', '_')}) %} 127 | {{ drop_relation_if_exists(new_data_relation) }} 128 | {% call statement('main') %} 129 | {{ get_create_table_as_sql(False, new_data_relation, sql) }} 130 | {% endcall %} 131 | {% call statement('delete_existing_data') %} 132 | delete from {{ existing_relation }} where ({{ unique_key }}) in (select {{ unique_key }} 133 | from {{ new_data_relation }}) 134 | {%- if incremental_predicates %} 135 | {% for predicate in incremental_predicates %} 136 | and {{ predicate }} 137 | {% endfor %} 138 | {%- endif -%}; 139 | {% endcall %} 140 | 141 | {%- set dest_columns = adapter.get_columns_in_relation(existing_relation) -%} 142 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 143 | {% call statement('insert_new_data') %} 144 | insert into {{ existing_relation}} select {{ dest_cols_csv}} from {{ new_data_relation }} 145 | {% endcall %} 146 | {% do adapter.drop_relation(new_data_relation) %} 147 | {% endmacro %} 148 | -------------------------------------------------------------------------------- /dbt/include/databend/macros/materializations/seed.sql: -------------------------------------------------------------------------------- 1 | {% macro basic_load_csv_rows(model, batch_size, agate_table) %} 2 | {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} 3 | {% set bindings = [] %} 4 | 5 | {% set statements = [] %} 6 | 7 | {% for chunk in agate_table.rows | batch(batch_size) %} 8 | {% set bindings = [] %} 9 | 10 | {% for row in chunk %} 11 | {% do bindings.extend(row) %} 12 | {% endfor %} 13 | 14 | {% set sql %} 15 | insert into {{ this.render() }} ({{ cols_sql }}) values 16 | {% for row in chunk -%} 17 | ({%- for column in agate_table.column_names -%} 18 | %s 19 | {%- if not loop.last%},{%- endif %} 20 | {%- endfor -%}) 21 | {%- if not loop.last%},{%- endif %} 22 | {%- endfor %} 23 | {% endset %} 24 | 25 | {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} 26 | 27 | {% if loop.index0 == 0 %} 28 | {% do statements.append(sql) %} 29 | {% endif %} 30 | {% endfor %} 31 | 32 | {# Return SQL so we can render it out into the compiled files #} 33 | {{ return(statements[0]) }} 34 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/databend/macros/materializations/snapshot.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendcloud/dbt-databend/798c733dc05196049b6dae4594024654c0b60fa5/dbt/include/databend/macros/materializations/snapshot.sql -------------------------------------------------------------------------------- /dbt/include/databend/macros/materializations/table.sql: -------------------------------------------------------------------------------- 1 | {% materialization table, adapter='databend' %} 2 | 3 | {%- set existing_relation = load_cached_relation(this) -%} 4 | {%- set target_relation = this.incorporate(type='table') -%} 5 | {%- set backup_relation = none -%} 6 | {%- set preexisting_backup_relation = none -%} 7 | {%- set preexisting_intermediate_relation = none -%} 8 | 9 | {% if existing_relation is not none %} 10 | {%- set backup_relation_type = existing_relation.type -%} 11 | {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} 12 | {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} 13 | -- {% if not existing_relation.can_exchange %} 14 | -- {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} 15 | -- {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} 16 | -- {% endif %} 17 | {% endif %} 18 | 19 | {% set grant_config = config.get('grants') %} 20 | 21 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 22 | 23 | -- drop the temp relations if they exist already in the database 24 | {{ drop_relation_if_exists(preexisting_intermediate_relation) }} 25 | {{ drop_relation_if_exists(preexisting_backup_relation) }} 26 | 27 | -- `BEGIN` happens here: 28 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 29 | 30 | {% if backup_relation is none %} 31 | {{ log('Creating new relation ' + target_relation.name )}} 32 | -- There is not existing relation, so we can just create 33 | {% call statement('main') -%} 34 | {{ get_create_table_as_sql(False, target_relation, sql) }} 35 | {%- endcall %} 36 | {% elif existing_relation.can_exchange %} 37 | -- We can do an atomic exchange, so no need for an intermediate 38 | {% call statement('main') -%} 39 | {{ get_create_table_as_sql(False, backup_relation, sql) }} 40 | {%- endcall %} 41 | {% do exchange_tables_atomic(backup_relation, existing_relation) %} 42 | {% else %} 43 | -- We have to use an intermediate and rename accordingly 44 | {% call statement('main') -%} 45 | {{ get_create_table_as_sql(False, intermediate_relation, sql) }} 46 | {%- endcall %} 47 | {{ adapter.rename_relation(existing_relation, backup_relation) }} 48 | {{ adapter.rename_relation(intermediate_relation, target_relation) }} 49 | {% endif %} 50 | 51 | -- cleanup 52 | {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} 53 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} 54 | 55 | {% do persist_docs(target_relation, model) %} 56 | 57 | {{ run_hooks(post_hooks, inside_transaction=True) }} 58 | 59 | {{ adapter.commit() }} 60 | 61 | {{ drop_relation_if_exists(backup_relation) }} 62 | 63 | {{ run_hooks(post_hooks, inside_transaction=False) }} 64 | 65 | {{ return({'relations': [target_relation]}) }} 66 | 67 | {% endmaterialization %} 68 | 69 | {% macro databend__insert_into(target_relation, sql) %} 70 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 71 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 72 | 73 | insert into {{ target_relation }} ({{ dest_cols_csv }}) 74 | {{ sql }} 75 | {%- endmacro %} 76 | -------------------------------------------------------------------------------- /dbt/include/databend/macros/utils.sql: -------------------------------------------------------------------------------- 1 | {% macro databend__any_value(expression) -%} 2 | any({{ expression }}) 3 | {%- endmacro %} 4 | 5 | {% macro databend__dateadd(datepart, interval, from_date_or_timestamp) %} 6 | date_add( 7 | {{ datepart }}, 8 | {{ interval }}, 9 | {{ from_date_or_timestamp }} 10 | ) 11 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/databend/profile_template.yml: -------------------------------------------------------------------------------- 1 | default: 2 | target: dev 3 | outputs: 4 | dev: 5 | type: databend 6 | host: [host] 7 | port: [port] 8 | schema: [schema(Your database)] 9 | user: [username] 10 | pass: [password] -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # install latest changes in dbt-core 2 | git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core 3 | git+https://github.com/dbt-labs/dbt-adapters.git 4 | git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter 5 | git+https://github.com/dbt-labs/dbt-common.git 6 | 7 | 8 | black==22.3.0 9 | bumpversion 10 | flake8 11 | flaky 12 | freezegun==0.3.12 13 | ipdb 14 | pip-tools 15 | pre-commit 16 | pytest 17 | pytest-dotenv 18 | pytest-logbook 19 | pytest-csv 20 | pytest-xdist 21 | pytz 22 | tox>=3.13 23 | twine 24 | wheel 25 | databend-py==0.4.9 26 | databend-sqlalchemy==0.3.2 27 | environs -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import find_namespace_packages, setup 3 | 4 | package_name = "dbt-databend-cloud" 5 | # make sure this always matches dbt/adapters/{adapter}/__version__.py 6 | package_version = "1.8.1" 7 | description = """The Databend adapter plugin for dbt""" 8 | 9 | setup( 10 | name=package_name, 11 | version=package_version, 12 | description=description, 13 | long_description=description, 14 | author="Databend Cloud Team", 15 | author_email="zhangzhihan@datafuselabs.com", 16 | url="https://github.com/databendcloud/dbt-databend.git", 17 | packages=find_namespace_packages(include=["dbt", "dbt.*"]), 18 | include_package_data=True, 19 | install_requires=[ 20 | "dbt-common>=1.0.4,<2.0", 21 | "dbt-adapters>=1.1.1,<2.0", 22 | # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency 23 | "dbt-core>=1.8.0", 24 | # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core 25 | "agate", 26 | "databend-sqlalchemy~=0.3.2", 27 | "agate", 28 | ], 29 | classifiers=[ 30 | "Development Status :: 5 - Production/Stable", 31 | "License :: OSI Approved :: Apache Software License", 32 | "Operating System :: Microsoft :: Windows", 33 | "Operating System :: MacOS :: MacOS X", 34 | "Operating System :: POSIX :: Linux", 35 | "Programming Language :: Python :: 3.10", 36 | "Programming Language :: Python :: 3.11", 37 | "Programming Language :: Python :: 3.1.x", 38 | ], 39 | python_requires=">=3.8", 40 | ) 41 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Testing dbt-databend-cloud 2 | 3 | ## Overview 4 | 5 | Here are the steps to run the tests: 6 | 1. Set up 7 | 2. Get config 8 | 3. Run tests 9 | 10 | ## Set up 11 | 12 | Make sure you have python environment, you can find the supported python version in setup.py. 13 | ```bash 14 | pip3 install -r requirements_dev.txt 15 | pip3 install . 16 | ``` 17 | 18 | ## Get config 19 | Config the configurations in `conftest.py`: 20 | 21 | ```python 22 | { 23 | "type": "databend", 24 | "host": "host", 25 | "port": 443, 26 | "user": "user", 27 | "pass": "pass", 28 | "schema": "your database", 29 | "secure": True, 30 | } 31 | ``` 32 | 33 | You can get the config information by the way in this [docs](https://docs.databend.com/using-databend-cloud/warehouses/connecting-a-warehouse). 34 | 35 | ## Run tests 36 | 37 | ```shell 38 | python -m pytest -s -vv tests/functional 39 | ``` 40 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/databendcloud/dbt-databend/798c733dc05196049b6dae4594024654c0b60fa5/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | # import os 4 | # import json 5 | 6 | # Import the fuctional fixtures as a plugin 7 | # Note: fixtures with session scope need to be local 8 | 9 | pytest_plugins = ["dbt.tests.fixtures.project"] 10 | 11 | 12 | # The profile dictionary, used to write out profiles.yml 13 | @pytest.fixture(scope="class") 14 | def dbt_profile_target(): 15 | return { 16 | "type": "databend", 17 | "host": "localhost", 18 | "port": 8000, 19 | "user": "databend", 20 | "pass": "databend", 21 | "schema": "default", 22 | "secure": False, 23 | } 24 | -------------------------------------------------------------------------------- /tests/functional/adapter/empty/test_empty.py: -------------------------------------------------------------------------------- 1 | from dbt.tests.adapter.empty.test_empty import BaseTestEmpty 2 | 3 | 4 | class TestDatabendEmpty(BaseTestEmpty): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/functional/adapter/incremental/test_incremental.py: -------------------------------------------------------------------------------- 1 | from dbt.tests.adapter.incremental.test_incremental_predicates import ( 2 | BaseIncrementalPredicates, 3 | ) 4 | from dbt.tests.adapter.incremental.test_incremental_unique_id import ( 5 | BaseIncrementalUniqueKey, 6 | ) 7 | from pytest import fixture, mark 8 | 9 | 10 | class TestIncrementalPredicatesDeleteInsertDatabend(BaseIncrementalPredicates): 11 | @fixture(scope='class') 12 | def project_config_update(self): 13 | return { 14 | 'models': { 15 | '+predicates': ['id != 2'], 16 | '+incremental_strategy': 'delete_insert', 17 | } 18 | } 19 | 20 | 21 | @mark.skip('No support for unique keys in default incremental strategy') 22 | class TestIncrementalUniqueKeyDatabend(BaseIncrementalUniqueKey): 23 | pass 24 | 25 | 26 | @mark.skip('No support for unique keys in default incremental strategy') 27 | class TestUniqueKeyDeleteInsertDatabend(BaseIncrementalUniqueKey): 28 | @fixture(scope='class') 29 | def project_config_update(self): 30 | return {'models': {'+incremental_strategy': 'delete_insert'}} 31 | -------------------------------------------------------------------------------- /tests/functional/adapter/statement_test/seeds.py: -------------------------------------------------------------------------------- 1 | seeds_csv = """ 2 | ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS 3 | 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168 4 | 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35 5 | 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243 6 | 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175 7 | 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136 8 | 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220 9 | 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64 10 | 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13 11 | 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186 12 | 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100 13 | 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67 14 | 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193 15 | 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5 16 | 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250 17 | 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245 18 | 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54 19 | 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96 20 | 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72 21 | 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174 22 | 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25 23 | 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253 24 | 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153 25 | 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201 26 | 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122 27 | 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95 28 | 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52 29 | 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26 30 | 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118 31 | 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28 32 | 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177 33 | 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233 34 | 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203 35 | 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149 36 | 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167 37 | 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110 38 | 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68 39 | 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89 40 | 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81 41 | 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15 42 | 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255 43 | 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140 44 | 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24 45 | 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249 46 | 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218 47 | 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198 48 | 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18 49 | 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238 50 | 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61 51 | 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21 52 | 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209 53 | 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87 54 | 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142 55 | 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126 56 | 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212 57 | 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194 58 | 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22 59 | 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60 60 | 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50 61 | 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222 62 | 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115 63 | 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155 64 | 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94 65 | 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106 66 | 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68 67 | 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41 68 | 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109 69 | 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77 70 | 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194 71 | 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135 72 | 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87 73 | 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44 74 | 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182 75 | 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241 76 | 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24 77 | 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214 78 | 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199 79 | 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41 80 | 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255 81 | 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144 82 | 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231 83 | 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188 84 | 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61 85 | 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30 86 | 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192 87 | 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232 88 | 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109 89 | 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156 90 | 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84 91 | 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235 92 | 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53 93 | 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221 94 | 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187 95 | 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57 96 | 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189 97 | 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180 98 | 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144 99 | 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117 100 | 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126 101 | 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244 102 | 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88 103 | """.lstrip() 104 | 105 | statement_expected_csv = """ 106 | SOURCE,VALUE 107 | matrix,100 108 | table,100 109 | """.lstrip() 110 | -------------------------------------------------------------------------------- /tests/functional/adapter/statement_test/test_statements.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dbt.tests.util import check_relations_equal, run_dbt 3 | from tests.functional.adapter.statement_test.seeds import seeds_csv, statement_expected_csv 4 | 5 | _STATEMENT_ACTUAL_SQL = """ 6 | -- {{ ref('seed') }} 7 | 8 | {%- call statement('test_statement', fetch_result=True) -%} 9 | 10 | select 11 | count(*) as "num_records" 12 | 13 | from {{ ref('seed') }} 14 | 15 | {%- endcall -%} 16 | 17 | {% set result = load_result('test_statement') %} 18 | 19 | {% set res_table = result['table'] %} 20 | {% set res_matrix = result['data'] %} 21 | 22 | {% set matrix_value = res_matrix[0][0] %} 23 | {% set table_value = res_table[0]['num_records'] %} 24 | 25 | select 'matrix' as source, {{ matrix_value }} as value 26 | union all 27 | select 'table' as source, {{ table_value }} as value 28 | """.lstrip() 29 | 30 | 31 | class TestStatements: 32 | @pytest.fixture(scope="class") 33 | def models(self): 34 | return {"statement_actual.sql": _STATEMENT_ACTUAL_SQL} 35 | 36 | @pytest.fixture(scope="class") 37 | def seeds(self): 38 | return { 39 | "seed.csv": seeds_csv, 40 | "statement_expected.csv": statement_expected_csv, 41 | } 42 | 43 | def test_databend_statements(self, project): 44 | seed_results = run_dbt(["seed"]) 45 | assert len(seed_results) == 2 46 | results = run_dbt() 47 | assert len(results) == 1 48 | 49 | db_with_schema = f"{project.database}.{project.test_schema}" 50 | check_relations_equal( 51 | project.adapter, 52 | [f"{db_with_schema}.STATEMENT_ACTUAL", f"{db_with_schema}.STATEMENT_EXPECTED"], 53 | ) 54 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_basic.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations 4 | from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests 5 | from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( 6 | BaseSingularTestsEphemeral, 7 | ) 8 | from dbt.tests.adapter.basic.test_empty import BaseEmpty 9 | from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral 10 | from dbt.tests.adapter.basic.test_incremental import BaseIncremental 11 | from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests 12 | from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate 13 | from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols 14 | from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp 15 | from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod 16 | from dbt.tests.util import check_relation_types, relation_from_name, run_dbt 17 | 18 | 19 | class TestSimpleMaterializationsDatabend(BaseSimpleMaterializations): 20 | pass 21 | 22 | 23 | # 24 | class TestEmptyDatabend(BaseEmpty): 25 | pass 26 | 27 | 28 | class TestBaseAdapterMethodDatabend(BaseAdapterMethod): 29 | pass 30 | 31 | 32 | # 33 | # 34 | # class TestEphemeralDatabend(BaseEphemeral): 35 | # pass 36 | 37 | 38 | class TestIncrementalDatabend(BaseIncremental): 39 | pass 40 | 41 | 42 | class TestDocsGenerateDatabend(BaseDocsGenerate): 43 | pass 44 | 45 | # CSV content with boolean column type. 46 | 47 | 48 | class TestCSVSeed: 49 | @pytest.fixture(scope="class") 50 | def seeds(self): 51 | return {"customer.csv": seeds_customer_csv, "empty.csv": seeds_empty_csv, "orders.csv": seeds_order_csv, 52 | "pay.csv": seeds_pay_csv} 53 | 54 | def test_seed(self, project): 55 | # seed command 56 | results = run_dbt(["seed"]) 57 | assert len(results) == 4 58 | 59 | 60 | seeds_pay_csv = """ 61 | 1,1,credit_card,1000 62 | 2,2,credit_card,2000 63 | 3,3,coupon,100 64 | 4,4,coupon,2500 65 | 5,5,bank_transfer,1700 66 | 6,6,credit_card,600 67 | 7,7,credit_card,1600 68 | 8,8,credit_card,2300 69 | 9,9,gift_card,2300 70 | 10,9,bank_transfer,0 71 | 11,10,bank_transfer,2600 72 | 12,11,credit_card,2700 73 | 13,12,credit_card,100 74 | 14,13,credit_card,500 75 | 15,13,bank_transfer,1400 76 | 16,14,bank_transfer,300 77 | 17,15,coupon,2200 78 | 18,16,credit_card,1000 79 | 19,17,bank_transfer,200 80 | 20,18,credit_card,500 81 | 21,18,credit_card,800 82 | 22,19,gift_card,600 83 | 23,20,bank_transfer,1500 84 | 24,21,credit_card,1200 85 | 25,22,bank_transfer,800 86 | 26,23,gift_card,2300 87 | 27,24,coupon,2600 88 | 28,25,bank_transfer,2000 89 | 29,25,credit_card,2200 90 | 30,25,coupon,1600 91 | 31,26,credit_card,3000 92 | 32,27,credit_card,2300 93 | 33,28,bank_transfer,1900 94 | 34,29,bank_transfer,1200 95 | 35,30,credit_card,1300 96 | 36,31,credit_card,1200 97 | 37,32,credit_card,300 98 | 38,33,credit_card,2200 99 | 39,34,bank_transfer,1500 100 | 40,35,credit_card,2900 101 | 41,36,bank_transfer,900 102 | 42,37,credit_card,2300 103 | 43,38,credit_card,1500 104 | 44,39,bank_transfer,800 105 | 45,40,credit_card,1400 106 | 46,41,credit_card,1700 107 | 47,42,coupon,1700 108 | 48,43,gift_card,1800 109 | 49,44,gift_card,1100 110 | 50,45,bank_transfer,500 111 | 51,46,bank_transfer,800 112 | 52,47,credit_card,2200 113 | 53,48,bank_transfer,300 114 | 54,49,credit_card,600 115 | 55,49,credit_card,900 116 | 56,50,credit_card,2600 117 | 57,51,credit_card,2900 118 | 58,51,credit_card,100 119 | 59,52,bank_transfer,1500 120 | 60,53,credit_card,300 121 | 61,54,credit_card,1800 122 | 62,54,bank_transfer,1100 123 | 63,55,credit_card,2900 124 | 64,56,credit_card,400 125 | 65,57,bank_transfer,200 126 | 66,58,coupon,1800 127 | 67,58,gift_card,600 128 | 68,59,gift_card,2800 129 | 69,60,credit_card,400 130 | 70,61,bank_transfer,1600 131 | 71,62,gift_card,1400 132 | 72,63,credit_card,2900 133 | 73,64,bank_transfer,2600 134 | 74,65,credit_card,0 135 | 75,66,credit_card,2800 136 | 76,67,bank_transfer,400 137 | 77,67,credit_card,1900 138 | 78,68,credit_card,1600 139 | 79,69,credit_card,1900 140 | 80,70,credit_card,2600 141 | 81,71,credit_card,500 142 | 82,72,credit_card,2900 143 | 83,73,bank_transfer,300 144 | 84,74,credit_card,3000 145 | 85,75,credit_card,1900 146 | 86,76,coupon,200 147 | 87,77,credit_card,0 148 | 88,77,bank_transfer,1900 149 | 89,78,bank_transfer,2600 150 | 90,79,credit_card,1800 151 | 91,79,credit_card,900 152 | 92,80,gift_card,300 153 | 93,81,coupon,200 154 | 94,82,credit_card,800 155 | 95,83,credit_card,100 156 | 96,84,bank_transfer,2500 157 | 97,85,bank_transfer,1700 158 | 98,86,coupon,2300 159 | 99,87,gift_card,3000 160 | 100,87,credit_card,2600 161 | 101,88,credit_card,2900 162 | 102,89,bank_transfer,2200 163 | 103,90,bank_transfer,200 164 | 104,91,credit_card,1900 165 | 105,92,bank_transfer,1500 166 | 106,92,coupon,200 167 | 107,93,gift_card,2600 168 | 108,94,coupon,700 169 | 109,95,coupon,2400 170 | 110,96,gift_card,1700 171 | 111,97,bank_transfer,1400 172 | 112,98,bank_transfer,1000 173 | 113,99,credit_card,2400 174 | """ 175 | 176 | seeds_customer_csv = """ 177 | id,first_name,last_name 178 | 1,Michael,P. 179 | 2,Shawn,M. 180 | 3,Kathleen,P. 181 | 4,Jimmy,C. 182 | 5,Katherine,R. 183 | 6,Sarah,R. 184 | 7,Martin,M. 185 | 8,Frank,R. 186 | 9,Jennifer,F. 187 | 10,Henry,W. 188 | 11,Fred,S. 189 | 12,Amy,D. 190 | 13,Kathleen,M. 191 | 14,Steve,F. 192 | 15,Teresa,H. 193 | 16,Amanda,H. 194 | 17,Kimberly,R. 195 | 18,Johnny,K. 196 | 19,Virginia,F. 197 | 20,Anna,A. 198 | 21,Willie,H. 199 | 22,Sean,H. 200 | 23,Mildred,A. 201 | 24,David,G. 202 | 25,Victor,H. 203 | 26,Aaron,R. 204 | 27,Benjamin,B. 205 | 28,Lisa,W. 206 | 29,Benjamin,K. 207 | 30,Christina,W. 208 | 31,Jane,G. 209 | 32,Thomas,O. 210 | 33,Katherine,M. 211 | 34,Jennifer,S. 212 | 35,Sara,T. 213 | 36,Harold,O. 214 | 37,Shirley,J. 215 | 38,Dennis,J. 216 | 39,Louise,W. 217 | 40,Maria,A. 218 | 41,Gloria,C. 219 | 42,Diana,S. 220 | 43,Kelly,N. 221 | 44,Jane,R. 222 | 45,Scott,B. 223 | 46,Norma,C. 224 | 47,Marie,P. 225 | 48,Lillian,C. 226 | 49,Judy,N. 227 | 50,Billy,L. 228 | 51,Howard,R. 229 | 52,Laura,F. 230 | 53,Anne,B. 231 | 54,Rose,M. 232 | 55,Nicholas,R. 233 | 56,Joshua,K. 234 | 57,Paul,W. 235 | 58,Kathryn,K. 236 | 59,Adam,A. 237 | 60,Norma,W. 238 | 61,Timothy,R. 239 | 62,Elizabeth,P. 240 | 63,Edward,G. 241 | 64,David,C. 242 | 65,Brenda,W. 243 | 66,Adam,W. 244 | 67,Michael,H. 245 | 68,Jesse,E. 246 | 69,Janet,P. 247 | 70,Helen,F. 248 | 71,Gerald,C. 249 | 72,Kathryn,O. 250 | 73,Alan,B. 251 | 74,Harry,A. 252 | 75,Andrea,H. 253 | 76,Barbara,W. 254 | 77,Anne,W. 255 | 78,Harry,H. 256 | 79,Jack,R. 257 | 80,Phillip,H. 258 | 81,Shirley,H. 259 | 82,Arthur,D. 260 | 83,Virginia,R. 261 | 84,Christina,R. 262 | 85,Theresa,M. 263 | 86,Jason,C. 264 | 87,Phillip,B. 265 | 88,Adam,T. 266 | 89,Margaret,J. 267 | 90,Paul,P. 268 | 91,Todd,W. 269 | 92,Willie,O. 270 | 93,Frances,R. 271 | 94,Gregory,H. 272 | 95,Lisa,P. 273 | 96,Jacqueline,A. 274 | 97,Shirley,D. 275 | 98,Nicole,M. 276 | 99,Mary,G. 277 | 100,Jean,M. 278 | """.lstrip() 279 | 280 | # CSV content with empty fields. 281 | seeds_empty_csv = """ 282 | key,val1,val2 283 | abc,1,1 284 | abc,1,0 285 | def,1,0 286 | hij,1,1 287 | hij,1, 288 | klm,1,0 289 | klm,1, 290 | """.lstrip() 291 | 292 | seeds_order_csv = """ 293 | id,user_id,order_date,status 294 | 1,1,2018-01-01,returned 295 | 2,3,2018-01-02,completed 296 | 3,94,2018-01-04,completed 297 | 4,50,2018-01-05,completed 298 | 5,64,2018-01-05,completed 299 | 6,54,2018-01-07,completed 300 | 7,88,2018-01-09,completed 301 | 8,2,2018-01-11,returned 302 | 9,53,2018-01-12,completed 303 | 10,7,2018-01-14,completed 304 | 11,99,2018-01-14,completed 305 | 12,59,2018-01-15,completed 306 | 13,84,2018-01-17,completed 307 | 14,40,2018-01-17,returned 308 | 15,25,2018-01-17,completed 309 | 16,39,2018-01-18,completed 310 | 17,71,2018-01-18,completed 311 | 18,64,2018-01-20,returned 312 | 19,54,2018-01-22,completed 313 | 20,20,2018-01-23,completed 314 | 21,71,2018-01-23,completed 315 | 22,86,2018-01-24,completed 316 | 23,22,2018-01-26,return_pending 317 | 24,3,2018-01-27,completed 318 | 25,51,2018-01-28,completed 319 | 26,32,2018-01-28,completed 320 | 27,94,2018-01-29,completed 321 | 28,8,2018-01-29,completed 322 | 29,57,2018-01-31,completed 323 | 30,69,2018-02-02,completed 324 | 31,16,2018-02-02,completed 325 | 32,28,2018-02-04,completed 326 | 33,42,2018-02-04,completed 327 | 34,38,2018-02-06,completed 328 | 35,80,2018-02-08,completed 329 | 36,85,2018-02-10,completed 330 | 37,1,2018-02-10,completed 331 | 38,51,2018-02-10,completed 332 | 39,26,2018-02-11,completed 333 | 40,33,2018-02-13,completed 334 | 41,99,2018-02-14,completed 335 | 42,92,2018-02-16,completed 336 | 43,31,2018-02-17,completed 337 | 44,66,2018-02-17,completed 338 | 45,22,2018-02-17,completed 339 | 46,6,2018-02-19,completed 340 | 47,50,2018-02-20,completed 341 | 48,27,2018-02-21,completed 342 | 49,35,2018-02-21,completed 343 | 50,51,2018-02-23,completed 344 | 51,71,2018-02-24,completed 345 | 52,54,2018-02-25,return_pending 346 | 53,34,2018-02-26,completed 347 | 54,54,2018-02-26,completed 348 | 55,18,2018-02-27,completed 349 | 56,79,2018-02-28,completed 350 | 57,93,2018-03-01,completed 351 | 58,22,2018-03-01,completed 352 | 59,30,2018-03-02,completed 353 | 60,12,2018-03-03,completed 354 | 61,63,2018-03-03,completed 355 | 62,57,2018-03-05,completed 356 | 63,70,2018-03-06,completed 357 | 64,13,2018-03-07,completed 358 | 65,26,2018-03-08,completed 359 | 66,36,2018-03-10,completed 360 | 67,79,2018-03-11,completed 361 | 68,53,2018-03-11,completed 362 | 69,3,2018-03-11,completed 363 | 70,8,2018-03-12,completed 364 | 71,42,2018-03-12,shipped 365 | 72,30,2018-03-14,shipped 366 | 73,19,2018-03-16,completed 367 | 74,9,2018-03-17,shipped 368 | 75,69,2018-03-18,completed 369 | 76,25,2018-03-20,completed 370 | 77,35,2018-03-21,shipped 371 | 78,90,2018-03-23,shipped 372 | 79,52,2018-03-23,shipped 373 | 80,11,2018-03-23,shipped 374 | 81,76,2018-03-23,shipped 375 | 82,46,2018-03-24,shipped 376 | 83,54,2018-03-24,shipped 377 | 84,70,2018-03-26,placed 378 | 85,47,2018-03-26,shipped 379 | 86,68,2018-03-26,placed 380 | 87,46,2018-03-27,placed 381 | 88,91,2018-03-27,shipped 382 | 89,21,2018-03-28,placed 383 | 90,66,2018-03-30,shipped 384 | 91,47,2018-03-31,placed 385 | 92,84,2018-04-02,placed 386 | 93,66,2018-04-03,placed 387 | 94,63,2018-04-03,placed 388 | 95,27,2018-04-04,placed 389 | 96,90,2018-04-06,placed 390 | 97,89,2018-04-07,placed 391 | 98,41,2018-04-07,placed 392 | 99,85,2018-04-09,placed 393 | """ 394 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_changing_relation_type.py: -------------------------------------------------------------------------------- 1 | from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator 2 | 3 | 4 | class TestDatabendChangeRelationTypes(BaseChangeRelationTypeValidator): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_list_relations_without_caching.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import json 4 | from dbt.tests.util import run_dbt, run_dbt_and_capture 5 | 6 | NUM_VIEWS = 100 7 | NUM_EXPECTED_RELATIONS = 1 + NUM_VIEWS 8 | 9 | TABLE_BASE_SQL = """ 10 | {{ config(materialized='table') }} 11 | 12 | select 1 as id 13 | """.lstrip() 14 | 15 | VIEW_X_SQL = """ 16 | select id from {{ ref('my_model_base') }} 17 | """.lstrip() 18 | 19 | MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING = """ 20 | {% macro validate_list_relations_without_caching(schema_relation) %} 21 | {% set relation_list_result = databend__list_relations_without_caching(schema_relation, max_iter=11, max_results_per_iter=10) %} 22 | {% set n_relations = relation_list_result | length %} 23 | {{ log("n_relations: " ~ n_relations) }} 24 | {% endmacro %} 25 | """ 26 | 27 | MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR = """ 28 | {% macro validate_list_relations_without_caching_raise_error(schema_relation) %} 29 | {{ databend__list_relations_without_caching(schema_relation, max_iter=33, max_results_per_iter=3) }} 30 | {% endmacro %} 31 | """ 32 | 33 | 34 | def parse_json_logs(json_log_output): 35 | parsed_logs = [] 36 | for line in json_log_output.split("\n"): 37 | try: 38 | log = json.loads(line) 39 | except ValueError: 40 | continue 41 | 42 | parsed_logs.append(log) 43 | 44 | return parsed_logs 45 | 46 | 47 | def find_result_in_parsed_logs(parsed_logs, result_name): 48 | return next( 49 | ( 50 | item["data"]["msg"] 51 | for item in parsed_logs 52 | if result_name in item["data"].get("msg", "msg") 53 | ), 54 | False, 55 | ) 56 | 57 | 58 | def find_exc_info_in_parsed_logs(parsed_logs, exc_info_name): 59 | return next( 60 | ( 61 | item["data"]["exc_info"] 62 | for item in parsed_logs 63 | if exc_info_name in item["data"].get("exc_info", "exc_info") 64 | ), 65 | False, 66 | ) 67 | 68 | 69 | class TestListRelationsWithoutCachingSingle: 70 | @pytest.fixture(scope="class") 71 | def models(self): 72 | my_models = {"my_model_base.sql": TABLE_BASE_SQL} 73 | for view in range(0, NUM_VIEWS): 74 | my_models.update({f"my_model_{view}.sql": VIEW_X_SQL}) 75 | 76 | return my_models 77 | 78 | @pytest.fixture(scope="class") 79 | def macros(self): 80 | return { 81 | "validate_list_relations_without_caching.sql": MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING, 82 | } 83 | 84 | def test__databend__list_relations_without_caching_termination(self, project): 85 | """ 86 | validates that we do NOT trigger pagination logic databend__list_relations_without_caching 87 | macro when there are fewer than max_results_per_iter relations in the target schema 88 | """ 89 | run_dbt(["run", "-s", "my_model_base"]) 90 | 91 | database = project.database 92 | schemas = project.created_schemas 93 | 94 | for schema in schemas: 95 | schema_relation = {"database": database, "schema": schema} 96 | kwargs = {"schema_relation": schema_relation} 97 | print(kwargs) -------------------------------------------------------------------------------- /tests/functional/adapter/test_simple_seed.py: -------------------------------------------------------------------------------- 1 | from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed 2 | 3 | 4 | class TestDatabendEmptySeed(BaseTestEmptySeed): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_timestamps.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps 3 | 4 | 5 | class TestCurrentTimestampDatabend(BaseCurrentTimestamps): 6 | @pytest.fixture(scope="class") 7 | def models(self): 8 | return { 9 | "get_current_timestamp.sql": "select NOW()" 10 | } 11 | 12 | @pytest.fixture(scope="class") 13 | def expected_schema(self): 14 | return {"now()": "Timestamp"} 15 | 16 | @pytest.fixture(scope="class") 17 | def expected_sql(self): 18 | return """select NOW()""" 19 | -------------------------------------------------------------------------------- /tests/functional/adapter/unit_testing/test_renamed_relations.py: -------------------------------------------------------------------------------- 1 | from dbt.adapters.databend.relation import DatabendRelation 2 | from dbt.adapters.contracts.relation import RelationType 3 | 4 | 5 | def test_renameable_relation(): 6 | relation = DatabendRelation.create( 7 | database=None, 8 | schema="my_schema", 9 | identifier="my_table", 10 | type=RelationType.Table, 11 | ) 12 | assert relation.renameable_relations == frozenset( 13 | ) 14 | -------------------------------------------------------------------------------- /tests/functional/adapter/unit_testing/test_unit_testing.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes 4 | from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity 5 | from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput 6 | 7 | 8 | class TestDatabendUnitTestingTypes(BaseUnitTestingTypes): 9 | @pytest.fixture 10 | def data_types(self): 11 | # sql_value, yaml_value 12 | return [ 13 | ["1", "1"], 14 | ["2.0", "2.0"], 15 | ["'12345'", "12345"], 16 | ["'string'", "string"], 17 | ["true", "true"], 18 | ["DATE '2020-01-02'", "2020-01-02"], 19 | ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], 20 | ["'2013-11-03 00:00:00-0'::TIMESTAMP", "2013-11-03 00:00:00-0"], 21 | ["3::VARIANT", "3"], 22 | ["TO_GEOMETRY('POINT(1820.12 890.56)')", "POINT(1820.12 890.56)"], 23 | [ 24 | "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", 25 | "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", 26 | ], 27 | ["['a','b','c']", "['a','b','c']"], 28 | ["[1,2,3]", "[1, 2, 3]"], 29 | ] 30 | 31 | 32 | class TestDatabendUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): 33 | pass 34 | 35 | 36 | class TestDatabendUnitTestInvalidInput(BaseUnitTestInvalidInput): 37 | pass 38 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = py37,py38,py39 4 | 5 | [testenv:{unit,py37,py38,py39,py}] 6 | description = unit testing 7 | skip_install = True 8 | passenv = DBT_* PYTEST_ADOPTS 9 | commands = {envpython} -m pytest {posargs} tests/unit 10 | deps = 11 | -rdev-requirements.txt 12 | -e. 13 | 14 | 15 | [testenv:{integration,py37,py38,py39,py}-{ databend }] 16 | description = adapter plugin integration testing 17 | skip_install = true 18 | passenv = DBT_* DATABEND_TEST_* PYTEST_ADOPTS 19 | commands = 20 | databend: {envpython} -m pytest -m profile_databend {posargs:test/integration} 21 | databend: {envpython} -m pytest {posargs} tests/functional 22 | deps = 23 | -rdev_requirements.txt 24 | -e. 25 | --------------------------------------------------------------------------------