├── .github
└── workflows
│ ├── incremental_test.yaml
│ └── test.yaml
├── .gitignore
├── CHANGELOG.md
├── CONTRIBUTING.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── dbt
├── adapters
│ └── databend
│ │ ├── __init__.py
│ │ ├── __version__.py
│ │ ├── column.py
│ │ ├── connections.py
│ │ ├── impl.py
│ │ └── relation.py
└── include
│ └── databend
│ ├── __init__.py
│ ├── dbt_project.yml
│ ├── macros
│ ├── adapters.sql
│ ├── adapters
│ │ ├── apply_grants.sql
│ │ └── relation.sql
│ ├── catalog.sql
│ ├── materializations
│ │ ├── incremental.sql
│ │ ├── seed.sql
│ │ ├── snapshot.sql
│ │ └── table.sql
│ └── utils.sql
│ └── profile_template.yml
├── dev-requirements.txt
├── setup.py
├── tests
├── README.md
├── __init__.py
├── conftest.py
└── functional
│ └── adapter
│ ├── empty
│ └── test_empty.py
│ ├── incremental
│ └── test_incremental.py
│ ├── statement_test
│ ├── seeds.py
│ └── test_statements.py
│ ├── test_basic.py
│ ├── test_changing_relation_type.py
│ ├── test_list_relations_without_caching.py
│ ├── test_simple_seed.py
│ ├── test_timestamps.py
│ └── unit_testing
│ ├── test_renamed_relations.py
│ └── test_unit_testing.py
└── tox.ini
/.github/workflows/incremental_test.yaml:
--------------------------------------------------------------------------------
1 | name: Incremental Test
2 |
3 | on:
4 | push:
5 | branches: [main]
6 | pull_request:
7 | branches: [main]
8 |
9 | jobs:
10 | test:
11 | runs-on: ubuntu-latest
12 | services:
13 | databend:
14 | image: datafuselabs/databend
15 | env:
16 | QUERY_DEFAULT_USER: databend
17 | QUERY_DEFAULT_PASSWORD: databend
18 | MINIO_ENABLED: true
19 | ports:
20 | - 8000:8000
21 | - 9000:9000
22 | steps:
23 | - name: Checkout
24 | uses: actions/checkout@v4
25 |
26 | - name: Setup Python
27 | uses: actions/setup-python@v5
28 | with:
29 | python-version: '3.11'
30 |
31 | - name: Pip Install
32 | run: |
33 | pip install pipenv
34 | pip install pytest
35 | pip install -r dev-requirements.txt
36 | pipenv install --dev --skip-lock
37 |
38 | - name: Verify Service Running
39 | run: |
40 | cid=$(docker ps -a | grep databend | cut -d' ' -f1)
41 | docker logs ${cid}
42 | curl -v http://localhost:8000/v1/health
43 |
44 | - name: dbt databend Incremental Test Suite
45 | run: |
46 | python -m pytest -s tests/functional/adapter/incremental/test_incremental.py
47 |
--------------------------------------------------------------------------------
/.github/workflows/test.yaml:
--------------------------------------------------------------------------------
1 | name: Ci Test
2 |
3 | on:
4 | push:
5 | branches: [main]
6 | pull_request:
7 | branches: [main]
8 |
9 | jobs:
10 | test:
11 | runs-on: ubuntu-latest
12 | services:
13 | databend:
14 | image: datafuselabs/databend
15 | env:
16 | QUERY_DEFAULT_USER: databend
17 | QUERY_DEFAULT_PASSWORD: databend
18 | MINIO_ENABLED: true
19 | ports:
20 | - 8000:8000
21 | - 9000:9000
22 | steps:
23 | - name: Checkout
24 | uses: actions/checkout@v4
25 |
26 | - name: Setup Python
27 | uses: actions/setup-python@v5
28 | with:
29 | python-version: '3.11'
30 |
31 | - name: Pip Install
32 | run: |
33 | pip install pipenv
34 | pip install pytest
35 | pip install -r dev-requirements.txt
36 | pipenv install --dev --skip-lock
37 |
38 | - name: Verify Service Running
39 | run: |
40 | cid=$(docker ps -a | grep databend | cut -d' ' -f1)
41 | docker logs ${cid}
42 | curl -v http://localhost:8000/v1/health
43 |
44 | - name: dbt databend Unit Test Suite
45 | run: |
46 | python -m pytest -s tests/functional/adapter/unit_testing/test_unit_testing.py
47 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 | .idea
6 | venv/
7 |
8 | # C extensions
9 | *.so
10 |
11 | # Distribution / packaging
12 | .Python
13 | build/
14 | develop-eggs/
15 | dist/
16 | downloads/
17 | eggs/
18 | .eggs/
19 | lib/
20 | lib64/
21 | parts/
22 | sdist/
23 | var/
24 | wheels/
25 | pip-wheel-metadata/
26 | share/python-wheels/
27 | *.egg-info/
28 | .installed.cfg
29 | *.egg
30 | MANIFEST
31 |
32 | # PyInstaller
33 | # Usually these files are written by a python script from a template
34 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
35 | *.manifest
36 | *.spec
37 |
38 | # Installer logs
39 | pip-log.txt
40 | pip-delete-this-directory.txt
41 |
42 | # Unit test / coverage reports
43 | htmlcov/
44 | .tox/
45 | .nox/
46 | .coverage
47 | .coverage.*
48 | .cache
49 | nosetests.xml
50 | coverage.xml
51 | *.cover
52 | *.py,cover
53 | .hypothesis/
54 | .pytest_cache/
55 | test.env
56 |
57 | # Translations
58 | *.mo
59 | *.pot
60 |
61 | # Django stuff:
62 | *.log
63 | local_settings.py
64 | db.sqlite3
65 | db.sqlite3-journal
66 |
67 | # Flask stuff:
68 | instance/
69 | .webassets-cache
70 |
71 | # Scrapy stuff:
72 | .scrapy
73 |
74 | # Sphinx documentation
75 | docs/_build/
76 |
77 | # PyBuilder
78 | target/
79 |
80 | # Jupyter Notebook
81 | .ipynb_checkpoints
82 |
83 | # IPython
84 | profile_default/
85 | ipython_config.py
86 |
87 | # pyenv
88 | .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
98 | __pypackages__/
99 |
100 | # Celery stuff
101 | celerybeat-schedule
102 | celerybeat.pid
103 |
104 | # SageMath parsed files
105 | *.sage.py
106 |
107 | # Environments
108 | .env
109 | .venv
110 | env/
111 | venv/
112 | ENV/
113 | env.bak/
114 | venv.bak/
115 |
116 | # Spyder project settings
117 | .spyderproject
118 | .spyproject
119 |
120 | # Rope project settings
121 | .ropeproject
122 |
123 | # mkdocs documentation
124 | /site
125 |
126 | # mypy
127 | .mypy_cache/
128 | .dmypy.json
129 | dmypy.json
130 |
131 | # Pyre type checker
132 | .pyre/
133 | logs/
134 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # dbt-databend Changelog
2 |
3 | - This file provides a full account of all changes to `dbt-databend`.
4 | - Changes are listed under the (pre)release in which they first appear. Subsequent releases include changes from previous releases.
5 | - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
6 |
7 | ## Previous Releases
8 | For information on prior major and minor releases, see their changelogs:
9 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing Guide
2 |
3 | Welcome to contribute to dbt-databend-cloud. Here are some guides for you.
4 |
5 | ## Ways to contribute
6 |
7 | We love to accept any kinds of contributions, such as:
8 |
9 | - **Report bugs:** if you find any bug in your use, you can make an issue for it.
10 | - **Answer questions:** check out the open issues to see if you can help answer questions.
11 | - **Suggest changes:** any suggestions which you can make this project better.
12 | - **Tackle an issue:** if you see an issue you would like to resolve, please feel free to fork the course and submit
13 | a PR (check out the instructions below).
14 |
15 | ## How to make a PR
16 |
17 | 1. Check out which [contributions](#ways-to-contribute) above you are willing to make
18 | 2. Fork this repository.
19 | 3. Code and pass [integration tests](test/README.md)
20 | 4. Commit your changes to your branch
21 | 5. Open a pull request to upstream
22 | 6. Request a review from the reviewers
23 |
24 | ## How to formate your code
25 |
26 | dbt-databend-cloud use black to fmt Python. To fmt your code, you need to:
27 |
28 | First, install black
29 | ```
30 | pip3 install black
31 | ```
32 | Then fmt
33 | ```
34 | black .
35 | ```
36 |
37 | You can also integrate with IntelliJ IDEA or other editors. See [here](https://black.readthedocs.io/en/stable/integrations/editors.html) for more details.
38 |
39 |
40 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | recursive-include dbt/include *.sql *.yml *.md
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | # dbt-databend-cloud
7 |
8 | 
9 | 
10 |
11 | The `dbt-databend-cloud` package contains all of the code enabling [dbt](https://getdbt.com) to work with
12 | [Databend Cloud](https://databend.rs/doc/cloud/).
13 |
14 | ## Table of Contents
15 | * [Installation](#installation)
16 | * [Supported features](#supported-features)
17 | * [Profile Configuration](#profile-configuration)
18 | * [Database User Privileges](#database-user-privileges)
19 | * [Running Tests](#running-tests)
20 | * [Example](#example)
21 | * [Contributing](#contributing)
22 |
23 | ## Installation
24 | Compile by source code.
25 |
26 | ```bash
27 | $ git clone https://github.com/databendcloud/dbt-databend.git
28 | $ cd dbt-databend
29 | $ pip install .
30 | ```
31 | Also, you can get it from pypi.
32 |
33 | ```bash
34 | $ pip install dbt-databend-cloud
35 | ```
36 | ## Supported features
37 |
38 | | ok | Feature |
39 | |:--:|:---------------------------:|
40 | | ✅ | Table materialization |
41 | | ✅ | View materialization |
42 | | ✅ | Incremental materialization |
43 | | ❌ | Ephemeral materialization |
44 | | ✅ | Seeds |
45 | | ✅ | Sources |
46 | | ✅ | Custom data tests |
47 | | ✅ | Docs generate |
48 | | ✅ | Snapshots |
49 | | ✅ | Connection retry |
50 |
51 | Note:
52 |
53 | * Databend does not support `Ephemeral` and `SnapShot`. You can find more detail [here](https://github.com/datafuselabs/databend/issues/8685)
54 |
55 | ## Profile Configuration
56 |
57 | Databend Cloud targets should be set up using the following configuration in your `profiles.yml` file.
58 |
59 | **Example entry for profiles.yml:**
60 |
61 | ```
62 | Your_Profile_Name:
63 | target: dev
64 | outputs:
65 | dev:
66 | type: databend
67 | host: [host]
68 | port: [port]
69 | schema: [schema(Your database)]
70 | user: [username]
71 | pass: [password]
72 | secure: [SSL]
73 | ```
74 |
75 | | Option | Description | Required? | Example |
76 | |--------|------------------------------------------------------|-----------|--------------------------------|
77 | | type | The specific adapter to use | Required | `databend` |
78 | | host | The server (hostname) to connect to | Required | `yourorg.databend.com` |
79 | | port | The port to use | Required | `443` |
80 | | schema | Specify the schema (database) to build models into | Required | `analytics` |
81 | | user | The username to use to connect to the server | Required | `dbt_admin` |
82 | | pass | The password to use for authenticating to the server | Required | `correct-horse-battery-staple` |
83 | | secure | The SSL of host (default as True) | Optional | `True` |
84 |
85 |
86 | Note:
87 |
88 | * You can find your host, user, pass information in this [docs](https://docs.databend.com/using-databend-cloud/warehouses/connecting-a-warehouse)
89 |
90 | ## Running Tests
91 |
92 | See [tests/README.md](tests/README.md) for details on running the integration tests.
93 |
94 | ## Example
95 |
96 | Click [here](https://github.com/databendcloud/dbt-databend/wiki/How-to-use-dbt-with-Databend-Cloud) to see a simple example about using dbt with dbt-databend-cloud.
97 |
98 | ## Contributing
99 |
100 | Welcome to contribute for dbt-databend-cloud. See [Contributing Guide](CONTRIBUTING.md) for more information.
101 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/__init__.py:
--------------------------------------------------------------------------------
1 | from dbt.adapters.databend.connections import DatabendConnectionManager # noqa
2 | from dbt.adapters.databend.connections import DatabendCredentials
3 | from dbt.adapters.databend.impl import DatabendAdapter
4 | from dbt.adapters.databend.column import DatabendColumn # noqa
5 | from dbt.adapters.databend.relation import DatabendRelation # noqa
6 |
7 | from dbt.adapters.base import AdapterPlugin
8 | from dbt.include import databend
9 |
10 |
11 | Plugin = AdapterPlugin(
12 | adapter=DatabendAdapter,
13 | credentials=DatabendCredentials,
14 | include_path=databend.PACKAGE_PATH,
15 | )
16 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/__version__.py:
--------------------------------------------------------------------------------
1 | version = "1.8.1"
2 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/column.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass
2 | from typing import TypeVar, Optional, Dict, Any
3 |
4 | from dbt.adapters.base.column import Column
5 | from dbt_common.exceptions import DbtRuntimeError
6 |
7 | Self = TypeVar("Self", bound="DatabendColumn")
8 |
9 |
10 | @dataclass
11 | class DatabendColumn(Column):
12 | @property
13 | def quoted(self) -> str:
14 | return '"{}"'.format(self.column)
15 |
16 | def is_string(self) -> bool:
17 | if self.dtype is None:
18 | return False
19 | return self.dtype.lower() in [
20 | "string",
21 | "varchar",
22 | ]
23 |
24 | def is_integer(self) -> bool:
25 | if self.dtype is None:
26 | return False
27 | return self.dtype.lower().startswith("int") or self.dtype.lower() in (
28 | "tinyint",
29 | "smallint",
30 | "bigint",
31 | )
32 |
33 | def is_numeric(self) -> bool:
34 | return False
35 |
36 | def is_float(self) -> bool:
37 | if self.dtype is None:
38 | return False
39 | return self.dtype.lower() in ("float", "double")
40 |
41 | def string_size(self) -> int:
42 | if not self.is_string():
43 | raise DbtRuntimeError(
44 | "Called string_size() on non-string field!"
45 | )
46 |
47 | if self.char_size is None:
48 | return 256
49 | else:
50 | return int(self.char_size)
51 |
52 | @classmethod
53 | def string_type(cls, size: int) -> str:
54 | return "VARCHAR"
55 |
56 | @classmethod
57 | def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str:
58 | return dtype
59 |
60 | def literal(self, value):
61 | return f"CAST({value} AS {self.dtype})"
62 |
63 | def can_expand_to(self, other_column: "Column") -> bool:
64 | return self.is_string() and other_column.is_string()
65 |
66 | def __repr__(self) -> str:
67 | return "".format(self.name, self.data_type)
68 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/connections.py:
--------------------------------------------------------------------------------
1 | from contextlib import contextmanager
2 | from dataclasses import dataclass
3 |
4 | import agate
5 | import dbt_common.exceptions # noqa
6 |
7 | from dbt.adapters.exceptions.connection import FailedToConnectError
8 | from dbt.adapters.contracts.connection import AdapterResponse, Connection, Credentials
9 | from dbt_common.clients.agate_helper import empty_table
10 | from dbt.adapters.sql import SQLConnectionManager as connection_cls
11 | from dbt.adapters.events.logging import AdapterLogger # type: ignore
12 | from dbt_common.events.functions import warn_or_error
13 | from dbt.adapters.events.types import AdapterEventWarning
14 | from dbt_common.ui import line_wrap_message, warning_tag
15 | from dbt_common.clients.agate_helper import empty_table
16 | from typing import Optional, Tuple, List, Any
17 | from databend_sqlalchemy import connector
18 |
19 | from dbt_common.exceptions import (
20 | DbtInternalError,
21 | DbtRuntimeError,
22 | DbtConfigError,
23 | )
24 |
25 | logger = AdapterLogger("databend")
26 |
27 |
28 | @dataclass
29 | class DatabendAdapterResponse(AdapterResponse):
30 | pass
31 |
32 |
33 | @dataclass
34 | class DatabendCredentials(Credentials):
35 | """
36 | Defines database specific credentials that get added to
37 | profiles.yml to connect to new adapter
38 | """
39 |
40 | host: Optional[str] = None
41 | port: Optional[int] = None
42 | database: Optional[str] = None
43 | username: Optional[str] = None
44 | password: Optional[str] = None
45 | schema: Optional[str] = None
46 | secure: Optional[bool] = None
47 |
48 | # Add credentials members here, like:
49 | # host: str
50 | # port: int
51 | # username: str
52 | # password: str
53 |
54 | _ALIASES = {"dbname": "database", "pass": "password", "user": "username"}
55 |
56 | def __init__(self, **kwargs):
57 | for k, v in kwargs.items():
58 | setattr(self, k, v)
59 | self.database = None
60 |
61 | @classmethod
62 | def __pre_deserialize__(cls, data):
63 | data = super().__pre_deserialize__(data)
64 | if "database" not in data:
65 | data["database"] = None
66 | return data
67 |
68 | def __post_init__(self):
69 | # databend classifies database and schema as the same thing
70 | self.database = None
71 | if self.database is not None and self.database != self.schema:
72 | raise DbtRuntimeError(
73 | f" schema: {self.schema} \n"
74 | f" database: {self.database} \n"
75 | f"On Databend, database must be omitted or have the same value as"
76 | f" schema."
77 | )
78 |
79 | @property
80 | def type(self):
81 | """Return name of adapter."""
82 | return "databend"
83 |
84 | @property
85 | def unique_field(self):
86 | """
87 | Hashed and included in anonymous telemetry to track adapter adoption.
88 | Pick a field that can uniquely identify one team/organization building with this adapter
89 | """
90 | return self.schema
91 |
92 | def _connection_keys(self):
93 | """
94 | List of keys to display in the `dbt debug` output.
95 | """
96 | return ("host", "port", "database", "schema", "user")
97 |
98 |
99 | @dataclass
100 | class DatabendAdapterResponse(AdapterResponse):
101 | query_id: str = ""
102 |
103 |
104 | class DatabendConnectionManager(connection_cls):
105 | TYPE = "databend"
106 |
107 | @contextmanager
108 | def exception_handler(self, sql: str):
109 | """
110 | Returns a context manager, that will handle exceptions raised
111 | from queries, catch, log, and raise dbt exceptions it knows how to handle.
112 | """
113 | try:
114 | yield
115 |
116 | except Exception as e:
117 | logger.debug("Error running SQL: {}".format(sql))
118 | logger.debug("Rolling back transaction.")
119 | self.rollback_if_open()
120 | raise DbtRuntimeError(str(e))
121 |
122 | # except for DML statements where explicitly defined
123 | def add_begin_query(self, *args, **kwargs):
124 | pass
125 |
126 | def add_commit_query(self, *args, **kwargs):
127 | pass
128 |
129 | def begin(self):
130 | pass
131 |
132 | def commit(self):
133 | pass
134 |
135 | def clear_transaction(self):
136 | pass
137 |
138 | @classmethod
139 | def open(cls, connection):
140 | """
141 | Receives a connection object and a Credentials object
142 | and moves it to the "open" state.
143 | """
144 | if connection.state == "open":
145 | logger.debug("Connection is already open, skipping open.")
146 | return connection
147 |
148 | credentials = connection.credentials
149 |
150 | try:
151 | if credentials.secure is None:
152 | credentials.secure = True
153 |
154 | if credentials.secure:
155 | handle = connector.connect(
156 | f"https://{credentials.username}:{credentials.password}@{credentials.host}:{credentials.port}/{credentials.schema}?secure=true "
157 | )
158 | else:
159 | handle = connector.connect(
160 | f"http://{credentials.username}:{credentials.password}@{credentials.host}:{credentials.port}/{credentials.schema}?secure=false "
161 | )
162 |
163 | except Exception as e:
164 | logger.debug("Error opening connection: {}".format(e))
165 | connection.handle = None
166 | connection.state = "fail"
167 | raise FailedToConnectError(str(e))
168 | connection.state = "open"
169 | connection.handle = handle
170 | return connection
171 |
172 | @classmethod
173 | def get_response(cls, cursor):
174 | return DatabendAdapterResponse(
175 | _message="{} {}".format("adapter response", cursor.rowcount),
176 | rows_affected=cursor.rowcount,
177 | )
178 |
179 | def execute(
180 | self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None
181 | ) -> Tuple[AdapterResponse, agate.Table]:
182 | # don't apply the query comment here
183 | # it will be applied after ';' queries are split
184 | _, cursor = self.add_query(sql, auto_begin)
185 | response = self.get_response(cursor)
186 | # table: rows, column_names=None, column_types=None, row_names=None
187 | if fetch:
188 | table = self.get_result_from_cursor(cursor, limit)
189 | else:
190 | table = dbt_common.clients.agate_helper.empty_table()
191 | return response, table
192 |
193 | def add_query(self, sql, auto_begin=False, bindings=None, abridge_sql_log=False):
194 | connection, cursor = super().add_query(
195 | sql, auto_begin, bindings=bindings, abridge_sql_log=abridge_sql_log
196 | )
197 |
198 | if cursor is None:
199 | conn = self.get_thread_connection()
200 | if conn is None or conn.name is None:
201 | conn_name = ""
202 | else:
203 | conn_name = conn.name
204 |
205 | raise Exception(
206 | "Tried to run an empty query on model '{}'. If you are "
207 | "conditionally running\nsql, eg. in a model hook, make "
208 | "sure your `else` clause contains valid sql!\n\n"
209 | "Provided SQL:\n{}".format(conn_name, sql)
210 | )
211 |
212 | return connection, cursor
213 |
214 | @classmethod
215 | def get_status(cls, _):
216 | """
217 | Returns connection status
218 | """
219 | return "OK"
220 |
221 | @classmethod
222 | def get_credentials(cls, credentials):
223 | """
224 | Returns Databend credentials
225 | """
226 | return credentials
227 |
228 | def cancel(self, connection):
229 | """
230 | Gets a connection object and attempts to cancel any ongoing queries.
231 | """
232 | connection_name = connection.name
233 | logger.debug("Cancelling query '{}'", connection_name)
234 | connection.handle.close()
235 | logger.debug("Cancel query '{}'", connection_name)
236 |
237 | @classmethod
238 | def process_results(cls, column_names, rows):
239 |
240 | return [dict(zip(column_names, row)) for row in rows]
241 |
242 | @classmethod
243 | def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Table:
244 | data: List[Any] = []
245 | column_names: List[str] = []
246 |
247 | if cursor.description is not None:
248 | column_names = [col[0] for col in cursor.description]
249 | if limit:
250 | rows = cursor.fetchmany(limit)
251 | else:
252 | rows = cursor.fetchall()
253 | data = cls.process_results(column_names, rows)
254 |
255 | return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names)
256 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/impl.py:
--------------------------------------------------------------------------------
1 | from concurrent.futures import Future
2 | from dataclasses import dataclass
3 | from typing import Callable, List, Optional, Set, Union, FrozenSet, Tuple
4 |
5 | import agate
6 | from dbt.adapters.base import AdapterConfig, available
7 | from dbt.adapters.base.impl import catch_as_completed
8 | from dbt.adapters.base.relation import InformationSchema
9 | from dbt.adapters.sql import SQLAdapter
10 |
11 | from dbt_common.clients.agate_helper import table_from_rows
12 | from dbt.adapters.events.logging import AdapterLogger
13 | from dbt.adapters.contracts.relation import RelationType
14 | from dbt_common.contracts.constraints import ConstraintType
15 | from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError
16 | from dbt_common.utils import filter_null_values
17 | from dbt_common.utils import executor
18 |
19 | import csv
20 | import io
21 | from dbt.adapters.databend.column import DatabendColumn
22 | from dbt.adapters.databend.connections import DatabendConnectionManager
23 | from dbt.adapters.databend.relation import DatabendRelation
24 |
25 | GET_CATALOG_MACRO_NAME = "get_catalog"
26 | LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching"
27 | LIST_SCHEMAS_MACRO_NAME = "list_schemas"
28 |
29 | logger = AdapterLogger("databend")
30 |
31 |
32 | def _expect_row_value(key: str, row: agate.Row):
33 | if key not in row.keys():
34 | raise DbtInternalError(
35 | f"Got a row without '{key}' column, columns: {row.keys()}"
36 | )
37 |
38 | return row[key]
39 |
40 |
41 | @dataclass
42 | class DatabendConfig(AdapterConfig):
43 | cluster_by: Optional[Union[List[str], str]] = None
44 |
45 |
46 | class DatabendAdapter(SQLAdapter):
47 | Relation = DatabendRelation
48 | Column = DatabendColumn
49 | ConnectionManager = DatabendConnectionManager
50 | AdapterSpecificConfigs = DatabendConfig
51 |
52 | @classmethod
53 | def date_function(cls):
54 | return "NOW()"
55 |
56 | @classmethod
57 | def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
58 | return "string"
59 |
60 | @classmethod
61 | def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
62 | decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
63 | return "float" if decimals else "int"
64 |
65 | @classmethod
66 | def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
67 | return "bool"
68 |
69 | @available
70 | def get_csv_data(self, table):
71 | csv_funcs = [c.csvify for c in table._column_types]
72 |
73 | buf = io.StringIO()
74 | writer = csv.writer(buf, lineterminator="\n")
75 |
76 | for row in table.rows:
77 | writer.writerow(tuple(csv_funcs[i](d) for i, d in enumerate(row)))
78 |
79 | return buf.getvalue()
80 |
81 | @classmethod
82 | def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
83 | return "timestamp"
84 |
85 | @classmethod
86 | def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str:
87 | return "date"
88 |
89 | @classmethod
90 | def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
91 | raise DbtRuntimeError(
92 | "`convert_time_type` is not implemented for this adapter!"
93 | )
94 |
95 | def quote(self, identifier):
96 | return "{}".format(identifier)
97 |
98 | def check_schema_exists(self, database, schema):
99 | results = self.execute_macro(
100 | LIST_SCHEMAS_MACRO_NAME, kwargs={"database": database}
101 | )
102 |
103 | exists = True if schema in [row[0] for row in results] else False
104 | return exists
105 |
106 | def list_relations_without_caching(
107 | self, schema_relation: DatabendRelation
108 | ) -> List[DatabendRelation]:
109 | kwargs = {"schema_relation": schema_relation}
110 | results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs)
111 |
112 | relations = []
113 | for row in results:
114 | if len(row) != 4:
115 | raise DbtRuntimeError(
116 | f"Invalid value from 'show table extended ...', "
117 | f"got {len(row)} values, expected 4"
118 | )
119 | _database, name, schema, type_info = row
120 | rel_type = RelationType.View if "view" in type_info else RelationType.Table
121 | relation = self.Relation.create(database=None, schema=schema, identifier=name, rt=rel_type)
122 | relations.append(relation)
123 |
124 | return relations
125 |
126 | @classmethod
127 | def _catalog_filter_table(
128 | cls, table: agate.Table, used_schemas: FrozenSet[Tuple[str, str]]
129 | ) -> agate.Table:
130 | table = table_from_rows(
131 | table.rows,
132 | table.column_names,
133 | text_only_columns=["table_schema", "table_name"],
134 | )
135 | return super()._catalog_filter_table(table, used_schemas)
136 |
137 | def get_relation(self, database: Optional[str], schema: str, identifier: str):
138 | # if not self.Relation.include_policy.database:
139 | # database = None
140 |
141 | return super().get_relation(database, schema, identifier)
142 |
143 | def parse_show_columns(
144 | self, _relation: DatabendRelation, raw_rows: List[agate.Row]
145 | ) -> List[DatabendColumn]:
146 | rows = [
147 | dict(zip(row._keys, row._values)) # pylint: disable=protected-access
148 | for row in raw_rows
149 | ]
150 |
151 | return [
152 | DatabendColumn(
153 | column=column["name"],
154 | dtype=column["type"],
155 | )
156 | for column in rows
157 | ]
158 |
159 | def get_columns_in_relation(
160 | self, relation: DatabendRelation
161 | ) -> List[DatabendColumn]:
162 | rows: List[agate.Row] = super().get_columns_in_relation(relation)
163 |
164 | return self.parse_show_columns(relation, rows)
165 |
166 | def _get_one_catalog(
167 | self,
168 | information_schema: InformationSchema,
169 | schemas: Set[str],
170 | used_schemas: FrozenSet[Tuple[str, str]],
171 | ) -> agate.Table:
172 | if len(schemas) != 1:
173 | DbtRuntimeError(
174 | f"Expected only one schema in databend _get_one_catalog, found {schemas}"
175 | )
176 |
177 | return super()._get_one_catalog(information_schema, schemas, used_schemas)
178 |
179 | def update_column_sql(
180 | self,
181 | dst_name: str,
182 | dst_column: str,
183 | clause: str,
184 | where_clause: Optional[str] = None,
185 | ) -> str:
186 | raise DbtInternalError(
187 | "`update_column_sql` is not implemented for this adapter!"
188 | )
189 |
190 | def run_sql_for_tests(self, sql, fetch, conn):
191 | cursor = conn.handle.cursor()
192 | try:
193 | cursor.execute(sql)
194 | if fetch == "one":
195 | if hasattr(cursor, "fetchone"):
196 | return cursor.fetchone()
197 | else:
198 | return cursor.fetchall()[0]
199 | elif fetch == "all":
200 | return cursor.fetchall()
201 | else:
202 | return
203 | except BaseException as exc:
204 | logger.error(sql)
205 | logger.error(exc)
206 | raise exc
207 | finally:
208 | conn.transaction_open = False
209 |
210 | def get_rows_different_sql(
211 | self,
212 | relation_a: DatabendRelation,
213 | relation_b: DatabendRelation,
214 | column_names: Optional[List[str]] = None,
215 | except_operator: str = "EXCEPT",
216 | ) -> str:
217 | names: List[str]
218 | if column_names is None:
219 | columns = self.get_columns_in_relation(relation_a)
220 | names = sorted((self.quote(c.name) for c in columns))
221 | else:
222 | names = sorted((self.quote(n) for n in column_names))
223 |
224 | alias_a = "A"
225 | alias_b = "B"
226 | columns_csv_a = ", ".join([f"{alias_a}.{name}" for name in names])
227 | columns_csv_b = ", ".join([f"{alias_b}.{name}" for name in names])
228 | join_condition = " AND ".join(
229 | [f"{alias_a}.{name} = {alias_b}.{name}" for name in names]
230 | )
231 | if len(names) == 0:
232 | return f"select 0, 0"
233 | first_column = names[0]
234 |
235 | # MySQL doesn't have an EXCEPT or MINUS operator, so we need to simulate it
236 | COLUMNS_EQUAL_SQL = """
237 | WITH
238 | a_except_b as (
239 | SELECT
240 | {columns_a}
241 | FROM {relation_a} as {alias_a}
242 | LEFT OUTER JOIN {relation_b} as {alias_b}
243 | ON {join_condition}
244 | WHERE {alias_b}.{first_column} is null
245 | ),
246 | b_except_a as (
247 | SELECT
248 | {columns_b}
249 | FROM {relation_b} as {alias_b}
250 | LEFT OUTER JOIN {relation_a} as {alias_a}
251 | ON {join_condition}
252 | WHERE {alias_a}.{first_column} is null
253 | ),
254 | diff_count as (
255 | SELECT
256 | 1 as id,
257 | COUNT(*) as num_missing FROM (
258 | SELECT * FROM a_except_b
259 | UNION ALL
260 | SELECT * FROM b_except_a
261 | ) as missing
262 | ),
263 | table_a as (
264 | SELECT COUNT(*) as num_rows FROM {relation_a}
265 | ),
266 | table_b as (
267 | SELECT COUNT(*) as num_rows FROM {relation_b}
268 | ),
269 | row_count_diff as (
270 | SELECT
271 | 1 as id,
272 | table_a.num_rows - table_b.num_rows as difference
273 | FROM table_a, table_b
274 | )
275 | SELECT
276 | row_count_diff.difference as row_count_difference,
277 | diff_count.num_missing as num_mismatched
278 | FROM row_count_diff
279 | INNER JOIN diff_count ON row_count_diff.id = diff_count.id
280 | """.strip()
281 |
282 | sql = COLUMNS_EQUAL_SQL.format(
283 | alias_a=alias_a,
284 | alias_b=alias_b,
285 | first_column=first_column,
286 | columns_a=columns_csv_a,
287 | columns_b=columns_csv_b,
288 | join_condition=join_condition,
289 | relation_a=str(relation_a),
290 | relation_b=str(relation_b),
291 | )
292 |
293 | return sql
294 |
295 | @property
296 | def default_python_submission_method(self):
297 | raise NotImplementedError("default_python_submission_method is not specified")
298 |
299 | @property
300 | def python_submission_helpers(self):
301 | raise NotImplementedError("python_submission_helpers is not specified")
302 |
--------------------------------------------------------------------------------
/dbt/adapters/databend/relation.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass, field
2 | from typing import Optional, TypeVar, Any, Type, Dict, Union, Iterator, Tuple, Set
3 |
4 | from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError, DbtInternalError
5 | from dbt.adapters.base.relation import BaseRelation, Policy
6 | from dbt.adapters.contracts.relation import (
7 | Path,
8 | RelationType,
9 | )
10 |
11 |
12 | @dataclass
13 | class DatabendQuotePolicy(Policy):
14 | database: bool = False
15 | schema: bool = False
16 | identifier: bool = False
17 |
18 |
19 | @dataclass
20 | class DatabendIncludePolicy(Policy):
21 | database: bool = False
22 | schema: bool = True
23 | identifier: bool = True
24 |
25 |
26 | Self = TypeVar("Self", bound="DatabendRelation")
27 |
28 |
29 | @dataclass(frozen=True, eq=False, repr=False)
30 | class DatabendRelation(BaseRelation):
31 | quote_policy: Policy = field(default_factory=lambda: DatabendQuotePolicy())
32 | include_policy: DatabendIncludePolicy = field(
33 | default_factory=lambda: DatabendIncludePolicy()
34 | )
35 | quote_character: str = ""
36 |
37 | def __post_init__(self):
38 | if self.database != self.schema and self.database:
39 | raise DbtDatabaseError(
40 | f" schema: {self.schema} \n"
41 | f" database: {self.database} \n"
42 | f"On Databend, database must be omitted or have the same value as"
43 | f" schema."
44 | )
45 |
46 | @classmethod
47 | def create(
48 | cls: Type[Self],
49 | database: Optional[str] = None,
50 | schema: Optional[str] = None,
51 | identifier: Optional[str] = None,
52 | rt: Optional[RelationType] = None,
53 | **kwargs,
54 | ) -> Self:
55 | database = None
56 | kwargs.update(
57 | {
58 | "path": {
59 | "database": database,
60 | "schema": schema,
61 | "identifier": identifier,
62 | },
63 | "type": rt,
64 | }
65 | )
66 | return cls.from_dict(kwargs)
67 |
68 | def render(self):
69 | if self.include_policy.database and self.include_policy.schema:
70 | raise DbtRuntimeError(
71 | "Got a databend relation with schema and database set to "
72 | "include, but only one can be set"
73 | )
74 | return super().render()
75 |
76 | @classmethod
77 | def get_path(
78 | cls, relation: BaseRelation, information_schema_view: Optional[str]
79 | ) -> Path:
80 | Path.database = None
81 | return Path(
82 | database=None,
83 | schema=relation.schema,
84 | identifier="INFORMATION_SCHEMA",
85 | )
86 |
87 | def matches(
88 | self,
89 | database: Optional[str] = None,
90 | schema: Optional[str] = None,
91 | identifier: Optional[str] = None,
92 | ):
93 | if database:
94 | raise DbtRuntimeError(
95 | f"Passed unexpected schema value {schema} to Relation.matches"
96 | )
97 | return self.schema == schema and self.identifier == identifier
98 |
--------------------------------------------------------------------------------
/dbt/include/databend/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | PACKAGE_PATH = os.path.dirname(__file__)
4 |
--------------------------------------------------------------------------------
/dbt/include/databend/dbt_project.yml:
--------------------------------------------------------------------------------
1 | name: dbt_databend
2 | version: 1.5.0
3 | config-version: 2
4 |
5 | macro-paths: ["macros"]
6 |
--------------------------------------------------------------------------------
/dbt/include/databend/macros/adapters.sql:
--------------------------------------------------------------------------------
1 | /* For examples of how to fill out the macros please refer to the postgres adapter and docs
2 | postgres adapter macros: https://github.com/dbt-labs/dbt-core/blob/main/plugins/postgres/dbt/include/postgres/macros/adapters.sql
3 | dbt docs: https://docs.getdbt.com/docs/contributing/building-a-new-adapter
4 | */
5 |
6 | {% macro databend__create_schema(relation) -%}
7 | '''Creates a new schema in the target database, if schema already exists, method is a no-op. '''
8 | {%- call statement('create_schema') -%}
9 | create database if not exists {{ relation.without_identifier().include(database=False) }}
10 | {% endcall %}
11 | {% endmacro %}
12 |
13 | {% macro databend__drop_relation(relation) -%}
14 | '''Deletes relatonship identifer between tables.'''
15 | /*
16 | 1. If database exists
17 | 2. Create a new schema if passed schema does not exist already
18 | */
19 | {% call statement('drop_relation', auto_begin=False) -%}
20 | drop {{ relation.type }} if exists {{ relation }}
21 | {%- endcall %}
22 | {% endmacro %}
23 |
24 | {% macro databend__drop_schema(relation) -%}
25 | '''drops a schema in a target database.'''
26 | /*
27 | 1. If database exists
28 | 2. search all calls of schema, and change include value to False, cascade it to backtrack
29 | */
30 | {%- call statement('drop_schema') -%}
31 | drop database if exists {{ relation.without_identifier().include(database=False) }}
32 | {%- endcall -%}
33 | {% endmacro %}
34 |
35 | {% macro databend__get_columns_in_relation(relation) -%}
36 | '''Returns a list of Columns in a table.'''
37 | /*
38 | 1. select as many values from column as needed
39 | 2. search relations to columns
40 | 3. where table name is equal to the relation identifier
41 | 4. if a relation schema exists and table schema is equal to the relation schema
42 | 5. order in whatever way you want to call.
43 | 6. create a table by loading result from call
44 | 7. return new table
45 | */
46 | {% call statement('get_columns_in_relation', fetch_result=True) %}
47 | select
48 | name,
49 | type,
50 | 0 as position
51 | from system.columns
52 | where
53 | table = '{{ relation.identifier }}'
54 | {% if relation.schema %}
55 | and database = '{{ relation.schema }}'
56 | {% endif %}
57 | {% endcall %}
58 | {% do return(load_result('get_columns_in_relation').table) %}
59 | {% endmacro %}
60 |
61 | -- Example of 2 of 3 required macros that do not come with a default implementation
62 |
63 | {% macro databend__list_relations_without_caching(schema_relation) -%}
64 | '''creates a table of relations withough using local caching.'''
65 | {% call statement('list_relations_without_caching', fetch_result=True) -%}
66 | select
67 | null as db,
68 | name as name,
69 | database as schema,
70 | if(engine = 'VIEW', 'view', 'table') as type
71 | from system.tables
72 | where database = '{{ schema_relation.schema }}'
73 | {% endcall %}
74 | {{ return(load_result('list_relations_without_caching').table) }}
75 | {% endmacro %}
76 |
77 | {% macro databend__list_schemas(database) -%}
78 | '''Returns a table of unique schemas.'''
79 | /*
80 | 1. search schemea by specific name
81 | 2. create a table with names
82 | */
83 | {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
84 | select name from system.databases
85 | {% endcall %}
86 | {{ return(load_result('list_schemas').table) }}
87 | {% endmacro %}
88 |
89 | {% macro databend__rename_relation(from_relation, to_relation) -%}
90 | '''Renames a relation in the database.'''
91 | /*
92 | 1. Search for a specific relation name
93 | 2. alter table by targeting specific name and passing in new name
94 | */
95 | {% call statement('drop_relation') %}
96 | drop {{ to_relation.type }} if exists {{ to_relation }}
97 | {% endcall %}
98 | {% call statement('rename_relation') %}
99 | rename table {{ from_relation }} to {{ to_relation }}
100 | {% endcall %}
101 | {% endmacro %}
102 |
103 | {% macro databend__truncate_relation(relation) -%}
104 | '''Removes all rows from a targeted set of tables.'''
105 | /*
106 | 1. grab all tables tied to the relation
107 | 2. remove rows from relations
108 | */
109 | {% call statement('truncate_relation') -%}
110 | truncate table {{ relation }}
111 | {%- endcall %}
112 | {% endmacro %}
113 |
114 |
115 | {% macro databend__make_temp_relation(base_relation, suffix) %}
116 | {% set tmp_identifier = base_relation.identifier ~ suffix %}
117 | {% set tmp_relation = base_relation.incorporate(
118 | path={"identifier": tmp_identifier, "schema": None}) -%}
119 | {% do return(tmp_relation) %}
120 | {% endmacro %}
121 |
122 | {% macro databend__generate_database_name(custom_database_name=none, node=none) -%}
123 | {% do return(None) %}
124 | {%- endmacro %}
125 |
126 | {% macro databend__current_timestamp() -%}
127 | NOW()
128 | {%- endmacro %}
129 |
130 | {% macro databend__create_table_as(temporary, relation, sql) -%}
131 | {%- set sql_header = config.get('sql_header', none) -%}
132 |
133 | {{ sql_header if sql_header is not none }}
134 |
135 | {% if temporary -%}
136 | create or replace transient table {{ relation.name }}
137 | {%- else %}
138 | create or replace table {{ relation.include(database=False) }}
139 | {{ cluster_by_clause(label="cluster by") }}
140 | {%- endif %}
141 | as {{ sql }}
142 | {%- endmacro %}
143 |
144 | {% macro databend__create_view_as(relation, sql) -%}
145 | {%- set sql_header = config.get('sql_header', none) -%}
146 |
147 | {{ sql_header if sql_header is not none }}
148 |
149 | create or replace view {{ relation.include(database=False) }}
150 | as {{ sql }}
151 | {%- endmacro %}
152 |
153 | {% macro databend__get_columns_in_query(select_sql) %}
154 | {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}
155 | select * from (
156 | {{ select_sql }}
157 | ) as __dbt_sbq
158 | limit 0
159 | {% endcall %}
160 |
161 | {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}
162 | {% endmacro %}
163 |
164 | {% macro cluster_by_clause(label) %}
165 | {%- set cols = config.get('cluster_by', validator=validation.any[list, basestring]) -%}
166 | {%- if cols is not none %}
167 | {%- if cols is string -%}
168 | {%- set cols = [cols] -%}
169 | {%- endif -%}
170 | {{ label }}
171 | {%- for item in cols -%}
172 | {{ item }}
173 | {%- if not loop.last -%},{%- endif -%}
174 | {%- endfor -%}
175 | {%- endif %}
176 | {%- endmacro -%}
--------------------------------------------------------------------------------
/dbt/include/databend/macros/adapters/apply_grants.sql:
--------------------------------------------------------------------------------
1 | {% macro databend__get_show_grant_sql(relation) %}
2 | {# Usually called from apply_grants. Should not be called directly. #}
3 | {{ adapter.raise_grant_error() }}
4 | {% endmacro %}
5 |
6 |
7 | {%- macro databend__get_grant_sql(relation, privilege, grantee) -%}
8 | {# Usually called from apply_grants. Should not be called directly. #}
9 | {{ adapter.raise_grant_error() }}
10 | {%- endmacro -%}
11 |
12 |
13 | {%- macro databend__get_revoke_sql(relation, privilege, grantee) -%}
14 | {# Usually called from apply_grants. Should not be called directly. #}
15 | {{ adapter.raise_grant_error() }}
16 | {%- endmacro -%}
17 |
18 |
19 | {% macro databend__copy_grants() %}
20 | {{ return(True) }}
21 | {% endmacro %}
22 |
23 |
24 | {% macro databend__apply_grants(relation, grant_config, should_revoke) %}
25 | {% if grant_config %}
26 | {{ adapter.raise_grant_error() }}
27 | {% endif %}
28 | {% endmacro %}
29 |
30 |
31 | {%- macro databend__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
32 | {# Databend does not support DCL statements yet #}
33 | {% if grant_config %}
34 | {{ adapter.raise_grant_error() }}
35 | {% endif %}
36 | {{ return([]) }}
37 | {%- endmacro %}
38 |
--------------------------------------------------------------------------------
/dbt/include/databend/macros/adapters/relation.sql:
--------------------------------------------------------------------------------
1 | {% macro databend__get_or_create_relation(database, schema, identifier, type) %}
2 | {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}
3 | {% if target_relation %}
4 | {% do return([true, target_relation]) %}
5 | {% endif %}
6 |
7 | {%- set new_relation = api.Relation.create(
8 | database=database,
9 | schema=schema,
10 | identifier=identifier,
11 | type=type
12 | ) -%}
13 | {% do return([false, new_relation]) %}
14 | {% endmacro %}
15 |
16 | {% macro databend__get_database(database) %}
17 | {% call statement('get_database', fetch_result=True) %}
18 | select name
19 | from system.databases
20 | where name = '{{ database }}'
21 | {% endcall %}
22 | {% do return(load_result('get_database').table) %}
23 | {% endmacro %}
--------------------------------------------------------------------------------
/dbt/include/databend/macros/catalog.sql:
--------------------------------------------------------------------------------
1 | {% macro databend__get_catalog(information_schema, schemas) -%}
2 | {%- call statement('catalog', fetch_result=True) -%}
3 | select
4 | null as table_database,
5 | tables.database as table_schema,
6 | tables.name as table_name,
7 | if(tables.engine = 'VIEW', 'view', 'table') as table_type,
8 | null as table_comment,
9 | null as column_name,
10 | 0 as column_index,
11 | null as column_type,
12 | null as column_comment,
13 | null as table_owner
14 | from system.tables
15 | where tables.database != 'system' and
16 | (
17 | {%- for schema in schemas -%}
18 | tables.database = '{{ schema }}'
19 | {%- if not loop.last %} or {% endif -%}
20 | {%- endfor -%}
21 | )
22 | order by tables.database, tables.name
23 | {%- endcall -%}
24 | {{ return(load_result('catalog').table) }}
25 | {%- endmacro %}
--------------------------------------------------------------------------------
/dbt/include/databend/macros/materializations/incremental.sql:
--------------------------------------------------------------------------------
1 | {% materialization incremental, adapter='databend' %}
2 |
3 | {%- set existing_relation = load_cached_relation(this) -%}
4 | {%- set target_relation = this.incorporate(type='table') -%}
5 |
6 | {%- set unique_key = config.get('unique_key') -%}
7 | {% if unique_key is not none and unique_key|length == 0 %}
8 | {% set unique_key = none %}
9 | {% endif %}
10 | {% if unique_key is iterable and (unique_key is not string and unique_key is not mapping) %}
11 | {% set unique_key = unique_key|join(', ') %}
12 | {% endif %}
13 | {%- set grant_config = config.get('grants') -%}
14 | {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}
15 | {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
16 |
17 | {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}
18 | {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}
19 | {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
20 | {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
21 | {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
22 |
23 | {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
24 | {{ drop_relation_if_exists(preexisting_backup_relation) }}
25 |
26 | {{ run_hooks(pre_hooks, inside_transaction=False) }}
27 | {{ run_hooks(pre_hooks, inside_transaction=True) }}
28 | {% set to_drop = [] %}
29 |
30 | {% if existing_relation is none %}
31 | -- No existing table, simply create a new one
32 | {% call statement('main') %}
33 | {{ get_create_table_as_sql(False, target_relation, sql) }}
34 | {% endcall %}
35 |
36 | {% elif full_refresh_mode %}
37 | -- Completely replacing the old table, so create a temporary table and then swap it
38 | {% call statement('main') %}
39 | {{ get_create_table_as_sql(False, intermediate_relation, sql) }}
40 | {% endcall %}
41 | {% set need_swap = true %}
42 |
43 | {% elif unique_key is none -%}
44 | {% call statement('main') %}
45 | {{ databend__insert_into(target_relation, sql) }}
46 | {% endcall %}
47 |
48 | {% else %}
49 | {% set schema_changes = 'ignore' %}
50 | {% set incremental_strategy = config.get('incremental_strategy') %}
51 | {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}
52 | {% if incremental_strategy != 'delete_insert' and incremental_predicates %}
53 | {% do exceptions.raise_compiler_error('Cannot apply incremental predicates with ' + incremental_strategy + ' strategy.') %}
54 | {% endif %}
55 | {% if incremental_strategy == 'delete_insert' %}
56 | {% do databend__incremental_delete_insert(existing_relation, unique_key, incremental_predicates) %}
57 | {% elif incremental_strategy == 'append' %}
58 | {% call statement('main') %}
59 | {{ databend__insert_into(target_relation, sql) }}
60 | {% endcall %}
61 | {% endif %}
62 | {% endif %}
63 |
64 | -- {% if need_swap %}
65 | -- {% if existing_relation.can_exchange %}
66 | -- {% do adapter.rename_relation(intermediate_relation, backup_relation) %}
67 | -- {% do exchange_tables_atomic(backup_relation, target_relation) %}
68 | -- {% else %}
69 | -- {% do adapter.rename_relation(target_relation, backup_relation) %}
70 | -- {% do adapter.rename_relation(intermediate_relation, target_relation) %}
71 | -- {% endif %}
72 | -- {% do to_drop.append(backup_relation) %}
73 | -- {% endif %}
74 |
75 | {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
76 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
77 |
78 | {% do persist_docs(target_relation, model) %}
79 |
80 | {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
81 | {% do create_indexes(target_relation) %}
82 | {% endif %}
83 |
84 | {{ run_hooks(post_hooks, inside_transaction=True) }}
85 |
86 | {% do adapter.commit() %}
87 |
88 | {% for rel in to_drop %}
89 | {% do adapter.drop_relation(rel) %}
90 | {% endfor %}
91 |
92 | {{ run_hooks(post_hooks, inside_transaction=False) }}
93 |
94 | {{ return({'relations': [target_relation]}) }}
95 |
96 | {%- endmaterialization %}
97 |
98 |
99 | {% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
100 |
101 | {%- set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) -%}
102 | {% if not schema_changes_dict['schema_changed'] %}
103 | {{ return }}
104 | {% endif %}
105 |
106 | {% if on_schema_change == 'fail' %}
107 | {% set fail_msg %}
108 | The source and target schemas on this incremental model are out of sync!
109 | They can be reconciled in several ways:
110 | - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.
111 | - Re-run the incremental model with `full_refresh: True` to update the target schema.
112 | - update the schema manually and re-run the process.
113 | {% endset %}
114 | {% do exceptions.raise_compiler_error(fail_msg) %}
115 | {{ return }}
116 | {% endif %}
117 |
118 | {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
119 |
120 | {% endmacro %}
121 |
122 |
123 |
124 | {% macro databend__incremental_delete_insert(existing_relation, unique_key, incremental_predicates) %}
125 | {% set new_data_relation = existing_relation.incorporate(path={"identifier": model['name']
126 | + '__dbt_new_data_' + invocation_id.replace('-', '_')}) %}
127 | {{ drop_relation_if_exists(new_data_relation) }}
128 | {% call statement('main') %}
129 | {{ get_create_table_as_sql(False, new_data_relation, sql) }}
130 | {% endcall %}
131 | {% call statement('delete_existing_data') %}
132 | delete from {{ existing_relation }} where ({{ unique_key }}) in (select {{ unique_key }}
133 | from {{ new_data_relation }})
134 | {%- if incremental_predicates %}
135 | {% for predicate in incremental_predicates %}
136 | and {{ predicate }}
137 | {% endfor %}
138 | {%- endif -%};
139 | {% endcall %}
140 |
141 | {%- set dest_columns = adapter.get_columns_in_relation(existing_relation) -%}
142 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
143 | {% call statement('insert_new_data') %}
144 | insert into {{ existing_relation}} select {{ dest_cols_csv}} from {{ new_data_relation }}
145 | {% endcall %}
146 | {% do adapter.drop_relation(new_data_relation) %}
147 | {% endmacro %}
148 |
--------------------------------------------------------------------------------
/dbt/include/databend/macros/materializations/seed.sql:
--------------------------------------------------------------------------------
1 | {% macro basic_load_csv_rows(model, batch_size, agate_table) %}
2 | {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
3 | {% set bindings = [] %}
4 |
5 | {% set statements = [] %}
6 |
7 | {% for chunk in agate_table.rows | batch(batch_size) %}
8 | {% set bindings = [] %}
9 |
10 | {% for row in chunk %}
11 | {% do bindings.extend(row) %}
12 | {% endfor %}
13 |
14 | {% set sql %}
15 | insert into {{ this.render() }} ({{ cols_sql }}) values
16 | {% for row in chunk -%}
17 | ({%- for column in agate_table.column_names -%}
18 | %s
19 | {%- if not loop.last%},{%- endif %}
20 | {%- endfor -%})
21 | {%- if not loop.last%},{%- endif %}
22 | {%- endfor %}
23 | {% endset %}
24 |
25 | {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
26 |
27 | {% if loop.index0 == 0 %}
28 | {% do statements.append(sql) %}
29 | {% endif %}
30 | {% endfor %}
31 |
32 | {# Return SQL so we can render it out into the compiled files #}
33 | {{ return(statements[0]) }}
34 | {% endmacro %}
--------------------------------------------------------------------------------
/dbt/include/databend/macros/materializations/snapshot.sql:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/databendcloud/dbt-databend/798c733dc05196049b6dae4594024654c0b60fa5/dbt/include/databend/macros/materializations/snapshot.sql
--------------------------------------------------------------------------------
/dbt/include/databend/macros/materializations/table.sql:
--------------------------------------------------------------------------------
1 | {% materialization table, adapter='databend' %}
2 |
3 | {%- set existing_relation = load_cached_relation(this) -%}
4 | {%- set target_relation = this.incorporate(type='table') -%}
5 | {%- set backup_relation = none -%}
6 | {%- set preexisting_backup_relation = none -%}
7 | {%- set preexisting_intermediate_relation = none -%}
8 |
9 | {% if existing_relation is not none %}
10 | {%- set backup_relation_type = existing_relation.type -%}
11 | {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
12 | {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
13 | -- {% if not existing_relation.can_exchange %}
14 | -- {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}
15 | -- {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}
16 | -- {% endif %}
17 | {% endif %}
18 |
19 | {% set grant_config = config.get('grants') %}
20 |
21 | {{ run_hooks(pre_hooks, inside_transaction=False) }}
22 |
23 | -- drop the temp relations if they exist already in the database
24 | {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
25 | {{ drop_relation_if_exists(preexisting_backup_relation) }}
26 |
27 | -- `BEGIN` happens here:
28 | {{ run_hooks(pre_hooks, inside_transaction=True) }}
29 |
30 | {% if backup_relation is none %}
31 | {{ log('Creating new relation ' + target_relation.name )}}
32 | -- There is not existing relation, so we can just create
33 | {% call statement('main') -%}
34 | {{ get_create_table_as_sql(False, target_relation, sql) }}
35 | {%- endcall %}
36 | {% elif existing_relation.can_exchange %}
37 | -- We can do an atomic exchange, so no need for an intermediate
38 | {% call statement('main') -%}
39 | {{ get_create_table_as_sql(False, backup_relation, sql) }}
40 | {%- endcall %}
41 | {% do exchange_tables_atomic(backup_relation, existing_relation) %}
42 | {% else %}
43 | -- We have to use an intermediate and rename accordingly
44 | {% call statement('main') -%}
45 | {{ get_create_table_as_sql(False, intermediate_relation, sql) }}
46 | {%- endcall %}
47 | {{ adapter.rename_relation(existing_relation, backup_relation) }}
48 | {{ adapter.rename_relation(intermediate_relation, target_relation) }}
49 | {% endif %}
50 |
51 | -- cleanup
52 | {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
53 | {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
54 |
55 | {% do persist_docs(target_relation, model) %}
56 |
57 | {{ run_hooks(post_hooks, inside_transaction=True) }}
58 |
59 | {{ adapter.commit() }}
60 |
61 | {{ drop_relation_if_exists(backup_relation) }}
62 |
63 | {{ run_hooks(post_hooks, inside_transaction=False) }}
64 |
65 | {{ return({'relations': [target_relation]}) }}
66 |
67 | {% endmaterialization %}
68 |
69 | {% macro databend__insert_into(target_relation, sql) %}
70 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
71 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
72 |
73 | insert into {{ target_relation }} ({{ dest_cols_csv }})
74 | {{ sql }}
75 | {%- endmacro %}
76 |
--------------------------------------------------------------------------------
/dbt/include/databend/macros/utils.sql:
--------------------------------------------------------------------------------
1 | {% macro databend__any_value(expression) -%}
2 | any({{ expression }})
3 | {%- endmacro %}
4 |
5 | {% macro databend__dateadd(datepart, interval, from_date_or_timestamp) %}
6 | date_add(
7 | {{ datepart }},
8 | {{ interval }},
9 | {{ from_date_or_timestamp }}
10 | )
11 | {% endmacro %}
--------------------------------------------------------------------------------
/dbt/include/databend/profile_template.yml:
--------------------------------------------------------------------------------
1 | default:
2 | target: dev
3 | outputs:
4 | dev:
5 | type: databend
6 | host: [host]
7 | port: [port]
8 | schema: [schema(Your database)]
9 | user: [username]
10 | pass: [password]
--------------------------------------------------------------------------------
/dev-requirements.txt:
--------------------------------------------------------------------------------
1 | # install latest changes in dbt-core
2 | git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
3 | git+https://github.com/dbt-labs/dbt-adapters.git
4 | git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
5 | git+https://github.com/dbt-labs/dbt-common.git
6 |
7 |
8 | black==22.3.0
9 | bumpversion
10 | flake8
11 | flaky
12 | freezegun==0.3.12
13 | ipdb
14 | pip-tools
15 | pre-commit
16 | pytest
17 | pytest-dotenv
18 | pytest-logbook
19 | pytest-csv
20 | pytest-xdist
21 | pytz
22 | tox>=3.13
23 | twine
24 | wheel
25 | databend-py==0.4.9
26 | databend-sqlalchemy==0.3.2
27 | environs
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from setuptools import find_namespace_packages, setup
3 |
4 | package_name = "dbt-databend-cloud"
5 | # make sure this always matches dbt/adapters/{adapter}/__version__.py
6 | package_version = "1.8.1"
7 | description = """The Databend adapter plugin for dbt"""
8 |
9 | setup(
10 | name=package_name,
11 | version=package_version,
12 | description=description,
13 | long_description=description,
14 | author="Databend Cloud Team",
15 | author_email="zhangzhihan@datafuselabs.com",
16 | url="https://github.com/databendcloud/dbt-databend.git",
17 | packages=find_namespace_packages(include=["dbt", "dbt.*"]),
18 | include_package_data=True,
19 | install_requires=[
20 | "dbt-common>=1.0.4,<2.0",
21 | "dbt-adapters>=1.1.1,<2.0",
22 | # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
23 | "dbt-core>=1.8.0",
24 | # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
25 | "agate",
26 | "databend-sqlalchemy~=0.3.2",
27 | "agate",
28 | ],
29 | classifiers=[
30 | "Development Status :: 5 - Production/Stable",
31 | "License :: OSI Approved :: Apache Software License",
32 | "Operating System :: Microsoft :: Windows",
33 | "Operating System :: MacOS :: MacOS X",
34 | "Operating System :: POSIX :: Linux",
35 | "Programming Language :: Python :: 3.10",
36 | "Programming Language :: Python :: 3.11",
37 | "Programming Language :: Python :: 3.1.x",
38 | ],
39 | python_requires=">=3.8",
40 | )
41 |
--------------------------------------------------------------------------------
/tests/README.md:
--------------------------------------------------------------------------------
1 | # Testing dbt-databend-cloud
2 |
3 | ## Overview
4 |
5 | Here are the steps to run the tests:
6 | 1. Set up
7 | 2. Get config
8 | 3. Run tests
9 |
10 | ## Set up
11 |
12 | Make sure you have python environment, you can find the supported python version in setup.py.
13 | ```bash
14 | pip3 install -r requirements_dev.txt
15 | pip3 install .
16 | ```
17 |
18 | ## Get config
19 | Config the configurations in `conftest.py`:
20 |
21 | ```python
22 | {
23 | "type": "databend",
24 | "host": "host",
25 | "port": 443,
26 | "user": "user",
27 | "pass": "pass",
28 | "schema": "your database",
29 | "secure": True,
30 | }
31 | ```
32 |
33 | You can get the config information by the way in this [docs](https://docs.databend.com/using-databend-cloud/warehouses/connecting-a-warehouse).
34 |
35 | ## Run tests
36 |
37 | ```shell
38 | python -m pytest -s -vv tests/functional
39 | ```
40 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/databendcloud/dbt-databend/798c733dc05196049b6dae4594024654c0b60fa5/tests/__init__.py
--------------------------------------------------------------------------------
/tests/conftest.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | # import os
4 | # import json
5 |
6 | # Import the fuctional fixtures as a plugin
7 | # Note: fixtures with session scope need to be local
8 |
9 | pytest_plugins = ["dbt.tests.fixtures.project"]
10 |
11 |
12 | # The profile dictionary, used to write out profiles.yml
13 | @pytest.fixture(scope="class")
14 | def dbt_profile_target():
15 | return {
16 | "type": "databend",
17 | "host": "localhost",
18 | "port": 8000,
19 | "user": "databend",
20 | "pass": "databend",
21 | "schema": "default",
22 | "secure": False,
23 | }
24 |
--------------------------------------------------------------------------------
/tests/functional/adapter/empty/test_empty.py:
--------------------------------------------------------------------------------
1 | from dbt.tests.adapter.empty.test_empty import BaseTestEmpty
2 |
3 |
4 | class TestDatabendEmpty(BaseTestEmpty):
5 | pass
6 |
--------------------------------------------------------------------------------
/tests/functional/adapter/incremental/test_incremental.py:
--------------------------------------------------------------------------------
1 | from dbt.tests.adapter.incremental.test_incremental_predicates import (
2 | BaseIncrementalPredicates,
3 | )
4 | from dbt.tests.adapter.incremental.test_incremental_unique_id import (
5 | BaseIncrementalUniqueKey,
6 | )
7 | from pytest import fixture, mark
8 |
9 |
10 | class TestIncrementalPredicatesDeleteInsertDatabend(BaseIncrementalPredicates):
11 | @fixture(scope='class')
12 | def project_config_update(self):
13 | return {
14 | 'models': {
15 | '+predicates': ['id != 2'],
16 | '+incremental_strategy': 'delete_insert',
17 | }
18 | }
19 |
20 |
21 | @mark.skip('No support for unique keys in default incremental strategy')
22 | class TestIncrementalUniqueKeyDatabend(BaseIncrementalUniqueKey):
23 | pass
24 |
25 |
26 | @mark.skip('No support for unique keys in default incremental strategy')
27 | class TestUniqueKeyDeleteInsertDatabend(BaseIncrementalUniqueKey):
28 | @fixture(scope='class')
29 | def project_config_update(self):
30 | return {'models': {'+incremental_strategy': 'delete_insert'}}
31 |
--------------------------------------------------------------------------------
/tests/functional/adapter/statement_test/seeds.py:
--------------------------------------------------------------------------------
1 | seeds_csv = """
2 | ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS
3 | 1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
4 | 2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
5 | 3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
6 | 4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
7 | 5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
8 | 6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
9 | 7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
10 | 8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
11 | 9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
12 | 10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100
13 | 11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67
14 | 12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193
15 | 13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5
16 | 14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250
17 | 15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245
18 | 16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54
19 | 17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96
20 | 18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72
21 | 19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174
22 | 20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25
23 | 21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253
24 | 22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153
25 | 23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201
26 | 24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122
27 | 25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95
28 | 26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52
29 | 27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26
30 | 28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118
31 | 29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28
32 | 30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177
33 | 31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233
34 | 32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203
35 | 33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149
36 | 34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167
37 | 35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110
38 | 36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68
39 | 37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89
40 | 38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81
41 | 39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15
42 | 40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255
43 | 41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140
44 | 42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24
45 | 43,Sean,Mason,smason16@icq.com,Male,159.219.155.249
46 | 44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218
47 | 45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198
48 | 46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18
49 | 47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238
50 | 48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61
51 | 49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21
52 | 50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209
53 | 51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87
54 | 52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142
55 | 53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126
56 | 54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212
57 | 55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194
58 | 56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22
59 | 57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60
60 | 58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50
61 | 59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222
62 | 60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115
63 | 61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155
64 | 62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94
65 | 63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106
66 | 64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68
67 | 65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41
68 | 66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109
69 | 67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77
70 | 68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194
71 | 69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135
72 | 70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87
73 | 71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44
74 | 72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182
75 | 73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241
76 | 74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24
77 | 75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214
78 | 76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199
79 | 77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41
80 | 78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255
81 | 79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144
82 | 80,Rose,King,rking27@ucoz.com,Female,212.123.168.231
83 | 81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188
84 | 82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61
85 | 83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30
86 | 84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192
87 | 85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232
88 | 86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109
89 | 87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156
90 | 88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84
91 | 89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235
92 | 90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53
93 | 91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221
94 | 92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187
95 | 93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57
96 | 94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189
97 | 95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180
98 | 96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144
99 | 97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117
100 | 98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126
101 | 99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244
102 | 100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88
103 | """.lstrip()
104 |
105 | statement_expected_csv = """
106 | SOURCE,VALUE
107 | matrix,100
108 | table,100
109 | """.lstrip()
110 |
--------------------------------------------------------------------------------
/tests/functional/adapter/statement_test/test_statements.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from dbt.tests.util import check_relations_equal, run_dbt
3 | from tests.functional.adapter.statement_test.seeds import seeds_csv, statement_expected_csv
4 |
5 | _STATEMENT_ACTUAL_SQL = """
6 | -- {{ ref('seed') }}
7 |
8 | {%- call statement('test_statement', fetch_result=True) -%}
9 |
10 | select
11 | count(*) as "num_records"
12 |
13 | from {{ ref('seed') }}
14 |
15 | {%- endcall -%}
16 |
17 | {% set result = load_result('test_statement') %}
18 |
19 | {% set res_table = result['table'] %}
20 | {% set res_matrix = result['data'] %}
21 |
22 | {% set matrix_value = res_matrix[0][0] %}
23 | {% set table_value = res_table[0]['num_records'] %}
24 |
25 | select 'matrix' as source, {{ matrix_value }} as value
26 | union all
27 | select 'table' as source, {{ table_value }} as value
28 | """.lstrip()
29 |
30 |
31 | class TestStatements:
32 | @pytest.fixture(scope="class")
33 | def models(self):
34 | return {"statement_actual.sql": _STATEMENT_ACTUAL_SQL}
35 |
36 | @pytest.fixture(scope="class")
37 | def seeds(self):
38 | return {
39 | "seed.csv": seeds_csv,
40 | "statement_expected.csv": statement_expected_csv,
41 | }
42 |
43 | def test_databend_statements(self, project):
44 | seed_results = run_dbt(["seed"])
45 | assert len(seed_results) == 2
46 | results = run_dbt()
47 | assert len(results) == 1
48 |
49 | db_with_schema = f"{project.database}.{project.test_schema}"
50 | check_relations_equal(
51 | project.adapter,
52 | [f"{db_with_schema}.STATEMENT_ACTUAL", f"{db_with_schema}.STATEMENT_EXPECTED"],
53 | )
54 |
--------------------------------------------------------------------------------
/tests/functional/adapter/test_basic.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
4 | from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
5 | from dbt.tests.adapter.basic.test_singular_tests_ephemeral import (
6 | BaseSingularTestsEphemeral,
7 | )
8 | from dbt.tests.adapter.basic.test_empty import BaseEmpty
9 | from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
10 | from dbt.tests.adapter.basic.test_incremental import BaseIncremental
11 | from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
12 | from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate
13 | from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
14 | from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
15 | from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
16 | from dbt.tests.util import check_relation_types, relation_from_name, run_dbt
17 |
18 |
19 | class TestSimpleMaterializationsDatabend(BaseSimpleMaterializations):
20 | pass
21 |
22 |
23 | #
24 | class TestEmptyDatabend(BaseEmpty):
25 | pass
26 |
27 |
28 | class TestBaseAdapterMethodDatabend(BaseAdapterMethod):
29 | pass
30 |
31 |
32 | #
33 | #
34 | # class TestEphemeralDatabend(BaseEphemeral):
35 | # pass
36 |
37 |
38 | class TestIncrementalDatabend(BaseIncremental):
39 | pass
40 |
41 |
42 | class TestDocsGenerateDatabend(BaseDocsGenerate):
43 | pass
44 |
45 | # CSV content with boolean column type.
46 |
47 |
48 | class TestCSVSeed:
49 | @pytest.fixture(scope="class")
50 | def seeds(self):
51 | return {"customer.csv": seeds_customer_csv, "empty.csv": seeds_empty_csv, "orders.csv": seeds_order_csv,
52 | "pay.csv": seeds_pay_csv}
53 |
54 | def test_seed(self, project):
55 | # seed command
56 | results = run_dbt(["seed"])
57 | assert len(results) == 4
58 |
59 |
60 | seeds_pay_csv = """
61 | 1,1,credit_card,1000
62 | 2,2,credit_card,2000
63 | 3,3,coupon,100
64 | 4,4,coupon,2500
65 | 5,5,bank_transfer,1700
66 | 6,6,credit_card,600
67 | 7,7,credit_card,1600
68 | 8,8,credit_card,2300
69 | 9,9,gift_card,2300
70 | 10,9,bank_transfer,0
71 | 11,10,bank_transfer,2600
72 | 12,11,credit_card,2700
73 | 13,12,credit_card,100
74 | 14,13,credit_card,500
75 | 15,13,bank_transfer,1400
76 | 16,14,bank_transfer,300
77 | 17,15,coupon,2200
78 | 18,16,credit_card,1000
79 | 19,17,bank_transfer,200
80 | 20,18,credit_card,500
81 | 21,18,credit_card,800
82 | 22,19,gift_card,600
83 | 23,20,bank_transfer,1500
84 | 24,21,credit_card,1200
85 | 25,22,bank_transfer,800
86 | 26,23,gift_card,2300
87 | 27,24,coupon,2600
88 | 28,25,bank_transfer,2000
89 | 29,25,credit_card,2200
90 | 30,25,coupon,1600
91 | 31,26,credit_card,3000
92 | 32,27,credit_card,2300
93 | 33,28,bank_transfer,1900
94 | 34,29,bank_transfer,1200
95 | 35,30,credit_card,1300
96 | 36,31,credit_card,1200
97 | 37,32,credit_card,300
98 | 38,33,credit_card,2200
99 | 39,34,bank_transfer,1500
100 | 40,35,credit_card,2900
101 | 41,36,bank_transfer,900
102 | 42,37,credit_card,2300
103 | 43,38,credit_card,1500
104 | 44,39,bank_transfer,800
105 | 45,40,credit_card,1400
106 | 46,41,credit_card,1700
107 | 47,42,coupon,1700
108 | 48,43,gift_card,1800
109 | 49,44,gift_card,1100
110 | 50,45,bank_transfer,500
111 | 51,46,bank_transfer,800
112 | 52,47,credit_card,2200
113 | 53,48,bank_transfer,300
114 | 54,49,credit_card,600
115 | 55,49,credit_card,900
116 | 56,50,credit_card,2600
117 | 57,51,credit_card,2900
118 | 58,51,credit_card,100
119 | 59,52,bank_transfer,1500
120 | 60,53,credit_card,300
121 | 61,54,credit_card,1800
122 | 62,54,bank_transfer,1100
123 | 63,55,credit_card,2900
124 | 64,56,credit_card,400
125 | 65,57,bank_transfer,200
126 | 66,58,coupon,1800
127 | 67,58,gift_card,600
128 | 68,59,gift_card,2800
129 | 69,60,credit_card,400
130 | 70,61,bank_transfer,1600
131 | 71,62,gift_card,1400
132 | 72,63,credit_card,2900
133 | 73,64,bank_transfer,2600
134 | 74,65,credit_card,0
135 | 75,66,credit_card,2800
136 | 76,67,bank_transfer,400
137 | 77,67,credit_card,1900
138 | 78,68,credit_card,1600
139 | 79,69,credit_card,1900
140 | 80,70,credit_card,2600
141 | 81,71,credit_card,500
142 | 82,72,credit_card,2900
143 | 83,73,bank_transfer,300
144 | 84,74,credit_card,3000
145 | 85,75,credit_card,1900
146 | 86,76,coupon,200
147 | 87,77,credit_card,0
148 | 88,77,bank_transfer,1900
149 | 89,78,bank_transfer,2600
150 | 90,79,credit_card,1800
151 | 91,79,credit_card,900
152 | 92,80,gift_card,300
153 | 93,81,coupon,200
154 | 94,82,credit_card,800
155 | 95,83,credit_card,100
156 | 96,84,bank_transfer,2500
157 | 97,85,bank_transfer,1700
158 | 98,86,coupon,2300
159 | 99,87,gift_card,3000
160 | 100,87,credit_card,2600
161 | 101,88,credit_card,2900
162 | 102,89,bank_transfer,2200
163 | 103,90,bank_transfer,200
164 | 104,91,credit_card,1900
165 | 105,92,bank_transfer,1500
166 | 106,92,coupon,200
167 | 107,93,gift_card,2600
168 | 108,94,coupon,700
169 | 109,95,coupon,2400
170 | 110,96,gift_card,1700
171 | 111,97,bank_transfer,1400
172 | 112,98,bank_transfer,1000
173 | 113,99,credit_card,2400
174 | """
175 |
176 | seeds_customer_csv = """
177 | id,first_name,last_name
178 | 1,Michael,P.
179 | 2,Shawn,M.
180 | 3,Kathleen,P.
181 | 4,Jimmy,C.
182 | 5,Katherine,R.
183 | 6,Sarah,R.
184 | 7,Martin,M.
185 | 8,Frank,R.
186 | 9,Jennifer,F.
187 | 10,Henry,W.
188 | 11,Fred,S.
189 | 12,Amy,D.
190 | 13,Kathleen,M.
191 | 14,Steve,F.
192 | 15,Teresa,H.
193 | 16,Amanda,H.
194 | 17,Kimberly,R.
195 | 18,Johnny,K.
196 | 19,Virginia,F.
197 | 20,Anna,A.
198 | 21,Willie,H.
199 | 22,Sean,H.
200 | 23,Mildred,A.
201 | 24,David,G.
202 | 25,Victor,H.
203 | 26,Aaron,R.
204 | 27,Benjamin,B.
205 | 28,Lisa,W.
206 | 29,Benjamin,K.
207 | 30,Christina,W.
208 | 31,Jane,G.
209 | 32,Thomas,O.
210 | 33,Katherine,M.
211 | 34,Jennifer,S.
212 | 35,Sara,T.
213 | 36,Harold,O.
214 | 37,Shirley,J.
215 | 38,Dennis,J.
216 | 39,Louise,W.
217 | 40,Maria,A.
218 | 41,Gloria,C.
219 | 42,Diana,S.
220 | 43,Kelly,N.
221 | 44,Jane,R.
222 | 45,Scott,B.
223 | 46,Norma,C.
224 | 47,Marie,P.
225 | 48,Lillian,C.
226 | 49,Judy,N.
227 | 50,Billy,L.
228 | 51,Howard,R.
229 | 52,Laura,F.
230 | 53,Anne,B.
231 | 54,Rose,M.
232 | 55,Nicholas,R.
233 | 56,Joshua,K.
234 | 57,Paul,W.
235 | 58,Kathryn,K.
236 | 59,Adam,A.
237 | 60,Norma,W.
238 | 61,Timothy,R.
239 | 62,Elizabeth,P.
240 | 63,Edward,G.
241 | 64,David,C.
242 | 65,Brenda,W.
243 | 66,Adam,W.
244 | 67,Michael,H.
245 | 68,Jesse,E.
246 | 69,Janet,P.
247 | 70,Helen,F.
248 | 71,Gerald,C.
249 | 72,Kathryn,O.
250 | 73,Alan,B.
251 | 74,Harry,A.
252 | 75,Andrea,H.
253 | 76,Barbara,W.
254 | 77,Anne,W.
255 | 78,Harry,H.
256 | 79,Jack,R.
257 | 80,Phillip,H.
258 | 81,Shirley,H.
259 | 82,Arthur,D.
260 | 83,Virginia,R.
261 | 84,Christina,R.
262 | 85,Theresa,M.
263 | 86,Jason,C.
264 | 87,Phillip,B.
265 | 88,Adam,T.
266 | 89,Margaret,J.
267 | 90,Paul,P.
268 | 91,Todd,W.
269 | 92,Willie,O.
270 | 93,Frances,R.
271 | 94,Gregory,H.
272 | 95,Lisa,P.
273 | 96,Jacqueline,A.
274 | 97,Shirley,D.
275 | 98,Nicole,M.
276 | 99,Mary,G.
277 | 100,Jean,M.
278 | """.lstrip()
279 |
280 | # CSV content with empty fields.
281 | seeds_empty_csv = """
282 | key,val1,val2
283 | abc,1,1
284 | abc,1,0
285 | def,1,0
286 | hij,1,1
287 | hij,1,
288 | klm,1,0
289 | klm,1,
290 | """.lstrip()
291 |
292 | seeds_order_csv = """
293 | id,user_id,order_date,status
294 | 1,1,2018-01-01,returned
295 | 2,3,2018-01-02,completed
296 | 3,94,2018-01-04,completed
297 | 4,50,2018-01-05,completed
298 | 5,64,2018-01-05,completed
299 | 6,54,2018-01-07,completed
300 | 7,88,2018-01-09,completed
301 | 8,2,2018-01-11,returned
302 | 9,53,2018-01-12,completed
303 | 10,7,2018-01-14,completed
304 | 11,99,2018-01-14,completed
305 | 12,59,2018-01-15,completed
306 | 13,84,2018-01-17,completed
307 | 14,40,2018-01-17,returned
308 | 15,25,2018-01-17,completed
309 | 16,39,2018-01-18,completed
310 | 17,71,2018-01-18,completed
311 | 18,64,2018-01-20,returned
312 | 19,54,2018-01-22,completed
313 | 20,20,2018-01-23,completed
314 | 21,71,2018-01-23,completed
315 | 22,86,2018-01-24,completed
316 | 23,22,2018-01-26,return_pending
317 | 24,3,2018-01-27,completed
318 | 25,51,2018-01-28,completed
319 | 26,32,2018-01-28,completed
320 | 27,94,2018-01-29,completed
321 | 28,8,2018-01-29,completed
322 | 29,57,2018-01-31,completed
323 | 30,69,2018-02-02,completed
324 | 31,16,2018-02-02,completed
325 | 32,28,2018-02-04,completed
326 | 33,42,2018-02-04,completed
327 | 34,38,2018-02-06,completed
328 | 35,80,2018-02-08,completed
329 | 36,85,2018-02-10,completed
330 | 37,1,2018-02-10,completed
331 | 38,51,2018-02-10,completed
332 | 39,26,2018-02-11,completed
333 | 40,33,2018-02-13,completed
334 | 41,99,2018-02-14,completed
335 | 42,92,2018-02-16,completed
336 | 43,31,2018-02-17,completed
337 | 44,66,2018-02-17,completed
338 | 45,22,2018-02-17,completed
339 | 46,6,2018-02-19,completed
340 | 47,50,2018-02-20,completed
341 | 48,27,2018-02-21,completed
342 | 49,35,2018-02-21,completed
343 | 50,51,2018-02-23,completed
344 | 51,71,2018-02-24,completed
345 | 52,54,2018-02-25,return_pending
346 | 53,34,2018-02-26,completed
347 | 54,54,2018-02-26,completed
348 | 55,18,2018-02-27,completed
349 | 56,79,2018-02-28,completed
350 | 57,93,2018-03-01,completed
351 | 58,22,2018-03-01,completed
352 | 59,30,2018-03-02,completed
353 | 60,12,2018-03-03,completed
354 | 61,63,2018-03-03,completed
355 | 62,57,2018-03-05,completed
356 | 63,70,2018-03-06,completed
357 | 64,13,2018-03-07,completed
358 | 65,26,2018-03-08,completed
359 | 66,36,2018-03-10,completed
360 | 67,79,2018-03-11,completed
361 | 68,53,2018-03-11,completed
362 | 69,3,2018-03-11,completed
363 | 70,8,2018-03-12,completed
364 | 71,42,2018-03-12,shipped
365 | 72,30,2018-03-14,shipped
366 | 73,19,2018-03-16,completed
367 | 74,9,2018-03-17,shipped
368 | 75,69,2018-03-18,completed
369 | 76,25,2018-03-20,completed
370 | 77,35,2018-03-21,shipped
371 | 78,90,2018-03-23,shipped
372 | 79,52,2018-03-23,shipped
373 | 80,11,2018-03-23,shipped
374 | 81,76,2018-03-23,shipped
375 | 82,46,2018-03-24,shipped
376 | 83,54,2018-03-24,shipped
377 | 84,70,2018-03-26,placed
378 | 85,47,2018-03-26,shipped
379 | 86,68,2018-03-26,placed
380 | 87,46,2018-03-27,placed
381 | 88,91,2018-03-27,shipped
382 | 89,21,2018-03-28,placed
383 | 90,66,2018-03-30,shipped
384 | 91,47,2018-03-31,placed
385 | 92,84,2018-04-02,placed
386 | 93,66,2018-04-03,placed
387 | 94,63,2018-04-03,placed
388 | 95,27,2018-04-04,placed
389 | 96,90,2018-04-06,placed
390 | 97,89,2018-04-07,placed
391 | 98,41,2018-04-07,placed
392 | 99,85,2018-04-09,placed
393 | """
394 |
--------------------------------------------------------------------------------
/tests/functional/adapter/test_changing_relation_type.py:
--------------------------------------------------------------------------------
1 | from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator
2 |
3 |
4 | class TestDatabendChangeRelationTypes(BaseChangeRelationTypeValidator):
5 | pass
6 |
--------------------------------------------------------------------------------
/tests/functional/adapter/test_list_relations_without_caching.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | import json
4 | from dbt.tests.util import run_dbt, run_dbt_and_capture
5 |
6 | NUM_VIEWS = 100
7 | NUM_EXPECTED_RELATIONS = 1 + NUM_VIEWS
8 |
9 | TABLE_BASE_SQL = """
10 | {{ config(materialized='table') }}
11 |
12 | select 1 as id
13 | """.lstrip()
14 |
15 | VIEW_X_SQL = """
16 | select id from {{ ref('my_model_base') }}
17 | """.lstrip()
18 |
19 | MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING = """
20 | {% macro validate_list_relations_without_caching(schema_relation) %}
21 | {% set relation_list_result = databend__list_relations_without_caching(schema_relation, max_iter=11, max_results_per_iter=10) %}
22 | {% set n_relations = relation_list_result | length %}
23 | {{ log("n_relations: " ~ n_relations) }}
24 | {% endmacro %}
25 | """
26 |
27 | MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING_RAISE_ERROR = """
28 | {% macro validate_list_relations_without_caching_raise_error(schema_relation) %}
29 | {{ databend__list_relations_without_caching(schema_relation, max_iter=33, max_results_per_iter=3) }}
30 | {% endmacro %}
31 | """
32 |
33 |
34 | def parse_json_logs(json_log_output):
35 | parsed_logs = []
36 | for line in json_log_output.split("\n"):
37 | try:
38 | log = json.loads(line)
39 | except ValueError:
40 | continue
41 |
42 | parsed_logs.append(log)
43 |
44 | return parsed_logs
45 |
46 |
47 | def find_result_in_parsed_logs(parsed_logs, result_name):
48 | return next(
49 | (
50 | item["data"]["msg"]
51 | for item in parsed_logs
52 | if result_name in item["data"].get("msg", "msg")
53 | ),
54 | False,
55 | )
56 |
57 |
58 | def find_exc_info_in_parsed_logs(parsed_logs, exc_info_name):
59 | return next(
60 | (
61 | item["data"]["exc_info"]
62 | for item in parsed_logs
63 | if exc_info_name in item["data"].get("exc_info", "exc_info")
64 | ),
65 | False,
66 | )
67 |
68 |
69 | class TestListRelationsWithoutCachingSingle:
70 | @pytest.fixture(scope="class")
71 | def models(self):
72 | my_models = {"my_model_base.sql": TABLE_BASE_SQL}
73 | for view in range(0, NUM_VIEWS):
74 | my_models.update({f"my_model_{view}.sql": VIEW_X_SQL})
75 |
76 | return my_models
77 |
78 | @pytest.fixture(scope="class")
79 | def macros(self):
80 | return {
81 | "validate_list_relations_without_caching.sql": MACROS__VALIDATE__DATABEND__LIST_RELATIONS_WITHOUT_CACHING,
82 | }
83 |
84 | def test__databend__list_relations_without_caching_termination(self, project):
85 | """
86 | validates that we do NOT trigger pagination logic databend__list_relations_without_caching
87 | macro when there are fewer than max_results_per_iter relations in the target schema
88 | """
89 | run_dbt(["run", "-s", "my_model_base"])
90 |
91 | database = project.database
92 | schemas = project.created_schemas
93 |
94 | for schema in schemas:
95 | schema_relation = {"database": database, "schema": schema}
96 | kwargs = {"schema_relation": schema_relation}
97 | print(kwargs)
--------------------------------------------------------------------------------
/tests/functional/adapter/test_simple_seed.py:
--------------------------------------------------------------------------------
1 | from dbt.tests.adapter.simple_seed.test_seed import BaseTestEmptySeed
2 |
3 |
4 | class TestDatabendEmptySeed(BaseTestEmptySeed):
5 | pass
6 |
--------------------------------------------------------------------------------
/tests/functional/adapter/test_timestamps.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps
3 |
4 |
5 | class TestCurrentTimestampDatabend(BaseCurrentTimestamps):
6 | @pytest.fixture(scope="class")
7 | def models(self):
8 | return {
9 | "get_current_timestamp.sql": "select NOW()"
10 | }
11 |
12 | @pytest.fixture(scope="class")
13 | def expected_schema(self):
14 | return {"now()": "Timestamp"}
15 |
16 | @pytest.fixture(scope="class")
17 | def expected_sql(self):
18 | return """select NOW()"""
19 |
--------------------------------------------------------------------------------
/tests/functional/adapter/unit_testing/test_renamed_relations.py:
--------------------------------------------------------------------------------
1 | from dbt.adapters.databend.relation import DatabendRelation
2 | from dbt.adapters.contracts.relation import RelationType
3 |
4 |
5 | def test_renameable_relation():
6 | relation = DatabendRelation.create(
7 | database=None,
8 | schema="my_schema",
9 | identifier="my_table",
10 | type=RelationType.Table,
11 | )
12 | assert relation.renameable_relations == frozenset(
13 | )
14 |
--------------------------------------------------------------------------------
/tests/functional/adapter/unit_testing/test_unit_testing.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes
4 | from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity
5 | from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput
6 |
7 |
8 | class TestDatabendUnitTestingTypes(BaseUnitTestingTypes):
9 | @pytest.fixture
10 | def data_types(self):
11 | # sql_value, yaml_value
12 | return [
13 | ["1", "1"],
14 | ["2.0", "2.0"],
15 | ["'12345'", "12345"],
16 | ["'string'", "string"],
17 | ["true", "true"],
18 | ["DATE '2020-01-02'", "2020-01-02"],
19 | ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"],
20 | ["'2013-11-03 00:00:00-0'::TIMESTAMP", "2013-11-03 00:00:00-0"],
21 | ["3::VARIANT", "3"],
22 | ["TO_GEOMETRY('POINT(1820.12 890.56)')", "POINT(1820.12 890.56)"],
23 | [
24 | "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}",
25 | "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}",
26 | ],
27 | ["['a','b','c']", "['a','b','c']"],
28 | ["[1,2,3]", "[1, 2, 3]"],
29 | ]
30 |
31 |
32 | class TestDatabendUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity):
33 | pass
34 |
35 |
36 | class TestDatabendUnitTestInvalidInput(BaseUnitTestInvalidInput):
37 | pass
38 |
--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | skipsdist = True
3 | envlist = py37,py38,py39
4 |
5 | [testenv:{unit,py37,py38,py39,py}]
6 | description = unit testing
7 | skip_install = True
8 | passenv = DBT_* PYTEST_ADOPTS
9 | commands = {envpython} -m pytest {posargs} tests/unit
10 | deps =
11 | -rdev-requirements.txt
12 | -e.
13 |
14 |
15 | [testenv:{integration,py37,py38,py39,py}-{ databend }]
16 | description = adapter plugin integration testing
17 | skip_install = true
18 | passenv = DBT_* DATABEND_TEST_* PYTEST_ADOPTS
19 | commands =
20 | databend: {envpython} -m pytest -m profile_databend {posargs:test/integration}
21 | databend: {envpython} -m pytest {posargs} tests/functional
22 | deps =
23 | -rdev_requirements.txt
24 | -e.
25 |
--------------------------------------------------------------------------------