├── .dockerignore ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── Dockerfile ├── LICENSE.txt ├── README.md ├── dbt ├── adapters │ └── sqlite │ │ ├── __init__.py │ │ ├── __version__.py │ │ ├── connections.py │ │ ├── impl.py │ │ └── relation.py └── include │ └── sqlite │ ├── __init__.py │ ├── dbt_project.yml │ ├── macros │ ├── adapters.sql │ ├── catalog.sql │ ├── core_overrides.sql │ ├── materializations │ │ ├── incremental │ │ │ └── incremental.sql │ │ ├── seed │ │ │ ├── helpers.sql │ │ │ └── seed.sql │ │ ├── snapshot │ │ │ ├── snapshot_merge.sql │ │ │ └── strategies.sql │ │ ├── table │ │ │ └── table.sql │ │ ├── test.sql │ │ └── view │ │ │ └── view.sql │ └── utils │ │ ├── any_value.sql │ │ ├── bool_or.sql │ │ ├── cast_bool_to_text.sql │ │ ├── dateadd.sql │ │ ├── datediff.sql │ │ ├── hash.sql │ │ ├── position.sql │ │ ├── right.sql │ │ └── timestamps.sql │ └── sample_profiles.yml ├── pytest.ini ├── run_tests.sh ├── run_tests_docker.sh ├── setup.py └── tests ├── conftest.py └── functional └── adapter ├── concurrency └── test_concurrency.py ├── ephemeral └── test_ephemeral.py ├── test_basic.py └── utils ├── test_data_types.py └── test_utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | *.log 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | 2 | name: dbt-sqlite 3 | 4 | on: [push] 5 | 6 | jobs: 7 | build: 8 | strategy: 9 | matrix: 10 | python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Build docker image 17 | run: docker build --build-arg PYTHON_VERSION=${{ matrix.python_version }} --tag dbt-sqlite:$GITHUB_SHA . 18 | - name: Run tests 19 | run: docker run dbt-sqlite:$GITHUB_SHA run_tests.sh 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python modules. 2 | *.pyc 3 | 4 | # Setuptools distribution folder. 5 | /dist/ 6 | /build 7 | 8 | # Python egg metadata, regenerated from source files by setuptools. 9 | /*.egg-info 10 | .vscode/settings.json 11 | env/ 12 | 13 | # sublime text 14 | *.sublime-* 15 | 16 | .idea/ 17 | 18 | *.log 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | ARG PYTHON_VERSION=3.12 3 | 4 | FROM python:${PYTHON_VERSION}-bullseye 5 | 6 | RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sqlite3 vim virtualenvwrapper wget 7 | 8 | WORKDIR /opt/dbt-sqlite 9 | 10 | RUN python3 -m pip install --upgrade pip 11 | 12 | RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so 13 | RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so 14 | RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/text.so 15 | 16 | WORKDIR /opt/dbt-sqlite/src 17 | 18 | COPY setup.py . 19 | COPY dbt ./dbt 20 | 21 | RUN pip install . 22 | 23 | RUN python3 -m pip install pytest pytest-dotenv dbt-tests-adapter~=1.10.4 24 | 25 | COPY run_tests.sh . 26 | COPY pytest.ini . 27 | COPY tests ./tests 28 | 29 | ENV TESTDATA=/opt/dbt-sqlite/testdata 30 | 31 | RUN mkdir $TESTDATA 32 | 33 | VOLUME /opt/dbt-sqlite/testdata 34 | 35 | WORKDIR /opt/dbt-sqlite/project 36 | 37 | ENV PATH=$PATH:/opt/dbt-sqlite/src 38 | 39 | VOLUME /opt/dbt-sqlite/project 40 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # dbt-sqlite 3 | 4 | A [SQLite](https://sqlite.org) adapter plugin for [dbt](https://www.getdbt.com/) (data build tool) 5 | 6 | Please read these docs carefully and use at your own risk. Issues and PRs welcome! 7 | 8 | ## The Use Case 9 | 10 | SQLite is an embedded SQL database. It comes included with most Python 11 | distributions and requires no installation or configuration. It can be 12 | a good choice if your project meets any of these criteria: 13 | 14 | - you store the database file on fast, local storage 15 | (not on a network drive) 16 | - the amount of data is relatively small (GBs, not TBs) 17 | - you're a data team of one with no need to share access to a database 18 | - your end goal is to export the results of your pipeline(s) into other 19 | systems for multi-user access or into BI/viz tools for analysis (i.e. 20 | you're doing ETL vs ELT) 21 | - your project is a proof of concept, to eventually be moved into 22 | another database or data warehouse platform 23 | - you want others to be able to deploy your data build without the 24 | overhead/cost of a full RDBMS or signing up for a data warehouse platform 25 | 26 | SQLite can be surprisingly fast, despite the query optimizer not being as 27 | sophisticated as other databases and data warehouse platforms. Tip: materialize 28 | your models as tables and create indexes in post-hooks to speed up filtering 29 | and joins. 30 | 31 | ## How to Use This 32 | 33 | Use the right version. Starting with the release of dbt-core 1.0.0, 34 | versions of dbt-sqlite are aligned to the same major+minor 35 | [version](https://semver.org/) of dbt-core. 36 | 37 | - versions 1.9.x of this adapter work with dbt-core 1.9.x 38 | - versions 1.6.x - 1.8.x are not supported: this adapter fell behind, and 39 | it's too much work to go back to support these versions 40 | - versions 1.5.x of this adapter work with dbt-core 1.5.x 41 | - versions 1.4.x of this adapter work with dbt-core 1.4.x 42 | - versions 1.3.x of this adapter work with dbt-core 1.3.x 43 | - versions 1.2.x of this adapter work with dbt-core 1.2.x 44 | - versions 1.1.x of this adapter work with dbt-core 1.1.x 45 | - versions 1.0.x of this adapter work with dbt-core 1.0.x 46 | - versions 0.2.x of this adapter work with dbt 0.20.x and 0.21.x 47 | - versions 0.1.x of this adapter work with dbt 0.19.x 48 | - versions 0.0.x of this adapter work with dbt 0.18.x 49 | 50 | Install this package: 51 | 52 | ``` 53 | # run this to install the latest version 54 | pip install dbt-sqlite 55 | 56 | # OR run this to install a specific version 57 | pip install dbt-sqlite==1.0.0 58 | ``` 59 | 60 | Create an entry in your `~/.dbt/profiles.yml` file with the following configuration: 61 | 62 | ```YAML 63 | dbt_sqlite: 64 | 65 | target: dev 66 | outputs: 67 | dev: 68 | type: sqlite 69 | 70 | # sqlite locks the whole db on writes so anything > 1 won't help 71 | threads: 1 72 | 73 | # value is arbitrary 74 | database: "database" 75 | 76 | # value of 'schema' must be defined in schema_paths below. in most cases, 77 | # this should be 'main' 78 | schema: 'main' 79 | 80 | # connect schemas to paths: at least one of these must be 'main' 81 | schemas_and_paths: 82 | main: '/my_project/data/etl.db' 83 | dataset: '/my_project/data/dataset_v1.db' 84 | 85 | # directory where all *.db files are attached as schema, using base filename 86 | # as schema name, and where new schema are created. this can overlap with the dirs of 87 | # files in schemas_and_paths as long as there's no conflicts. 88 | schema_directory: '/my_project/data' 89 | 90 | # optional: list of file paths of SQLite extensions to load. see README for more details. 91 | extensions: 92 | - "/path/to/sqlean/crypto.so" 93 | - "/path/to/sqlean/math.so" 94 | - "/path/to/sqlean/text.so" 95 | 96 | ``` 97 | 98 | Set `profile: 'dbt_sqlite'` in your project's `dbt_project.yml` file. 99 | 100 | ## Notes 101 | 102 | - There is no 'database' portion of relation names in SQLite so it gets 103 | stripped from the output of `ref()` and from SQL everywhere. It still 104 | needs to be set in the configuration and is used by dbt internally. 105 | 106 | - Schema are implemented as attached database files. (SQLite conflates databases 107 | and schemas.) 108 | 109 | - SQLite automatically assigns 'main' to the file you initially connect to, 110 | so this must be defined in your profile. Other schemas defined in your profile 111 | get attached when database connection is created. 112 | 113 | - If dbt needs to create a new schema, it will be created in `schema_directory` 114 | as `schema_name.db`. Dropping a schema results in dropping all its relations, 115 | detaching the database file from the session, and deleting the file. 116 | 117 | - Schema names are stored in view definitions, so when you access a non-'main' 118 | database file outside dbt, you'll need to attach it using the same name, or 119 | the views won't work. 120 | 121 | - SQLite does not allow views in one schema (i.e. database file) to reference 122 | objects in another schema. You'll get this error from SQLite: "view [someview] 123 | cannot reference objects in database [somedatabase]". You must set 124 | `materialized='table'` in models that reference other schemas. 125 | 126 | - Materializations are simplified: they drop and re-create the model, instead of 127 | doing the backup-and-swap-in new model that the other dbt database adapters 128 | support. This choice was made because SQLite doesn't support `DROP ... CASCADE` 129 | or `ALTER VIEW` or provide information about relation dependencies in something 130 | information_schema-like. These limitations make it really difficult to make the 131 | backup-and-swap-in functionality work properly. Given how SQLite aggressively 132 | [locks](https://sqlite.org/lockingv3.html) the database anyway, it's probably 133 | not worth the effort. 134 | 135 | - It's often idiomatic with dbt to use plentiful CASTs. The results of CASTs in 136 | SQLite are tricky and depend on how the model is materialized. In a nutshell, 137 | using table materializations gives better results. 138 | 139 | - When materialized as a view, the resulting column type from any CAST (or 140 | any expression) will always be empty. The SQLite adapter will regard this 141 | column type as 'UNKNOWN'. 142 | 143 | - When materialized as a table, a CAST will result in the specified type for 144 | INT, REAL, TEXT; casts to NUMERIC and BOOLEAN result in a 'NUM' column type. 145 | 146 | - To get the best fidelity to your seed data, declare all the column types as TEXT 147 | in your [seed configurations](https://docs.getdbt.com/reference/seed-configs) 148 | and create a model to do the casts and conversions. 149 | 150 | 151 | ## SQLite Extensions 152 | 153 | These modules from SQLean are needed for certain functionality to work: 154 | - `crypto`: provides `md5` function needed for snapshots 155 | - `math`: provides `ceil` and `floor` needed for the datediff macro to work 156 | - `text`: provides `split_part` function 157 | 158 | Precompiled binaries are available for download from the [SQLean github repository page](https://github.com/nalgeon/sqlean). 159 | You can also compile them yourself if you want. Note that some modules depend on other libraries 160 | (`math` for example depends on GLIBC); if an extension fails to load, you may want to try building it yourself. 161 | 162 | Point to these module files in your profile config as shown in the example above. 163 | 164 | Mac OS seems to ship with [SQLite libraries that do not have support for loading extensions compiled in](https://docs.python.org/3/library/sqlite3.html#f1), 165 | so this won't work "out of the box." Accordingly, snapshots won't work. 166 | If you need snapshot functionality, you'll need to compile SQLite/python 167 | or find a python distribution for Mac OS with this support. 168 | 169 | ## Development Notes / TODOs 170 | 171 | ... 172 | 173 | ### Publishing a release to PyPI 174 | 175 | Because I forget... 176 | 177 | ``` 178 | # assumes ~/.pypirc is already set up 179 | 180 | workon dbt-sqlite-devel 181 | 182 | vi dbt/adapters/sqlite/__version__.py # update version 183 | vi setup.py # update dbt-core dependency if appropriate 184 | 185 | # start clean 186 | rm -rf dist/ build/ *.egg-info 187 | 188 | # make sure tools are up to date 189 | python -m pip install --upgrade build setuptools wheel twine 190 | 191 | # build 192 | python -m build 193 | 194 | # upload to PyPI 195 | python -m twine upload dist/* 196 | 197 | git commit 198 | git tag vXXX 199 | git push --tags 200 | 201 | # go to github and "Draft a new release" 202 | ``` 203 | 204 | ## Running Tests 205 | 206 | This runs the test suite and cleans up after itself: 207 | ``` 208 | ./run_tests_docker.sh 209 | ``` 210 | 211 | To run tests interactively and be able to examine test artifacts: 212 | ``` 213 | docker build . -t dbt-sqlite 214 | 215 | docker run --rm -it dbt-sqlite bash 216 | 217 | # see output for the locations of artifacts 218 | run_tests.sh -s 219 | ``` 220 | 221 | 222 | ## Credits 223 | 224 | Inspired by this initial work by stephen1000: https://github.com/stephen1000/dbt_sqlite 225 | 226 | https://github.com/jwills/dbt-duckdb/ - useful for ideas on working with 227 | another embedded database 228 | 229 | https://github.com/fishtown-analytics/dbt-spark/ - spark also has two-part 230 | relation names (no 'database') 231 | -------------------------------------------------------------------------------- /dbt/adapters/sqlite/__init__.py: -------------------------------------------------------------------------------- 1 | from dbt.adapters.sqlite.connections import SQLiteConnectionManager 2 | from dbt.adapters.sqlite.connections import SQLiteCredentials 3 | from dbt.adapters.sqlite.impl import SQLiteAdapter 4 | 5 | from dbt.adapters.base import AdapterPlugin 6 | from dbt.include import sqlite 7 | 8 | 9 | Plugin = AdapterPlugin( 10 | adapter=SQLiteAdapter, 11 | credentials=SQLiteCredentials, 12 | include_path=sqlite.PACKAGE_PATH) 13 | -------------------------------------------------------------------------------- /dbt/adapters/sqlite/__version__.py: -------------------------------------------------------------------------------- 1 | version = '1.9.1' 2 | -------------------------------------------------------------------------------- /dbt/adapters/sqlite/connections.py: -------------------------------------------------------------------------------- 1 | 2 | from contextlib import contextmanager 3 | from dataclasses import dataclass, field 4 | import glob 5 | import os.path 6 | import sqlite3 7 | from socket import gethostname 8 | from typing import Optional, Tuple, Any, Dict, List, Union 9 | 10 | from dbt.adapters.contracts.connection import Credentials 11 | from dbt.adapters.sql import SQLConnectionManager 12 | from dbt.adapters.contracts.connection import AdapterResponse 13 | from dbt.adapters.contracts.connection import Connection 14 | from dbt.adapters.events.logging import AdapterLogger 15 | from dbt.adapters.exceptions.connection import FailedToConnectError 16 | from dbt_common.exceptions import ( 17 | DbtDatabaseError, 18 | DbtRuntimeError 19 | ) 20 | 21 | logger = AdapterLogger("SQLite") 22 | 23 | 24 | @dataclass 25 | class SQLiteCredentials(Credentials): 26 | """ Required connections for a SQLite connection""" 27 | 28 | schemas_and_paths: Dict[str, str] 29 | schema_directory: str 30 | extensions: List[str] = field(default_factory=list) 31 | 32 | @property 33 | def type(self): 34 | return "sqlite" 35 | 36 | @property 37 | def unique_field(self): 38 | """ 39 | Hashed and included in anonymous telemetry to track adapter adoption. 40 | Pick a field that can uniquely identify one team/organization building with this adapter 41 | """ 42 | return gethostname() 43 | 44 | def _connection_keys(self): 45 | """ Keys to show when debugging """ 46 | return ["database", "schema", "schemas_and_paths", "schema_directory" ] 47 | 48 | 49 | class SQLiteConnectionManager(SQLConnectionManager): 50 | TYPE = "sqlite" 51 | 52 | @classmethod 53 | def open(cls, connection: Connection): 54 | if connection.state == "open": 55 | logger.debug("Connection is already open, skipping open.") 56 | return connection 57 | 58 | credentials: SQLiteCredentials = connection.credentials 59 | 60 | schemas_and_paths = {} 61 | for schema, path in credentials.schemas_and_paths.items(): 62 | # Make .db file path absolute 63 | schemas_and_paths[schema] = os.path.abspath(path) 64 | 65 | try: 66 | attached = [] 67 | if 'main' in schemas_and_paths: 68 | handle: sqlite3.Connection = sqlite3.connect(schemas_and_paths['main']) 69 | attached.append(schemas_and_paths['main']) 70 | else: 71 | raise FailedToConnectError("at least one schema must be called 'main'") 72 | 73 | if len(credentials.extensions) > 0: 74 | handle.enable_load_extension(True) 75 | 76 | for ext_path in credentials.extensions: 77 | handle.load_extension(ext_path) 78 | 79 | cursor = handle.cursor() 80 | 81 | for schema in set(schemas_and_paths.keys()) - set(['main']): 82 | path = schemas_and_paths[schema] 83 | cursor.execute(f"attach '{path}' as '{schema}'") 84 | attached.append(schema) 85 | 86 | for path in glob.glob(os.path.join(credentials.schema_directory, "*.db")): 87 | abs_path = os.path.abspath(path) 88 | 89 | # if file was already attached from being defined in schemas_and_paths, ignore it 90 | if not abs_path in schemas_and_paths.values(): 91 | schema = os.path.basename(path)[:-3] 92 | 93 | # has schema name been used already? 94 | if schema not in attached: 95 | cursor.execute(f"attach '{path}' as '{schema}'") 96 | else: 97 | raise FailedToConnectError( 98 | f"found {path} while scanning schema_directory, but cannot attach it as '{schema}' " + 99 | f"because that schema name is already defined in schemas_and_paths. " + 100 | f"fix your ~/.dbt/profiles.yml file") 101 | 102 | # # uncomment these lines to print out SQL: this only happens if statement is successful 103 | # handle.set_trace_callback(print) 104 | # sqlite3.enable_callback_tracebacks(True) 105 | 106 | connection.state = "open" 107 | connection.handle = handle 108 | 109 | return connection 110 | except sqlite3.Error as e: 111 | logger.debug( 112 | "Got an error when attempting to open a sqlite3 connection: '%s'", e 113 | ) 114 | connection.handle = None 115 | connection.state = "fail" 116 | 117 | raise FailedToConnectError(str(e)) 118 | except Exception as e: 119 | print(f"Unknown error opening SQLite connection: {e}") 120 | raise e 121 | 122 | @classmethod 123 | def get_status(cls, cursor: sqlite3.Cursor): 124 | return f"OK"# {cursor.rowcount}" 125 | 126 | 127 | def get_response(cls, cursor) -> AdapterResponse: 128 | """ 129 | new to support dbt 0.19: this method replaces get_response 130 | """ 131 | message = 'OK' 132 | rows = cursor.rowcount 133 | return AdapterResponse( 134 | _message=message, 135 | rows_affected=rows 136 | ) 137 | 138 | 139 | def cancel(self, connection): 140 | """ cancel ongoing queries """ 141 | 142 | logger.debug("Cancelling queries") 143 | try: 144 | connection.handle.interrupt() 145 | except sqlite3.Error: 146 | pass 147 | logger.debug("Queries canceled") 148 | 149 | @contextmanager 150 | def exception_handler(self, sql: str): 151 | try: 152 | yield 153 | except sqlite3.DatabaseError as e: 154 | self.release() 155 | logger.debug("sqlite3 error: {}".format(str(e))) 156 | raise DbtDatabaseError(str(e)) 157 | except Exception as e: 158 | logger.debug("Error running SQL: {}".format(sql)) 159 | logger.debug("Rolling back transaction.") 160 | self.release() 161 | raise DbtRuntimeError(str(e)) 162 | 163 | def add_query( 164 | self, 165 | sql: str, 166 | auto_begin: bool = True, 167 | bindings: Optional[Any] = None, 168 | abridge_sql_log: bool = False 169 | ) -> Tuple[Connection, Any]: 170 | """ 171 | sqlite3's cursor.execute() doesn't like None as the 172 | bindings argument, so substitute an empty dict 173 | """ 174 | if not bindings: 175 | bindings = {} 176 | 177 | return super().add_query(sql=sql, auto_begin=auto_begin, bindings=bindings, abridge_sql_log=abridge_sql_log) 178 | 179 | @classmethod 180 | def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: 181 | # TODO: figure out how to implement this 182 | return "UNKNOWN" 183 | -------------------------------------------------------------------------------- /dbt/adapters/sqlite/impl.py: -------------------------------------------------------------------------------- 1 | 2 | import datetime 3 | import decimal 4 | import os 5 | import os.path 6 | from typing import List, Optional, Set 7 | 8 | import agate 9 | from dbt.adapters.base import available 10 | from dbt.adapters.base.relation import BaseRelation, InformationSchema 11 | from dbt.adapters.sql import SQLAdapter 12 | from dbt.adapters.sqlite import SQLiteConnectionManager 13 | from dbt.adapters.sqlite.relation import SQLiteRelation 14 | from dbt_common.exceptions import NotImplementedError 15 | from dbt.contracts.graph.manifest import Manifest 16 | 17 | 18 | class SQLiteAdapter(SQLAdapter): 19 | ConnectionManager = SQLiteConnectionManager 20 | 21 | Relation = SQLiteRelation 22 | 23 | @classmethod 24 | def date_function(cls): 25 | return 'date()' 26 | 27 | # sqlite reports the exact string (including case) used when declaring a column of a certain type. 28 | # the types here should correspond to affinities recognized by SQLite. 29 | # see https://www.sqlite.org/datatype3.html 30 | 31 | @classmethod 32 | def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str: 33 | return "TEXT" 34 | 35 | @classmethod 36 | def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: 37 | decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] 38 | return "REAL" if decimals else "INT" 39 | 40 | @classmethod 41 | def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: 42 | return "INT" 43 | 44 | @classmethod 45 | def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str: 46 | return "TEXT" 47 | 48 | @classmethod 49 | def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: 50 | return "TEXT" 51 | 52 | @classmethod 53 | def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: 54 | return "TEXT" 55 | 56 | def get_live_relation_type(self, relation): 57 | """ 58 | returns the type of relation (table, view) from the live database 59 | """ 60 | sql = f"SELECT type as data_type FROM { relation.schema }.sqlite_master WHERE name = '{relation.identifier}'" 61 | result = self.connections.execute(sql, fetch=True) 62 | data_type = result[1].rows[0][0] 63 | return data_type 64 | 65 | def rename_relation(self, from_relation, to_relation): 66 | """ 67 | Override method instead of calling the macro in adapters.sql 68 | because renaming views is complicated 69 | """ 70 | self.cache_renamed(from_relation, to_relation) 71 | 72 | existing_relation_type = from_relation.type 73 | 74 | if existing_relation_type == 'table': 75 | 76 | self.connections.execute(f"ALTER TABLE {from_relation} RENAME TO {to_relation.identifier}") 77 | 78 | elif existing_relation_type == 'view': 79 | 80 | result = self.connections.execute(f""" 81 | SELECT sql FROM {from_relation.schema}.sqlite_master 82 | WHERE type = 'view' and name = '{from_relation.identifier}' 83 | """, fetch=True) 84 | 85 | definition = result[1].rows[0][0] 86 | 87 | self.connections.execute(f"DROP VIEW {from_relation}") 88 | 89 | self.connections.execute(f"DROP VIEW IF EXISTS {to_relation}") 90 | 91 | new_definition = definition.replace(from_relation.identifier, f"{to_relation}", 1) 92 | 93 | self.connections.execute(new_definition) 94 | 95 | else: 96 | raise NotImplementedError( 97 | f"I don't know how to rename this type of relation: {from_relation.type}," + 98 | f" from: {from_relation}, to: {to_relation}") 99 | 100 | def get_columns_in_relation(self, relation): 101 | _, results = self.connections.execute(f"pragma {relation.schema}.table_info({relation.identifier})", fetch=True) 102 | 103 | new_rows = [] 104 | for row in results: 105 | new_row = [ 106 | row[1], 107 | row[2] or 'UNKNOWN', 108 | None, 109 | None, 110 | None 111 | ] 112 | new_rows.append(new_row) 113 | 114 | column_names = [ 115 | 'column_name', 116 | 'data_type', 117 | 'character_maximum_length', 118 | 'numeric_precision', 119 | 'numeric_scale' 120 | ] 121 | 122 | table = agate.Table(new_rows, column_names) 123 | 124 | kwargs = { 125 | 'table': table 126 | } 127 | 128 | result = self.execute_macro( 129 | 'sql_convert_columns_in_relation', 130 | kwargs=kwargs 131 | ) 132 | return result 133 | 134 | def _get_one_catalog( 135 | self, 136 | information_schema: InformationSchema, 137 | schemas: Set[str], 138 | manifest: Manifest, 139 | ) -> agate.Table: 140 | """ 141 | bad form to override this method but... 142 | """ 143 | 144 | # this does N+1 queries but there doesn't seem to be 145 | # any other way to do this 146 | 147 | rows = [] 148 | for schema in schemas: 149 | 150 | schema_obj = self.Relation.create(database=information_schema.database, schema=schema) 151 | results = self.list_relations_without_caching(schema_obj) 152 | 153 | if len(results) > 0: 154 | for relation_row in results: 155 | name = relation_row.name 156 | relation_type = str(relation_row.type) 157 | 158 | table_info = self.connections.execute( 159 | f"pragma {schema}.table_info({name})", fetch=True) 160 | 161 | for table_row in table_info[1]: 162 | rows.append([ 163 | information_schema.database, 164 | schema, 165 | name, 166 | relation_type, 167 | '', 168 | '', 169 | table_row['name'], 170 | table_row['cid'], 171 | table_row['type'] or 'UNKNOWN', 172 | '' 173 | ]) 174 | 175 | column_names = [ 176 | 'table_database', 177 | 'table_schema', 178 | 'table_name', 179 | 'table_type', 180 | 'table_comment', 181 | 'table_owner', 182 | 'column_name', 183 | 'column_index', 184 | 'column_type', 185 | 'column_comment' 186 | ] 187 | table = agate.Table(rows, column_names) 188 | 189 | results = self._catalog_filter_table(table, manifest) 190 | return results 191 | 192 | def get_rows_different_sql( 193 | self, 194 | relation_a: BaseRelation, 195 | relation_b: BaseRelation, 196 | column_names: Optional[List[str]] = None, 197 | except_operator: str = 'EXCEPT', 198 | ) -> str: 199 | # This method only really exists for test reasons. 200 | names: List[str] 201 | if column_names is None: 202 | columns = self.get_columns_in_relation(relation_a) 203 | names = sorted((self.quote(c.name) for c in columns)) 204 | else: 205 | names = sorted((self.quote(n) for n in column_names)) 206 | columns_csv = ', '.join(names) 207 | 208 | # difference from base class: sqlite requires SELECTs around UNION 209 | # queries 210 | COLUMNS_EQUAL_SQL = ''' 211 | with diff_count as ( 212 | SELECT 213 | 1 as id, 214 | COUNT(*) as num_missing FROM ( 215 | SELECT * FROM 216 | (SELECT {columns} FROM {relation_a} {except_op} 217 | SELECT {columns} FROM {relation_b}) t1 218 | UNION ALL 219 | SELECT * FROM 220 | (SELECT {columns} FROM {relation_b} {except_op} 221 | SELECT {columns} FROM {relation_a}) t2 222 | ) as a 223 | ), table_a as ( 224 | SELECT COUNT(*) as num_rows FROM {relation_a} 225 | ), table_b as ( 226 | SELECT COUNT(*) as num_rows FROM {relation_b} 227 | ), row_count_diff as ( 228 | select 229 | 1 as id, 230 | table_a.num_rows - table_b.num_rows as difference 231 | from table_a, table_b 232 | ) 233 | select 234 | row_count_diff.difference as row_count_difference, 235 | diff_count.num_missing as num_mismatched 236 | from row_count_diff 237 | join diff_count using (id) 238 | '''.strip() 239 | 240 | sql = COLUMNS_EQUAL_SQL.format( 241 | columns=columns_csv, 242 | relation_a=str(relation_a), 243 | relation_b=str(relation_b), 244 | except_op=except_operator, 245 | ) 246 | 247 | return sql 248 | 249 | def _transform_seed_value(self, value): 250 | new_value = value 251 | if isinstance(value, decimal.Decimal): 252 | new_value = str(value) 253 | return new_value 254 | 255 | @available 256 | def transform_seed_row(self, row): 257 | """ 258 | sqlite3 chokes on Decimal values (emitted by agate) in 259 | bound values so convert those to strings. there may be other 260 | types that need to be added here. 261 | 262 | This is the error that comes up: 263 | "Error binding parameter 0 - probably unsupported type." 264 | 265 | see dbt.clients.agate_helper.build_type_tester() for the 266 | TypeTester passed to agate when parsing CSVs. 267 | 268 | """ 269 | return [self._transform_seed_value(value) for value in row] 270 | 271 | def timestamp_add_sql( 272 | self, add_to: str, number: int = 1, interval: str = 'hour' 273 | ) -> str: 274 | return f"DATETIME({add_to}, '{number} {interval}')" 275 | 276 | def drop_schema(self, relation: BaseRelation) -> None: 277 | super().drop_schema(relation) 278 | 279 | # never detach main 280 | if relation.schema != 'main': 281 | if self.check_schema_exists(relation.database, relation.schema): 282 | self.connections.execute(f"DETACH DATABASE {relation.schema}") 283 | 284 | if relation.schema in self.config.credentials.schemas_and_paths: 285 | path = self.config.credentials.schemas_and_paths[relation.schema] 286 | else: 287 | path = os.path.join(self.config.credentials.schema_directory, relation.schema + ".db") 288 | os.remove(path) 289 | -------------------------------------------------------------------------------- /dbt/adapters/sqlite/relation.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | 3 | from dbt.adapters.base.relation import BaseRelation, Policy 4 | 5 | 6 | @dataclass 7 | class SQLiteQuotePolicy(Policy): 8 | database: bool = False 9 | schema: bool = False 10 | identifier: bool = True 11 | 12 | 13 | @dataclass 14 | class SQLiteIncludePolicy(Policy): 15 | database: bool = False 16 | schema: bool = True 17 | identifier: bool = True 18 | 19 | 20 | @dataclass(frozen=True, eq=False, repr=False) 21 | class SQLiteRelation(BaseRelation): 22 | quote_policy: SQLiteQuotePolicy = field(default_factory=SQLiteQuotePolicy) 23 | include_policy: SQLiteIncludePolicy = field(default_factory=SQLiteIncludePolicy) 24 | -------------------------------------------------------------------------------- /dbt/include/sqlite/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | PACKAGE_PATH = os.path.dirname(__file__) 3 | -------------------------------------------------------------------------------- /dbt/include/sqlite/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | name: dbt_sqlite 3 | version: 1.0 4 | config-version: 2 5 | 6 | macro-paths: ["macros"] 7 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/adapters.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__list_schemas(database) %} 2 | {% call statement('list_schemas', fetch_result=True) %} 3 | pragma database_list 4 | {% endcall %} 5 | {% set results = load_result('list_schemas').table %} 6 | {{ return(results.select(['name']).rename(column_names = {'name': 'schema'})) }} 7 | {% endmacro %} 8 | 9 | {% macro sqlite__create_schema(relation, auto_begin=False) %} 10 | {% set path = [ adapter.config.credentials.schema_directory, relation.without_identifier().include(database=False) | string + '.db' ] | join('/') %} 11 | {%- call statement('create_schema') -%} 12 | attach database '{{ path }}' as {{ relation.without_identifier().include(database=False) }} 13 | {%- endcall -%} 14 | {% endmacro %} 15 | 16 | {% macro sqlite__drop_schema(relation) -%} 17 | {# drop all tables in the schema; detaching happens in the adapter class #} 18 | 19 | {% set relations_in_schema = list_relations_without_caching(relation.without_identifier()) %} 20 | 21 | {% for row in relations_in_schema %} 22 | {%- call statement('drop_relation_in_schema') -%} 23 | drop {{ row.data_type}} {{ row.schema }}.{{ row.name }} 24 | {%- endcall -%} 25 | {% endfor %} 26 | {% endmacro %} 27 | 28 | {% macro sqlite__drop_relation(relation) -%} 29 | {% call statement('drop_relation', auto_begin=False) -%} 30 | drop {{ relation.type }} if exists {{ relation }} 31 | {%- endcall %} 32 | {% endmacro %} 33 | 34 | {% macro sqlite__truncate_relation(relation) -%} 35 | {% call statement('truncate_relation') -%} 36 | delete from {{ relation }} 37 | {%- endcall %} 38 | {% endmacro %} 39 | 40 | {% macro sqlite__check_schema_exists(information_schema, schema) -%} 41 | {% if schema in list_schemas(database).columns[0].values() %} 42 | {% call statement('check_schema_exists', fetch_result=True) %} 43 | SELECT 1 as schema_exist 44 | {% endcall %} 45 | {{ return(load_result('check_schema_exists').table) }} 46 | {% else %} 47 | {% call statement('check_schema_exists', fetch_result=True) %} 48 | SELECT 0 as schema_exist 49 | {% endcall %} 50 | {{ return(load_result('check_schema_exists').table) }} 51 | {% endif %} 52 | {% endmacro %} 53 | 54 | {% macro sqlite__list_relations_without_caching(schema_relation) %} 55 | 56 | {% set schemas = list_schemas(schema_relation.database).columns[0].values() %} 57 | 58 | {% if schema_relation.schema in schemas %} 59 | {% call statement('list_relations_without_caching', fetch_result=True) %} 60 | SELECT 61 | '{{ schema_relation.database }}' as database 62 | ,name 63 | ,'{{ schema_relation.schema }}' AS schema 64 | ,type as data_type 65 | FROM 66 | {{ schema_relation.schema }}.sqlite_master 67 | WHERE 68 | name NOT LIKE 'sqlite_%' 69 | {% endcall %} 70 | 71 | {{ return(load_result('list_relations_without_caching').table) }} 72 | {% else %} 73 | {% call statement('empty_table', fetch_result=True) %} 74 | SELECT null as database, null as name, null as schema, null as data_type WHERE 1=0 75 | {% endcall %} 76 | 77 | {{ return(load_result('empty_table').table) }} 78 | {% endif %} 79 | {% endmacro %} 80 | 81 | {% macro sqlite__create_table_as(temporary, relation, sql) -%} 82 | {% set contract_config = config.get('contract') %} 83 | {% if contract_config.enforced %} 84 | {{exceptions.warn("Model contracts cannot be enforced by sqlite!")}} 85 | {% endif %} 86 | create {% if temporary -%} 87 | temporary 88 | {%- endif %} table {{ relation }} 89 | as 90 | {{ sql }} 91 | {% endmacro %} 92 | 93 | {% macro sqlite__create_view_as(relation, sql, auto_begin=False) -%} 94 | {% set contract_config = config.get('contract') %} 95 | {% if contract_config.enforced %} 96 | {{exceptions.warn("Model contracts cannot be enforced by sqlite!")}} 97 | {% endif %} 98 | create view {{ relation }} as 99 | {{ sql }}; 100 | {%- endmacro %} 101 | 102 | {% macro sqlite__rename_relation(from_relation, to_relation) -%} 103 | {# no-op #} 104 | {# see SQLiteAdapter.rename_relation() #} 105 | {% endmacro %} 106 | 107 | {# 108 | the only allowable schema for temporary tables in SQLite is 'temp', so set 109 | that here when making the relation and everything else should Just Work 110 | #} 111 | {% macro sqlite__make_temp_relation(base_relation, suffix) %} 112 | {% set tmp_identifier = base_relation.identifier ~ suffix %} 113 | {% set tmp_relation = base_relation.incorporate( 114 | path={"schema": "temp", "identifier": tmp_identifier}) -%} 115 | 116 | {% do return(tmp_relation) %} 117 | {% endmacro %} 118 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/catalog.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro sqlite__get_catalog(information_schema, schemas) -%} 3 | {# no-op #} 4 | {# see SQLiteAdapter._get_one_catalog() #} 5 | {% endmacro %} 6 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/core_overrides.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro ref(model_name) %} 3 | 4 | {# override to strip off database, and return only schema.table #} 5 | 6 | {% set rel = builtins.ref(model_name) %} 7 | {% do return(rel.include(database=False)) %} 8 | 9 | {% endmacro %} 10 | 11 | {% macro source(source_name, model_name) %} 12 | 13 | {# override to strip off database, and return only schema.table #} 14 | 15 | {% set rel = builtins.source(source_name, model_name) %} 16 | {% do return(rel.include(database=False)) %} 17 | 18 | {% endmacro %} -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/incremental/incremental.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro sqlite_incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name="main") %} 3 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 4 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 5 | 6 | {%- if unique_key is not none -%} 7 | {% call statement('sqlite_incremental_upsert') -%} 8 | delete 9 | from {{ target_relation }} 10 | where ({{ unique_key }}) in ( 11 | select ({{ unique_key }}) 12 | from {{ tmp_relation }} 13 | ); 14 | {%- endcall %} 15 | {%- endif %} 16 | 17 | {# difference here is sqlite doesn't want parens around the select query #} 18 | insert into {{ target_relation }} ({{ dest_cols_csv }}) 19 | select {{ dest_cols_csv }} 20 | from {{ tmp_relation }} 21 | ; 22 | {%- endmacro %} 23 | 24 | {# 25 | the incremental materialization was overhauled in dbt 1.3.0 to make it easier to override 26 | adapter-specific pieces, but it's hard to use it as intended in our case, because it renames 27 | models, which is difficult to handle so we just avoid it 28 | 29 | also, we call sqlite_incremental_upsert b/c the syntax for insert differs 30 | #} 31 | {% materialization incremental, adapter='sqlite' -%} 32 | 33 | {% set unique_key = config.get('unique_key') %} 34 | 35 | {% set target_relation = this.incorporate(type='table') %} 36 | {% set existing_relation = load_relation(this) %} 37 | {% set tmp_relation = make_temp_relation(this) %} 38 | 39 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 40 | 41 | -- `BEGIN` happens here: 42 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 43 | 44 | {% set to_drop = [] %} 45 | {% if existing_relation is none %} 46 | {% set build_sql = create_table_as(False, target_relation, sql) %} 47 | {% elif existing_relation.is_view or should_full_refresh() %} 48 | {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} 49 | {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} 50 | {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} 51 | {% do adapter.drop_relation(backup_relation) %} 52 | 53 | {# this is simplified from macro in core: don't try to make a backup #} 54 | {% do adapter.drop_relation(existing_relation) %} 55 | 56 | {% set build_sql = create_table_as(False, target_relation, sql) %} 57 | {% else %} 58 | {% set tmp_relation = make_temp_relation(target_relation) %} 59 | {% do run_query(create_table_as(True, tmp_relation, sql)) %} 60 | {% do adapter.expand_target_column_types( 61 | from_relation=tmp_relation, 62 | to_relation=target_relation) %} 63 | {% set build_sql = sqlite_incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %} 64 | {% endif %} 65 | 66 | {% call statement("main") %} 67 | {{ build_sql }} 68 | {% endcall %} 69 | 70 | {% do persist_docs(target_relation, model) %} 71 | 72 | {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %} 73 | {% do create_indexes(target_relation) %} 74 | {% endif %} 75 | 76 | {{ run_hooks(post_hooks, inside_transaction=True) }} 77 | 78 | -- `COMMIT` happens here 79 | {% do adapter.commit() %} 80 | 81 | {% for rel in to_drop %} 82 | {% do adapter.drop_relation(rel) %} 83 | {% endfor %} 84 | 85 | {{ run_hooks(post_hooks, inside_transaction=False) }} 86 | 87 | {{ return({'relations': [target_relation]}) }} 88 | 89 | {%- endmaterialization %} 90 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/seed/helpers.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__get_binding_char() %} 2 | {{ return('?') }} 3 | {% endmacro %} 4 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/seed/seed.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro sqlite__load_csv_rows(model, agate_table) %} 3 | {% set batch_size = 10000 %} 4 | {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} 5 | {% set bindings = [] %} 6 | 7 | {% set statements = [] %} 8 | 9 | {% for chunk in agate_table.rows | batch(batch_size) %} 10 | {% set bindings = [] %} 11 | 12 | {% for row in chunk %} 13 | {# transform rows so sqlite is happy with data types #} 14 | {% set processed_row = adapter.transform_seed_row(row) %} 15 | {% do bindings.extend(processed_row) %} 16 | {% endfor %} 17 | 18 | {% set sql %} 19 | insert into {{ this.schema }}.{{ this.identifier}} ({{ cols_sql }}) values 20 | {% for row in chunk -%} 21 | ({%- for column in agate_table.column_names -%} 22 | {{ get_binding_char() }} 23 | {%- if not loop.last%},{%- endif %} 24 | {%- endfor -%}) 25 | {%- if not loop.last%},{%- endif %} 26 | {%- endfor %} 27 | {% endset %} 28 | 29 | {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} 30 | 31 | {% if loop.index0 == 0 %} 32 | {% do statements.append(sql) %} 33 | {% endif %} 34 | {% endfor %} 35 | 36 | {# Return SQL so we can render it out into the compiled files #} 37 | {{ return(statements[0]) }} 38 | {% endmacro %} 39 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/snapshot/snapshot_merge.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro sqlite__snapshot_merge_sql(target, source, insert_cols) -%} 3 | {%- set insert_cols_csv = insert_cols | join(', ') -%} 4 | 5 | {% set update_sql %} 6 | update {{ target }} 7 | set dbt_valid_to = ( 8 | SELECT 9 | DBT_INTERNAL_SOURCE.dbt_valid_to 10 | from {{ source }} as DBT_INTERNAL_SOURCE 11 | where DBT_INTERNAL_SOURCE.dbt_scd_id = {{ target }}.dbt_scd_id 12 | and DBT_INTERNAL_SOURCE.dbt_change_type = 'update' 13 | ) 14 | WHERE dbt_valid_to is null; 15 | {% endset %} 16 | 17 | {# 18 | TODO: this is a hack: this macro is supposed to return a SQL string 19 | but we're executing a query here. this happens to work and it avoids having 20 | to override the snapshot_merge macro to properly execute two separate 21 | statements but it's horrible. 22 | #} 23 | {% do adapter.add_query(update_sql, auto_begin=False) %} 24 | 25 | insert into {{ target }} ({{ insert_cols_csv }}) 26 | select {% for column in insert_cols -%} 27 | DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %} 28 | {%- endfor %} 29 | from {{ source }} as DBT_INTERNAL_SOURCE 30 | where DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'; 31 | 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/snapshot/strategies.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__snapshot_hash_arguments(args) -%} 2 | hex(md5({%- for arg in args -%} 3 | coalesce(cast({{ arg }} as varchar ), '') 4 | {% if not loop.last %} || '|' || {% endif %} 5 | {%- endfor -%})) 6 | {%- endmacro %} 7 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/table/table.sql: -------------------------------------------------------------------------------- 1 | {% materialization table, adapter='sqlite' %} 2 | {%- set identifier = model['alias'] -%} 3 | 4 | {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} 5 | {%- set target_relation = api.Relation.create(identifier=identifier, 6 | schema=schema, 7 | database=database, 8 | type='table') -%} 9 | 10 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 11 | 12 | -- `BEGIN` happens here: 13 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 14 | 15 | {% if old_relation is not none %} 16 | {{ adapter.drop_relation(old_relation) }} 17 | {% endif %} 18 | 19 | -- build model 20 | {% call statement('main') -%} 21 | {{ create_table_as(False, target_relation, sql) }} 22 | {%- endcall %} 23 | 24 | {{ run_hooks(post_hooks, inside_transaction=True) }} 25 | 26 | {% do persist_docs(target_relation, model) %} 27 | 28 | -- `COMMIT` happens here 29 | {{ adapter.commit() }} 30 | 31 | {{ run_hooks(post_hooks, inside_transaction=False) }} 32 | 33 | {{ return({'relations': [target_relation]}) }} 34 | {% endmaterialization %} 35 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/test.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} 2 | select 3 | {{ fail_calc }} as failures, 4 | case when {{ fail_calc }} {{ warn_if }} 5 | then 'true' else 'false' end as should_warn, 6 | case when {{ fail_calc }} {{ error_if }} 7 | then 'true' else 'false' end as should_error 8 | from ( 9 | {{ main_sql }} 10 | {{ "limit " ~ limit if limit != none }} 11 | ) dbt_internal_test 12 | {%- endmacro %} 13 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/materializations/view/view.sql: -------------------------------------------------------------------------------- 1 | {%- materialization view, adapter='sqlite' -%} 2 | 3 | {%- set identifier = model['alias'] -%} 4 | 5 | {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} 6 | {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, 7 | type='view') -%} 8 | 9 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 10 | 11 | -- `BEGIN` happens here: 12 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 13 | 14 | {% if old_relation is not none %} 15 | {{ adapter.drop_relation(old_relation) }} 16 | {% endif %} 17 | 18 | -- build model 19 | {% call statement('main') -%} 20 | {{ create_view_as(target_relation, sql) }} 21 | {%- endcall %} 22 | 23 | {% do persist_docs(target_relation, model) %} 24 | 25 | {{ run_hooks(post_hooks, inside_transaction=True) }} 26 | 27 | {{ adapter.commit() }} 28 | 29 | {{ run_hooks(post_hooks, inside_transaction=False) }} 30 | 31 | {{ return({'relations': [target_relation]}) }} 32 | 33 | {%- endmaterialization -%} 34 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/any_value.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__any_value(expression) -%} 2 | 3 | min({{ expression }}) 4 | 5 | {%- endmacro %} 6 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/bool_or.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__bool_or(expression) -%} 2 | 3 | max({{ expression }}) 4 | 5 | {%- endmacro %} 6 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/cast_bool_to_text.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__cast_bool_to_text(field) %} 2 | case 3 | when {{ field }} = 0 then 'false' 4 | when {{ field }} = 1 then 'true' 5 | end 6 | {% endmacro %} 7 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/dateadd.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__dateadd(datepart, interval, from_date_or_timestamp) %} 2 | -- If provided a DATETIME, returns a DATETIME 3 | -- If provided a DATE, returns a DATE 4 | 5 | CASE 6 | -- Matches DATETIME type based on ISO-8601 7 | WHEN {{ from_date_or_timestamp }} LIKE '%:%' OR ({{ from_date_or_timestamp }} LIKE '%T%' AND {{ from_date_or_timestamp }} LIKE '%Z%') THEN 8 | CASE 9 | WHEN LOWER({{ datepart }}) = 'second' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' seconds') 10 | WHEN LOWER({{ datepart }}) = 'minute' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' minutes') 11 | WHEN LOWER({{ datepart }}) = 'hour' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' hours') 12 | WHEN LOWER({{ datepart }}) = 'day' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' days') 13 | WHEN LOWER({{ datepart }}) = 'week' THEN datetime({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 7) || ' days') 14 | WHEN LOWER({{ datepart }}) = 'month' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' months') 15 | WHEN LOWER({{ datepart }}) = 'quarter' THEN datetime({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 3) || ' months') 16 | WHEN LOWER({{ datepart }}) = 'year' THEN datetime({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' years') 17 | ELSE NULL 18 | END 19 | -- Matches DATE type based on ISO-8601 20 | WHEN {{ from_date_or_timestamp }} LIKE '%-%' AND {{ from_date_or_timestamp }} NOT LIKE '%T%' AND {{ from_date_or_timestamp }} NOT LIKE '% %' THEN 21 | CASE 22 | WHEN LOWER({{ datepart }}) IN ('second', 'minute', 'hour') THEN date({{ from_date_or_timestamp }}) 23 | WHEN LOWER({{ datepart }}) = 'day' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' days') 24 | WHEN LOWER({{ datepart }}) = 'week' THEN date({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 7) || ' days') 25 | WHEN LOWER({{ datepart }}) = 'month' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' months') 26 | WHEN LOWER({{ datepart }}) = 'quarter' THEN date({{ from_date_or_timestamp }}, '+' || ({{ interval }} * 3) || ' months') 27 | WHEN LOWER({{ datepart }}) = 'year' THEN date({{ from_date_or_timestamp }}, '+' || {{ interval }} || ' years') 28 | ELSE NULL 29 | END 30 | ELSE 31 | NULL 32 | END 33 | {% endmacro %} 34 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/datediff.sql: -------------------------------------------------------------------------------- 1 | 2 | {# TODO: fully implement this and rename #} 3 | {# adapted from postgresql #} 4 | {% macro sqlite__datediff_broken(first_date, second_date, datepart) -%} 5 | 6 | {% if datepart == 'year' %} 7 | (strftime('%Y', {{second_date}}) - strftime('%Y', {{first_date}})) 8 | {# 9 | {% elif datepart == 'quarter' %} 10 | ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) 11 | #} 12 | {% elif datepart == 'month' %} 13 | (({{ datediff(first_date, second_date, 'year') }} * 12 + strftime('%m', {{second_date}})) - strftime('%m', {{first_date}})) 14 | {% elif datepart == 'day' %} 15 | (floor(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 86400) + 16 | case when {{second_date}} <= strftime('%Y-%m-%d 23:59:59.999999', {{first_date}}) then -1 else 0 end) 17 | {% elif datepart == 'week' %} 18 | ({{ datediff(first_date, second_date, 'day') }} / 7 + case 19 | when strftime('%w', {{first_date}}) <= strftime('%w', {{second_date}}) then 20 | case when {{first_date}} <= {{second_date}} then 0 else -1 end 21 | else 22 | case when {{first_date}} <= {{second_date}} then 1 else 0 end 23 | end) 24 | {% elif datepart == 'hour' %} 25 | {# ({{ datediff(first_date, second_date, 'day') }} * 24 + strftime("%H", {{second_date}}) - strftime("%H", {{first_date}})) #} 26 | (ceil(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 3600)) 27 | {% elif datepart == 'minute' %} 28 | {# ({{ datediff(first_date, second_date, 'hour') }} * 60 + strftime("%M", {{second_date}}) - strftime("%M", {{first_date}})) #} 29 | (ceil(cast(strftime('%s', {{second_date}}) - strftime('%s', {{first_date}}) as real) / 60)) 30 | {% elif datepart == 'second' %} 31 | (strftime('%s', {{second_date}}) - strftime('%s', {{first_date}})) 32 | {# 33 | {% elif datepart == 'millisecond' %} 34 | ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) 35 | {% elif datepart == 'microsecond' %} 36 | ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) 37 | #} 38 | {% else %} 39 | {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in sqlite: {!r}".format(datepart)) }} 40 | {% endif %} 41 | 42 | {%- endmacro %} 43 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/hash.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__hash(field) -%} 2 | case 3 | when {{ field }} is not null 4 | then lower(hex(md5(cast({{ field }} as {{ api.Column.translate_type('string') }})))) 5 | end 6 | {%- endmacro %} 7 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/position.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__position(substring_text, string_text) %} 2 | 3 | instr({{ string_text }}, {{ substring_text }}) 4 | 5 | {%- endmacro -%} 6 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/right.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__right(string_text, length_expression) %} 2 | case 3 | when {{ length_expression }} <> 0 then 4 | substr( 5 | {{ string_text }}, 6 | -1 * {{ length_expression }} 7 | ) 8 | else '' 9 | end 10 | {%- endmacro -%} 11 | -------------------------------------------------------------------------------- /dbt/include/sqlite/macros/utils/timestamps.sql: -------------------------------------------------------------------------------- 1 | {% macro sqlite__current_timestamp() -%} 2 | datetime() 3 | {%- endmacro %} 4 | 5 | {% macro sqlite__snapshot_string_as_time(timestamp) -%} 6 | {# just return the string; SQLite doesn''t have a timestamp data type per se #} 7 | {{ return("'" + timestamp|string + "'") }} 8 | {%- endmacro %} 9 | 10 | {% macro sqlite__snapshot_get_time() -%} 11 | datetime() 12 | {%- endmacro %} 13 | -------------------------------------------------------------------------------- /dbt/include/sqlite/sample_profiles.yml: -------------------------------------------------------------------------------- 1 | default: 2 | outputs: 3 | 4 | dev: 5 | type: sqlite 6 | threads: 1 7 | database: 8 | schema: 'main' 9 | schemas_and_paths: 10 | main: '/my_project/data/etl.db' 11 | schema_directory: '/my_project/data' 12 | # optional: list of file paths of SQLite extensions to load. see README for more details. 13 | # extensions: 14 | # - '/path/to/sqlean/crypto.so' 15 | # - '/path/to/sqlean/math.so' 16 | # - '/path/to/sqlean/text.so' 17 | 18 | prod: 19 | type: sqlite 20 | threads: 1 21 | database: 22 | schema: 'main' 23 | schemas_and_paths: 24 | main: '/my_project/data/etl.db' 25 | schema_directory: '/my_project/data' 26 | # optional: list of file paths of SQLite extensions to load. see README for more details. 27 | # extensions: 28 | # - '/path/to/sqlean/crypto.so' 29 | # - '/path/to/sqlean/math.so' 30 | # - '/path/to/sqlean/text.so' 31 | 32 | target: dev 33 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning 4 | ignore:unclosed file .*:ResourceWarning 5 | env_files = 6 | test.env # uses pytest-dotenv plugin 7 | # this allows you to store env vars for database connection in a file named test.env 8 | # rather than passing them in every CLI command, or setting in `PYTEST_ADDOPTS` 9 | # be sure to add "test.env" to .gitignore as well! 10 | testpaths = 11 | tests/functional # name per convention 12 | -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 6 | cd $SCRIPT_DIR 7 | 8 | # Leaving the database file between runs of pytest can mess up subsequent test runs. 9 | # Since this runs in a fresh container each time, it's not an issue. 10 | 11 | pytest tests/functional $@ 12 | 13 | #### 14 | 15 | # dbt-sqlite overrides some stuff pertaining to 'docs generate' 16 | # so exercise it using jaffle_shop repo 17 | 18 | # dbt-sqlite overrides some stuff pertaining to 'docs generate' 19 | # so exercise it using jaffle_shop repo 20 | 21 | cd $HOME 22 | 23 | git clone --depth 1 https://github.com/dbt-labs/jaffle_shop.git 24 | 25 | cd jaffle_shop 26 | 27 | mkdir -p /tmp/jaffle_shop 28 | 29 | mkdir -p $HOME/.dbt 30 | 31 | if [ -f $HOME/.dbt/profiles.yml ]; then 32 | echo "ERROR: profiles.yml already exists, refusing to overwrite it" 33 | exit 1 34 | fi 35 | 36 | cat >> $HOME/.dbt/profiles.yml </__version__.py 12 | def _get_plugin_version(): 13 | _version_path = os.path.join( 14 | this_directory, 'dbt', 'adapters', 'sqlite', '__version__.py' 15 | ) 16 | with open(_version_path) as f: 17 | line = f.read().strip() 18 | delim = '"' if '"' in line else "'" 19 | return line.split(delim)[1] 20 | 21 | 22 | package_name = "dbt-sqlite" 23 | package_version = _get_plugin_version() 24 | description = """A SQLite adapter plugin for dbt (data build tool)""" 25 | long_description = "Please see the github repository for detailed information" 26 | 27 | setup( 28 | name=package_name, 29 | version=package_version, 30 | description=description, 31 | long_description=long_description, 32 | author='Jeff Chiu', 33 | author_email='jeff@codefork.com', 34 | url='https://github.com/codeforkjeff/dbt-sqlite', 35 | packages=[ 36 | 'dbt.adapters.sqlite', 37 | 'dbt.include.sqlite', 38 | ], 39 | package_data={ 40 | 'dbt.include.sqlite': [ 41 | 'macros/*.sql', 42 | 'macros/**/*.sql', 43 | 'macros/**/**/*.sql', 44 | 'dbt_project.yml', 45 | 'sample_profiles.yml', 46 | ] 47 | }, 48 | install_requires=[ 49 | "dbt-core>=1.9.0", 50 | "dbt-core<2", 51 | "dbt-adapters~=1.13.1" 52 | ], 53 | classifiers=[ 54 | 'Development Status :: 4 - Beta', 55 | 56 | 'License :: OSI Approved :: Apache Software License', 57 | 58 | 'Operating System :: Microsoft :: Windows', 59 | 'Operating System :: MacOS :: MacOS X', 60 | 'Operating System :: POSIX :: Linux' 61 | ] 62 | ) 63 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | # Import the standard functional fixtures as a plugin 5 | # Note: fixtures with session scope need to be local 6 | pytest_plugins = ["dbt.tests.fixtures.project"] 7 | 8 | # The profile dictionary, used to write out profiles.yml 9 | # dbt will supply a unique schema per test, so we do not specify 'schema' here 10 | @pytest.fixture(scope="class") 11 | def dbt_profile_target(): 12 | return { 13 | 'type': 'sqlite', 14 | 'threads': 1, 15 | 'database': 'adapter_test', 16 | 'schema': 'main', 17 | 'schemas_and_paths': { 18 | 'main': '/opt/dbt-sqlite/testdata/adapter_test.db' 19 | }, 20 | 'schema_directory': '/opt/dbt-sqlite/testdata', 21 | 'extensions' : [ 22 | "/opt/dbt-sqlite/crypto.so", 23 | "/opt/dbt-sqlite/math.so", 24 | "/opt/dbt-sqlite/text.so" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /tests/functional/adapter/concurrency/test_concurrency.py: -------------------------------------------------------------------------------- 1 | from dbt.tests.util import ( 2 | run_dbt, 3 | check_relations_equal, 4 | rm_file, 5 | write_file 6 | ) 7 | from dbt.tests.adapter.concurrency.test_concurrency import ( 8 | BaseConcurrency, 9 | seeds__update_csv 10 | ) 11 | 12 | 13 | class TestConncurenncySqlite(BaseConcurrency): 14 | 15 | def test_conncurrency_sqlite(self, project): 16 | run_dbt(["seed", "--select", "seed"]) 17 | results = run_dbt(["run"], expect_pass=False) 18 | assert len(results) == 7 19 | check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"]) 20 | check_relations_equal(project.adapter, ["SEED", "DEP"]) 21 | check_relations_equal(project.adapter, ["SEED", "TABLE_A"]) 22 | check_relations_equal(project.adapter, ["SEED", "TABLE_B"]) 23 | 24 | rm_file(project.project_root, "seeds", "seed.csv") 25 | write_file(seeds__update_csv, project.project_root + '/seeds', "seed.csv") 26 | results = run_dbt(["run"], expect_pass=False) 27 | assert len(results) == 7 28 | check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"]) 29 | check_relations_equal(project.adapter, ["SEED", "DEP"]) 30 | check_relations_equal(project.adapter, ["SEED", "TABLE_A"]) 31 | check_relations_equal(project.adapter, ["SEED", "TABLE_B"]) 32 | -------------------------------------------------------------------------------- /tests/functional/adapter/ephemeral/test_ephemeral.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti 3 | from dbt.tests.util import run_dbt, check_relations_equal 4 | 5 | 6 | @pytest.mark.skip("started failing with dbt-core 1.9.0, not sure what's going on here") 7 | class TestEphemeralMultiSqlite(BaseEphemeralMulti): 8 | 9 | def test_ephemeral_multi_sqlite(self, project): 10 | run_dbt(["seed"]) 11 | results = run_dbt(["run"]) 12 | assert len(results) == 3 13 | check_relations_equal(project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"]) 14 | -------------------------------------------------------------------------------- /tests/functional/adapter/test_basic.py: -------------------------------------------------------------------------------- 1 | 2 | import glob 3 | import os 4 | 5 | import pytest 6 | 7 | from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog, no_stats, expected_references_catalog 8 | from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations 9 | from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests 10 | from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral 11 | from dbt.tests.adapter.basic.test_empty import BaseEmpty 12 | from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral 13 | from dbt.tests.adapter.basic.test_incremental import BaseIncremental 14 | from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange 15 | from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests 16 | from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols 17 | from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp 18 | from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod 19 | from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences, models__schema_yml, models__readme_md 20 | 21 | 22 | class TestSimpleMaterializationsSqlite(BaseSimpleMaterializations): 23 | pass 24 | 25 | 26 | class TestSingularTestsSqlite(BaseSingularTests): 27 | pass 28 | 29 | 30 | class TestSingularTestsEphemeralSqlite(BaseSingularTestsEphemeral): 31 | pass 32 | 33 | 34 | class TestEmptySqlite(BaseEmpty): 35 | pass 36 | 37 | 38 | class TestEphemeralSqlite(BaseEphemeral): 39 | pass 40 | 41 | 42 | class TestIncrementalSqlite(BaseIncremental): 43 | pass 44 | 45 | 46 | class TestBaseIncrementalNotSchemaChangeSqlite(BaseIncrementalNotSchemaChange): 47 | pass 48 | 49 | 50 | class TestGenericTestsSqlite(BaseGenericTests): 51 | pass 52 | 53 | 54 | class TestSnapshotCheckColsSqlite(BaseSnapshotCheckCols): 55 | pass 56 | 57 | 58 | class TestSnapshotTimestampSqlite(BaseSnapshotTimestamp): 59 | pass 60 | 61 | 62 | class TestBaseAdapterMethodSqlite(BaseAdapterMethod): 63 | pass 64 | 65 | 66 | class TestDocsGenerateSqlite(BaseDocsGenerate): 67 | """ 68 | Change underlying test to avoid having views referencing views in other schemas, which is a no-no in sqlite. 69 | """ 70 | 71 | models__model_sql = """ 72 | {{ 73 | config( 74 | materialized='table', 75 | ) 76 | }} 77 | 78 | select * from {{ ref('seed') }} 79 | """ 80 | 81 | models__second_model_sql = """ 82 | {{ 83 | config( 84 | materialized='table', 85 | schema='test', 86 | ) 87 | }} 88 | 89 | select * from {{ ref('seed') }} 90 | """ 91 | 92 | @pytest.fixture(scope="class") 93 | def models(self): 94 | # replace models with 95 | return { 96 | "schema.yml": models__schema_yml, 97 | "second_model.sql": self.models__second_model_sql, 98 | "readme.md": models__readme_md, 99 | "model.sql": self.models__model_sql, 100 | } 101 | 102 | @pytest.fixture(scope="class") 103 | def expected_catalog(self, project): 104 | expected_catalog = base_expected_catalog( 105 | project, 106 | role=None, 107 | id_type="INT", 108 | text_type="TEXT", 109 | time_type="TEXT", 110 | view_type="view", 111 | table_type="table", 112 | model_stats=no_stats(), 113 | seed_stats=no_stats(), 114 | ) 115 | 116 | # patch 117 | expected_catalog['nodes']['model.test.model']['metadata']['type']='table' 118 | expected_catalog['nodes']['model.test.second_model']['metadata']['type']='table' 119 | 120 | return expected_catalog 121 | 122 | 123 | @pytest.mark.skip("TODO: not sure why 'index' values are off by 1") 124 | class TestDocsGenReferencesSqlite(BaseDocsGenReferences): 125 | 126 | @pytest.fixture(scope="class") 127 | def expected_catalog(self, project): 128 | return expected_references_catalog( 129 | project, 130 | role=None, 131 | id_type="INT", 132 | text_type="TEXT", 133 | time_type="TEXT", 134 | bigint_type="bigint", 135 | view_type="view", 136 | table_type="table", 137 | model_stats=no_stats(), 138 | #seed_stats=no_stats(), 139 | #view_summary_stats=no_stats(), 140 | ) 141 | -------------------------------------------------------------------------------- /tests/functional/adapter/utils/test_data_types.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt 3 | from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean 4 | from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat 5 | from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt 6 | from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric 7 | from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString 8 | from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp 9 | 10 | # These tests compare the resulting column types of CASTs against the types 11 | # inferred by agate when loading seeds. 12 | # 13 | # There's a Column class in dbt-core that's used by the default adapter implementation 14 | # of methods like [adapter].type_timestamp() to get a type for CASTs. 15 | # 16 | # Some quirks of SQLite that make these tests challenging: 17 | # 18 | # - a CAST seems to always result in an empty type (i.e. no type affinity) in views, 19 | # but not in a CREATE TABLE AS. So we tweak the tests to materialize models as tables. 20 | # 21 | # - CASTs to an unrecognized type will result in the type being 'NUM' which is a bit 22 | # mysterious. 23 | 24 | class TestTypeBigInt(BaseTypeBigInt): 25 | pass 26 | 27 | 28 | # users should imlement boolean columns as INT with values of 0 or 1 29 | @pytest.mark.skip("boolean not supported in SQLite") 30 | class TestTypeBoolean(BaseTypeBoolean): 31 | pass 32 | 33 | 34 | class TestTypeFloat(BaseTypeFloat): 35 | 36 | models__actual_sql = """ 37 | {{ config(materialized='table') }} 38 | 39 | select cast('1.2345' as {{ type_float() }}) as float_col 40 | """ 41 | 42 | @pytest.fixture(scope="class") 43 | def models(self): 44 | return {"actual.sql": self.interpolate_macro_namespace(self.models__actual_sql, "type_float")} 45 | 46 | 47 | class TestTypeInt(BaseTypeInt): 48 | 49 | models__actual_sql = """ 50 | {{ config(materialized='table') }} 51 | 52 | select cast('12345678' as {{ type_int() }}) as int_col 53 | """ 54 | 55 | @pytest.fixture(scope="class") 56 | def models(self): 57 | return {"actual.sql": self.interpolate_macro_namespace(self.models__actual_sql, "type_int")} 58 | 59 | 60 | class TestTypeNumeric(BaseTypeNumeric): 61 | 62 | models__actual_sql = """ 63 | {{ config(materialized='table') }} 64 | 65 | select cast('1.2345' as {{ type_numeric() }}) as numeric_col 66 | """ 67 | 68 | @pytest.fixture(scope="class") 69 | def models(self): 70 | return {"actual.sql": self.interpolate_macro_namespace(self.models__actual_sql, "type_numeric")} 71 | 72 | def numeric_fixture_type(self): 73 | return "NUM" 74 | 75 | 76 | class TestTypeString(BaseTypeString): 77 | 78 | models__actual_sql = """ 79 | {{ config(materialized='table') }} 80 | 81 | select cast('Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.' 82 | as {{ type_string() }}) as string_col 83 | """ 84 | 85 | @pytest.fixture(scope="class") 86 | def models(self): 87 | return {"actual.sql": self.interpolate_macro_namespace(self.models__actual_sql, "type_string")} 88 | 89 | 90 | # casting to TIMESTAMP results in an 'NUM' type which truncates the original value 91 | # to only the year portion. It's up to the user to properly deal with timestamps 92 | # values from source tables. 93 | @pytest.mark.skip("timestamp not supported in SQLite") 94 | class TestTypeTimestamp(BaseTypeTimestamp): 95 | pass 96 | -------------------------------------------------------------------------------- /tests/functional/adapter/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dbt.tests.adapter.utils.base_utils import BaseUtils 3 | from dbt.tests.adapter.utils.fixture_datediff import ( 4 | seeds__data_datediff_csv, 5 | models__test_datediff_yml, 6 | ) 7 | from dbt.tests.adapter.utils.fixture_dateadd import ( 8 | seeds__data_dateadd_csv, 9 | models__test_dateadd_yml, 10 | ) 11 | from dbt.tests.adapter.utils.test_any_value import BaseAnyValue 12 | from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend 13 | from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat 14 | from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct 15 | from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr 16 | from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText 17 | from dbt.tests.adapter.utils.test_concat import BaseConcat 18 | from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive 19 | from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd 20 | #from dbt.tests.adapter.utils.test_datediff import BaseDateDiff 21 | from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc 22 | from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote 23 | from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesBackslash 24 | from dbt.tests.adapter.utils.test_except import BaseExcept 25 | from dbt.tests.adapter.utils.test_hash import BaseHash 26 | from dbt.tests.adapter.utils.test_intersect import BaseIntersect 27 | from dbt.tests.adapter.utils.test_last_day import BaseLastDay 28 | from dbt.tests.adapter.utils.test_length import BaseLength 29 | from dbt.tests.adapter.utils.test_listagg import BaseListagg 30 | from dbt.tests.adapter.utils.test_position import BasePosition 31 | from dbt.tests.adapter.utils.test_replace import BaseReplace 32 | from dbt.tests.adapter.utils.test_right import BaseRight 33 | from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast 34 | from dbt.tests.adapter.utils.test_split_part import BaseSplitPart 35 | from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral 36 | from dbt.tests.adapter.utils.test_equals import BaseEquals 37 | from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare, BaseNullCompare 38 | from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod 39 | 40 | 41 | class TestAnyValue(BaseAnyValue): 42 | pass 43 | 44 | 45 | @pytest.mark.skip("arrays not supported in SQLite") 46 | class TestArrayAppend(BaseArrayAppend): 47 | pass 48 | 49 | 50 | @pytest.mark.skip("arrays not supported in SQLite") 51 | class TestArrayConcat(BaseArrayConcat): 52 | pass 53 | 54 | 55 | @pytest.mark.skip("arrays not supported in SQLite") 56 | class TestArrayConstruct(BaseArrayConstruct): 57 | pass 58 | 59 | 60 | class TestBoolOr(BaseBoolOr): 61 | pass 62 | 63 | 64 | class TestCastBoolToText(BaseCastBoolToText): 65 | pass 66 | 67 | 68 | class TestConcat(BaseConcat): 69 | pass 70 | 71 | 72 | @pytest.mark.skip("timestamps not supported in SQLite") 73 | class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): 74 | pass 75 | 76 | class BaseDateAdd(BaseUtils): 77 | 78 | models__test_dateadd_sql = """ 79 | with data as ( 80 | select * from {{ ref('data_dateadd') }} 81 | ) 82 | 83 | select 84 | {{ dateadd('datepart', 'interval_length', 'from_time') }} AS actual, 85 | result as expected 86 | from data 87 | """ 88 | 89 | @pytest.fixture(scope="class") 90 | def seeds(self): 91 | return {"data_dateadd.csv": seeds__data_dateadd_csv} 92 | 93 | @pytest.fixture(scope="class") 94 | def models(self): 95 | return { 96 | "test_dateadd.yml": models__test_dateadd_yml, 97 | "test_dateadd.sql": self.interpolate_macro_namespace( 98 | self.models__test_dateadd_sql, "dateadd" 99 | ), 100 | } 101 | 102 | 103 | class TestDateAdd(BaseDateAdd): 104 | pass 105 | 106 | 107 | class BaseDateDiff(BaseUtils): 108 | 109 | models__test_datediff_sql = """ 110 | with data as ( 111 | 112 | select * from {{ ref('data_datediff') }} 113 | 114 | ) 115 | 116 | select 117 | 118 | case 119 | when datepart = 'second' then {{ datediff('first_date', 'second_date', 'second') }} 120 | when datepart = 'minute' then {{ datediff('first_date', 'second_date', 'minute') }} 121 | when datepart = 'hour' then {{ datediff('first_date', 'second_date', 'hour') }} 122 | when datepart = 'day' then {{ datediff('first_date', 'second_date', 'day') }} 123 | when datepart = 'week' then {{ datediff('first_date', 'second_date', 'week') }} 124 | when datepart = 'month' then {{ datediff('first_date', 'second_date', 'month') }} 125 | when datepart = 'year' then {{ datediff('first_date', 'second_date', 'year') }} 126 | else null 127 | end as actual, 128 | result as expected 129 | 130 | from data 131 | 132 | -- Also test correct casting of literal values. 133 | 134 | -- sqlite implementation of datediff doesn't support microsecond or quarter 135 | 136 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected 137 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected 138 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected 139 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected 140 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected 141 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected 142 | union all select {{ datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected 143 | """ 144 | 145 | 146 | @pytest.fixture(scope="class") 147 | def seeds(self): 148 | return {"data_datediff.csv": seeds__data_datediff_csv} 149 | 150 | @pytest.fixture(scope="class") 151 | def models(self): 152 | return { 153 | "test_datediff.yml": models__test_datediff_yml, 154 | "test_datediff.sql": self.interpolate_macro_namespace( 155 | self.models__test_datediff_sql, "datediff" 156 | ), 157 | } 158 | 159 | 160 | @pytest.mark.skip("TODO: implement datediff") 161 | class TestDateDiff(BaseDateDiff): 162 | pass 163 | 164 | 165 | @pytest.mark.skip("TODO: implement date_trunc") 166 | class TestDateTrunc(BaseDateTrunc): 167 | pass 168 | 169 | 170 | class TestEscapeSingleQuotes(BaseEscapeSingleQuotesQuote): 171 | pass 172 | 173 | 174 | class TestExcept(BaseExcept): 175 | pass 176 | 177 | 178 | class TestHash(BaseHash): 179 | pass 180 | 181 | 182 | class TestIntersect(BaseIntersect): 183 | pass 184 | 185 | 186 | @pytest.mark.skip("TODO: implement lastday") 187 | class TestLastDay(BaseLastDay): 188 | pass 189 | 190 | 191 | class TestLength(BaseLength): 192 | pass 193 | 194 | 195 | @pytest.mark.skip("TODO: implement listagg") 196 | class TestListagg(BaseListagg): 197 | pass 198 | 199 | 200 | class TestPosition(BasePosition): 201 | pass 202 | 203 | 204 | class TestReplace(BaseReplace): 205 | pass 206 | 207 | 208 | class TestRight(BaseRight): 209 | pass 210 | 211 | 212 | class TestSafeCast(BaseSafeCast): 213 | pass 214 | 215 | @pytest.mark.skip("TODO: implement split_part, either using sqlite>=3.8.3 for WITH RECURSIVE support, or possibly sooner using jinja and agate tables") 216 | class TestSplitPart(BaseSplitPart): 217 | pass 218 | 219 | class TestStringLiteral(BaseStringLiteral): 220 | pass 221 | 222 | class TestEquals(BaseEquals): 223 | pass 224 | 225 | class TestMixedNullCompare(BaseMixedNullCompare): 226 | pass 227 | 228 | class TestNullCompare(BaseNullCompare): 229 | pass 230 | 231 | class TestValidateSqlMethod(BaseValidateSqlMethod): 232 | pass --------------------------------------------------------------------------------