├── .dockerignore ├── .github └── workflows │ ├── code-quality.yml │ └── docker-tests.yml ├── .gitignore ├── DEVELOPERS.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docker-compose.yaml ├── docs └── superset.png ├── pyproject.toml ├── release.sh ├── src ├── examples │ ├── __init__.py │ ├── hello_world.py │ ├── psycopg2_connect.py │ ├── server_utilisation.py │ ├── sqlalchemy_orm.py │ └── sqlalchemy_raw.py ├── qdb_superset │ ├── __init__.py │ └── db_engine_specs │ │ ├── __init__.py │ │ └── questdb.py └── questdb_connect │ ├── __init__.py │ ├── common.py │ ├── compilers.py │ ├── dialect.py │ ├── dml.py │ ├── identifier_preparer.py │ ├── inspector.py │ ├── keywords_functions.py │ ├── table_engine.py │ └── types.py └── tests ├── __init__.py ├── conftest.py ├── test_dialect.py ├── test_examples.py ├── test_superset.py ├── test_types.py └── test_username.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .git 3 | .github 4 | .gitignore 5 | .pytest_cache 6 | .dockerignore 7 | .python-version 8 | .questdb_data 9 | venv 10 | Dockerfile 11 | questdb_connect.egg-info 12 | superset_toolkit 13 | Makefile 14 | dist 15 | -------------------------------------------------------------------------------- /.github/workflows/code-quality.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Code Quality 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | permissions: 13 | contents: read 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python 3.10 21 | uses: actions/setup-python@v3 22 | with: 23 | python-version: "3.10" 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install . 28 | pip install '.[test]' 29 | - name: ruff 30 | run: | 31 | python -m ruff check src/questdb_connect 32 | python -m ruff check src/examples 33 | python -m ruff check tests 34 | -------------------------------------------------------------------------------- /.github/workflows/docker-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Docker Image 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | permissions: 13 | contents: read 14 | 15 | jobs: 16 | docker: 17 | runs-on: ubuntu-latest 18 | services: 19 | registry: 20 | image: registry:2 21 | ports: 22 | - 5000:5000 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3.6.1 28 | with: 29 | # network=host driver-opt needed to push to local registry 30 | driver-opts: network=host 31 | - name: Run compose 32 | uses: isbang/compose-action@v2.0.1 33 | with: 34 | compose-file: "docker-compose.yaml" 35 | down-flags: "--volumes" 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache 2 | .questdb_data 3 | .python-version 4 | venv 5 | .idea 6 | .env 7 | *.pyc 8 | *.pyd 9 | questdb_connect.egg-info 10 | __pycache__ 11 | dist 12 | build 13 | -------------------------------------------------------------------------------- /DEVELOPERS.md: -------------------------------------------------------------------------------- 1 | ## Developer installation 2 | 3 | If you want to contribute to this repository, you need to setup a local virtual `venv` environment: 4 | 5 | ```shell 6 | python3 -m venv venv 7 | source venv/bin/activate 8 | pip install -U pip 9 | pip install -e . 10 | pip install -e '.[test]' 11 | ``` 12 | 13 | _questdb-connect_ does not have dependencies to other modules, it relies on the user to have installed 14 | **psycopg2** and **SQLAlchemy**. When developing however, installing the `.[test]` dependencies takes 15 | care of this. 16 | 17 | [QuestDB 7.1.2](https://github.com/questdb/questdb/releases/tag/7.1.2), or higher, is required because 18 | it has support for `implicit cast String -> Long256`, and must be up and running. 19 | 20 | You can develop in your preferred IDE, and run `make test` in a terminal to check linting 21 | (with [ruff](https://github.com/charliermarsh/ruff/)) and run the tests. Before pushing a 22 | commit to the main branch, make sure you build the docker container and that the container 23 | tests pass: 24 | 25 | ```shell 26 | make test 27 | make 28 | make docker-test 29 | ``` 30 | 31 | Note: `make` by itself builds the docker image, then you can call `make docker-test` to run 32 | the tests in docker. `make test` runs the tests locally and it is quicker, however CI only 33 | runs the docker version. 34 | 35 | ## Install/Run Apache Superset from repo 36 | 37 | These are instructions to have a running superset suitable for development. 38 | 39 | You need to clone [superset's](https://github.com/apache/superset) repository. You can follow the 40 | [instructions](https://superset.apache.org/docs/installation/installing-superset-from-scratch/), 41 | which roughly equate to (depending on your environment): 42 | 43 | - Edit file `docker/pythonpath_dev/superset_config.py` to define yout _SECRET_KEY_: 44 | ```python 45 | SUPERSET_SECRET_KEY="yourParticularSecretKeyAndMakeSureItIsSecureUnlikeThisOne" 46 | SECRET_KEY=SUPERSET_SECRET_KEY 47 | ``` 48 | - Create file `docker/requirements-local.txt` and add: 49 | ```shell 50 | questdb-connect= 51 | ``` 52 | 53 | - And then run Apache Superset in developer mode (this takes a while): 54 | ```shell 55 | docker-compose up 56 | ``` 57 | - Open a browser [http://localhost:8088](http://localhost:8088) 58 | 59 | To update `questdb-connect`: 60 | 61 | 1. `docker-compose down -v` 62 | 2. Update the version in file `./docker/requirements-local.txt` 63 | 3. `docker-compose up` 64 | 65 | While running, the server will reload on modification of the Python and JavaScript source code. 66 | 67 | This will be the URI for QuestDB: 68 | 69 | ```shell 70 | questdb://admin:quest@host.docker.internal:8812/main 71 | ``` 72 | 73 | ## Build questdb-connect wheel and publish it 74 | 75 | Follow the guidelines 76 | in [https://packaging.python.org/en/latest/tutorials/packaging-projects/](https://packaging.python.org/en/latest/tutorials/packaging-projects/). 77 | 78 | ```shell 79 | python3 -m pip install --upgrade build 80 | python3 -m pip install --upgrade twine 81 | 82 | python3 -m build 83 | python3 -m twine upload dist/* 84 | ``` 85 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim-buster 2 | ENV ARCHITECTURE=x64 3 | ENV PYTHONDONTWRITEBYTECODE 1 # Keeps Python from generating .pyc files in the container 4 | ENV PYTHONUNBUFFERED 1 # Turns off buffering for easier container logging 5 | ENV SQLALCHEMY_SILENCE_UBER_WARNING 1 # because we really should upgrade to SQLAlchemy 2.x 6 | ENV QUESTDB_CONNECT_HOST "host.docker.internal" 7 | 8 | RUN apt-get -y update 9 | RUN apt-get -y upgrade 10 | RUN apt-get -y --no-install-recommends install syslog-ng ca-certificates vim procps unzip less tar gzip iputils-ping gcc build-essential 11 | RUN apt-get clean 12 | RUN rm -rf /var/lib/apt/lists/* 13 | 14 | COPY . /app 15 | WORKDIR /app 16 | RUN pip install -U pip && pip install psycopg2-binary 'SQLAlchemy<=1.4.47' . 17 | CMD ["python", "src/examples/sqlalchemy_orm.py"] 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .DEFAULT_GOAL := docker-build 3 | 4 | IMAGE ?= questdb/questdb-connect 5 | SHORT_COMMIT_HASH ?= $(shell git rev-parse --short HEAD) 6 | BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) 7 | 8 | DB_HOSTNAME ?= 127.0.0.1 9 | 10 | ifeq (, $(shell which docker)) 11 | $(error "No docker in $(PATH), consider checking out https://docker.io/ for info") 12 | endif 13 | 14 | ifeq (, $(shell which curl)) 15 | $(error "No curl in $(PATH)") 16 | endif 17 | 18 | docker-build: 19 | docker build -t questdb/questdb-connect:latest . 20 | 21 | docker-run: 22 | docker run -it questdb/questdb-connect:latest 23 | 24 | docker-push: 25 | docker push questdb/questdb-connect:latest 26 | 27 | compose-up: 28 | docker-compose up 29 | 30 | compose-down: 31 | docker-compose down --remove-orphans 32 | echo "y" | docker container prune 33 | echo "y" | docker volume prune 34 | 35 | docker-test: 36 | docker run -e QUESTDB_CONNECT_HOST='host.docker.internal' -e SQLALCHEMY_SILENCE_UBER_WARNING=1 questdb/questdb-connect:latest 37 | 38 | test: 39 | python3 -m pytest 40 | python3 -m black src 41 | python3 -m ruff check src/questdb_connect --fix 42 | python3 -m ruff check src/examples --fix 43 | python3 -m ruff check tests --fix 44 | 45 | -include ../Mk/phonies 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | QuestDB Logo 3 | 4 |

5 | 6 | QuestDB community Slack channel 7 | 8 | 9 | ## QuestDB Connect 10 | 11 | This repository contains the official implementation of QuestDB's dialect for [SQLAlchemy](https://www.sqlalchemy.org/), 12 | as well as an engine specification for [Apache Superset](https://github.com/apache/superset/), using 13 | [psycopg2](https://www.psycopg.org/) for database connectivity. 14 | 15 | The Python module is available here: 16 | 17 | 18 | PyPi 19 | https://pypi.org/project/questdb-connect/ 20 | 21 |

22 | 23 | _Psycopg2_ is a widely used and trusted Python module for connecting to, and working with, QuestDB and other 24 | PostgreSQL databases. 25 | 26 | _SQLAlchemy_ is a SQL toolkit and ORM library for Python. It provides a high-level API for communicating with 27 | relational databases, including schema creation and modification. The ORM layer abstracts away the complexities 28 | of the database, allowing developers to work with Python objects instead of raw SQL statements. 29 | 30 | _Apache Superset_ is an open-source business intelligence web application that enables users to visualize and 31 | explore data through customizable dashboards and reports. It provides a rich set of data visualizations, including 32 | charts, tables, and maps. 33 | 34 | ## Requirements 35 | 36 | * **Python from 3.9 to 3.11** (superset itself use version _3.9.x_) 37 | * **Psycopg2** `('psycopg2-binary~=2.9.6')` 38 | * **SQLAlchemy** `('SQLAlchemy>=1.4')` 39 | 40 | You need to install these packages because questdb-connect depends on them. Note that `questdb-connect` v1.1 41 | is compatible with both `SQLAlchemy` v1.4 and v2.0 while `questdb-connect` v1.0 is compatible with `SQLAlchemy` v1.4 only. 42 | 43 | ## Versions 0.0.X 44 | 45 | These are versions released for testing purposes. 46 | 47 | ## Installation 48 | 49 | You can install this package using pip: 50 | 51 | ```shell 52 | pip install questdb-connect 53 | ``` 54 | 55 | ## SQLAlchemy Sample Usage 56 | 57 | Use the QuestDB dialect by specifying it in your SQLAlchemy connection string: 58 | 59 | ```shell 60 | questdb://admin:quest@localhost:8812/main 61 | questdb://admin:quest@host.docker.internal:8812/main 62 | ``` 63 | 64 | From that point on use standard SQLAlchemy. Example with raw SQL API: 65 | ```python 66 | import datetime 67 | import time 68 | import uuid 69 | from sqlalchemy import create_engine, text 70 | 71 | def main(): 72 | engine = create_engine('questdb://admin:quest@localhost:8812/main') 73 | 74 | with engine.begin() as connection: 75 | # Create the table 76 | connection.execute(text(""" 77 | CREATE TABLE IF NOT EXISTS signal ( 78 | source SYMBOL, 79 | value DOUBLE, 80 | ts TIMESTAMP, 81 | uuid UUID 82 | ) TIMESTAMP(ts) PARTITION BY HOUR WAL; 83 | """)) 84 | 85 | # Insert 2 rows 86 | connection.execute(text(""" 87 | INSERT INTO signal (source, value, ts, uuid) VALUES 88 | (:source1, :value1, :ts1, :uuid1), 89 | (:source2, :value2, :ts2, :uuid2) 90 | """), { 91 | 'source1': 'coconut', 'value1': 16.88993244, 'ts1': datetime.datetime.utcnow(), 'uuid1': uuid.uuid4(), 92 | 'source2': 'banana', 'value2': 3.14159265, 'ts2': datetime.datetime.utcnow(), 'uuid2': uuid.uuid4() 93 | }) 94 | 95 | # WAL is applied asynchronously, so we need to wait for it to be applied before querying 96 | time.sleep(1) 97 | 98 | # Start a new transaction 99 | with engine.begin() as connection: 100 | # Query the table for rows where value > 10 101 | result = connection.execute( 102 | text("SELECT source, value, ts, uuid FROM signal WHERE value > :value"), 103 | {'value': 10} 104 | ) 105 | for row in result: 106 | print(row.source, row.value, row.ts, row.uuid) 107 | 108 | 109 | if __name__ == '__main__': 110 | main() 111 | ``` 112 | 113 | Alternatively, you can use the ORM API: 114 | ```python 115 | import datetime 116 | import uuid 117 | import time 118 | from questdb_connect import Symbol, PartitionBy, UUID, Double, Timestamp, QDBTableEngine 119 | from sqlalchemy import Column, MetaData, create_engine, text 120 | from sqlalchemy.orm import declarative_base, sessionmaker 121 | 122 | Base = declarative_base(metadata=MetaData()) 123 | 124 | 125 | class Signal(Base): 126 | # Stored in a QuestDB table 'signal'. The tables has WAL enabled, is partitioned by hour, designated timestamp is 'ts' 127 | __tablename__ = 'signal' 128 | __table_args__ = (QDBTableEngine(None, 'ts', PartitionBy.HOUR, is_wal=True),) 129 | source = Column(Symbol) 130 | value = Column(Double) 131 | ts = Column(Timestamp) 132 | uuid = Column(UUID, primary_key=True) 133 | 134 | def __repr__(self): 135 | return f"Signal(source={self.source}, value={self.value}, ts={self.ts}, uuid={self.uuid})" 136 | 137 | 138 | def main(): 139 | engine = create_engine('questdb://admin:quest@localhost:8812/main') 140 | 141 | # Create the table 142 | Base.metadata.create_all(engine) 143 | Session = sessionmaker(bind=engine) 144 | session = Session() 145 | 146 | # Insert 2 rows 147 | session.add(Signal( 148 | source='coconut', 149 | value=16.88993244, 150 | ts=datetime.datetime.utcnow(), 151 | uuid=uuid.uuid4() 152 | )) 153 | 154 | session.add(Signal( 155 | source='banana', 156 | value=3.14159265, 157 | ts=datetime.datetime.utcnow(), 158 | uuid=uuid.uuid4() 159 | )) 160 | session.commit() 161 | 162 | # WAL is applied asynchronously, so we need to wait for it to be applied before querying 163 | time.sleep(1) 164 | 165 | # Query the table for rows where value > 10 166 | signals = session.query(Signal).filter(Signal.value > 10).all() 167 | for signal in signals: 168 | print(signal.source, signal.value, signal.ts, signal.uuid) 169 | 170 | 171 | if __name__ == '__main__': 172 | main() 173 | ``` 174 | ORM (Object-Relational Mapping) API is not recommended for QuestDB due to its fundamental 175 | design differences from traditional transactional databases. While ORMs excel at managing 176 | relationships and complex object mappings in systems like PostgreSQL or MySQL, QuestDB is 177 | specifically optimized for time-series data operations and high-performance ingestion. It 178 | intentionally omits certain SQL features that ORMs typically rely on, such as generated 179 | columns, foreign keys, and complex joins, in favor of time-series-specific optimizations. 180 | 181 | For optimal performance and to fully leverage QuestDB's capabilities, we strongly recommend 182 | using the raw SQL API, which allows direct interaction with QuestDB's time-series-focused 183 | query engine and provides better control over time-based operations. 184 | 185 | ## Primary Key Considerations 186 | 187 | QuestDB differs from traditional relational databases in its handling of data uniqueness. While most databases enforce 188 | primary keys to guarantee unique record identification, QuestDB operates differently due to its time-series optimized 189 | architecture. 190 | 191 | When using SQLAlchemy with QuestDB: 192 | - You can define primary keys in your SQLAlchemy models, but QuestDB won't enforce uniqueness for individual columns 193 | - Duplicate rows with identical primary key values can exist in the database 194 | - Data integrity must be managed at the application level 195 | - QuestDB support [deduplication](https://questdb.io/docs/concept/deduplication/) during ingestion to avoid data duplication, this can be enabled in the table creation 196 | 197 | ### Recommended Approaches 198 | 199 | 1. **Composite Keys + QuestDB Deduplication** 200 | 201 | Composite keys can be used to define uniqueness based on multiple columns. This approach: 202 | - Can combine timestamp with any number of additional columns 203 | - Works with QuestDB's deduplication capabilities 204 | - Useful for scenarios where uniqueness is defined by multiple attributes 205 | - Common combinations might include: 206 | * timestamp + device_id + metric_type 207 | * timestamp + location + sensor_id 208 | * timestamp + instrument_id + exchange + side 209 | 210 | Deduplication is often enabled in QuestDB regardless of the primary key definition since 211 | it's required to avoid data duplication during ingestion. 212 | 213 | Example: 214 | ```python 215 | from questdb_connect import QDBTableEngine, PartitionBy, Double, Timestamp, Symbol 216 | class Measurement(Base): 217 | __tablename__ = 'signal' 218 | __table_args__ = (QDBTableEngine(None, 'timestamp', PartitionBy.HOUR, is_wal=True),) 219 | timestamp = Column(Timestamp, primary_key=True) 220 | sensor_id = Column(Symbol, primary_key=True) 221 | location = Column(Symbol, primary_key=True) 222 | value = Column(Double) 223 | ``` 224 | 225 | 226 | Choose your approach based on your data model and whether you need to leverage QuestDB's deduplication capabilities. 227 | 228 | 2. **UUID-based Identification** 229 | 230 | UUIDs are ideal for QuestDB applications because they: 231 | - Are globally unique across distributed systems 232 | - Can be generated client-side without database coordination 233 | - Work well with high-throughput data ingestion 234 | 235 | Example: 236 | ```python 237 | from questdb_connect import Symbol, PartitionBy, UUID, Double, Timestamp, QDBTableEngine 238 | class Signal(Base): 239 | __tablename__ = 'signal' 240 | __table_args__ = (QDBTableEngine(None, 'ts', PartitionBy.HOUR, is_wal=True),) 241 | source = Column(Symbol) 242 | value = Column(Double) 243 | ts = Column(Timestamp) 244 | uuid = Column(UUID, primary_key=True) 245 | # other columns... 246 | ``` 247 | 248 | ## Superset Installation 249 | This repository also contains an engine specification for Apache Superset, which allows you to connect 250 | to QuestDB from within the Superset interface. 251 | 252 | Apache Superset 253 | 254 | 255 | Follow the official [QuestDB Superset guide](https://questdb.io/docs/third-party-tools/superset/) available on the 256 | QuestDB website to install and configure the QuestDB engine in Superset. 257 | 258 | ## Contributing 259 | 260 | This package is open-source, contributions are welcome. If you find a bug or would like to request a feature, 261 | please open an issue on the GitHub repository. Have a look at the instructions for [developers](DEVELOPERS.md) 262 | if you would like to push a PR. 263 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | questdb: 3 | image: questdb/questdb:latest 4 | container_name: questdb_connect 5 | ports: 6 | - "8812:8812" 7 | - "9000:9000" 8 | - "9009:9009" 9 | networks: 10 | - questdb 11 | volumes: 12 | - ./.questdb_data:/root/.questdb/db 13 | 14 | questdb-connect: 15 | container_name: questdb_connect_tests 16 | build: . 17 | environment: 18 | QUESTDB_CONNECT_HOST: "questdb" 19 | depends_on: 20 | - questdb 21 | networks: 22 | - questdb 23 | 24 | networks: 25 | questdb: -------------------------------------------------------------------------------- /docs/superset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/questdb/questdb-connect/9456033982fc2091adc7824d1769111084a2afb1/docs/superset.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | # https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ 3 | name = 'questdb-connect' 4 | version = '1.1.5' # Standalone production version (with engine) 5 | # version = '0.0.113' # testing version 6 | authors = [{ name = 'questdb.io', email = 'support@questdb.io' }] 7 | description = "SqlAlchemy library" 8 | readme = "README.md" 9 | requires-python = ">=3.9" 10 | classifiers = [ 11 | 'Intended Audience :: Developers', 12 | 'Topic :: Database', 13 | 'License :: OSI Approved :: Apache Software License', 14 | 'Programming Language :: Python :: 3.9', 15 | 'Programming Language :: Python :: 3.10', 16 | 'Programming Language :: Python :: 3.11', 17 | ] 18 | dependencies = [] 19 | 20 | [project.urls] 21 | 'Homepage' = "https://github.com/questdb/questdb-connect/" 22 | 'Bug Tracker' = "https://github.com/questdb/questdb-connect/issues/" 23 | 'QuestDB GitHub' = "https://github.com/questdb/questdb/" 24 | 'QuestDB Docs' = "https://questdb.io/docs/" 25 | 26 | [project.entry-points.'sqlalchemy.dialects'] 27 | questdb = 'questdb_connect.dialect:QuestDBDialect' 28 | 29 | [project.entry-points.'superset.db_engine_specs'] 30 | questdb = 'qdb_superset.db_engine_specs.questdb:QuestDbEngineSpec' 31 | 32 | [project.optional-dependencies] 33 | test = [ 34 | 'psycopg2-binary~=2.9.6', 35 | 'SQLAlchemy>=1.4, <2', 36 | 'apache-superset>=3.0.0', 37 | 'sqlparse==0.4.4', 38 | 'pytest~=7.3.0', 39 | 'pytest_mock~=3.11.1', 40 | 'black~=23.3.0', 41 | 'ruff~=0.0.269', 42 | ] 43 | 44 | [tool.ruff] 45 | # https://github.com/charliermarsh/ruff#configuration 46 | select = ["PL", "RUF", "TCH", "TID", "PT", "C4", "B", "S", "I"] 47 | line-length = 120 48 | exclude = [ 49 | ".pytest_cache", 50 | ".questdb_data", 51 | ".git", 52 | ".ruff_cache", 53 | "venv", 54 | "dist", 55 | "questdb_connect.egg-info", 56 | ] 57 | 58 | [tool.ruff.pylint] 59 | max-branches = 20 60 | max-args = 10 61 | 62 | [tool.ruff.per-file-ignores] 63 | 'tests/test_dialect.py' = ['S101', 'PLR2004'] 64 | 'tests/test_types.py' = ['S101'] 65 | 'tests/test_superset.py' = ['S101'] 66 | 'tests/conftest.py' = ['S608'] 67 | 'src/examples/sqlalchemy_raw.py' = ['S608'] 68 | 'src/examples/server_utilisation.py' = ['S311'] 69 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | rm -rf dist 5 | python3 -m build 6 | twine upload -r questdb-connect dist/* 7 | -------------------------------------------------------------------------------- /src/examples/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | CONNECTION_ATTRS = { 4 | "host": os.environ.get("QUESTDB_CONNECT_HOST", "localhost"), 5 | "port": int(os.environ.get("QUESTDB_CONNECT_PORT", "8812")), 6 | "username": os.environ.get("QUESTDB_CONNECT_USER", "admin"), 7 | "password": os.environ.get("QUESTDB_CONNECT_PASSWORD", "quest"), 8 | } 9 | -------------------------------------------------------------------------------- /src/examples/hello_world.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | os.environ.setdefault("SQLALCHEMY_SILENCE_UBER_WARNING", "1") 5 | 6 | import questdb_connect as qdbc 7 | from sqlalchemy import Column, MetaData, create_engine, insert 8 | from sqlalchemy.orm import declarative_base 9 | 10 | Base = declarative_base(metadata=MetaData()) 11 | 12 | 13 | class Signal(Base): 14 | __tablename__ = "signal" 15 | __table_args__ = ( 16 | qdbc.QDBTableEngine("signal", "ts", qdbc.PartitionBy.HOUR, is_wal=True), 17 | ) 18 | source = Column(qdbc.Symbol(capacity=1024, cache=False)) 19 | value = Column(qdbc.Double) 20 | ts = Column(qdbc.Timestamp, primary_key=True) 21 | 22 | 23 | def main(): 24 | engine = create_engine("questdb://localhost:8812/main") 25 | try: 26 | Base.metadata.create_all(engine) 27 | with engine.connect() as conn: 28 | conn.execute( 29 | insert(Signal).values( 30 | source="coconut", value=16.88993244, ts=datetime.datetime.utcnow() 31 | ) 32 | ) 33 | finally: 34 | if engine: 35 | engine.dispose() 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /src/examples/psycopg2_connect.py: -------------------------------------------------------------------------------- 1 | """ 2 | This code connects to a running QuestDB instance and displays information 3 | about the tables and partitions in the database. It uses two SQL statements 4 | to retrieve this information: "tables()" and "table_partitions()". The output 5 | is formatted and printed to the console. 6 | """ 7 | import json 8 | 9 | from questdb_connect import connect 10 | 11 | from examples import CONNECTION_ATTRS 12 | 13 | 14 | def print_partition(row): 15 | p_index, p_by, _, min_ts, max_ts, num_rows, _, p_size, *_ = row 16 | print( 17 | f" - Partition {p_index} by {p_by} [{min_ts}, {max_ts}] {num_rows} rows {p_size}" 18 | ) 19 | 20 | 21 | def print_table(row): 22 | table_id, table_name, ts_column, p_by, _, _, is_wal, dir_name, is_dedup = row 23 | msg = ", ".join( 24 | ( 25 | f"Table id:{table_id}", 26 | f"name:{table_name}", 27 | f"ts-col:{ts_column}", 28 | f"partition-by:{p_by}", 29 | f"is-wal:{is_wal}", 30 | f"dir-name:{dir_name}", 31 | f"is-dedup:{is_dedup}", 32 | ) 33 | ) 34 | print(msg) 35 | 36 | 37 | def print_server_info(dsn_parameters): 38 | print(f"QuestDB server information: {json.dumps(dsn_parameters, indent=4)}") 39 | 40 | 41 | def main(): 42 | with connect(**CONNECTION_ATTRS) as conn: 43 | print_server_info(conn.get_dsn_parameters()) 44 | with conn.cursor() as tables_cur, conn.cursor() as partitions_cur: 45 | tables_cur.execute("tables()") 46 | for table_row in tables_cur.fetchall(): 47 | print_table(table_row) 48 | partitions_cur.execute(f"table_partitions('{table_row[1]}')") 49 | for partition_row in partitions_cur.fetchall(): 50 | print_partition(partition_row) 51 | 52 | 53 | if __name__ == "__main__": 54 | main() 55 | -------------------------------------------------------------------------------- /src/examples/server_utilisation.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import enum 3 | import os 4 | import random 5 | import time 6 | 7 | os.environ.setdefault("SQLALCHEMY_SILENCE_UBER_WARNING", "1") 8 | 9 | import questdb_connect as qdbc 10 | from sqlalchemy import Column, MetaData, create_engine 11 | from sqlalchemy.orm import Session, declarative_base 12 | 13 | 14 | class BaseEnum(enum.Enum): 15 | @classmethod 16 | def rand(cls): 17 | return cls._value2member_map_[random.randint(0, len(cls._member_map_) - 1)] 18 | 19 | 20 | class Nodes(BaseEnum): 21 | NODE0 = 0 22 | NODE1 = 1 23 | 24 | 25 | class Metrics(BaseEnum): 26 | CPU = 0 27 | RAM = 1 28 | HDD0 = 2 29 | HDD1 = 3 30 | NETWORK = 4 31 | 32 | 33 | Base = declarative_base(metadata=MetaData()) 34 | 35 | 36 | class NodeMetrics(Base): 37 | __tablename__ = "node_metrics" 38 | __table_args__ = ( 39 | qdbc.QDBTableEngine("node_metrics", "ts", qdbc.PartitionBy.HOUR, is_wal=True), 40 | ) 41 | source = Column(qdbc.Symbol) # Nodes 42 | attr_name = Column(qdbc.Symbol) # Metrics 43 | attr_value = Column(qdbc.Double) 44 | ts = Column(qdbc.Timestamp, primary_key=True) 45 | 46 | 47 | def main(duration_sec: float = 10.0): 48 | end_time = time.time() + max(duration_sec - 0.5, 2.0) 49 | engine = create_engine("questdb://localhost:8812/main") 50 | session = Session(engine) 51 | max_batch_size = 3000 52 | try: 53 | Base.metadata.drop_all(engine) 54 | Base.metadata.create_all(engine) 55 | batch_size = 0 56 | while time.time() < end_time: 57 | node = Nodes.rand() 58 | session.add( 59 | NodeMetrics( 60 | source=node.name, 61 | attr_name=Metrics.rand().name, 62 | attr_value=random.random() * node.value * random.randint(1, 100), 63 | ts=datetime.datetime.utcnow(), 64 | ) 65 | ) 66 | batch_size += 1 67 | if batch_size > max_batch_size: 68 | session.commit() 69 | batch_size = 0 70 | if batch_size > 0: 71 | session.commit() 72 | finally: 73 | if session: 74 | session.close() 75 | if engine: 76 | engine.dispose() 77 | 78 | 79 | if __name__ == "__main__": 80 | main() 81 | -------------------------------------------------------------------------------- /src/examples/sqlalchemy_orm.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | import time 5 | 6 | os.environ.setdefault("SQLALCHEMY_SILENCE_UBER_WARNING", "1") 7 | 8 | import questdb_connect as qdbc 9 | from sqlalchemy import Column, MetaData, insert, text 10 | from sqlalchemy.orm import declarative_base 11 | 12 | from examples import CONNECTION_ATTRS 13 | 14 | Base = declarative_base(metadata=MetaData()) 15 | table_name = "all_types" 16 | 17 | 18 | class MyTable(Base): 19 | __tablename__ = table_name 20 | __table_args__ = ( 21 | qdbc.QDBTableEngine(table_name, "col_ts", qdbc.PartitionBy.DAY, is_wal=True), 22 | ) 23 | col_boolean = Column(qdbc.Boolean) 24 | col_byte = Column(qdbc.Byte) 25 | col_short = Column(qdbc.Short) 26 | col_int = Column(qdbc.Int) 27 | col_long = Column(qdbc.Long) 28 | col_float = Column(qdbc.Float) 29 | col_double = Column(qdbc.Double) 30 | col_symbol = Column(qdbc.Symbol) 31 | col_string = Column(qdbc.String) 32 | col_char = Column(qdbc.Char) 33 | col_uuid = Column(qdbc.UUID) 34 | col_date = Column(qdbc.Date) 35 | col_ts = Column(qdbc.Timestamp, primary_key=True) 36 | col_geohash = Column(qdbc.GeohashInt) 37 | col_long256 = Column(qdbc.Long256) 38 | col_varchar = Column(qdbc.Varchar) 39 | 40 | 41 | def main(): 42 | # obtain the engine, which we will dispose of at the end in the finally 43 | engine = qdbc.create_engine(**CONNECTION_ATTRS) 44 | try: 45 | # delete any previous existing 'all_types' table 46 | while True: 47 | try: 48 | Base.metadata.drop_all(engine) 49 | break 50 | except Exception as see: 51 | if "Connection refused" in str(see) or ( 52 | hasattr(see, "orig") and "Connection refused" in str(see.orig) 53 | ): 54 | print(f"awaiting for QuestDB to start") 55 | time.sleep(10) 56 | else: 57 | raise see 58 | 59 | # create the 'all_types' table 60 | Base.metadata.create_all(engine) 61 | 62 | # connect with QuestDB 63 | with engine.connect() as conn: 64 | # insert a fully populated row 65 | now = datetime.datetime(2023, 4, 22, 18, 10, 10, 765123) 66 | conn.execute( 67 | insert(MyTable).values( 68 | col_boolean=True, 69 | col_byte=8, 70 | col_short=12, 71 | col_int=13, 72 | col_long=14, 73 | col_float=15.234, 74 | col_double=16.88993244, 75 | col_symbol="coconut", 76 | col_string="banana", 77 | col_char="C", 78 | col_uuid="6d5eb038-63d1-4971-8484-30c16e13de5b", 79 | col_date=now.date(), 80 | col_ts=now, 81 | col_geohash="dfvgsj2vptwu", 82 | col_long256="0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a", 83 | ) 84 | ) 85 | conn.commit() 86 | columns = [col.name for col in MyTable.__table__.columns] 87 | while True: 88 | rs = conn.execute(text("all_types")) 89 | if rs.rowcount: 90 | print(f"rows: {rs.rowcount}") 91 | for row in rs: 92 | print(json.dumps(dict(zip(columns, map(str, row))), indent=4)) 93 | break 94 | finally: 95 | if engine: 96 | engine.dispose() 97 | 98 | 99 | if __name__ == "__main__": 100 | main() 101 | -------------------------------------------------------------------------------- /src/examples/sqlalchemy_raw.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sqla 2 | 3 | 4 | def main(): 5 | print(f"SqlAlchemy {sqla.__version__}") 6 | 7 | table_name = "sqlalchemy_raw" 8 | engine = sqla.create_engine("questdb://localhost:8812/main", echo=True, future=True) 9 | try: 10 | with engine.connect() as conn: 11 | conn.execute(sqla.text(f"DROP TABLE IF EXISTS {table_name}")) 12 | conn.execute( 13 | sqla.text(f"CREATE TABLE IF NOT EXISTS {table_name} (x int, y int)") 14 | ) 15 | conn.execute( 16 | sqla.text(f"INSERT INTO {table_name} (x, y) VALUES (:x, :y)"), 17 | [{"x": 1, "y": 1}, {"x": 2, "y": 4}], 18 | ) 19 | conn.commit() 20 | 21 | result = conn.execute( 22 | sqla.text(f"SELECT x, y FROM {table_name} WHERE y > :y"), {"y": 2} 23 | ) 24 | for row in result: 25 | print(f"x: {row.x} y: {row.y}") 26 | 27 | result = conn.execute(sqla.text(f"SELECT x, y FROM {table_name}")) 28 | for dict_row in result.mappings(): 29 | x = dict_row["x"] 30 | y = dict_row["y"] 31 | print(f"x: {x} y: {y}") 32 | finally: 33 | if engine: 34 | engine.dispose() 35 | 36 | 37 | if __name__ == "__main__": 38 | main() 39 | -------------------------------------------------------------------------------- /src/qdb_superset/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/qdb_superset/db_engine_specs/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/qdb_superset/db_engine_specs/questdb.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | from datetime import datetime 5 | from typing import Any 6 | 7 | import questdb_connect.types as qdbc_types 8 | from flask_babel import gettext as __ 9 | from marshmallow import fields, Schema 10 | from questdb_connect.common import remove_public_schema 11 | from sqlalchemy.engine.base import Engine 12 | from sqlalchemy.engine.reflection import Inspector 13 | from sqlalchemy.sql.expression import text, TextClause 14 | from sqlalchemy.types import TypeEngine 15 | import logging 16 | 17 | # Configure the logging 18 | logging.basicConfig(level=logging.ERROR) 19 | logger = logging.getLogger(__name__) 20 | 21 | from superset.db_engine_specs.base import ( 22 | BaseEngineSpec, 23 | BasicParametersMixin, 24 | BasicParametersType, 25 | ) 26 | from superset import sql_parse 27 | from superset.utils import core as utils 28 | from superset.utils.core import GenericDataType 29 | 30 | 31 | class QuestDbParametersSchema(Schema): 32 | username = fields.String( 33 | metadata={"description": __("username")}, 34 | dump_default="admin", 35 | load_default="admin", 36 | ) 37 | password = fields.String( 38 | metadata={"description": __("password")}, 39 | dump_default="quest", 40 | load_default="quest", 41 | ) 42 | host = fields.String( 43 | metadata={"description": __("host")}, 44 | dump_default="host.docker.internal", 45 | load_default="host.docker.internal", 46 | ) 47 | port = fields.Integer( 48 | metadata={"description": __("port")}, 49 | dump_default="8812", 50 | load_default="8812", 51 | ) 52 | database = fields.String( 53 | metadata={"description": __("database")}, 54 | dump_default="main", 55 | load_default="main", 56 | ) 57 | 58 | 59 | class QuestDbEngineSpec(BaseEngineSpec, BasicParametersMixin): 60 | engine = "questdb" 61 | engine_name = "QuestDB" 62 | default_driver = "psycopg2" 63 | encryption_parameters = {"sslmode": "prefer"} 64 | sqlalchemy_uri_placeholder = "questdb://username:password@host:port/database" 65 | parameters_schema = QuestDbParametersSchema() 66 | time_groupby_inline = False 67 | allows_hidden_cc_in_orderby = True 68 | time_secondary_columns = True 69 | try_remove_schema_from_table_name = True 70 | max_column_name_length = 120 71 | supports_dynamic_schema = False 72 | top_keywords: set[str] = set({}) 73 | # https://en.wikipedia.org/wiki/ISO_8601#Durations 74 | # https://questdb.io/docs/reference/function/date-time/#date_trunc 75 | _time_grain_expressions = { 76 | None: "{col}", 77 | "PT1S": "DATE_TRUNC('second', {col})", 78 | "PT1M": "DATE_TRUNC('minute', {col})", 79 | "PT1H": "DATE_TRUNC('hour', {col})", 80 | "P1D": "DATE_TRUNC('day', {col})", 81 | "P1W": "DATE_TRUNC('week', {col})", 82 | "P1M": "DATE_TRUNC('month', {col})", 83 | "P1Y": "DATE_TRUNC('year', {col})", 84 | "P3M": "DATE_TRUNC('quarter', {col})", 85 | } 86 | column_type_mappings = ( 87 | ( 88 | re.compile("^BOOLEAN$", re.IGNORECASE), 89 | qdbc_types.Boolean, 90 | GenericDataType.BOOLEAN, 91 | ), 92 | (re.compile("^BYTE$", re.IGNORECASE), qdbc_types.Byte, GenericDataType.NUMERIC), 93 | ( 94 | re.compile("^SHORT$", re.IGNORECASE), 95 | qdbc_types.Short, 96 | GenericDataType.NUMERIC, 97 | ), 98 | (re.compile("^CHAR$", re.IGNORECASE), qdbc_types.Char, GenericDataType.STRING), 99 | (re.compile("^INT$", re.IGNORECASE), qdbc_types.Int, GenericDataType.NUMERIC), 100 | (re.compile("^LONG$", re.IGNORECASE), qdbc_types.Long, GenericDataType.NUMERIC), 101 | ( 102 | re.compile("^DATE$", re.IGNORECASE), 103 | qdbc_types.Date, 104 | GenericDataType.TEMPORAL, 105 | ), 106 | ( 107 | re.compile("^TIMESTAMP$", re.IGNORECASE), 108 | qdbc_types.Timestamp, 109 | GenericDataType.TEMPORAL, 110 | ), 111 | ( 112 | re.compile("^FLOAT$", re.IGNORECASE), 113 | qdbc_types.Float, 114 | GenericDataType.NUMERIC, 115 | ), 116 | ( 117 | re.compile("^DOUBLE$", re.IGNORECASE), 118 | qdbc_types.Double, 119 | GenericDataType.NUMERIC, 120 | ), 121 | ( 122 | re.compile("^STRING$", re.IGNORECASE), 123 | qdbc_types.String, 124 | GenericDataType.STRING, 125 | ), 126 | ( 127 | re.compile("^VARCHAR$", re.IGNORECASE), 128 | qdbc_types.Varchar, 129 | GenericDataType.STRING, 130 | ), 131 | ( 132 | re.compile("^SYMBOL$", re.IGNORECASE), 133 | qdbc_types.Symbol, 134 | GenericDataType.STRING, 135 | ), 136 | ( 137 | re.compile("^LONG256$", re.IGNORECASE), 138 | qdbc_types.Long256, 139 | GenericDataType.STRING, 140 | ), 141 | ( 142 | re.compile(r"^GEOHASH\(\d+[b|c]\)$", re.IGNORECASE), 143 | qdbc_types.GeohashLong, 144 | GenericDataType.STRING, 145 | ), 146 | (re.compile("^UUID$", re.IGNORECASE), qdbc_types.UUID, GenericDataType.STRING), 147 | ( 148 | re.compile("^LONG128$", re.IGNORECASE), 149 | qdbc_types.Long128, 150 | GenericDataType.STRING, 151 | ), 152 | (re.compile("^IPV4$", re.IGNORECASE), qdbc_types.IPv4, GenericDataType.STRING), 153 | ) 154 | 155 | @classmethod 156 | def build_sqlalchemy_uri( 157 | cls, 158 | parameters: BasicParametersType, 159 | encrypted_extra: dict[str, str] | None = None, 160 | ) -> str: 161 | host = parameters.get("host") 162 | port = parameters.get("port") 163 | username = parameters.get("username") 164 | password = parameters.get("password") 165 | database = parameters.get("database") 166 | return f"questdb://{username}:{password}@{host}:{port}/{database}" 167 | 168 | @classmethod 169 | def get_default_schema_for_query(cls, database, query) -> str | None: 170 | """Return the default schema for a given query.""" 171 | return None 172 | 173 | @classmethod 174 | def epoch_to_dttm(cls) -> str: 175 | """SQL expression that converts epoch (seconds) to datetime that can be used 176 | in a query. The reference column should be denoted as `{col}` in the return 177 | expression, e.g. "FROM_UNIXTIME({col})" 178 | :return: SQL Expression 179 | """ 180 | return "{col} * 1000000" 181 | 182 | @classmethod 183 | def convert_dttm( 184 | cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None 185 | ) -> str | None: 186 | """Convert a Python `datetime` object to a SQL expression. 187 | :param target_type: The target type of expression 188 | :param dttm: The datetime object 189 | :return: The SQL expression 190 | """ 191 | type_u = target_type.upper() 192 | if type_u == "DATE": 193 | return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" 194 | if type_u in ("DATETIME", "TIMESTAMP"): 195 | dttm_formatted = dttm.isoformat(sep="T", timespec="microseconds") 196 | return f"TO_TIMESTAMP('{dttm_formatted}', 'yyyy-MM-ddTHH:mm:ss.SSSUUU')" 197 | return None 198 | 199 | @classmethod 200 | def get_datatype(cls, type_code: Any) -> str | None: 201 | """Change column type code from cursor description to string representation. 202 | :param type_code: Type code from cursor description 203 | :return: String representation of type code 204 | """ 205 | if isinstance(type_code, str) and type_code: 206 | return type_code.upper() 207 | return str(type_code) 208 | 209 | @classmethod 210 | def get_column_spec( 211 | cls, 212 | native_type: str | None, 213 | db_extra: dict[str, Any] | None = None, 214 | source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, 215 | ) -> utils.ColumnSpec | None: 216 | """Get generic type related specs regarding a native column type. 217 | :param native_type: Native database type 218 | :param db_extra: The database extra object 219 | :param source: Type coming from the database table or cursor description 220 | :return: ColumnSpec object 221 | """ 222 | sqla_type = qdbc_types.resolve_type_from_name(native_type) 223 | if not sqla_type: 224 | return BaseEngineSpec.get_column_spec(native_type, db_extra, source) 225 | name_u = sqla_type.__visit_name__ 226 | generic_type = GenericDataType.STRING 227 | if name_u == "BOOLEAN": 228 | generic_type = GenericDataType.BOOLEAN 229 | elif name_u in ("BYTE", "SHORT", "INT", "LONG", "FLOAT", "DOUBLE"): 230 | generic_type = GenericDataType.NUMERIC 231 | elif name_u in ( 232 | "SYMBOL", 233 | "STRING", 234 | "VARCHAR", 235 | "CHAR", 236 | "LONG256", 237 | "UUID", 238 | "LONG128", 239 | "IPV4", 240 | ): 241 | generic_type = GenericDataType.STRING 242 | elif name_u in ("DATE", "TIMESTAMP"): 243 | generic_type = GenericDataType.TEMPORAL 244 | elif "GEOHASH" in name_u and "(" in name_u and ")" in name_u: 245 | generic_type = GenericDataType.STRING 246 | return utils.ColumnSpec( 247 | sqla_type, 248 | generic_type, 249 | generic_type == GenericDataType.TEMPORAL, 250 | ) 251 | 252 | @classmethod 253 | def get_sqla_column_type( 254 | cls, 255 | native_type: str | None, 256 | db_extra: dict[str, Any] | None = None, 257 | source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, 258 | ) -> TypeEngine | None: 259 | """Converts native database type to sqlalchemy column type. 260 | :param native_type: Native database type 261 | :param db_extra: The database extra object 262 | :param source: Type coming from the database table or cursor description 263 | :return: ColumnSpec object 264 | """ 265 | resolved = qdbc_types.resolve_type_from_name(native_type) 266 | return resolved.impl if resolved else None 267 | 268 | @classmethod 269 | def select_star( # pylint: disable=too-many-arguments 270 | cls, 271 | database: Any, 272 | table_name: str, 273 | engine: Engine, 274 | schema: str | None = None, 275 | limit: int = 100, 276 | show_cols: bool = False, 277 | indent: bool = True, 278 | latest_partition: bool = True, 279 | cols: list[dict[str, Any]] | None = None, 280 | ) -> str: 281 | """Generate a "SELECT * from table_name" query with appropriate limit. 282 | :param database: Database instance 283 | :param table_name: Table name, unquoted 284 | :param engine: SqlAlchemy Engine instance 285 | :param schema: Schema, unquoted 286 | :param limit: limit to impose on query 287 | :param show_cols: Show columns in query; otherwise use "*" 288 | :param indent: Add indentation to query 289 | :param latest_partition: Only query the latest partition 290 | :param cols: Columns to include in query 291 | :return: SQL query 292 | """ 293 | return super().select_star( 294 | database, 295 | table_name, 296 | engine, 297 | None, 298 | limit, 299 | show_cols, 300 | indent, 301 | latest_partition, 302 | cols, 303 | ) 304 | 305 | @classmethod 306 | def get_allow_cost_estimate(cls, extra: dict[str, Any]) -> bool: 307 | return False 308 | 309 | @classmethod 310 | def get_view_names( 311 | cls, 312 | database, 313 | inspector: Inspector, 314 | schema: str | None, 315 | ) -> set[str]: 316 | return set() 317 | 318 | @classmethod 319 | def get_text_clause(cls, clause: str) -> TextClause: 320 | """ 321 | SQLAlchemy wrapper to ensure text clauses are escaped properly 322 | 323 | :param clause: string clause with potentially unescaped characters 324 | :return: text clause with escaped characters 325 | """ 326 | if cls.allows_escaped_colons: 327 | clause = clause.replace(":", "\\:") 328 | return text(remove_public_schema(clause)) 329 | 330 | @classmethod 331 | def execute( # pylint: disable=unused-argument 332 | cls, 333 | cursor: Any, 334 | query: str, 335 | **kwargs: Any, 336 | ) -> None: 337 | """Execute a SQL query 338 | :param cursor: Cursor instance 339 | :param query: Query to execute 340 | :param kwargs: kwargs to be passed to cursor.execute() 341 | :return: 342 | """ 343 | try: 344 | sql = sql_parse.strip_comments_from_sql(query) 345 | cursor.execute(sql) 346 | except Exception as ex: 347 | # Log the exception with traceback 348 | logger.exception( 349 | "An error occurred, query(%s): %s\nerror: %s", type(query), query, ex 350 | ) 351 | raise cls.get_dbapi_mapped_exception(ex) from ex 352 | -------------------------------------------------------------------------------- /src/questdb_connect/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import psycopg2 5 | 6 | from questdb_connect.common import PartitionBy, remove_public_schema 7 | from questdb_connect.compilers import QDBDDLCompiler, QDBSQLCompiler 8 | from questdb_connect.dialect import ( 9 | QuestDBDialect, 10 | connection_uri, 11 | create_engine, 12 | create_superset_engine, 13 | ) 14 | from questdb_connect.dml import QDBSelect, select 15 | from questdb_connect.identifier_preparer import QDBIdentifierPreparer 16 | from questdb_connect.inspector import QDBInspector 17 | from questdb_connect.keywords_functions import get_functions_list, get_keywords_list 18 | from questdb_connect.table_engine import QDBTableEngine 19 | from questdb_connect.types import ( 20 | QUESTDB_TYPES, 21 | UUID, 22 | Boolean, 23 | Byte, 24 | Char, 25 | Date, 26 | Double, 27 | Float, 28 | GeohashByte, 29 | GeohashInt, 30 | GeohashLong, 31 | GeohashShort, 32 | Int, 33 | IPv4, 34 | Long, 35 | Long128, 36 | Long256, 37 | QDBTypeMixin, 38 | Short, 39 | String, 40 | Symbol, 41 | Timestamp, 42 | Varchar, 43 | geohash_class, 44 | geohash_type_name, 45 | resolve_type_from_name, 46 | ) 47 | 48 | # ===== DBAPI ===== 49 | # https://peps.python.org/pep-0249/ 50 | 51 | apilevel = "2.0" 52 | threadsafety = 2 53 | paramstyle = "pyformat" 54 | 55 | __all__ = ( 56 | "select", 57 | "QDBSelect", 58 | ) 59 | 60 | 61 | class Error(Exception): 62 | pass 63 | 64 | 65 | class Cursor(psycopg2.extensions.cursor): 66 | def execute(self, query, vars=None): 67 | """execute(query, vars=None) -- Execute query with bound vars.""" 68 | return super().execute(remove_public_schema(query), vars) 69 | 70 | 71 | def cursor_factory(*args, **kwargs): 72 | return Cursor(*args, **kwargs) 73 | 74 | 75 | def connect(**kwargs): 76 | host = kwargs.get("host") or "127.0.0.1" 77 | port = kwargs.get("port") or 8812 78 | user = kwargs.get("user") or "admin" 79 | password = kwargs.get("password") or "quest" 80 | database = kwargs.get("database") or "main" 81 | conn = psycopg2.connect( 82 | cursor_factory=cursor_factory, 83 | host=host, 84 | port=port, 85 | user=user, 86 | password=password, 87 | database=database, 88 | ) 89 | # retrieve and cache function names and keywords lists 90 | get_keywords_list(conn) 91 | get_functions_list(conn) 92 | return conn 93 | -------------------------------------------------------------------------------- /src/questdb_connect/common.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import re 3 | 4 | 5 | class PartitionBy(enum.Enum): 6 | DAY = 0 7 | MONTH = 1 8 | YEAR = 2 9 | NONE = 3 10 | HOUR = 4 11 | WEEK = 5 12 | 13 | 14 | def remove_public_schema(query): 15 | if isinstance(query, str) and query and "public" in query: 16 | return re.sub(_PUBLIC_SCHEMA_FILTER, "", query) 17 | return query 18 | 19 | 20 | def quote_identifier(identifier: str): 21 | if not identifier: 22 | return None 23 | first = 0 24 | last = len(identifier) 25 | if identifier[first] in _QUOTES: 26 | first += 1 27 | if identifier[last - 1] in _QUOTES: 28 | last -= 1 29 | return f'"{identifier[first:last]}"' 30 | 31 | 32 | _PUBLIC_SCHEMA_FILTER = re.compile( 33 | r"(')?(public(?(1)\1|)\.)", re.IGNORECASE | re.MULTILINE 34 | ) 35 | _QUOTES = ("'", '"') 36 | -------------------------------------------------------------------------------- /src/questdb_connect/compilers.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import sqlalchemy 4 | from sqlalchemy.sql.base import elements 5 | 6 | from .common import quote_identifier, remove_public_schema 7 | from .types import QDBTypeMixin 8 | 9 | 10 | class QDBDDLCompiler(sqlalchemy.sql.compiler.DDLCompiler, abc.ABC): 11 | def visit_create_schema(self, create, **kw): 12 | raise Exception("QuestDB does not support SCHEMAS, there is only 'public'") 13 | 14 | def visit_drop_schema(self, drop, **kw): 15 | raise Exception("QuestDB does not support SCHEMAS, there is only 'public'") 16 | 17 | def visit_create_table(self, create, **kw): 18 | table = create.element 19 | create_table = f"CREATE TABLE {quote_identifier(table.fullname)} (" 20 | create_table += ", ".join( 21 | [self.get_column_specification(c.element) for c in create.columns] 22 | ) 23 | return create_table + ") " + table.engine.get_table_suffix() 24 | 25 | def get_column_specification(self, column: sqlalchemy.Column, **_): 26 | if not isinstance(column.type, QDBTypeMixin): 27 | raise sqlalchemy.exc.ArgumentError( 28 | "Column type is not a valid QuestDB type" 29 | ) 30 | return column.type.column_spec(column.name) 31 | 32 | 33 | class QDBSQLCompiler(sqlalchemy.sql.compiler.SQLCompiler, abc.ABC): 34 | # Maximum value for 64-bit signed integer (2^63 - 1) 35 | BIGINT_MAX = 9223372036854775807 36 | 37 | def visit_sample_by(self, sample_by, **kw): 38 | """Compile a SAMPLE BY clause.""" 39 | text = "" 40 | 41 | # Basic SAMPLE BY 42 | if sample_by.unit: 43 | text = f"SAMPLE BY {sample_by.value}{sample_by.unit}" 44 | else: 45 | text = f"SAMPLE BY {sample_by.value}" 46 | 47 | if sample_by.from_timestamp: 48 | # Format datetime to ISO format that QuestDB expects 49 | text += f" FROM '{sample_by.from_timestamp.isoformat()}'" 50 | if sample_by.to_timestamp: 51 | text += f" TO '{sample_by.to_timestamp.isoformat()}'" 52 | 53 | # Add FILL if specified 54 | if sample_by.fill is not None: 55 | if isinstance(sample_by.fill, str): 56 | text += f" FILL({sample_by.fill})" 57 | else: 58 | text += f" FILL({sample_by.fill:g})" 59 | 60 | # Add ALIGN TO clause 61 | text += f" ALIGN TO {sample_by.align_to}" 62 | 63 | # Add TIME ZONE if specified 64 | if sample_by.timezone: 65 | text += f" TIME ZONE '{sample_by.timezone}'" 66 | 67 | # Add WITH OFFSET if specified 68 | if sample_by.offset: 69 | text += f" WITH OFFSET '{sample_by.offset}'" 70 | 71 | return text 72 | 73 | def group_by_clause(self, select, **kw): 74 | """Customize GROUP BY to also render SAMPLE BY.""" 75 | text = "" 76 | 77 | # Add SAMPLE BY first if present 78 | if _has_sample_by(select): 79 | text += " " + self.process(select._sample_by_clause, **kw) 80 | 81 | # Use parent's GROUP BY implementation 82 | group_by_text = super().group_by_clause(select, **kw) 83 | if group_by_text: 84 | text += group_by_text 85 | 86 | return text 87 | 88 | def visit_select(self, select, **kw): 89 | """Add SAMPLE BY support to the standard SELECT compilation.""" 90 | 91 | # If we have SAMPLE BY but no GROUP BY, 92 | # add a dummy GROUP BY clause to trigger the rendering 93 | if ( 94 | _has_sample_by(select) 95 | and not select._group_by_clauses 96 | ): 97 | select = select._clone() 98 | select._group_by_clauses = [elements.TextClause("")] 99 | 100 | text = super().visit_select(select, **kw) 101 | return text 102 | 103 | def _is_safe_for_fast_insert_values_helper(self): 104 | return True 105 | 106 | def visit_textclause(self, textclause, add_to_result_map=None, **kw): 107 | textclause.text = remove_public_schema(textclause.text) 108 | return super().visit_textclause(textclause, add_to_result_map, **kw) 109 | 110 | def limit_clause(self, select, **kw): 111 | """ 112 | Generate QuestDB-style LIMIT clause from SQLAlchemy select statement. 113 | QuestDB supports arbitrary expressions in LIMIT clause. 114 | """ 115 | text = "" 116 | limit = select._limit_clause 117 | offset = select._offset_clause 118 | 119 | if limit is None and offset is None: 120 | return text 121 | 122 | text += "\n LIMIT " 123 | 124 | # Handle cases based on presence of limit and offset 125 | if limit is not None and offset is not None: 126 | # Convert LIMIT x OFFSET y to LIMIT y,y+x 127 | lower_bound = self.process(offset, **kw) 128 | limit_val = self.process(limit, **kw) 129 | text += f"{lower_bound},{lower_bound} + {limit_val}" 130 | 131 | elif limit is not None: 132 | text += self.process(limit, **kw) 133 | 134 | elif offset is not None: 135 | # If only offset is specified, use max bigint as upper bound 136 | text += f"{self.process(offset, **kw)},{self.BIGINT_MAX}" 137 | 138 | return text 139 | 140 | def _has_sample_by(select): 141 | return hasattr(select, '_sample_by_clause') and select._sample_by_clause is not None -------------------------------------------------------------------------------- /src/questdb_connect/dialect.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import sqlalchemy 4 | from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 5 | from sqlalchemy.sql.compiler import GenericTypeCompiler 6 | 7 | from .compilers import QDBDDLCompiler, QDBSQLCompiler 8 | from .identifier_preparer import QDBIdentifierPreparer 9 | from .inspector import QDBInspector 10 | 11 | # ===== SQLAlchemy Dialect ====== 12 | # https://docs.sqlalchemy.org/en/14/ apache-superset requires SQLAlchemy 1.4 13 | 14 | 15 | def connection_uri( 16 | host: str, port: str, username: str, password: str, database: str = "main" 17 | ): 18 | return f"questdb://{username}:{password}@{host}:{port}/{database}" 19 | 20 | 21 | def create_engine( 22 | host: str, port: str, username: str, password: str, database: str = "main" 23 | ): 24 | return sqlalchemy.create_engine( 25 | connection_uri(host, port, username, password, database), 26 | future=True, 27 | hide_parameters=False, 28 | implicit_returning=False, 29 | isolation_level="REPEATABLE READ", 30 | ) 31 | 32 | 33 | def create_superset_engine( 34 | host: str, port: str, username: str, password: str, database: str = "main" 35 | ): 36 | return sqlalchemy.create_engine( 37 | connection_uri(host, port, username, password, database), 38 | future=False, 39 | hide_parameters=False, 40 | implicit_returning=True, 41 | isolation_level="REPEATABLE READ", 42 | ) 43 | 44 | 45 | class QuestDBDialect(PGDialect_psycopg2, abc.ABC): 46 | name = "questdb" 47 | psycopg2_version = (2, 9) 48 | default_schema_name = "public" 49 | statement_compiler = QDBSQLCompiler 50 | ddl_compiler = QDBDDLCompiler 51 | type_compiler = GenericTypeCompiler 52 | inspector = QDBInspector 53 | preparer = QDBIdentifierPreparer 54 | supports_schemas = False 55 | supports_statement_cache = False 56 | supports_server_side_cursors = False 57 | supports_native_boolean = True 58 | supports_views = False 59 | supports_empty_insert = False 60 | supports_multivalues_insert = True 61 | supports_comments = True 62 | inline_comments = False 63 | postfetch_lastrowid = False 64 | non_native_boolean_check_constraint = False 65 | max_identifier_length = 255 66 | _user_defined_max_identifier_length = 255 67 | _has_native_hstore = False 68 | supports_is_distinct_from = False 69 | 70 | @classmethod 71 | def dbapi(cls): 72 | import questdb_connect as dbapi 73 | 74 | return dbapi 75 | 76 | def get_schema_names(self, conn, **kw): 77 | return ["public"] 78 | 79 | def get_table_names(self, conn, schema=None, **kw): 80 | return [row.table_name for row in self._exec(conn, "SHOW tables")] 81 | 82 | def has_table(self, conn, table_name, schema=None, **kw): 83 | return table_name in set(self.get_table_names(conn, schema)) 84 | 85 | @sqlalchemy.engine.reflection.cache 86 | def get_columns(self, conn, table_name, schema=None, **kw): 87 | return self.inspector.format_table_columns( 88 | table_name, self._exec(conn, f"table_columns('{table_name}')") 89 | ) 90 | 91 | def get_pk_constraint(self, conn, table_name, schema=None, **kw): 92 | return [] 93 | 94 | def get_foreign_keys( 95 | self, 96 | conn, 97 | table_name, 98 | schema=None, 99 | postgresql_ignore_search_path=False, 100 | **kw, 101 | ): 102 | return [] 103 | 104 | def get_temp_table_names(self, conn, **kw): 105 | return [] 106 | 107 | def get_view_names(self, conn, schema=None, **kw): 108 | return [] 109 | 110 | def get_temp_view_names(self, conn, schema=None, **kw): 111 | return [] 112 | 113 | def get_view_definition(self, conn, view_name, schema=None, **kw): 114 | pass 115 | 116 | def get_indexes(self, conn, table_name, schema=None, **kw): 117 | return [] 118 | 119 | def get_unique_constraints(self, conn, table_name, schema=None, **kw): 120 | return [] 121 | 122 | def get_check_constraints(self, conn, table_name, schema=None, **kw): 123 | return [] 124 | 125 | def has_sequence(self, conn, sequence_name, schema=None, **_kw): 126 | return False 127 | 128 | def do_begin_twophase(self, conn, xid): 129 | raise NotImplementedError 130 | 131 | def do_prepare_twophase(self, conn, xid): 132 | raise NotImplementedError 133 | 134 | def do_rollback_twophase(self, conn, xid, is_prepared=True, recover=False): 135 | raise NotImplementedError 136 | 137 | def do_commit_twophase(self, conn, xid, is_prepared=True, recover=False): 138 | raise NotImplementedError 139 | 140 | def do_recover_twophase(self, conn): 141 | raise NotImplementedError 142 | 143 | def set_isolation_level(self, dbapi_conn, level): 144 | pass 145 | 146 | def get_isolation_level(self, dbapi_conn): 147 | return None 148 | 149 | def _exec(self, conn, sql_query): 150 | return conn.execute(sqlalchemy.text(sql_query)) 151 | -------------------------------------------------------------------------------- /src/questdb_connect/dml.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any, Optional, Sequence, Union 4 | 5 | from sqlalchemy import select as sa_select 6 | from sqlalchemy.sql import ClauseElement 7 | from sqlalchemy.sql import Select as StandardSelect 8 | 9 | if TYPE_CHECKING: 10 | from datetime import date, datetime 11 | 12 | from sqlalchemy.sql.visitors import Visitable 13 | 14 | 15 | class SampleByClause(ClauseElement): 16 | """Represents the QuestDB SAMPLE BY clause.""" 17 | 18 | __visit_name__ = "sample_by" 19 | stringify_dialect = "questdb" 20 | 21 | def __init__( 22 | self, 23 | value: Union[int, float], 24 | unit: Optional[str] = None, 25 | fill: Optional[Union[str, float]] = None, 26 | align_to: str = "CALENDAR", # default per docs 27 | timezone: Optional[str] = None, 28 | offset: Optional[str] = None, 29 | from_timestamp: Optional[Union[datetime, date]] = None, 30 | to_timestamp: Optional[Union[datetime, date]] = None 31 | ): 32 | self.value = value 33 | self.unit = unit.lower() if unit else None 34 | self.fill = fill 35 | self.align_to = align_to.upper() 36 | self.timezone = timezone 37 | self.offset = offset 38 | self.from_timestamp = from_timestamp 39 | self.to_timestamp = to_timestamp 40 | 41 | def __str__(self) -> str: 42 | if self.unit: 43 | return f"SAMPLE BY {self.value}{self.unit}" 44 | return f"SAMPLE BY {self.value}" 45 | 46 | def get_children(self, **kwargs: Any) -> Sequence[Visitable]: 47 | return [] 48 | 49 | 50 | class QDBSelect(StandardSelect): 51 | """QuestDB-specific implementation of SELECT. 52 | 53 | Adds methods for QuestDB-specific syntaxes such as SAMPLE BY. 54 | 55 | The :class:`_questdb.QDBSelect` object is created using the 56 | :func:`sqlalchemy.dialects.questdb.select` function. 57 | """ 58 | 59 | stringify_dialect = "questdb" 60 | _sample_by_clause: Optional[SampleByClause] = None 61 | 62 | def get_children(self, **kwargs: Any) -> Sequence[Visitable]: 63 | children = super().get_children(**kwargs) 64 | if self._sample_by_clause is not None: 65 | children = [*children, self._sample_by_clause] 66 | return children 67 | 68 | def sample_by( 69 | self, 70 | value: Union[int, float], 71 | unit: Optional[str] = None, 72 | fill: Optional[Union[str, float]] = None, 73 | align_to: str = "CALENDAR", 74 | timezone: Optional[str] = None, 75 | offset: Optional[str] = None, 76 | from_timestamp: Optional[Union[datetime, date]] = None, 77 | to_timestamp: Optional[Union[datetime, date]] = None, 78 | ) -> QDBSelect: 79 | """Add a SAMPLE BY clause. 80 | 81 | :param value: time interval value 82 | :param unit: 's' for seconds, 'm' for minutes, 'h' for hours, etc. 83 | :param fill: fill strategy - NONE, NULL, PREV, LINEAR, or constant value 84 | :param align_to: CALENDAR or FIRST OBSERVATION 85 | :param timezone: Optional timezone for calendar alignment 86 | :param offset: Optional offset in format '+/-HH:mm' 87 | :param from_timestamp: Optional start timestamp for the sample 88 | :param to_timestamp: Optional end timestamp for the sample 89 | """ 90 | 91 | # Create a copy of our object with _generative 92 | s = self.__class__.__new__(self.__class__) 93 | s.__dict__ = self.__dict__.copy() 94 | 95 | # Set the sample by clause 96 | s._sample_by_clause = SampleByClause( 97 | value, unit, fill, align_to, timezone, offset, from_timestamp, to_timestamp 98 | ) 99 | return s 100 | 101 | 102 | def select(*entities: Any, **kwargs: Any) -> QDBSelect: 103 | """Construct a QuestDB-specific variant :class:`_questdb.Select` construct. 104 | 105 | .. container:: inherited_member 106 | 107 | The :func:`sqlalchemy.dialects.questdb.select` function creates 108 | a :class:`sqlalchemy.dialects.questdb.Select`. This class is based 109 | on the dialect-agnostic :class:`_sql.Select` construct which may 110 | be constructed using the :func:`_sql.select` function in 111 | SQLAlchemy Core. 112 | 113 | The :class:`_questdb.Select` construct includes additional method 114 | :meth:`_questdb.Select.sample_by` for QuestDB's SAMPLE BY clause. 115 | """ 116 | stmt = sa_select(*entities, **kwargs) 117 | # Convert the SQLAlchemy Select into our QDBSelect 118 | qdbs = QDBSelect.__new__(QDBSelect) 119 | qdbs.__dict__ = stmt.__dict__.copy() 120 | return qdbs -------------------------------------------------------------------------------- /src/questdb_connect/identifier_preparer.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import sqlalchemy 4 | 5 | from .common import quote_identifier 6 | 7 | 8 | def _none(_ignore): 9 | return None 10 | 11 | 12 | _special_chars = { 13 | "(", 14 | ")", 15 | "[", 16 | "[]", 17 | "{", 18 | "}", 19 | "'", 20 | '"', 21 | ":", 22 | ";", 23 | ".", 24 | "!", 25 | "%", 26 | "&", 27 | "*", 28 | "$", 29 | "@", 30 | "~", 31 | "^", 32 | "-", 33 | "?", 34 | "/", 35 | "\\", 36 | " ", 37 | "\t", 38 | "\r", 39 | "\n", 40 | } 41 | 42 | 43 | def _has_special_char(_value): 44 | for candidate in _value: 45 | if candidate in _special_chars: 46 | return True 47 | return False 48 | 49 | 50 | class QDBIdentifierPreparer(sqlalchemy.sql.compiler.IdentifierPreparer, abc.ABC): 51 | schema_for_object = staticmethod(_none) 52 | 53 | def __init__( 54 | self, 55 | dialect, 56 | initial_quote='"', 57 | final_quote=None, 58 | escape_quote='"', 59 | quote_case_sensitive_collations=False, 60 | omit_schema=True, 61 | ): 62 | super().__init__( 63 | dialect=dialect, 64 | initial_quote=initial_quote, 65 | final_quote=final_quote, 66 | escape_quote=escape_quote, 67 | quote_case_sensitive_collations=quote_case_sensitive_collations, 68 | omit_schema=omit_schema, 69 | ) 70 | 71 | def quote_identifier(self, value): 72 | return quote_identifier(value) 73 | 74 | def _requires_quotes(self, _value): 75 | return _value and _has_special_char(_value) 76 | 77 | def format_schema(self, name): 78 | """Prepare a quoted schema name.""" 79 | return "" 80 | 81 | def format_table(self, table, use_schema=True, name=None): 82 | """Prepare a quoted table and schema name.""" 83 | return quote_identifier(name if name else table.name) 84 | -------------------------------------------------------------------------------- /src/questdb_connect/inspector.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import psycopg2 4 | import sqlalchemy 5 | 6 | from .common import PartitionBy 7 | from .table_engine import QDBTableEngine 8 | from .types import resolve_type_from_name 9 | 10 | 11 | class QDBInspector(sqlalchemy.engine.reflection.Inspector, abc.ABC): 12 | def reflecttable( 13 | self, 14 | table, 15 | include_columns, 16 | exclude_columns=(), 17 | resolve_fks=True, 18 | _extend_on=None, 19 | ): 20 | # backward compatibility SQLAlchemy 1.3 21 | return self.reflect_table( 22 | table, include_columns, exclude_columns, resolve_fks, _extend_on 23 | ) 24 | 25 | def reflect_table( 26 | self, 27 | table, 28 | include_columns=None, 29 | exclude_columns=None, 30 | resolve_fks=False, 31 | _extend_on=None, 32 | _reflect_info=None, 33 | ): 34 | table_name = table.name 35 | try: 36 | result_set = self.bind.execute( 37 | sqlalchemy.text( 38 | "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE table_name = :tn" 39 | ), 40 | {"tn": table_name}, 41 | ) 42 | except psycopg2.DatabaseError: 43 | # older version 44 | result_set = self.bind.execute( 45 | sqlalchemy.text( 46 | "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE name = :tn" 47 | ), 48 | {"tn": table_name}, 49 | ) 50 | if not result_set: 51 | self._panic_table(table_name) 52 | table_attrs = result_set.first() 53 | if table_attrs: 54 | col_ts_name = table_attrs[0] 55 | partition_by = PartitionBy[table_attrs[1]] 56 | is_wal = True if table_attrs[2] else False 57 | else: 58 | col_ts_name = None 59 | partition_by = PartitionBy.NONE 60 | is_wal = True 61 | dedup_upsert_keys = [] 62 | for row in self.bind.execute( 63 | sqlalchemy.text( 64 | 'SELECT "column", "type", "upsertKey" FROM table_columns(:tn)' 65 | ), 66 | {"tn": table_name}, 67 | ): 68 | col_name = row[0] 69 | if include_columns and col_name not in include_columns: 70 | continue 71 | if exclude_columns and col_name in exclude_columns: 72 | continue 73 | if row[2]: # upsertKey 74 | dedup_upsert_keys.append(col_name) 75 | col_type = resolve_type_from_name(row[1]) 76 | table.append_column( 77 | sqlalchemy.Column( 78 | col_name, 79 | col_type, 80 | primary_key=( 81 | col_ts_name and col_ts_name.upper() == col_name.upper() 82 | ), 83 | ) 84 | ) 85 | table.engine = QDBTableEngine( 86 | table_name, 87 | col_ts_name, 88 | partition_by, 89 | is_wal, 90 | tuple(dedup_upsert_keys) if dedup_upsert_keys else None, 91 | ) 92 | table.metadata = sqlalchemy.MetaData() 93 | 94 | def get_columns(self, table_name, schema=None, **kw): 95 | result_set = self.bind.execute( 96 | sqlalchemy.text('SELECT "column", "type" FROM table_columns(:tn)'), 97 | {"tn": table_name}, 98 | ) 99 | return self.format_table_columns(table_name, result_set) 100 | 101 | def get_schema_names(self): 102 | return ["public"] 103 | 104 | def format_table_columns(self, table_name, result_set): 105 | if not result_set: 106 | self._panic_table(table_name) 107 | return [ 108 | { 109 | "name": row[0], 110 | "type": resolve_type_from_name(row[1])(), 111 | "nullable": True, 112 | "autoincrement": False, 113 | } 114 | for row in result_set 115 | ] 116 | 117 | def _panic_table(self, table_name): 118 | raise sqlalchemy.orm.exc.NoResultFound(f"Table '{table_name}' does not exist") 119 | -------------------------------------------------------------------------------- /src/questdb_connect/keywords_functions.py: -------------------------------------------------------------------------------- 1 | def get_keywords_list(conn=None): 2 | return __initialize_list( 3 | conn, "SELECT keyword FROM keywords()", __keywords, __default_keywords 4 | ) 5 | 6 | 7 | def get_functions_list(conn=None): 8 | return __initialize_list( 9 | conn, "SELECT name FROM functions()", __func_names, __default_func_names 10 | ) 11 | 12 | 13 | def __initialize_list(conn, sql_stmt, target_list, default_target_list): 14 | if not target_list: 15 | try: 16 | with conn.cursor() as functions_cur: 17 | functions_cur.execute(sql_stmt) 18 | for func_row in functions_cur.fetchall(): 19 | target_list.append(func_row[0]) 20 | except Exception as _ignore: 21 | target_list.extend(default_target_list) 22 | return target_list 23 | 24 | 25 | __func_names = [] 26 | __default_func_names = [ 27 | "abs", 28 | "acos", 29 | "all_tables", 30 | "and", 31 | "asin", 32 | "atan", 33 | "atan2", 34 | "avg", 35 | "base64", 36 | "between", 37 | "build", 38 | "case", 39 | "cast", 40 | "ceil", 41 | "ceiling", 42 | "coalesce", 43 | "concat", 44 | "cos", 45 | "cot", 46 | "count", 47 | "count_distinct", 48 | "current_database", 49 | "current_schema", 50 | "current_schemas", 51 | "current_user", 52 | "date_trunc", 53 | "dateadd", 54 | "datediff", 55 | "day", 56 | "day_of_week", 57 | "day_of_week_sunday_first", 58 | "days_in_month", 59 | "degrees", 60 | "dump_memory_usage", 61 | "dump_thread_stacks", 62 | "extract", 63 | "first", 64 | "floor", 65 | "flush_query_cache", 66 | "format_type", 67 | "haversine_dist_deg", 68 | "hour", 69 | "ilike", 70 | "information_schema._pg_expandarray", 71 | "isOrdered", 72 | "is_leap_year", 73 | "ksum", 74 | "last", 75 | "left", 76 | "length", 77 | "like", 78 | "list", 79 | "log", 80 | "long_sequence", 81 | "lower", 82 | "lpad", 83 | "ltrim", 84 | "make_geohash", 85 | "max", 86 | "memory_metrics", 87 | "micros", 88 | "millis", 89 | "min", 90 | "minute", 91 | "month", 92 | "not", 93 | "now", 94 | "nsum", 95 | "nullif", 96 | "pg_advisory_unlock_all", 97 | "pg_attrdef", 98 | "pg_attribute", 99 | "pg_catalog.age", 100 | "pg_catalog.current_database", 101 | "pg_catalog.current_schema", 102 | "pg_catalog.current_schemas", 103 | "pg_catalog.pg_attrdef", 104 | "pg_catalog.pg_attribute", 105 | "pg_catalog.pg_class", 106 | "pg_catalog.pg_database", 107 | "pg_catalog.pg_description", 108 | "pg_catalog.pg_get_expr", 109 | "pg_catalog.pg_get_keywords", 110 | "pg_catalog.pg_get_partkeydef", 111 | "pg_catalog.pg_get_userbyid", 112 | "pg_catalog.pg_index", 113 | "pg_catalog.pg_inherits", 114 | "pg_catalog.pg_is_in_recovery", 115 | "pg_catalog.pg_locks", 116 | "pg_catalog.pg_namespace", 117 | "pg_catalog.pg_roles", 118 | "pg_catalog.pg_shdescription", 119 | "pg_catalog.pg_table_is_visible", 120 | "pg_catalog.pg_type", 121 | "pg_catalog.txid_current", 122 | "pg_catalog.version", 123 | "pg_class", 124 | "pg_database", 125 | "pg_description", 126 | "pg_get_expr", 127 | "pg_get_keywords", 128 | "pg_get_partkeydef", 129 | "pg_index", 130 | "pg_inherits", 131 | "pg_is_in_recovery", 132 | "pg_locks", 133 | "pg_namespace", 134 | "pg_postmaster_start_time", 135 | "pg_proc", 136 | "pg_range", 137 | "pg_roles", 138 | "pg_type", 139 | "position", 140 | "power", 141 | "radians", 142 | "reader_pool", 143 | "regexp_replace", 144 | "replace", 145 | "right", 146 | "rnd_bin", 147 | "rnd_boolean", 148 | "rnd_byte", 149 | "rnd_char", 150 | "rnd_date", 151 | "rnd_double", 152 | "rnd_float", 153 | "rnd_geohash", 154 | "rnd_int", 155 | "rnd_log", 156 | "rnd_long", 157 | "rnd_long256", 158 | "rnd_short", 159 | "rnd_str", 160 | "rnd_symbol", 161 | "rnd_timestamp", 162 | "rnd_uuid4", 163 | "round", 164 | "round_down", 165 | "round_half_even", 166 | "round_up", 167 | "row_number", 168 | "rpad", 169 | "rtrim", 170 | "second", 171 | "session_user", 172 | "simulate_crash", 173 | "sin", 174 | "size_pretty", 175 | "split_part", 176 | "sqrt", 177 | "starts_with", 178 | "stddev_samp", 179 | "string_agg", 180 | "strpos", 181 | "substring", 182 | "sum", 183 | "switch", 184 | "sysdate", 185 | "systimestamp", 186 | "table_columns", 187 | "table_partitions", 188 | "table_writer_metrics", 189 | "tables", 190 | "tan", 191 | "timestamp_ceil", 192 | "timestamp_floor", 193 | "timestamp_sequence", 194 | "timestamp_shuffle", 195 | "to_char", 196 | "to_date", 197 | "to_long128", 198 | "to_lowercase", 199 | "to_pg_date", 200 | "to_str", 201 | "to_timestamp", 202 | "to_timezone", 203 | "to_uppercase", 204 | "to_utc", 205 | "touch", 206 | "trim", 207 | "txid_current", 208 | "typeOf", 209 | "upper", 210 | "version", 211 | "wal_tables", 212 | "week_of_year", 213 | "year", 214 | ] 215 | __keywords = [] 216 | __default_keywords = [ 217 | "add", 218 | "all", 219 | "alter", 220 | "and", 221 | "as", 222 | "asc", 223 | "asof", 224 | "backup", 225 | "between", 226 | "by", 227 | "cache", 228 | "capacity", 229 | "case", 230 | "cast", 231 | "column", 232 | "columns", 233 | "copy", 234 | "create", 235 | "cross", 236 | "database", 237 | "default", 238 | "delete", 239 | "desc", 240 | "distinct", 241 | "drop", 242 | "else", 243 | "end", 244 | "except", 245 | "exists", 246 | "fill", 247 | "foreign", 248 | "from", 249 | "grant", 250 | "group", 251 | "header", 252 | "if", 253 | "in", 254 | "index", 255 | "inner", 256 | "insert", 257 | "intersect", 258 | "into", 259 | "isolation", 260 | "join", 261 | "key", 262 | "latest", 263 | "limit", 264 | "lock", 265 | "lt", 266 | "nan", 267 | "natural", 268 | "nocache", 269 | "none", 270 | "not", 271 | "null", 272 | "on", 273 | "only", 274 | "or", 275 | "order", 276 | "outer", 277 | "over", 278 | "partition", 279 | "primary", 280 | "references", 281 | "rename", 282 | "repair", 283 | "right", 284 | "sample", 285 | "select", 286 | "show", 287 | "splice", 288 | "system", 289 | "table", 290 | "tables", 291 | "then", 292 | "to", 293 | "transaction", 294 | "truncate", 295 | "type", 296 | "union", 297 | "unlock", 298 | "update", 299 | "values", 300 | "when", 301 | "where", 302 | "with", 303 | "writer", 304 | ] 305 | -------------------------------------------------------------------------------- /src/questdb_connect/table_engine.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | import sqlalchemy 4 | 5 | from .common import PartitionBy, quote_identifier 6 | 7 | 8 | class QDBTableEngine( 9 | sqlalchemy.sql.base.SchemaEventTarget, sqlalchemy.sql.visitors.Traversible 10 | ): 11 | def __init__( 12 | self, 13 | table_name: str, 14 | ts_col_name: str, 15 | partition_by: PartitionBy = PartitionBy.DAY, 16 | is_wal: bool = True, 17 | dedup_upsert_keys: typing.Optional[typing.Tuple[str]] = None, 18 | ): 19 | sqlalchemy.sql.visitors.Traversible.__init__(self) 20 | self.name = table_name 21 | self.ts_col_name = ts_col_name 22 | self.partition_by = partition_by 23 | self.is_wal = is_wal 24 | self.dedup_upsert_keys = dedup_upsert_keys 25 | self.compiled = None 26 | 27 | def get_table_suffix(self): 28 | if self.compiled is None: 29 | self.compiled = "" 30 | has_ts = self.ts_col_name is not None 31 | is_partitioned = self.partition_by and self.partition_by != PartitionBy.NONE 32 | if has_ts: 33 | self.compiled += f'TIMESTAMP("{self.ts_col_name}")' 34 | if is_partitioned: 35 | if not has_ts: 36 | raise sqlalchemy.exc.ArgumentError( 37 | None, 38 | "Designated timestamp must be specified for partitioned table", 39 | ) 40 | self.compiled += f" PARTITION BY {self.partition_by.name}" 41 | if self.is_wal: 42 | if not is_partitioned: 43 | raise sqlalchemy.exc.ArgumentError( 44 | None, "WAL table requires designated timestamp and partition by" 45 | ) 46 | if self.is_wal: 47 | self.compiled += " WAL" 48 | if self.dedup_upsert_keys: 49 | self.compiled += " DEDUP UPSERT KEYS(" 50 | self.compiled += ",".join( 51 | map(quote_identifier, self.dedup_upsert_keys) 52 | ) 53 | self.compiled += ")" 54 | else: 55 | if self.dedup_upsert_keys: 56 | raise sqlalchemy.exc.ArgumentError( 57 | None, "DEDUP only applies to WAL tables" 58 | ) 59 | self.compiled += " BYPASS WAL" 60 | return self.compiled 61 | 62 | def _set_parent(self, parent, **_kwargs): 63 | parent.engine = self 64 | -------------------------------------------------------------------------------- /src/questdb_connect/types.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import sqlalchemy 4 | 5 | from .common import quote_identifier 6 | 7 | _GEOHASH_BYTE_MAX = 8 8 | _GEOHASH_SHORT_MAX = 16 9 | _GEOHASH_INT_MAX = 32 10 | _GEOHASH_LONG_BITS = 60 11 | _TYPE_CACHE = { 12 | # key: '__visit_name__' of the implementor of QDBTypeMixin 13 | # value: implementor class itself 14 | } 15 | 16 | 17 | def geohash_type_name(bits): 18 | if not isinstance(bits, int) or bits < 0 or bits > _GEOHASH_LONG_BITS: 19 | raise sqlalchemy.exc.ArgumentError( 20 | f"geohash precision must be int [0, {_GEOHASH_LONG_BITS}]" 21 | ) 22 | if 0 < bits <= _GEOHASH_BYTE_MAX: 23 | return "GEOHASH(8b)" 24 | elif _GEOHASH_BYTE_MAX < bits <= _GEOHASH_SHORT_MAX: 25 | return "GEOHASH(3c)" 26 | elif _GEOHASH_SHORT_MAX < bits <= _GEOHASH_INT_MAX: 27 | return "GEOHASH(6c)" 28 | return "GEOHASH(12c)" 29 | 30 | 31 | def geohash_class(bits): 32 | if not isinstance(bits, int) or bits < 0 or bits > _GEOHASH_LONG_BITS: 33 | raise sqlalchemy.exc.ArgumentError( 34 | f"geohash precision must be int [0, {_GEOHASH_LONG_BITS}]" 35 | ) 36 | if 0 < bits <= _GEOHASH_BYTE_MAX: 37 | return GeohashByte 38 | elif _GEOHASH_BYTE_MAX < bits <= _GEOHASH_SHORT_MAX: 39 | return GeohashShort 40 | elif _GEOHASH_SHORT_MAX < bits <= _GEOHASH_INT_MAX: 41 | return GeohashInt 42 | return GeohashLong 43 | 44 | 45 | class QDBTypeMixin(sqlalchemy.types.TypeDecorator): 46 | __visit_name__ = "QDBTypeMixin" 47 | impl = sqlalchemy.types.String 48 | cache_ok = True 49 | 50 | @classmethod 51 | def matches_type_name(cls, type_name): 52 | return cls if type_name == cls.__visit_name__ else None 53 | 54 | def column_spec(self, column_name): 55 | return f"{quote_identifier(column_name)} {self.__visit_name__}" 56 | 57 | def compile(self, dialect=None): 58 | return self.__visit_name__ 59 | 60 | 61 | class Boolean(QDBTypeMixin): 62 | __visit_name__ = "BOOLEAN" 63 | impl = sqlalchemy.types.Boolean 64 | type_code = 1 65 | 66 | 67 | class Byte(QDBTypeMixin): 68 | __visit_name__ = "BYTE" 69 | impl = sqlalchemy.types.Integer 70 | type_code = 2 71 | 72 | 73 | class Short(QDBTypeMixin): 74 | __visit_name__ = "SHORT" 75 | type_code = 3 76 | impl = sqlalchemy.types.Integer 77 | 78 | 79 | class Char(QDBTypeMixin): 80 | __visit_name__ = "CHAR" 81 | type_code = 4 82 | 83 | 84 | class Int(QDBTypeMixin): 85 | __visit_name__ = "INT" 86 | type_code = 5 87 | impl = sqlalchemy.types.Integer 88 | 89 | 90 | class Long(QDBTypeMixin): 91 | __visit_name__ = "LONG" 92 | type_code = 6 93 | impl = sqlalchemy.types.Integer 94 | 95 | 96 | class Date(QDBTypeMixin): 97 | __visit_name__ = "DATE" 98 | type_code = 7 99 | impl = sqlalchemy.types.Date 100 | 101 | 102 | class Timestamp(QDBTypeMixin): 103 | __visit_name__ = "TIMESTAMP" 104 | type_code = 8 105 | impl = sqlalchemy.types.DateTime 106 | 107 | 108 | class Float(QDBTypeMixin): 109 | __visit_name__ = "FLOAT" 110 | type_code = 9 111 | impl = sqlalchemy.types.Float 112 | 113 | 114 | class Double(QDBTypeMixin): 115 | __visit_name__ = "DOUBLE" 116 | type_code = 10 117 | impl = sqlalchemy.types.Float 118 | 119 | 120 | class String(QDBTypeMixin): 121 | __visit_name__ = "STRING" 122 | type_code = 11 123 | 124 | 125 | class Symbol(QDBTypeMixin): 126 | """ 127 | QuestDB SYMBOL type implementation with support for capacity and cache parameters. 128 | 129 | Example usage: 130 | source = Column(Symbol(capacity=128, cache=True)) 131 | """ 132 | __visit_name__ = "SYMBOL" 133 | type_code = 12 134 | 135 | def __init__( 136 | self, 137 | capacity: Optional[int] = None, 138 | cache: Optional[bool] = None, 139 | *args, **kwargs 140 | ): 141 | super().__init__(*args, **kwargs) 142 | self.capacity = capacity 143 | self.cache = cache 144 | 145 | def compile(self, dialect=None): 146 | params = [] 147 | 148 | if self.capacity is not None: 149 | params.append(f"CAPACITY {self.capacity}") 150 | if self.cache is not None: 151 | params.append("CACHE" if self.cache else "NOCACHE") 152 | 153 | if params: 154 | return f"{self.__visit_name__} {' '.join(params)}" 155 | return self.__visit_name__ 156 | 157 | def column_spec(self, column_name): 158 | return f"{quote_identifier(column_name)} {self.compile()}" 159 | 160 | 161 | class Long256(QDBTypeMixin): 162 | __visit_name__ = "LONG256" 163 | type_code = 13 164 | 165 | 166 | class GeohashByte(QDBTypeMixin): 167 | __visit_name__ = geohash_type_name(8) 168 | type_code = 14 169 | 170 | 171 | class GeohashShort(QDBTypeMixin): 172 | __visit_name__ = geohash_type_name(16) 173 | type_code = 15 174 | 175 | 176 | class GeohashInt(QDBTypeMixin): 177 | __visit_name__ = geohash_type_name(32) 178 | type_code = 16 179 | 180 | 181 | class GeohashLong(QDBTypeMixin): 182 | __visit_name__ = geohash_type_name(60) 183 | type_code = 17 184 | 185 | 186 | class UUID(QDBTypeMixin): 187 | __visit_name__ = "UUID" 188 | type_code = 19 189 | 190 | 191 | class Long128(QDBTypeMixin): 192 | __visit_name__ = "LONG128" 193 | type_code = 24 194 | 195 | 196 | class IPv4(QDBTypeMixin): 197 | __visit_name__ = "IPV4" 198 | type_code = 26 199 | 200 | class Varchar(QDBTypeMixin): 201 | __visit_name__ = "VARCHAR" 202 | type_code = 27 203 | 204 | 205 | QUESTDB_TYPES = [ 206 | Boolean, 207 | Byte, 208 | Short, 209 | Char, 210 | Int, 211 | Long, 212 | Date, 213 | Timestamp, 214 | Float, 215 | Double, 216 | String, 217 | Symbol, 218 | Long256, 219 | GeohashByte, 220 | GeohashInt, 221 | GeohashShort, 222 | GeohashLong, 223 | UUID, 224 | Long128, 225 | IPv4, 226 | Varchar, 227 | ] 228 | 229 | 230 | def resolve_type_from_name(type_name): 231 | if not type_name: 232 | return None 233 | type_class = _TYPE_CACHE.get(type_name) 234 | if not type_class: 235 | for candidate_class in QUESTDB_TYPES: 236 | type_class = candidate_class.matches_type_name(type_name) 237 | if type_class: 238 | _TYPE_CACHE[type_name] = type_class 239 | break 240 | elif ( 241 | "GEOHASH" in type_name.upper() and "(" in type_name and ")" in type_name 242 | ): 243 | open_p = type_name.index("(") 244 | close_p = type_name.index(")") 245 | description = type_name[open_p + 1 : close_p] 246 | g_size = int(description[:-1]) 247 | if description[-1] in ("C", "c"): 248 | g_size *= 5 249 | type_class = geohash_class(g_size) 250 | break 251 | return type_class 252 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from typing import NamedTuple 4 | 5 | import pytest 6 | import questdb_connect as qdbc 7 | from sqlalchemy import Column, MetaData, text 8 | from sqlalchemy.orm import declarative_base 9 | 10 | os.environ.setdefault('SQLALCHEMY_SILENCE_UBER_WARNING', '1') 11 | 12 | ALL_TYPES_TABLE_NAME = 'all_types_table' 13 | METRICS_TABLE_NAME = 'metrics_table' 14 | 15 | 16 | class TestConfig(NamedTuple): 17 | host: str 18 | port: str 19 | username: str 20 | password: str 21 | database: str 22 | __test__ = True 23 | 24 | 25 | @pytest.fixture(scope='session', autouse=True, name='test_config') 26 | def test_config_fixture() -> TestConfig: 27 | return TestConfig( 28 | host=os.environ.get('QUESTDB_CONNECT_HOST', 'localhost'), 29 | port=os.environ.get('QUESTDB_CONNECT_PORT', '8812'), 30 | username=os.environ.get('QUESTDB_CONNECT_USER', 'admin'), 31 | password=os.environ.get('QUESTDB_CONNECT_PASSWORD', 'quest'), 32 | database=os.environ.get('QUESTDB_CONNECT_DATABASE', 'main') 33 | ) 34 | 35 | 36 | @pytest.fixture(scope='module', name='test_engine') 37 | def test_engine_fixture(test_config: TestConfig): 38 | engine = None 39 | try: 40 | engine = qdbc.create_engine( 41 | test_config.host, 42 | test_config.port, 43 | test_config.username, 44 | test_config.password, 45 | test_config.database) 46 | return engine 47 | finally: 48 | if engine: 49 | engine.dispose() 50 | del engine 51 | 52 | @pytest.fixture(scope='module', name='superset_test_engine') 53 | def test_superset_engine_fixture(test_config: TestConfig): 54 | engine = None 55 | try: 56 | engine = qdbc.create_superset_engine( 57 | test_config.host, 58 | test_config.port, 59 | test_config.username, 60 | test_config.password, 61 | test_config.database) 62 | return engine 63 | finally: 64 | if engine: 65 | engine.dispose() 66 | del engine 67 | 68 | @pytest.fixture(autouse=True, name='test_model') 69 | def test_model_fixture(test_engine): 70 | Base = declarative_base(metadata=MetaData()) 71 | 72 | class TableModel(Base): 73 | __tablename__ = ALL_TYPES_TABLE_NAME 74 | __table_args__ = (qdbc.QDBTableEngine(ALL_TYPES_TABLE_NAME, 'col_ts', qdbc.PartitionBy.DAY, is_wal=True),) 75 | col_boolean = Column('col_boolean', qdbc.Boolean) 76 | col_byte = Column('col_byte', qdbc.Byte) 77 | col_short = Column('col_short', qdbc.Short) 78 | col_int = Column('col_int', qdbc.Int) 79 | col_long = Column('col_long', qdbc.Long) 80 | col_float = Column('col_float', qdbc.Float) 81 | col_double = Column('col_double', qdbc.Double) 82 | col_symbol = Column('col_symbol', qdbc.Symbol) 83 | col_string = Column('col_string', qdbc.String) 84 | col_char = Column('col_char', qdbc.Char) 85 | col_uuid = Column('col_uuid', qdbc.UUID) 86 | col_date = Column('col_date', qdbc.Date) 87 | col_ts = Column('col_ts', qdbc.Timestamp, primary_key=True) 88 | col_geohash = Column('col_geohash', qdbc.GeohashInt) 89 | col_long256 = Column('col_long256', qdbc.Long256) 90 | col_varchar = Column('col_varchar', qdbc.Varchar) 91 | 92 | Base.metadata.drop_all(test_engine) 93 | Base.metadata.create_all(test_engine) 94 | return TableModel 95 | 96 | 97 | @pytest.fixture(autouse=True, name='test_metrics') 98 | def test_metrics_fixture(test_engine): 99 | Base = declarative_base(metadata=MetaData()) 100 | 101 | class TableMetrics(Base): 102 | __tablename__ = METRICS_TABLE_NAME 103 | __table_args__ = ( 104 | qdbc.QDBTableEngine( 105 | METRICS_TABLE_NAME, 106 | 'ts', 107 | qdbc.PartitionBy.HOUR, 108 | is_wal=True, 109 | dedup_upsert_keys=('source', 'attr_name', 'ts') 110 | ), 111 | ) 112 | source = Column(qdbc.Symbol) 113 | attr_name = Column(qdbc.Symbol) 114 | attr_value = Column(qdbc.Double) 115 | ts = Column(qdbc.Timestamp, primary_key=True) 116 | 117 | Base.metadata.drop_all(test_engine) 118 | Base.metadata.create_all(test_engine) 119 | return TableMetrics 120 | 121 | 122 | def collect_select_all(session, expected_rows) -> str: 123 | session.commit() 124 | while True: 125 | rs = session.execute(text(f'select * from public.{ALL_TYPES_TABLE_NAME} order by 1 asc')) 126 | if rs.rowcount == expected_rows: 127 | return '\n'.join(str(row) for row in rs) 128 | 129 | def wait_until_table_is_ready(test_engine, table_name, expected_rows, timeout=10): 130 | """ 131 | Wait until a table has the expected number of rows, with timeout. 132 | Args: 133 | test_engine: SQLAlchemy engine 134 | table_name: Name of the table to check 135 | expected_rows: Expected number of rows 136 | timeout: Maximum time to wait in seconds (default: 10 seconds) 137 | Returns: 138 | bool: True if table is ready, False if timeout occurred 139 | Raises: 140 | sqlalchemy.exc.SQLAlchemyError: If there's a database error 141 | """ 142 | start_time = time.time() 143 | 144 | while time.time() - start_time < timeout: 145 | with test_engine.connect() as conn: 146 | result = conn.execute(text(f'SELECT count(*) FROM {table_name}')) 147 | row = result.fetchone() 148 | if row and row[0] == expected_rows: 149 | return True 150 | 151 | print(f'Waiting for table {table_name} to have {expected_rows} rows, current: {row[0] if row else 0}') 152 | time.sleep(0.01) # Wait 10ms between checks 153 | return False 154 | 155 | 156 | def wait_until_table_is_ready(test_engine, table_name, expected_rows, timeout=10): 157 | """ 158 | Wait until a table has the expected number of rows, with timeout. 159 | 160 | Args: 161 | test_engine: SQLAlchemy engine 162 | table_name: Name of the table to check 163 | expected_rows: Expected number of rows 164 | timeout: Maximum time to wait in seconds (default: 10 seconds) 165 | 166 | Returns: 167 | bool: True if table is ready, False if timeout occurred 168 | 169 | Raises: 170 | sqlalchemy.exc.SQLAlchemyError: If there's a database error 171 | """ 172 | start_time = time.time() 173 | 174 | while time.time() - start_time < timeout: 175 | with test_engine.connect() as conn: 176 | result = conn.execute(text(f'SELECT count(*) FROM {table_name}')) 177 | row = result.fetchone() 178 | if row and row[0] == expected_rows: 179 | return True 180 | 181 | print(f'Waiting for table {table_name} to have {expected_rows} rows, current: {row[0] if row else 0}') 182 | time.sleep(0.01) # Wait 10ms between checks 183 | return False 184 | 185 | 186 | def collect_select_all_raw_connection(test_engine, expected_rows) -> str: 187 | conn = test_engine.raw_connection() 188 | try: 189 | while True: 190 | with conn.cursor() as cursor: 191 | cursor.execute(f'select * from public.{ALL_TYPES_TABLE_NAME} order by 1 asc') 192 | if cursor.rowcount == expected_rows: 193 | return '\n'.join(str(row) for row in cursor) 194 | finally: 195 | if conn: 196 | conn.close() 197 | -------------------------------------------------------------------------------- /tests/test_dialect.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import questdb_connect 4 | import questdb_connect as qdbc 5 | import sqlalchemy as sqla 6 | from sqlalchemy.orm import Session 7 | 8 | from tests.conftest import ( 9 | ALL_TYPES_TABLE_NAME, 10 | METRICS_TABLE_NAME, 11 | collect_select_all, 12 | collect_select_all_raw_connection, 13 | wait_until_table_is_ready, 14 | ) 15 | 16 | 17 | def test_sample_by_in_subquery(test_engine, test_model): 18 | """Test SAMPLE BY usage within subqueries.""" 19 | base_ts = datetime.datetime(2023, 4, 12, 0, 0, 0) 20 | session = Session(test_engine) 21 | try: 22 | # Insert test data - one row every minute for 2 hours 23 | num_rows = 120 # 2 hours * 60 minutes 24 | models = [ 25 | test_model( 26 | col_boolean=True, 27 | col_byte=8, 28 | col_short=12, 29 | col_int=idx, 30 | col_long=14, 31 | col_float=15.234, 32 | col_double=16.88993244, 33 | col_symbol='coconut', 34 | col_string='banana', 35 | col_char='C', 36 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 37 | col_date=base_ts.date(), 38 | col_ts=base_ts + datetime.timedelta(minutes=idx), 39 | col_geohash='dfvgsj2vptwu', 40 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 41 | col_varchar='pineapple' 42 | ) for idx in range(num_rows) 43 | ] 44 | session.bulk_save_objects(models) 45 | session.commit() 46 | 47 | metadata = sqla.MetaData() 48 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 49 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, num_rows) 50 | 51 | with test_engine.connect() as conn: 52 | # Subquery with SAMPLE BY 53 | subq = ( 54 | questdb_connect.select( 55 | table.c.col_ts, 56 | sqla.func.avg(table.c.col_int).label('avg_int') 57 | ) 58 | .sample_by(30, 'm') # 30 minute samples in subquery 59 | .subquery() 60 | ) 61 | 62 | # Main query selecting from subquery with extra conditions 63 | query = ( 64 | questdb_connect.select( 65 | subq.c.col_ts, 66 | subq.c.avg_int 67 | ) 68 | .where(subq.c.avg_int > 30) 69 | .order_by(subq.c.col_ts) 70 | ) 71 | 72 | result = conn.execute(query) 73 | rows = result.fetchall() 74 | 75 | # Should only get samples from second half of the data 76 | # where averages are > 30 77 | assert len(rows) == 3 # expecting 2 30-min samples > 30 78 | assert all(row.avg_int > 30 for row in rows) 79 | assert rows[0].avg_int < rows[1].avg_int # ordered by timestamp 80 | 81 | # Test nested aggregation 82 | outer_query = ( 83 | questdb_connect.select( 84 | sqla.func.sum(subq.c.avg_int).label('total_avg') 85 | ) 86 | .select_from(subq) 87 | ) 88 | 89 | result = conn.execute(outer_query) 90 | row = result.fetchone() 91 | # Sum of all 30-min sample averages 92 | assert row.total_avg == 238; 93 | 94 | finally: 95 | if session: 96 | session.close() 97 | 98 | def test_sample_by_clause(test_engine, test_model): 99 | """Test SAMPLE BY clause functionality.""" 100 | base_ts = datetime.datetime(2023, 4, 12, 0, 0, 0) 101 | session = Session(test_engine) 102 | try: 103 | # Insert test data - one row every minute for 2 hours 104 | num_rows = 120 # 2 hours * 60 minutes 105 | models = [ 106 | test_model( 107 | col_boolean=True, 108 | col_byte=8, 109 | col_short=12, 110 | col_int=idx, 111 | col_long=14, 112 | col_float=15.234, 113 | col_double=16.88993244, 114 | col_symbol='coconut', 115 | col_string='banana', 116 | col_char='C', 117 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 118 | col_date=base_ts.date(), 119 | # Add idx minutes to base timestamp 120 | col_ts=base_ts + datetime.timedelta(minutes=idx), 121 | col_geohash='dfvgsj2vptwu', 122 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 123 | col_varchar='pineapple' 124 | ) for idx in range(num_rows) 125 | ] 126 | session.bulk_save_objects(models) 127 | session.commit() 128 | 129 | metadata = sqla.MetaData() 130 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 131 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, num_rows) 132 | 133 | with test_engine.connect() as conn: 134 | # Simple SAMPLE BY 135 | query = ( 136 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 137 | .sample_by(30, 'm') # 30 minute samples 138 | ) 139 | result = conn.execute(query) 140 | rows = result.fetchall() 141 | assert len(rows) == 4 # 2 hours should give us 4 30-minute samples 142 | 143 | # Verify sample averages 144 | # First 30 min should average 0-29, second 30-59, etc. 145 | expected_averages = [14.5, 44.5, 74.5, 104.5] # (min+max)/2 for each 30-min period 146 | for row, expected_avg in zip(rows, expected_averages): 147 | assert abs(row.avg_int - expected_avg) < 0.1 148 | 149 | # SAMPLE BY with ORDER BY 150 | query = ( 151 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 152 | .sample_by(1, 'h') # 1 hour samples 153 | .order_by(sqla.desc('avg_int')) 154 | ) 155 | result = conn.execute(query) 156 | rows = result.fetchall() 157 | assert len(rows) == 2 # 2 one-hour samples 158 | assert rows[0].avg_int > rows[1].avg_int # Descending order 159 | 160 | # SAMPLE BY with WHERE clause 161 | query = ( 162 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 163 | .where(table.c.col_int > 30) 164 | .sample_by(1, 'h') 165 | ) 166 | result = conn.execute(query) 167 | rows = result.fetchall() 168 | assert len(rows) == 2 169 | assert all(row.avg_int > 30 for row in rows) 170 | 171 | # SAMPLE BY with LIMIT 172 | query = ( 173 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 174 | .sample_by(15, 'm') # 15 minute samples 175 | .limit(3) 176 | ) 177 | result = conn.execute(query) 178 | rows = result.fetchall() 179 | assert len(rows) == 3 # Should limit to first 3 samples 180 | finally: 181 | if session: 182 | session.close() 183 | 184 | def test_insert(test_engine, test_model): 185 | with test_engine.connect() as conn: 186 | assert test_engine.dialect.has_table(conn, ALL_TYPES_TABLE_NAME) 187 | assert not test_engine.dialect.has_table(conn, 'scorchio') 188 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 189 | now_date = now.date() 190 | expected = ("(True, 8, 12, 13, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 191 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 192 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 193 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')\n" 194 | "(True, 8, 12, 13, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 195 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 196 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 197 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')") 198 | insert_stmt = sqla.insert(test_model).values( 199 | col_boolean=True, 200 | col_byte=8, 201 | col_short=12, 202 | col_int=13, 203 | col_long=14, 204 | col_float=15.234, 205 | col_double=16.88993244, 206 | col_symbol='coconut', 207 | col_string='banana', 208 | col_char='C', 209 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 210 | col_date=now_date, 211 | col_ts=now, 212 | col_geohash='dfvgsj2vptwu', 213 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 214 | col_varchar='pineapple' 215 | ) 216 | conn.execute(insert_stmt) 217 | conn.execute(sqla.insert(test_model), { 218 | 'col_boolean': True, 219 | 'col_byte': 8, 220 | 'col_short': 12, 221 | 'col_int': 13, 222 | 'col_long': 14, 223 | 'col_float': 15.234, 224 | 'col_double': 16.88993244, 225 | 'col_symbol': 'coconut', 226 | 'col_string': 'banana', 227 | 'col_char': 'C', 228 | 'col_uuid': '6d5eb038-63d1-4971-8484-30c16e13de5b', 229 | 'col_date': now_date, 230 | 'col_ts': now, 231 | 'col_geohash': 'dfvgsj2vptwu', 232 | 'col_long256': '0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 233 | 'col_varchar': 'pineapple' 234 | }) 235 | assert collect_select_all(conn, expected_rows=2) == expected 236 | assert collect_select_all_raw_connection(test_engine, expected_rows=2) == expected 237 | 238 | 239 | def test_inspect_1(test_engine, test_model): 240 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 241 | now_date = now.date() 242 | session = Session(test_engine) 243 | try: 244 | session.add(test_model( 245 | col_boolean=True, 246 | col_byte=8, 247 | col_short=12, 248 | col_int=0, 249 | col_long=14, 250 | col_float=15.234, 251 | col_double=16.88993244, 252 | col_symbol='coconut', 253 | col_string='banana', 254 | col_char='C', 255 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 256 | col_date=now_date, 257 | col_ts=now, 258 | col_geohash='dfvgsj2vptwu', 259 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 260 | col_varchar='pineapple' 261 | )) 262 | session.commit() 263 | finally: 264 | if session: 265 | session.close() 266 | metadata = sqla.MetaData() 267 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 268 | table_columns = str([(col.name, col.type, col.primary_key) for col in table.columns]) 269 | assert table_columns == str([ 270 | ('col_boolean', qdbc.Boolean(), False), 271 | ('col_byte', qdbc.Byte(), False), 272 | ('col_short', qdbc.Short(), False), 273 | ('col_int', qdbc.Int(), False), 274 | ('col_long', qdbc.Long(), False), 275 | ('col_float', qdbc.Float(), False), 276 | ('col_double', qdbc.Double(), False), 277 | ('col_symbol', qdbc.Symbol(), False), 278 | ('col_string', qdbc.String(), False), 279 | ('col_char', qdbc.Char(), False), 280 | ('col_uuid', qdbc.UUID(), False), 281 | ('col_date', qdbc.Date(), False), 282 | ('col_ts', qdbc.Timestamp(), True), 283 | ('col_geohash', qdbc.GeohashInt(), False), 284 | ('col_long256', qdbc.Long256(), False), 285 | ('col_varchar', qdbc.Varchar(), False), 286 | ]) 287 | 288 | 289 | def test_inspect_2(test_engine, test_metrics): 290 | metadata = sqla.MetaData() 291 | table = sqla.Table(METRICS_TABLE_NAME, metadata, autoload_with=test_engine) 292 | table_columns = str([(col.name, col.type, col.primary_key) for col in table.columns]) 293 | assert table_columns == str([ 294 | ('source', qdbc.Symbol(), False), 295 | ('attr_name', qdbc.Symbol(), False), 296 | ('attr_value', qdbc.Double(), False), 297 | ('ts', qdbc.Timestamp(), True), 298 | ]) 299 | 300 | 301 | def test_multiple_insert(test_engine, test_model): 302 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 303 | now_date = now.date() 304 | session = Session(test_engine) 305 | num_rows = 3 306 | expected = ("(True, 8, 12, 0, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 307 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 308 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 309 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')\n" 310 | "(True, 8, 12, 1, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 311 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 312 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 313 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')\n" 314 | "(True, 8, 12, 2, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 315 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 316 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 317 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')") 318 | try: 319 | for idx in range(num_rows): 320 | session.add(test_model( 321 | col_boolean=True, 322 | col_byte=8, 323 | col_short=12, 324 | col_int=idx, 325 | col_long=14, 326 | col_float=15.234, 327 | col_double=16.88993244, 328 | col_symbol='coconut', 329 | col_string='banana', 330 | col_char='C', 331 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 332 | col_date=now_date, 333 | col_ts=now, 334 | col_geohash='dfvgsj2vptwu', 335 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 336 | col_varchar='pineapple' 337 | )) 338 | session.commit() 339 | assert collect_select_all(session, expected_rows=num_rows) == expected 340 | finally: 341 | if session: 342 | session.close() 343 | assert collect_select_all_raw_connection(test_engine, expected_rows=num_rows) == expected 344 | 345 | 346 | def test_bulk_insert(test_engine, test_model): 347 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 348 | now_date = now.date() 349 | session = Session(test_engine) 350 | num_rows = 3 351 | expected = ("(True, 8, 12, 0, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 352 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 353 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 354 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')\n" 355 | "(True, 8, 12, 1, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 356 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 357 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 358 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')\n" 359 | "(True, 8, 12, 2, 14, 15.234, 16.88993244, 'coconut', 'banana', 'C', " 360 | "UUID('6d5eb038-63d1-4971-8484-30c16e13de5b'), datetime.datetime(2023, 4, 12, " 361 | "0, 0), datetime.datetime(2023, 4, 12, 23, 55, 59, 342380), 'dfvgsj', " 362 | "'0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 'pineapple')") 363 | models = [ 364 | test_model( 365 | col_boolean=True, 366 | col_byte=8, 367 | col_short=12, 368 | col_int=idx, 369 | col_long=14, 370 | col_float=15.234, 371 | col_double=16.88993244, 372 | col_symbol='coconut', 373 | col_string='banana', 374 | col_char='C', 375 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 376 | col_date=now_date, 377 | col_ts=now, 378 | col_geohash='dfvgsj2vptwu', 379 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 380 | col_varchar='pineapple' 381 | ) for idx in range(num_rows) 382 | ] 383 | try: 384 | session.bulk_save_objects(models) 385 | session.commit() 386 | assert collect_select_all(session, expected_rows=num_rows) == expected 387 | finally: 388 | if session: 389 | session.close() 390 | assert collect_select_all_raw_connection(test_engine, expected_rows=num_rows) == expected 391 | 392 | 393 | def test_sample_by_from_to(test_engine, test_model): 394 | """Test SAMPLE BY with FROM-TO extension.""" 395 | base_ts = datetime.datetime(2023, 4, 12, 0, 0, 0) 396 | day_before = base_ts - datetime.timedelta(days=1) 397 | day_after = base_ts + datetime.timedelta(days=1) 398 | session = Session(test_engine) 399 | try: 400 | num_rows = 6 # 6 hours only 401 | models = [ 402 | test_model( 403 | col_int=idx, 404 | col_ts=base_ts + datetime.timedelta(hours=idx), 405 | ) for idx in range(num_rows) 406 | ] 407 | 408 | session.bulk_save_objects(models) 409 | session.commit() 410 | 411 | metadata = sqla.MetaData() 412 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 413 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, len(models)) 414 | 415 | with test_engine.connect() as conn: 416 | # Test FROM-TO with FILL 417 | query = ( 418 | questdb_connect.select( 419 | table.c.col_ts, 420 | sqla.func.avg(table.c.col_int).label('avg_int') 421 | ) 422 | .sample_by( 423 | 1, 'h', 424 | fill="NULL", 425 | from_timestamp=day_before, # day before data starts 426 | to_timestamp=day_after # day after data ends 427 | ) 428 | ) 429 | result = conn.execute(query) 430 | rows = result.fetchall() 431 | 432 | assert len(rows) == 48 # 48 hours in total 433 | 434 | # First rows should be NULL (before our data starts) 435 | assert rows[0].avg_int is None 436 | assert rows[1].avg_int is None 437 | assert rows[2].avg_int is None 438 | assert rows[3].avg_int is None 439 | 440 | # Middle rows should have data 441 | assert any(row.avg_int is not None for row in rows[4:-4]) 442 | 443 | # Last rows should be NULL (after our data ends) 444 | assert rows[-4].avg_int is None 445 | assert rows[-3].avg_int is None 446 | assert rows[-2].avg_int is None 447 | assert rows[-1].avg_int is None 448 | 449 | # Test FROM only 450 | query = ( 451 | questdb_connect.select( 452 | table.c.col_ts, 453 | sqla.func.avg(table.c.col_int).label('avg_int') 454 | ) 455 | .sample_by( 456 | 1, 'h', 457 | fill="NULL", 458 | from_timestamp=day_before # day before data starts 459 | ) 460 | ) 461 | result = conn.execute(query) 462 | rows = result.fetchall() 463 | 464 | # First rows should be NULL 465 | assert rows[0].avg_int is None 466 | assert rows[1].avg_int is None 467 | assert rows[2].avg_int is None 468 | assert rows[3].avg_int is None 469 | 470 | # Test TO only 471 | query = ( 472 | questdb_connect.select( 473 | table.c.col_ts, 474 | sqla.func.avg(table.c.col_int).label('avg_int') 475 | ) 476 | .sample_by( 477 | 1, 'h', 478 | fill="NULL", 479 | to_timestamp=day_after # day after data ends 480 | ) 481 | ) 482 | result = conn.execute(query) 483 | rows = result.fetchall() 484 | 485 | # Last rows should be NULL 486 | assert rows[-4].avg_int is None 487 | assert rows[-3].avg_int is None 488 | assert rows[-2].avg_int is None 489 | assert rows[-1].avg_int is None 490 | 491 | finally: 492 | if session: 493 | session.close() 494 | 495 | def test_plain_select_core_api(test_engine, test_model): 496 | """ 497 | Test plain select with core API. Plain select means select implementation from sqlalchemy.sql.selectable, 498 | not from questdb_connect. 499 | """ 500 | 501 | session = Session(test_engine) 502 | try: 503 | num_rows = 3 504 | models = [ 505 | test_model( 506 | col_int=idx, 507 | col_ts=datetime.datetime(2023, 4, 12, 0, 0, 0) + datetime.timedelta(hours=idx), 508 | ) for idx in range(num_rows) 509 | ] 510 | session.bulk_save_objects(models) 511 | session.commit() 512 | 513 | metadata = sqla.MetaData() 514 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 515 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, len(models)) 516 | 517 | with test_engine.connect() as conn: 518 | query = ( 519 | # important: use sqla.select, not questdb_connect.select! 520 | sqla.select(table.c.col_ts, table.c.col_int) 521 | ) 522 | result = conn.execute(query) 523 | rows = result.fetchall() 524 | assert len(rows) == 3 525 | finally: 526 | if session: 527 | session.close() 528 | 529 | def test_sample_by_options(test_engine, test_model): 530 | """Test SAMPLE BY with ALIGN TO and FILL options.""" 531 | base_ts = datetime.datetime(2023, 4, 12, 0, 0, 0) 532 | session = Session(test_engine) 533 | try: 534 | # Insert test data - one row every hour for a day 535 | num_rows = 24 536 | models = [ 537 | test_model( 538 | col_int=idx, 539 | col_ts=base_ts + datetime.timedelta(hours=idx), 540 | ) for idx in range(num_rows) 541 | ] 542 | # Add some gaps by removing every 3rd record 543 | models = [m for i, m in enumerate(models) if i % 3 != 0] 544 | 545 | session.bulk_save_objects(models) 546 | session.commit() 547 | 548 | metadata = sqla.MetaData() 549 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 550 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, len(models)) 551 | 552 | with test_engine.connect() as conn: 553 | # Test FILL(NULL) 554 | query = ( 555 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 556 | .sample_by(15, 'm', fill="NULL") 557 | ) 558 | result = conn.execute(query) 559 | rows = result.fetchall() 560 | assert len(rows) == 89 561 | # Should have NULLs for missing data points 562 | assert any(row.avg_int is None for row in rows) 563 | 564 | 565 | # Test FILL(PREV) 566 | query = ( 567 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 568 | .sample_by(15, 'm', fill="PREV") 569 | ) 570 | result = conn.execute(query) 571 | rows = result.fetchall() 572 | assert all(row.avg_int is not None for row in rows) 573 | 574 | # Test FILL with constant 575 | query = ( 576 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 577 | .sample_by(15, 'm', fill=999.99) 578 | .limit(10) 579 | ) 580 | result = conn.execute(query) 581 | rows = result.fetchall() 582 | assert any(row.avg_int == 999.99 for row in rows) 583 | 584 | # Test ALIGN TO FIRST OBSERVATION 585 | query = ( 586 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 587 | .sample_by(15, 'm', align_to="FIRST OBSERVATION") 588 | .limit(10) 589 | ) 590 | result = conn.execute(query) 591 | first_row = result.fetchone() 592 | # First timestamp should match our first data point 593 | assert first_row.col_ts == models[0].col_ts 594 | 595 | # Test with timezone 596 | query = ( 597 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 598 | .sample_by(1, 'd', align_to="CALENDAR", timezone="Europe/Prague") 599 | ) 600 | result = conn.execute(query) 601 | rows = result.fetchall() 602 | # First row should be at midnight Prague time, that is 22:00 UTC the previous day 603 | assert rows[0].col_ts.hour == 22 604 | assert rows[1].col_ts.hour == 22 605 | 606 | # Test with offset 607 | query = ( 608 | questdb_connect.select(table.c.col_ts, sqla.func.avg(table.c.col_int).label('avg_int')) 609 | .sample_by(1, 'd', align_to="CALENDAR", offset="02:00") 610 | ) 611 | result = conn.execute(query) 612 | rows = result.fetchall() 613 | # First row should start at 02:00 614 | assert rows[0].col_ts.hour == 2 615 | 616 | finally: 617 | if session: 618 | session.close() 619 | 620 | def test_dialect_get_schema_names(test_engine): 621 | dialect = qdbc.QuestDBDialect() 622 | with test_engine.connect() as conn: 623 | assert dialect.get_schema_names(conn) == ["public"] 624 | 625 | 626 | def test_dialect_get_table_names(test_engine): 627 | dialect = qdbc.QuestDBDialect() 628 | with test_engine.connect() as conn: 629 | table_names = dialect.get_table_names(conn, schema="public") 630 | assert table_names == dialect.get_table_names(conn) 631 | assert len(table_names) > 0 632 | 633 | 634 | def test_dialect_has_table(test_engine): 635 | dialect = qdbc.QuestDBDialect() 636 | with test_engine.connect() as conn: 637 | for table_name in dialect.get_table_names(conn): 638 | if not dialect.has_table(conn, table_name): 639 | raise AssertionError() 640 | if not dialect.has_table(conn, table_name, schema="public"): 641 | raise AssertionError() 642 | if not dialect.has_table(conn, table_name, schema="questdb", kw={'key': "value"}): 643 | raise AssertionError() 644 | 645 | 646 | def test_functions(test_engine): 647 | with test_engine.connect() as conn: 648 | sql = sqla.text("SELECT name FROM functions()") 649 | expected = [row[0] for row in conn.execute(sql).fetchall()] 650 | assert qdbc.get_functions_list() == expected 651 | 652 | 653 | def test_keywords(test_engine): 654 | with test_engine.connect() as conn: 655 | sql = sqla.text("SELECT keyword FROM keywords()") 656 | expected = [row[0] for row in conn.execute(sql).fetchall()] 657 | assert qdbc.get_keywords_list() == expected 658 | 659 | 660 | def test_limit_clause_basic(test_engine, test_model): 661 | """Test basic LIMIT clause functionality.""" 662 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 663 | now_date = now.date() 664 | session = Session(test_engine) 665 | num_rows = 10 666 | 667 | try: 668 | # Insert test data 669 | models = [ 670 | test_model( 671 | col_boolean=True, 672 | col_byte=8, 673 | col_short=12, 674 | col_int=idx, # Using idx to make rows distinct and ordered 675 | col_long=14, 676 | col_float=15.234, 677 | col_double=16.88993244, 678 | col_symbol='coconut', 679 | col_string='banana', 680 | col_char='C', 681 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 682 | col_date=now_date, 683 | col_ts=now, 684 | col_geohash='dfvgsj2vptwu', 685 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 686 | col_varchar='pineapple' 687 | ) for idx in range(num_rows) 688 | ] 689 | session.bulk_save_objects(models) 690 | session.commit() 691 | 692 | metadata = sqla.MetaData() 693 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 694 | 695 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, num_rows) 696 | 697 | with test_engine.connect() as conn: 698 | # simple LIMIT 699 | query = sqla.select(table).limit(5) 700 | result = conn.execute(query) 701 | rows = result.fetchall() 702 | assert len(rows) == 5 703 | assert rows[0].col_int == 0 704 | assert rows[-1].col_int == 4 705 | 706 | # LIMIT with OFFSET 707 | query = sqla.select(table).limit(3).offset(2) 708 | result = conn.execute(query) 709 | rows = result.fetchall() 710 | assert len(rows) == 3 711 | assert rows[0].col_int == 2 712 | assert rows[-1].col_int == 4 713 | 714 | # OFFSET only 715 | query = sqla.select(table).offset(8) 716 | result = conn.execute(query) 717 | rows = result.fetchall() 718 | assert len(rows) == 2 719 | assert rows[0].col_int == 8 720 | assert rows[-1].col_int == 9 721 | 722 | # LIMIT 0 723 | query = sqla.select(table).limit(0) 724 | result = conn.execute(query) 725 | rows = result.fetchall() 726 | assert len(rows) == 0 727 | 728 | # LIMIT 0 and offset 729 | query = sqla.select(table).limit(0).offset(1) 730 | result = conn.execute(query) 731 | rows = result.fetchall() 732 | assert len(rows) == 0 733 | 734 | finally: 735 | if session: 736 | session.close() 737 | 738 | 739 | def test_limit_clause_with_binds_and_expressions(test_engine, test_model): 740 | """Test LIMIT clause with bind parameters and expressions.""" 741 | # Setup test data 742 | now = datetime.datetime(2023, 4, 12, 23, 55, 59, 342380) 743 | now_date = now.date() 744 | session = Session(test_engine) 745 | num_rows = 10 746 | 747 | try: 748 | # Insert test data 749 | models = [ 750 | test_model( 751 | col_boolean=True, 752 | col_byte=8, 753 | col_short=12, 754 | col_int=idx, # Using idx to make rows distinct and ordered 755 | col_long=14, 756 | col_float=15.234, 757 | col_double=16.88993244, 758 | col_symbol='coconut', 759 | col_string='banana', 760 | col_char='C', 761 | col_uuid='6d5eb038-63d1-4971-8484-30c16e13de5b', 762 | col_date=now_date, 763 | col_ts=now, 764 | col_geohash='dfvgsj2vptwu', 765 | col_long256='0xa3b400fcf6ed707d710d5d4e672305203ed3cc6254d1cefe313e4a465861f42a', 766 | col_varchar='pineapple' 767 | ) for idx in range(num_rows) 768 | ] 769 | session.bulk_save_objects(models) 770 | session.commit() 771 | 772 | metadata = sqla.MetaData() 773 | table = sqla.Table(ALL_TYPES_TABLE_NAME, metadata, autoload_with=test_engine) 774 | 775 | wait_until_table_is_ready(test_engine, ALL_TYPES_TABLE_NAME, num_rows) 776 | 777 | with test_engine.connect() as conn: 778 | # simple bindparam 779 | result = conn.execute( 780 | sqla.select(table).limit(sqla.bindparam('limit_val')), 781 | {"limit_val": 5} 782 | ) 783 | rows = result.fetchall() 784 | assert len(rows) == 5 785 | assert rows[0].col_int == 0 786 | assert rows[-1].col_int == 4 787 | 788 | # bindparam with expressions 789 | result = conn.execute( 790 | sqla.select(table).limit(sqla.bindparam('base_limit') * 2), 791 | {"base_limit": 3} 792 | ) 793 | rows = result.fetchall() 794 | assert len(rows) == 6 795 | assert rows[0].col_int == 0 796 | assert rows[-1].col_int == 5 797 | 798 | # multiple bindparams in expression 799 | result = conn.execute( 800 | sqla.select(table).limit( 801 | sqla.bindparam('limit_val') 802 | ).offset( 803 | sqla.bindparam('offset_val') 804 | ), 805 | { 806 | "limit_val": 3, 807 | "offset_val": 2 808 | } 809 | ) 810 | rows = result.fetchall() 811 | assert len(rows) == 3 812 | assert rows[0].col_int == 2 813 | assert rows[-1].col_int == 4 814 | 815 | # bindparam with type specification 816 | from sqlalchemy import Integer 817 | result = conn.execute( 818 | sqla.select(table).limit( 819 | sqla.bindparam('limit_val', type_=Integer) + 1 820 | ), 821 | {"limit_val": 4} 822 | ) 823 | rows = result.fetchall() 824 | assert len(rows) == 5 825 | assert rows[0].col_int == 0 826 | assert rows[-1].col_int == 4 827 | 828 | # text() and bindparam 829 | from sqlalchemy import text 830 | result = conn.execute( 831 | text("SELECT * FROM all_types_table LIMIT :lo, :hi"), 832 | {"lo": 3, "hi": 8} 833 | ) 834 | rows = result.fetchall() 835 | assert len(rows) == 5 836 | assert rows[0].col_int == 3 837 | assert rows[-1].col_int == 7 838 | 839 | finally: 840 | if session: 841 | session.close() 842 | -------------------------------------------------------------------------------- /tests/test_examples.py: -------------------------------------------------------------------------------- 1 | from examples import hello_world, psycopg2_connect, server_utilisation, sqlalchemy_orm, sqlalchemy_raw 2 | 3 | 4 | def test_hello_world(): 5 | hello_world.main() 6 | 7 | 8 | def test_psycopg2_connect(): 9 | psycopg2_connect.main() 10 | 11 | 12 | def test_server_utilisation(): 13 | server_utilisation.main(duration_sec=2.0) 14 | 15 | 16 | def test_sqlalchemy_orm(): 17 | sqlalchemy_orm.main() 18 | 19 | 20 | def test_sqlalchemy_raw(): 21 | sqlalchemy_raw.main() 22 | -------------------------------------------------------------------------------- /tests/test_superset.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest import mock 3 | 4 | import pytest 5 | from qdb_superset.db_engine_specs.questdb import QuestDbEngineSpec 6 | from questdb_connect.types import QUESTDB_TYPES, Timestamp 7 | from sqlalchemy import column, literal_column 8 | from sqlalchemy.types import TypeEngine 9 | 10 | 11 | def test_build_sqlalchemy_uri(): 12 | assert QuestDbEngineSpec.build_sqlalchemy_uri( 13 | { 14 | "host": "localhost", 15 | "port": "8812", 16 | "username": "admin", 17 | "password": "quest", 18 | "database": "main", 19 | } 20 | ) == "questdb://admin:quest@localhost:8812/main" 21 | 22 | 23 | def test_default_schema_for_query(): 24 | assert QuestDbEngineSpec.get_default_schema_for_query("main", None) == None 25 | 26 | 27 | def test_get_text_clause(): 28 | sql_clause = "SELECT * FROM public.mytable t1" 29 | sql_clause += " JOIN public.myclient t2 ON t1.id = t2.id" 30 | expected_clause = "SELECT * FROM mytable t1 JOIN myclient t2 ON t1.id = t2.id" 31 | actual_clause = str(QuestDbEngineSpec.get_text_clause(sql_clause)) 32 | print(f"sql: {sql_clause}, ex: {expected_clause}, ac: {actual_clause}") 33 | assert expected_clause == actual_clause 34 | 35 | 36 | def test_epoch_to_dttm(): 37 | assert QuestDbEngineSpec.epoch_to_dttm() == "{col} * 1000000" 38 | 39 | 40 | @pytest.mark.parametrize( 41 | ("target_type", "expected_result", "dttm"), 42 | [ 43 | ( 44 | "Date", 45 | "TO_DATE('2023-04-28', 'YYYY-MM-DD')", 46 | datetime.datetime(2023, 4, 28, 23, 55, 59, 281567), 47 | ), 48 | ( 49 | "DateTime", 50 | "TO_TIMESTAMP('2023-04-28T23:55:59.281567', 'yyyy-MM-ddTHH:mm:ss.SSSUUU')", 51 | datetime.datetime(2023, 4, 28, 23, 55, 59, 281567), 52 | ), 53 | ( 54 | "TimeStamp", 55 | "TO_TIMESTAMP('2023-04-28T23:55:59.281567', 'yyyy-MM-ddTHH:mm:ss.SSSUUU')", 56 | datetime.datetime(2023, 4, 28, 23, 55, 59, 281567), 57 | ), 58 | ("UnknownType", None, datetime.datetime(2023, 4, 28, 23, 55, 59, 281567)), 59 | ], 60 | ) 61 | def test_convert_dttm(target_type, expected_result, dttm) -> None: 62 | # datetime(year, month, day, hour, minute, second, microsecond) 63 | for target in ( 64 | target_type, 65 | target_type.upper(), 66 | target_type.lower(), 67 | target_type.capitalize(), 68 | ): 69 | assert QuestDbEngineSpec.convert_dttm( 70 | target_type=target, dttm=dttm 71 | ) == expected_result 72 | 73 | 74 | def test_get_datatype(): 75 | assert QuestDbEngineSpec.get_datatype("int") == "INT" 76 | assert QuestDbEngineSpec.get_datatype(["int"]) == "['int']" 77 | 78 | 79 | def test_get_column_spec(): 80 | for native_type in QUESTDB_TYPES: 81 | column_spec = QuestDbEngineSpec.get_column_spec(native_type.__visit_name__) 82 | assert native_type == column_spec.sqla_type 83 | assert native_type != Timestamp or column_spec.is_dttm 84 | 85 | 86 | def test_get_sqla_column_type(): 87 | for native_type in QUESTDB_TYPES: 88 | column_type = QuestDbEngineSpec.get_sqla_column_type(native_type.__visit_name__) 89 | assert isinstance(column_type, TypeEngine.__class__) 90 | 91 | 92 | def test_get_allow_cost_estimate(): 93 | assert not QuestDbEngineSpec.get_allow_cost_estimate(extra=None) 94 | 95 | 96 | def test_get_view_names(): 97 | assert set() == QuestDbEngineSpec.get_view_names("main", None, None) 98 | 99 | 100 | def test_get_table_names(): 101 | inspector = mock.Mock() 102 | inspector.get_table_names = mock.Mock( 103 | return_value=["public.table", "table_2", '"public.table_3"'] 104 | ) 105 | pg_result = QuestDbEngineSpec.get_table_names( 106 | database=mock.ANY, schema="public", inspector=inspector 107 | ) 108 | assert {"table", '"public.table_3"', "table_2"} == pg_result 109 | 110 | 111 | def test_time_exp_literal_no_grain(superset_test_engine): 112 | col = literal_column("COALESCE(a, b)") 113 | expr = QuestDbEngineSpec.get_timestamp_expr(col, None, None) 114 | result = str(expr.compile(None, dialect=superset_test_engine.dialect)) 115 | assert "COALESCE(a, b)" == result 116 | 117 | 118 | def test_time_ex_lowr_col_no_grain(superset_test_engine): 119 | col = column("lower_case") 120 | expr = QuestDbEngineSpec.get_timestamp_expr(col, None, None) 121 | result = str(expr.compile(None, dialect=superset_test_engine.dialect)) 122 | assert "lower_case" == result 123 | 124 | 125 | def test_execute_sql_statement(superset_test_engine) -> None: 126 | query = """ 127 | select * from tables() 128 | LIMIT 1001 129 | """ 130 | with superset_test_engine.connect() as cursor: 131 | rs = QuestDbEngineSpec.execute(cursor, query) 132 | print (rs) 133 | -------------------------------------------------------------------------------- /tests/test_types.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | import questdb_connect as qdbc 4 | from questdb_connect.common import quote_identifier 5 | 6 | 7 | def test_resolve_type_from_name(): 8 | for type_class in qdbc.QUESTDB_TYPES: 9 | resolved_class = qdbc.resolve_type_from_name(type_class.__visit_name__) 10 | assert type_class.__visit_name__ == resolved_class.__visit_name__ 11 | assert isinstance(type_class(), resolved_class) 12 | assert isinstance(resolved_class(), type_class) 13 | 14 | for n in range(1, 61): 15 | g_name = qdbc.geohash_type_name(n) 16 | g_class = qdbc.resolve_type_from_name(g_name) 17 | assert isinstance(g_class(), qdbc.geohash_class(n)) 18 | 19 | 20 | def test_symbol_type(): 21 | # Test basic Symbol without parameters 22 | symbol = qdbc.Symbol() 23 | assert symbol.__visit_name__ == "SYMBOL" 24 | assert symbol.compile() == "SYMBOL" 25 | assert symbol.column_spec("test_col") == "\"test_col\" SYMBOL" 26 | 27 | # Test Symbol with capacity 28 | symbol_cap = qdbc.Symbol(capacity=128) 29 | assert symbol_cap.compile() == "SYMBOL CAPACITY 128" 30 | assert symbol_cap.column_spec("test_col") == "\"test_col\" SYMBOL CAPACITY 128" 31 | 32 | # Test Symbol with cache true 33 | symbol_cache = qdbc.Symbol(cache=True) 34 | assert symbol_cache.compile() == "SYMBOL CACHE" 35 | assert symbol_cache.column_spec("test_col") == "\"test_col\" SYMBOL CACHE" 36 | 37 | # Test Symbol with cache false 38 | symbol_nocache = qdbc.Symbol(cache=False) 39 | assert symbol_nocache.compile() == "SYMBOL NOCACHE" 40 | assert symbol_nocache.column_spec("test_col") == "\"test_col\" SYMBOL NOCACHE" 41 | 42 | # Test Symbol with both parameters 43 | symbol_full = qdbc.Symbol(capacity=256, cache=True) 44 | assert symbol_full.compile() == "SYMBOL CAPACITY 256 CACHE" 45 | assert symbol_full.column_spec("test_col") == "\"test_col\" SYMBOL CAPACITY 256 CACHE" 46 | 47 | # Test inheritance and type resolution 48 | assert isinstance(symbol, qdbc.QDBTypeMixin) 49 | resolved_class = qdbc.resolve_type_from_name("SYMBOL") 50 | assert resolved_class.__visit_name__ == "SYMBOL" 51 | assert isinstance(symbol, resolved_class) 52 | 53 | # Test that parameters don't affect type resolution 54 | symbol_with_params = qdbc.Symbol(capacity=128, cache=True) 55 | assert isinstance(symbol_with_params, resolved_class) 56 | assert isinstance(resolved_class(), type(symbol_with_params)) 57 | 58 | 59 | def test_symbol_backward_compatibility(): 60 | """Verify that the parametrized Symbol type maintains backward compatibility with older code.""" 61 | # Test all the ways Symbol type could be previously instantiated 62 | symbol1 = qdbc.Symbol 63 | symbol2 = qdbc.Symbol() 64 | 65 | # Check that both work in column definitions 66 | from sqlalchemy import Column, MetaData, Table 67 | 68 | metadata = MetaData() 69 | test_table = Table( 70 | 'test_table', 71 | metadata, 72 | Column('old_style1', symbol1), # Old style: direct class reference 73 | Column('old_style2', symbol2), # Old style: basic instantiation 74 | ) 75 | 76 | # Verify type resolution still works 77 | for column in test_table.columns: 78 | # Check inheritance 79 | assert isinstance(column.type, qdbc.QDBTypeMixin) 80 | 81 | # Check type resolution 82 | resolved_class = qdbc.resolve_type_from_name("SYMBOL") 83 | assert isinstance(column.type, resolved_class) 84 | 85 | # Check SQL generation matches old behavior 86 | assert column.type.compile() == "SYMBOL" 87 | assert column.type.column_spec(column.name) == f"{quote_identifier(column.name)} SYMBOL" 88 | 89 | def test_symbol_type_in_column(): 90 | # Test Symbol type in Column definition 91 | from sqlalchemy import Column, MetaData, Table 92 | 93 | metadata = MetaData() 94 | 95 | # Create a test table with different Symbol column variations 96 | test_table = Table( 97 | 'test_table', 98 | metadata, 99 | Column('basic_symbol', qdbc.Symbol()), 100 | Column('symbol_with_capacity', qdbc.Symbol(capacity=128)), 101 | Column('symbol_with_cache', qdbc.Symbol(cache=True)), 102 | Column('symbol_with_nocache', qdbc.Symbol(cache=False)), 103 | Column('symbol_full', qdbc.Symbol(capacity=256, cache=True)) 104 | ) 105 | 106 | # Get the create table SQL (implementation-dependent) 107 | # This part might need adjustment based on your actual SQL compilation logic 108 | for column in test_table.columns: 109 | assert isinstance(column.type, qdbc.Symbol) 110 | assert isinstance(column.type, qdbc.QDBTypeMixin) 111 | 112 | if column.name == 'basic_symbol': 113 | assert column.type.compile() == "SYMBOL" 114 | elif column.name == 'symbol_with_capacity': 115 | assert column.type.compile() == "SYMBOL CAPACITY 128" 116 | elif column.name == 'symbol_with_cache': 117 | assert column.type.compile() == "SYMBOL CACHE" 118 | elif column.name == 'symbol_with_nocache': 119 | assert column.type.compile() == "SYMBOL NOCACHE" 120 | elif column.name == 'symbol_full': 121 | assert column.type.compile() == "SYMBOL CAPACITY 256 CACHE" 122 | 123 | 124 | def test_superset_default_mappings(): 125 | default_column_type_mappings = ( 126 | (re.compile("^BOOLEAN$", re.IGNORECASE), qdbc.Boolean), 127 | (re.compile("^BYTE$", re.IGNORECASE), qdbc.Byte), 128 | (re.compile("^SHORT$", re.IGNORECASE), qdbc.Short), 129 | (re.compile("^CHAR$", re.IGNORECASE), qdbc.Char), 130 | (re.compile("^INT$", re.IGNORECASE), qdbc.Int), 131 | (re.compile("^LONG$", re.IGNORECASE), qdbc.Long), 132 | (re.compile("^DATE$", re.IGNORECASE), qdbc.Date), 133 | (re.compile("^TIMESTAMP$", re.IGNORECASE), qdbc.Timestamp), 134 | (re.compile("^FLOAT$", re.IGNORECASE), qdbc.Float), 135 | (re.compile("^DOUBLE$", re.IGNORECASE), qdbc.Double), 136 | (re.compile("^STRING$", re.IGNORECASE), qdbc.String), 137 | (re.compile("^SYMBOL$", re.IGNORECASE), qdbc.Symbol), 138 | (re.compile("^LONG256$", re.IGNORECASE), qdbc.Long256), 139 | (re.compile("^UUID$", re.IGNORECASE), qdbc.UUID), 140 | (re.compile("^LONG118$", re.IGNORECASE), qdbc.UUID), 141 | (re.compile("^IPV4$", re.IGNORECASE), qdbc.IPv4), 142 | (re.compile("^VARCHAR$", re.IGNORECASE), qdbc.Varchar), 143 | ) 144 | for type_class in qdbc.QUESTDB_TYPES: 145 | for pattern, _expected_type in default_column_type_mappings: 146 | matching_name = pattern.match(type_class.__visit_name__) 147 | if matching_name: 148 | print(f"match: {matching_name}, type_class: {type_class}") 149 | resolved_class = qdbc.resolve_type_from_name(matching_name.group(0)) 150 | assert type_class.__visit_name__ == resolved_class.__visit_name__ 151 | assert isinstance(type_class(), resolved_class) 152 | assert isinstance(resolved_class(), type_class) 153 | break 154 | geohash_pattern = re.compile(r"^GEOHASH\(\d+[b|c]\)$", re.IGNORECASE) 155 | for n in range(1, 61): 156 | g_name = qdbc.geohash_type_name(n) 157 | matching_name = geohash_pattern.match(g_name).group(0) 158 | assert matching_name == g_name 159 | g_class = qdbc.resolve_type_from_name(g_name) 160 | assert isinstance(g_class(), qdbc.geohash_class(n)) 161 | -------------------------------------------------------------------------------- /tests/test_username.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from _pytest.outcomes import fail 3 | from psycopg2 import OperationalError 4 | from sqlalchemy import create_engine 5 | 6 | 7 | def test_user(test_engine, test_model): 8 | engine = create_engine("questdb://admin:quest@localhost:8812/qdb") 9 | engine.connect() 10 | 11 | engine = create_engine("questdb://user1:quest@localhost:8812/qdb") 12 | with pytest.raises(OperationalError) as exc_info: 13 | engine.connect() 14 | if not str(exc_info.value).__contains__("ERROR: invalid username/password"): 15 | fail() 16 | --------------------------------------------------------------------------------