├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ └── bug_report.md ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── actions.yml │ ├── docs.yml │ └── sa-versions.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.md ├── CONTRIBUTING.rst ├── LICENSE ├── README.rst ├── clickhouse_sqlalchemy ├── __init__.py ├── alembic │ ├── __init__.py │ ├── comparators.py │ ├── dialect.py │ ├── operations.py │ ├── renderers.py │ └── toimpl.py ├── drivers │ ├── __init__.py │ ├── asynch │ │ ├── __init__.py │ │ ├── base.py │ │ └── connector.py │ ├── base.py │ ├── compilers │ │ ├── __init__.py │ │ ├── ddlcompiler.py │ │ ├── sqlcompiler.py │ │ └── typecompiler.py │ ├── http │ │ ├── __init__.py │ │ ├── base.py │ │ ├── connector.py │ │ ├── escaper.py │ │ ├── exceptions.py │ │ ├── transport.py │ │ └── utils.py │ ├── native │ │ ├── __init__.py │ │ ├── base.py │ │ └── connector.py │ ├── reflection.py │ └── util.py ├── engines │ ├── __init__.py │ ├── base.py │ ├── mergetree.py │ ├── misc.py │ ├── replicated.py │ └── util.py ├── exceptions.py ├── ext │ ├── __init__.py │ ├── clauses.py │ └── declarative.py ├── orm │ ├── __init__.py │ ├── query.py │ └── session.py ├── sql │ ├── __init__.py │ ├── ddl.py │ ├── functions.py │ ├── schema.py │ └── selectable.py └── types │ ├── __init__.py │ ├── common.py │ ├── ip.py │ └── nested.py ├── docs ├── Makefile ├── _static │ └── css │ │ └── custom.css ├── _templates │ └── layout.html ├── changelog.rst ├── conf.py ├── connection.rst ├── contents.rst.inc ├── contributing.rst ├── development.rst ├── features.rst ├── index.rst ├── installation.rst ├── license.rst ├── migrations.rst ├── quickstart.rst └── types.rst ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── alembic │ ├── __init__.py │ └── test_default_schema_comparators.py ├── config.py ├── docker-compose.yml ├── drivers │ ├── __init__.py │ ├── asynch │ │ ├── __init__.py │ │ ├── test_base.py │ │ ├── test_cursor.py │ │ ├── test_insert.py │ │ └── test_select.py │ ├── http │ │ ├── __init__.py │ │ ├── test_cursor.py │ │ ├── test_escaping.py │ │ ├── test_select.py │ │ ├── test_stream.py │ │ ├── test_transport.py │ │ └── test_utils.py │ ├── native │ │ ├── __init__.py │ │ ├── test_base.py │ │ ├── test_cursor.py │ │ ├── test_insert.py │ │ └── test_select.py │ ├── test_clickhouse_dialect.py │ └── test_util.py ├── engines │ ├── __init__.py │ ├── test_compilation.py │ ├── test_parse_columns.py │ └── test_reflection.py ├── ext │ ├── __init__.py │ └── test_declative.py ├── functions │ ├── __init__.py │ ├── test_count.py │ ├── test_extract.py │ ├── test_has.py │ └── test_if.py ├── log.py ├── orm │ ├── __init__.py │ └── test_select.py ├── session.py ├── sql │ ├── __init__.py │ ├── test_case.py │ ├── test_delete.py │ ├── test_functions.py │ ├── test_ilike.py │ ├── test_insert.py │ ├── test_is_distinct_from.py │ ├── test_lambda.py │ ├── test_limit.py │ ├── test_regexp_match.py │ ├── test_schema.py │ ├── test_selectable.py │ └── test_update.py ├── test_compiler.py ├── test_ddl.py ├── test_reflection.py ├── testcase.py ├── types │ ├── __init__.py │ ├── test_boolean.py │ ├── test_date32.py │ ├── test_datetime.py │ ├── test_datetime64.py │ ├── test_enum16.py │ ├── test_enum8.py │ ├── test_int128.py │ ├── test_int256.py │ ├── test_ip.py │ ├── test_json.py │ └── test_numeric.py └── util.py └── testsrequire.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | patreon: xzkostyan 2 | 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Minimal piece of Python code that reproduces the problem. 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Versions** 20 | 21 | - Version of package with the problem. 22 | - Python version. 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Keep GitHub Actions up to date with GitHub's Dependabot... 2 | # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot 3 | # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem 4 | version: 2 5 | updates: 6 | - package-ecosystem: github-actions 7 | directory: / 8 | groups: 9 | github-actions: 10 | patterns: 11 | - "*" # Group all Actions updates into a single larger pull request 12 | schedule: 13 | interval: weekly 14 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | - fixes # 7 | 8 | 14 | 15 | Checklist: 16 | 17 | - [ ] Add tests that demonstrate the correct behavior of the change. Tests should fail without the change. 18 | - [ ] Add or update relevant docs, in the docs folder and in code. 19 | - [ ] Ensure PR doesn't contain untouched code reformatting: spaces, etc. 20 | - [ ] Run `flake8` and fix issues. 21 | - [ ] Run `pytest` no tests failed. See https://clickhouse-sqlalchemy.readthedocs.io/en/latest/development.html. 22 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: build 3 | jobs: 4 | tests: 5 | runs-on: ubuntu-22.04 6 | strategy: 7 | matrix: 8 | python-version: 9 | - "3.7" 10 | - "3.8" 11 | - "3.9" 12 | - "3.10" 13 | - "3.11" 14 | - "3.12" 15 | include: 16 | - clickhouse-version: 18.14.9 17 | clickhouse-org: yandex 18 | - clickhouse-version: 19.3.5 19 | clickhouse-org: yandex 20 | - clickhouse-version: 22.5.1.2079 21 | clickhouse-org: clickhouse 22 | - clickhouse-version: 23.8.4.69 23 | clickhouse-org: clickhouse 24 | services: 25 | clickhouse-server: 26 | image: ${{ matrix.clickhouse-org }}/clickhouse-server:${{ matrix.clickhouse-version }} 27 | ports: 28 | - 8123:8123 29 | - 9000:9000 30 | 31 | name: ${{ matrix.python-version }} CH=${{ matrix.clickhouse-version }} 32 | steps: 33 | - uses: actions/checkout@v4 34 | - name: Set up Python 35 | uses: actions/setup-python@v5 36 | with: 37 | python-version: ${{ matrix.python-version }} 38 | architecture: x64 39 | - name: Install test requirements 40 | run: | 41 | pip install --upgrade pip setuptools wheel 42 | pip install flake8 flake8-print coverage 43 | python testsrequire.py 44 | python setup.py develop 45 | # Limit each test time execution. 46 | pip install pytest-timeout 47 | - name: Run flake8 48 | run: flake8 49 | - name: Run tests 50 | run: coverage run --source=clickhouse_sqlalchemy -m pytest --timeout=10 -v 51 | timeout-minutes: 2 52 | - name: Set up Python for coverage submission 53 | if: ${{ matrix.python-version == '2.7' }} 54 | uses: actions/setup-python@v5 55 | with: 56 | python-version: 3.8 57 | architecture: x64 58 | - name: Install coveralls 59 | run: | 60 | # Newer coveralls do not work with github actions. 61 | pip install 'coveralls<3.0.0' 62 | - name: Upload coverage 63 | run: coveralls 64 | env: 65 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 66 | COVERALLS_PARALLEL: true 67 | COVERALLS_FLAG_NAME: ${{ matrix.python-version }} CH=${{ matrix.clickhouse-version }} 68 | 69 | coveralls-finished: 70 | name: Indicate completion to coveralls.io 71 | needs: tests 72 | runs-on: ubuntu-latest 73 | steps: 74 | - name: Finished 75 | uses: coverallsapp/github-action@v2.3.3 76 | with: 77 | github-token: ${{ secrets.GITHUB_TOKEN }} 78 | parallel-finished: true 79 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: build-docs 3 | jobs: 4 | tests: 5 | runs-on: ubuntu-22.04 6 | name: Build docs 7 | steps: 8 | - uses: actions/checkout@v4 9 | - name: Set up Python 10 | uses: actions/setup-python@v5 11 | with: 12 | python-version: 3.12 13 | architecture: x64 14 | - name: Update tools 15 | run: pip install --upgrade pip setuptools wheel 16 | - name: Install sphinx 17 | run: pip install sphinx 18 | - name: Install package 19 | run: pip install -e . 20 | - name: Build docs 21 | run: cd docs && make html 22 | -------------------------------------------------------------------------------- /.github/workflows/sa-versions.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: "SQLAlchemy >=2.0.0 versions" 3 | jobs: 4 | tests: 5 | runs-on: ubuntu-20.04 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | python-version: 10 | - "3.12" 11 | clickhouse-version: 12 | - 23.8.4.69 13 | sa-version: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36] 14 | services: 15 | clickhouse-server: 16 | image: clickhouse/clickhouse-server:${{ matrix.clickhouse-version }} 17 | ports: 18 | - 8123:8123 19 | - 9000:9000 20 | 21 | name: ${{ matrix.python-version }} SA=2.0.${{ matrix.sa-version }} 22 | steps: 23 | - uses: actions/checkout@v4 24 | - name: Set up Python 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | architecture: x64 29 | - name: Install test requirements 30 | run: | 31 | pip install --upgrade pip setuptools wheel 32 | python testsrequire.py 33 | python setup.py develop 34 | # Limit each test time execution. 35 | pip install pytest-timeout 36 | - name: Install SQLAlchemy 37 | run: pip install sqlalchemy==2.0.${{ matrix.sa-version }} 38 | - name: Run tests 39 | run: pytest --timeout=10 -v 40 | timeout-minutes: 2 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .idea 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # IPython Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | 82 | # virtualenv 83 | venv/ 84 | ENV/ 85 | 86 | # Spyder project settings 87 | .spyderproject 88 | 89 | # Rope project settings 90 | .ropeproject 91 | 92 | # PyCharm project settings 93 | .idea/ 94 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.11" 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | python: 12 | install: 13 | - method: pip 14 | path: . 15 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | How to Contribute 2 | ================= 3 | 4 | #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. 5 | #. Fork `the repository `_ on GitHub to start making your changes to the **master** branch (or branch off of it). 6 | #. Write a test which shows that the bug was fixed or that the feature works as expected. 7 | #. Send a pull request and bug the maintainer until it gets merged and published. 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is the MIT license: http://www.opensource.org/licenses/mit-license.php 2 | 3 | Copyright (c) 2017 by Konstantin Lebedev. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ClickHouse SQLAlchemy 2 | ===================== 3 | 4 | ClickHouse dialect for SQLAlchemy to `ClickHouse database `_. 5 | 6 | 7 | .. image:: https://img.shields.io/pypi/v/clickhouse-sqlalchemy.svg 8 | :target: https://pypi.org/project/clickhouse-sqlalchemy 9 | 10 | .. image:: https://coveralls.io/repos/github/xzkostyan/clickhouse-sqlalchemy/badge.svg?branch=master 11 | :target: https://coveralls.io/github/xzkostyan/clickhouse-sqlalchemy?branch=master 12 | 13 | .. image:: https://img.shields.io/pypi/l/clickhouse-sqlalchemy.svg 14 | :target: https://pypi.org/project/clickhouse-sqlalchemy 15 | 16 | .. image:: https://img.shields.io/pypi/pyversions/clickhouse-sqlalchemy.svg 17 | :target: https://pypi.org/project/clickhouse-sqlalchemy 18 | 19 | .. image:: https://img.shields.io/pypi/dm/clickhouse-sqlalchemy.svg 20 | :target: https://pypi.org/project/clickhouse-sqlalchemy 21 | 22 | .. image:: https://github.com/xzkostyan/clickhouse-sqlalchemy/actions/workflows/actions.yml/badge.svg 23 | :target: https://github.com/xzkostyan/clickhouse-sqlalchemy/actions/workflows/actions.yml 24 | 25 | 26 | Documentation 27 | ============= 28 | 29 | Documentation is available at https://clickhouse-sqlalchemy.readthedocs.io. 30 | 31 | 32 | Usage 33 | ===== 34 | 35 | Supported interfaces: 36 | 37 | - **native** [recommended] (TCP) via `clickhouse-driver ` 38 | - **async native** (TCP) via `asynch ` 39 | - **http** via requests 40 | 41 | Define table 42 | 43 | .. code-block:: python 44 | 45 | from sqlalchemy import create_engine, Column, MetaData 46 | 47 | from clickhouse_sqlalchemy import ( 48 | Table, make_session, get_declarative_base, types, engines 49 | ) 50 | 51 | uri = 'clickhouse+native://localhost/default' 52 | 53 | engine = create_engine(uri) 54 | session = make_session(engine) 55 | metadata = MetaData(bind=engine) 56 | 57 | Base = get_declarative_base(metadata=metadata) 58 | 59 | class Rate(Base): 60 | day = Column(types.Date, primary_key=True) 61 | value = Column(types.Int32) 62 | 63 | __table_args__ = ( 64 | engines.Memory(), 65 | ) 66 | 67 | Rate.__table__.create() 68 | 69 | 70 | Insert some data 71 | 72 | .. code-block:: python 73 | 74 | from datetime import date, timedelta 75 | 76 | from sqlalchemy import func 77 | 78 | today = date.today() 79 | rates = [ 80 | {'day': today - timedelta(i), 'value': 200 - i} 81 | for i in range(100) 82 | ] 83 | 84 | 85 | And query inserted data 86 | 87 | .. code-block:: python 88 | 89 | session.execute(Rate.__table__.insert(), rates) 90 | 91 | session.query(func.count(Rate.day)) \ 92 | .filter(Rate.day > today - timedelta(20)) \ 93 | .scalar() 94 | 95 | 96 | License 97 | ======= 98 | 99 | ClickHouse SQLAlchemy is distributed under the `MIT license 100 | `_. 101 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .ext.declarative import get_declarative_base 3 | from .orm.session import make_session 4 | from .sql import Table, MaterializedView, select 5 | 6 | 7 | VERSION = (0, 3, 2) 8 | __version__ = '.'.join(str(x) for x in VERSION) 9 | 10 | 11 | __all__ = ( 12 | 'get_declarative_base', 13 | 'make_session', 14 | 'Table', 'MaterializedView', 'select' 15 | ) 16 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/alembic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/alembic/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/alembic/dialect.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import func, Column, types as sqltypes 2 | 3 | try: 4 | from alembic.ddl import impl 5 | from alembic.ddl.base import ( 6 | compiles, ColumnComment, format_table_name, format_column_name 7 | ) 8 | except ImportError: 9 | raise RuntimeError('alembic must be installed') 10 | 11 | from clickhouse_sqlalchemy import types, engines 12 | from clickhouse_sqlalchemy.sql.ddl import DropTable 13 | from .comparators import compare_mat_view 14 | from .renderers import ( 15 | render_attach_mat_view, render_detach_mat_view, 16 | render_create_mat_view, render_drop_mat_view 17 | ) 18 | from .toimpl import ( 19 | create_mat_view, attach_mat_view 20 | ) 21 | 22 | 23 | class ClickHouseDialectImpl(impl.DefaultImpl): 24 | __dialect__ = 'clickhouse' 25 | transactional_ddl = False 26 | 27 | def drop_table(self, table): 28 | table.dispatch.before_drop( 29 | table, self.connection, checkfirst=False, _ddl_runner=self 30 | ) 31 | 32 | self._exec(DropTable(table)) 33 | 34 | table.dispatch.after_drop( 35 | table, self.connection, checkfirst=False, _ddl_runner=self 36 | ) 37 | 38 | 39 | def patch_alembic_version(context, **kwargs): 40 | migration_context = context._proxy._migration_context 41 | version = migration_context._version 42 | 43 | dt = Column('dt', types.DateTime, server_default=func.now()) 44 | version_num = Column('version_num', types.String, primary_key=True) 45 | version.append_column(dt) 46 | version.append_column(version_num, replace_existing=True) 47 | 48 | if 'cluster' in kwargs: 49 | cluster = kwargs['cluster'] 50 | version.engine = engines.ReplicatedReplacingMergeTree( 51 | kwargs['table_path'], kwargs['replica_name'], 52 | version=dt, order_by=func.tuple() 53 | ) 54 | version.kwargs['clickhouse_cluster'] = cluster 55 | else: 56 | version.engine = engines.ReplacingMergeTree( 57 | version=dt, order_by=func.tuple() 58 | ) 59 | 60 | 61 | def include_object(object, name, type_, reflected, compare_to): 62 | # skip inner matview tables in autogeneration. 63 | if type_ == 'table' and object.info.get('mv_storage'): 64 | return False 65 | 66 | return True 67 | 68 | 69 | @compiles(ColumnComment, 'clickhouse') 70 | def visit_column_comment(element, compiler, **kw): 71 | ddl = "ALTER TABLE {table_name} COMMENT COLUMN {column_name} {comment}" 72 | comment = ( 73 | compiler.sql_compiler.render_literal_value( 74 | element.comment or '', sqltypes.String() 75 | ) 76 | ) 77 | 78 | return ddl.format( 79 | table_name=format_table_name( 80 | compiler, element.table_name, element.schema 81 | ), 82 | column_name=format_column_name(compiler, element.column_name), 83 | comment=comment, 84 | ) 85 | 86 | 87 | __all__ = ( 88 | 'ClickHouseDialectImpl', 'compare_mat_view', 89 | 'render_attach_mat_view', 'render_detach_mat_view', 90 | 'render_create_mat_view', 'render_drop_mat_view', 91 | 'create_mat_view', 'attach_mat_view' 92 | ) 93 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/alembic/operations.py: -------------------------------------------------------------------------------- 1 | from alembic.operations import Operations, MigrateOperation 2 | from alembic.operations.ops import ExecuteSQLOp 3 | 4 | 5 | @Operations.register_operation('create_mat_view') 6 | class CreateMatViewOp(MigrateOperation): 7 | def __init__(self, name, selectable, engine, *columns, **kwargs): 8 | self.name = name 9 | self.selectable = selectable 10 | self.engine = engine 11 | self.columns = columns 12 | self.kwargs = kwargs 13 | 14 | @classmethod 15 | def create_mat_view(cls, operations, name, selectable, engine, *columns, 16 | **kwargs): 17 | """Issue a "CREATE MATERIALIZED VIEW" instruction.""" 18 | 19 | op = CreateMatViewOp(name, selectable, engine, *columns, **kwargs) 20 | return operations.invoke(op) 21 | 22 | def reverse(self): 23 | return DropMatViewOp( 24 | self.name, self.selectable, self.engine, *self.columns, 25 | **self.kwargs 26 | ) 27 | 28 | 29 | @Operations.register_operation('create_mat_view_to_table') 30 | class CreateMatViewToTableOp(MigrateOperation): 31 | def __init__(self, name, selectable, inner_name, **kwargs): 32 | self.name = name 33 | self.selectable = selectable 34 | self.inner_name = inner_name 35 | self.kwargs = kwargs 36 | 37 | @classmethod 38 | def create_mat_view_to_table(cls, operations, name, selectable, inner_name, 39 | **kwargs): 40 | """Issue a "CREATE MATERIALIZED VIEW" instruction wit "TO" clause.""" 41 | 42 | op = CreateMatViewToTableOp(name, selectable, inner_name, **kwargs) 43 | return operations.invoke(op) 44 | 45 | def reverse(self): 46 | return DropMatViewToTableOp( 47 | self.name, self.selectable, self.inner_name, **self.kwargs 48 | ) 49 | 50 | 51 | @Operations.register_operation('drop_mat_view_to_table') 52 | class DropMatViewToTableOp(MigrateOperation): 53 | def __init__(self, name, old_selectable, inner_name, **kwargs): 54 | self.name = name 55 | self.old_selectable = old_selectable 56 | self.inner_name = inner_name 57 | self.kwargs = kwargs 58 | 59 | @classmethod 60 | def drop_mat_view_to_table(cls, operations, name, **kwargs): 61 | """Issue a "DROP VIEW" instruction.""" 62 | 63 | sql = 'DROP VIEW ' 64 | if kwargs.get('if_exists'): 65 | sql += 'IF EXISTS ' 66 | 67 | sql += name 68 | 69 | if kwargs.get('on_cluster'): 70 | sql += ' ON CLUSTER ' + kwargs['on_cluster'] 71 | 72 | op = ExecuteSQLOp(sql) 73 | return operations.invoke(op) 74 | 75 | def reverse(self): 76 | return CreateMatViewToTableOp( 77 | self.name, self.old_selectable, self.inner_name, **self.kwargs 78 | ) 79 | 80 | 81 | @Operations.register_operation('drop_mat_view') 82 | class DropMatViewOp(MigrateOperation): 83 | def __init__(self, name, selectable, engine, *columns, **kwargs): 84 | self.name = name 85 | self.selectable = selectable 86 | self.engine = engine 87 | self.columns = columns 88 | self.kwargs = kwargs 89 | 90 | @classmethod 91 | def drop_mat_view(cls, operations, name, **kwargs): 92 | """Issue a "DROP VIEW" instruction.""" 93 | 94 | sql = 'DROP VIEW ' 95 | if kwargs.get('if_exists'): 96 | sql += 'IF EXISTS ' 97 | 98 | sql += name 99 | 100 | if kwargs.get('on_cluster'): 101 | sql += ' ON CLUSTER ' + kwargs['on_cluster'] 102 | 103 | op = ExecuteSQLOp(sql) 104 | return operations.invoke(op) 105 | 106 | def reverse(self): 107 | return CreateMatViewOp( 108 | self.name, self.selectable, self.engine, *self.columns, 109 | **self.kwargs 110 | ) 111 | 112 | 113 | @Operations.register_operation('attach_mat_view') 114 | class AttachMatViewOp(MigrateOperation): 115 | def __init__(self, name, selectable, engine, *columns, **kwargs): 116 | self.name = name 117 | self.selectable = selectable 118 | self.engine = engine 119 | self.columns = columns 120 | self.kwargs = kwargs 121 | 122 | @classmethod 123 | def attach_mat_view(cls, operations, name, selectable, engine, *columns, 124 | **kwargs): 125 | """Issue a "ATTACH MATERIALIZED VIEW" instruction.""" 126 | 127 | op = AttachMatViewOp(name, selectable, engine, *columns, **kwargs) 128 | return operations.invoke(op) 129 | 130 | def reverse(self): 131 | return DetachMatViewOp( 132 | self.name, self.selectable, self.engine, *self.columns, 133 | **self.kwargs 134 | ) 135 | 136 | 137 | @Operations.register_operation('detach_mat_view') 138 | class DetachMatViewOp(MigrateOperation): 139 | def __init__(self, name, old_selectable, engine, *columns, **kwargs): 140 | self.name = name 141 | self.old_selectable = old_selectable 142 | self.engine = engine 143 | self.columns = columns 144 | self.kwargs = kwargs 145 | 146 | @classmethod 147 | def detach_mat_view(cls, operations, name, **kwargs): 148 | """Issue a "DETACH VIEW" instruction.""" 149 | 150 | sql = 'DETACH VIEW ' 151 | 152 | if kwargs.get('if_exists'): 153 | sql += 'IF EXISTS ' 154 | 155 | sql += name 156 | 157 | if kwargs.get('on_cluster'): 158 | sql += ' ON CLUSTER ' + kwargs['on_cluster'] 159 | 160 | if kwargs.get('permanently'): 161 | sql += ' PERMANENTLY' 162 | 163 | op = ExecuteSQLOp(sql) 164 | return operations.invoke(op) 165 | 166 | def reverse(self): 167 | return AttachMatViewOp( 168 | self.name, self.old_selectable, self.engine, *self.columns, 169 | **self.kwargs 170 | ) 171 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/alembic/renderers.py: -------------------------------------------------------------------------------- 1 | from alembic.autogenerate import render 2 | from alembic.autogenerate import renderers 3 | 4 | from . import operations 5 | 6 | indent = ' ' * 4 7 | 8 | 9 | def escape(x): 10 | return x.replace("'", "\\'") 11 | 12 | 13 | @renderers.dispatch_for(operations.CreateMatViewOp) 14 | def render_create_mat_view(autogen_context, op): 15 | columns = [ 16 | col 17 | for col in [ 18 | render._render_column(col, autogen_context) for col in op.columns 19 | ] 20 | if col 21 | ] 22 | 23 | templ = ( 24 | "{prefix}create_mat_view(\n" 25 | "{indent}'{name}',\n" 26 | "{indent}'{selectable}',\n" 27 | "{indent}'{engine}',\n" 28 | "{indent}{columns}\n" 29 | ")" 30 | ) 31 | 32 | join_indent = ("'\n" + indent + "'") 33 | return templ.format( 34 | prefix=render._alembic_autogenerate_prefix(autogen_context), 35 | name=op.name, 36 | selectable=join_indent.join(escape(op.selectable).split('\n')), 37 | engine=join_indent.join(escape(op.engine.strip()).split('\n')), 38 | columns=(',\n' + indent).join(str(arg) for arg in columns), 39 | indent=indent 40 | ) 41 | 42 | 43 | @renderers.dispatch_for(operations.DropMatViewOp) 44 | def render_drop_mat_view(autogen_context, op): 45 | return ( 46 | render._alembic_autogenerate_prefix(autogen_context) + 47 | "drop_mat_view('" + op.name + "')" 48 | ) 49 | 50 | 51 | @renderers.dispatch_for(operations.CreateMatViewToTableOp) 52 | def render_create_mat_view_to_table(autogen_context, op): 53 | templ = ( 54 | "{prefix}create_mat_view_to_table(\n" 55 | "{indent}'{name}',\n" 56 | "{indent}'{selectable}',\n" 57 | "{indent}'{inner_name}'\n" 58 | ")" 59 | ) 60 | 61 | join_indent = ("'\n" + indent + "'") 62 | return templ.format( 63 | prefix=render._alembic_autogenerate_prefix(autogen_context), 64 | name=op.name, 65 | selectable=join_indent.join(escape(op.selectable).split('\n')), 66 | inner_name=op.inner_name, 67 | indent=indent 68 | ) 69 | 70 | 71 | @renderers.dispatch_for(operations.DropMatViewToTableOp) 72 | def render_drop_mat_view_to_table(autogen_context, op): 73 | return ( 74 | render._alembic_autogenerate_prefix(autogen_context) + 75 | "drop_mat_view_to_table('" + op.name + "')" 76 | ) 77 | 78 | 79 | @renderers.dispatch_for(operations.AttachMatViewOp) 80 | def render_attach_mat_view(autogen_context, op): 81 | columns = [ 82 | col 83 | for col in [ 84 | render._render_column(col, autogen_context) for col in op.columns 85 | ] 86 | if col 87 | ] 88 | 89 | templ = ( 90 | "{prefix}attach_mat_view(\n" 91 | "{indent}'{name}',\n" 92 | "{indent}'{selectable}',\n" 93 | "{indent}'{engine}',\n" 94 | "{indent}{columns}\n" 95 | ")" 96 | ) 97 | 98 | join_indent = ("'\n" + indent + "'") 99 | return templ.format( 100 | prefix=render._alembic_autogenerate_prefix(autogen_context), 101 | name=op.name, 102 | selectable=join_indent.join(escape(op.selectable).split('\n')), 103 | engine=join_indent.join(escape(op.engine.strip()).split('\n')), 104 | columns=(',\n' + indent).join(str(arg) for arg in columns), 105 | indent=indent 106 | ) 107 | 108 | 109 | @renderers.dispatch_for(operations.DetachMatViewOp) 110 | def render_detach_mat_view(autogen_context, op): 111 | return ( 112 | render._alembic_autogenerate_prefix(autogen_context) + 113 | "detach_mat_view('" + op.name + "')" 114 | ) 115 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/alembic/toimpl.py: -------------------------------------------------------------------------------- 1 | from alembic.operations import Operations 2 | from sqlalchemy.sql.ddl import CreateColumn 3 | 4 | from . import operations 5 | 6 | 7 | @Operations.implementation_for(operations.CreateMatViewOp) 8 | def create_mat_view(operations, operation): 9 | impl = operations.impl 10 | ddl_compiler = impl.dialect.ddl_compiler(impl.dialect, None) 11 | 12 | text = 'CREATE MATERIALIZED VIEW ' 13 | 14 | if operation.kwargs.get('if_not_exists'): 15 | text += 'IF NOT EXISTS ' 16 | 17 | text += operation.name 18 | 19 | if operation.kwargs.get('on_cluster'): 20 | text += ' ON CLUSTER ' + operation.kwargs['on_cluster'] 21 | 22 | text += ' (' + ', '.join( 23 | ddl_compiler.process(CreateColumn(c)) for c in operation.columns 24 | ) + ') ' 25 | 26 | text += 'ENGINE = ' + operation.engine 27 | 28 | if operation.kwargs.get('populate'): 29 | text += ' POPULATE' 30 | 31 | text += ' AS ' + operation.selectable 32 | 33 | operations.execute(text) 34 | 35 | 36 | @Operations.implementation_for(operations.CreateMatViewToTableOp) 37 | def create_mat_view_to_table(operations, operation): 38 | text = 'CREATE MATERIALIZED VIEW ' 39 | 40 | if operation.kwargs.get('if_not_exists'): 41 | text += 'IF NOT EXISTS ' 42 | 43 | text += operation.name 44 | 45 | if operation.kwargs.get('on_cluster'): 46 | text += ' ON CLUSTER ' + operation.kwargs['on_cluster'] 47 | 48 | text += ' TO ' + operation.inner_name 49 | 50 | if operation.kwargs.get('populate'): 51 | text += ' POPULATE' 52 | 53 | text += ' AS ' + operation.selectable 54 | 55 | operations.execute(text) 56 | 57 | 58 | @Operations.implementation_for(operations.AttachMatViewOp) 59 | def attach_mat_view(operations, operation): 60 | impl = operations.impl 61 | ddl_compiler = impl.dialect.ddl_compiler(impl.dialect, None) 62 | 63 | text = 'ATTACH MATERIALIZED VIEW ' 64 | 65 | if operation.kwargs.get('if_not_exists'): 66 | text += 'IF NOT EXISTS ' 67 | 68 | text += operation.name + ' ' 69 | 70 | if operation.kwargs.get('on_cluster'): 71 | text += ' ON CLUSTER ' + operation.kwargs['on_cluster'] 72 | 73 | text += ' (' + ', '.join( 74 | ddl_compiler.process(CreateColumn(c)) for c in operation.columns 75 | ) + ') ' 76 | 77 | text += 'ENGINE = ' + operation.engine + ' AS ' + operation.selectable 78 | 79 | operations.execute(text) 80 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | from . import base 2 | from .http import base as http_driver 3 | 4 | base.dialect = http_driver.dialect 5 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/asynch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/drivers/asynch/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/asynch/base.py: -------------------------------------------------------------------------------- 1 | import asynch 2 | 3 | from sqlalchemy.sql.elements import TextClause 4 | from sqlalchemy.pool import AsyncAdaptedQueuePool 5 | 6 | from .connector import AsyncAdapt_asynch_dbapi 7 | from ..native.base import ClickHouseDialect_native, ClickHouseExecutionContext 8 | 9 | # Export connector version 10 | VERSION = (0, 0, 1, None) 11 | 12 | 13 | class ClickHouseAsynchExecutionContext(ClickHouseExecutionContext): 14 | def create_server_side_cursor(self): 15 | return self.create_default_cursor() 16 | 17 | 18 | class ClickHouseDialect_asynch(ClickHouseDialect_native): 19 | driver = 'asynch' 20 | execution_ctx_cls = ClickHouseAsynchExecutionContext 21 | 22 | is_async = True 23 | supports_statement_cache = True 24 | supports_server_side_cursors = True 25 | 26 | @classmethod 27 | def import_dbapi(cls): 28 | return AsyncAdapt_asynch_dbapi(asynch) 29 | 30 | @classmethod 31 | def get_pool_class(cls, url): 32 | return AsyncAdaptedQueuePool 33 | 34 | def _execute(self, connection, sql, scalar=False, **kwargs): 35 | if isinstance(sql, str): 36 | # Makes sure the query will go through the 37 | # `ClickHouseExecutionContext` logic. 38 | sql = TextClause(sql) 39 | f = connection.scalar if scalar else connection.execute 40 | return f(sql, kwargs) 41 | 42 | def do_execute(self, cursor, statement, parameters, context=None): 43 | cursor.execute(statement, parameters, context) 44 | 45 | def do_executemany(self, cursor, statement, parameters, context=None): 46 | cursor.executemany(statement, parameters, context) 47 | 48 | 49 | dialect = ClickHouseDialect_asynch 50 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/asynch/connector.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from sqlalchemy.engine.interfaces import AdaptedConnection 4 | from sqlalchemy.util.concurrency import await_only 5 | 6 | 7 | class AsyncAdapt_asynch_cursor: 8 | __slots__ = ( 9 | '_adapt_connection', 10 | '_connection', 11 | 'await_', 12 | '_cursor', 13 | '_rows' 14 | ) 15 | 16 | def __init__(self, adapt_connection): 17 | self._adapt_connection = adapt_connection 18 | self._connection = adapt_connection._connection # noqa 19 | self.await_ = adapt_connection.await_ 20 | 21 | cursor = self._connection.cursor() 22 | 23 | self._cursor = self.await_(cursor.__aenter__()) 24 | self._rows = [] 25 | 26 | @property 27 | def _execute_mutex(self): 28 | return self._adapt_connection._execute_mutex # noqa 29 | 30 | @property 31 | def description(self): 32 | return self._cursor.description 33 | 34 | @property 35 | def rowcount(self): 36 | return self._cursor.rowcount 37 | 38 | @property 39 | def arraysize(self): 40 | return self._cursor.arraysize 41 | 42 | @arraysize.setter 43 | def arraysize(self, value): 44 | self._cursor.arraysize = value 45 | 46 | @property 47 | def lastrowid(self): 48 | return self._cursor.lastrowid 49 | 50 | def close(self): 51 | # note we aren't actually closing the cursor here, 52 | # we are just letting GC do it. to allow this to be async 53 | # we would need the Result to change how it does "Safe close cursor". 54 | self._rows[:] = [] # noqa 55 | 56 | def execute(self, operation, params=None, context=None): 57 | return self.await_(self._execute_async(operation, params, context)) 58 | 59 | async def _execute_async(self, operation, params, context): 60 | async with self._execute_mutex: 61 | result = await self._cursor.execute( 62 | operation, 63 | args=params, 64 | context=context 65 | ) 66 | 67 | self._rows = list(await self._cursor.fetchall()) 68 | return result 69 | 70 | def executemany(self, operation, params=None, context=None): 71 | return self.await_(self._executemany_async(operation, params, context)) 72 | 73 | async def _executemany_async(self, operation, params, context): 74 | async with self._execute_mutex: 75 | return await self._cursor.executemany( 76 | operation, 77 | args=params, 78 | context=context 79 | ) 80 | 81 | def setinputsizes(self, *args): 82 | pass 83 | 84 | def setoutputsizes(self, *args): 85 | pass 86 | 87 | def __iter__(self): 88 | while self._rows: 89 | yield self._rows.pop(0) 90 | 91 | def fetchone(self): 92 | if self._rows: 93 | return self._rows.pop(0) 94 | else: 95 | return None 96 | 97 | def fetchmany(self, size=None): 98 | if size is None: 99 | size = self.arraysize 100 | 101 | retval = self._rows[0:size] 102 | self._rows[:] = self._rows[size:] 103 | return retval 104 | 105 | def fetchall(self): 106 | retval = self._rows[:] 107 | self._rows[:] = [] 108 | return retval 109 | 110 | 111 | class AsyncAdapt_asynch_dbapi: 112 | def __init__(self, asynch): 113 | self.asynch = asynch 114 | self.paramstyle = 'pyformat' 115 | self._init_dbapi_attributes() 116 | 117 | class Error(Exception): 118 | pass 119 | 120 | def _init_dbapi_attributes(self): 121 | for name in ( 122 | 'ServerException', 123 | 'UnexpectedPacketFromServerError', 124 | 'LogicalError', 125 | 'UnknownTypeError', 126 | 'ChecksumDoesntMatchError', 127 | 'TypeMismatchError', 128 | 'UnknownCompressionMethod', 129 | 'TooLargeStringSize', 130 | 'NetworkError', 131 | 'SocketTimeoutError', 132 | 'UnknownPacketFromServerError', 133 | 'CannotParseUuidError', 134 | 'CannotParseDomainError', 135 | 'PartiallyConsumedQueryError', 136 | 'ColumnException', 137 | 'ColumnTypeMismatchException', 138 | 'StructPackException', 139 | 'InterfaceError', 140 | 'DatabaseError', 141 | 'ProgrammingError', 142 | 'NotSupportedError', 143 | ): 144 | setattr(self, name, getattr(self.asynch.errors, name)) 145 | 146 | def connect(self, *args, **kwargs) -> 'AsyncAdapt_asynch_connection': 147 | return AsyncAdapt_asynch_connection( 148 | self, 149 | await_only(self.asynch.connect(*args, **kwargs)) 150 | ) 151 | 152 | 153 | class AsyncAdapt_asynch_connection(AdaptedConnection): 154 | await_ = staticmethod(await_only) 155 | __slots__ = ('dbapi', '_execute_mutex') 156 | 157 | def __init__(self, dbapi, connection): 158 | self.dbapi = dbapi 159 | self._connection = connection 160 | self._execute_mutex = asyncio.Lock() 161 | 162 | def ping(self, reconnect): 163 | return self.await_(self._ping_async()) 164 | 165 | async def _ping_async(self): 166 | async with self._execute_mutex: 167 | return await self._connection.ping() 168 | 169 | def character_set_name(self): 170 | return self._connection.character_set_name() 171 | 172 | def autocommit(self, value): 173 | self.await_(self._connection.autocommit(value)) 174 | 175 | def cursor(self, server_side=False): 176 | return AsyncAdapt_asynch_cursor(self) 177 | 178 | def rollback(self): 179 | self.await_(self._connection.rollback()) 180 | 181 | def commit(self): 182 | self.await_(self._connection.commit()) 183 | 184 | def close(self): 185 | self.await_(self._connection.close()) 186 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/compilers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/drivers/compilers/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/compilers/typecompiler.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql import compiler, type_api 2 | from sqlalchemy.sql.ddl import CreateColumn 3 | 4 | 5 | class ClickHouseTypeCompiler(compiler.GenericTypeCompiler): 6 | def visit_string(self, type_, **kw): 7 | if type_.length is None: 8 | return 'String' 9 | else: 10 | return 'FixedString(%s)' % type_.length 11 | 12 | def visit_array(self, type_, **kw): 13 | item_type = type_api.to_instance(type_.item_type) 14 | return 'Array(%s)' % self.process(item_type, **kw) 15 | 16 | def visit_nullable(self, type_, **kw): 17 | nested_type = type_api.to_instance(type_.nested_type) 18 | return 'Nullable(%s)' % self.process(nested_type, **kw) 19 | 20 | def visit_lowcardinality(self, type_, **kw): 21 | nested_type = type_api.to_instance(type_.nested_type) 22 | return "LowCardinality(%s)" % self.process(nested_type, **kw) 23 | 24 | def visit_int8(self, type_, **kw): 25 | return 'Int8' 26 | 27 | def visit_uint8(self, type_, **kw): 28 | return 'UInt8' 29 | 30 | def visit_int16(self, type_, **kw): 31 | return 'Int16' 32 | 33 | def visit_uint16(self, type_, **kw): 34 | return 'UInt16' 35 | 36 | def visit_int32(self, type_, **kw): 37 | return 'Int32' 38 | 39 | def visit_uint32(self, type_, **kw): 40 | return 'UInt32' 41 | 42 | def visit_int64(self, type_, **kw): 43 | return 'Int64' 44 | 45 | def visit_uint64(self, type_, **kw): 46 | return 'UInt64' 47 | 48 | def visit_int128(self, type_, **kw): 49 | return 'Int128' 50 | 51 | def visit_uint128(self, type_, **kw): 52 | return 'UInt128' 53 | 54 | def visit_int256(self, type_, **kw): 55 | return 'Int256' 56 | 57 | def visit_uint256(self, type_, **kw): 58 | return 'UInt256' 59 | 60 | def visit_date(self, type_, **kw): 61 | return 'Date' 62 | 63 | def visit_date32(self, type_, **kw): 64 | return 'Date32' 65 | 66 | def visit_datetime(self, type_, **kw): 67 | if type_.timezone: 68 | return "DateTime('%s')" % type_.timezone 69 | else: 70 | return 'DateTime' 71 | 72 | def visit_datetime64(self, type_, **kw): 73 | if type_.timezone: 74 | return "DateTime64(%s, '%s')" % (type_.precision, type_.timezone) 75 | else: 76 | return 'DateTime64(%s)' % type_.precision 77 | 78 | def visit_float32(self, type_, **kw): 79 | return 'Float32' 80 | 81 | def visit_float64(self, type_, **kw): 82 | return 'Float64' 83 | 84 | def visit_numeric(self, type_, **kw): 85 | return 'Decimal(%s, %s)' % (type_.precision, type_.scale) 86 | 87 | def visit_boolean(self, type_, **kw): 88 | return 'Bool' 89 | 90 | def visit_json(self, type_, **kw): 91 | return 'JSON' 92 | 93 | def visit_nested(self, nested, **kwargs): 94 | ddl_compiler = self.dialect.ddl_compiler(self.dialect, None) 95 | cols_create = [ 96 | ddl_compiler.visit_create_column(CreateColumn(nested_child)) 97 | for nested_child in nested.columns 98 | ] 99 | return 'Nested(%s)' % ', '.join(cols_create) 100 | 101 | def _render_enum(self, db_type, type_, **kw): 102 | choices = ( 103 | "'%s' = %d" % 104 | (x.name.replace("'", r"\'"), x.value) for x in type_.enum_class 105 | ) 106 | return '%s(%s)' % (db_type, ', '.join(choices)) 107 | 108 | def visit_enum(self, type_, **kw): 109 | return self._render_enum('Enum', type_, **kw) 110 | 111 | def visit_enum8(self, type_, **kw): 112 | return self._render_enum('Enum8', type_, **kw) 113 | 114 | def visit_enum16(self, type_, **kw): 115 | return self._render_enum('Enum16', type_, **kw) 116 | 117 | def visit_uuid(self, type_, **kw): 118 | return 'UUID' 119 | 120 | def visit_ipv4(self, type_, **kw): 121 | return 'IPv4' 122 | 123 | def visit_ipv6(self, type_, **kw): 124 | return 'IPv6' 125 | 126 | def visit_tuple(self, type_, **kw): 127 | cols = [] 128 | is_named_type = all([ 129 | isinstance(nested_type, tuple) and len(nested_type) == 2 130 | for nested_type in type_.nested_types 131 | ]) 132 | if is_named_type: 133 | for nested_type in type_.nested_types: 134 | name = nested_type[0] 135 | name_type = nested_type[1] 136 | inner_type = self.process( 137 | type_api.to_instance(name_type), 138 | **kw 139 | ) 140 | cols.append( 141 | f'{name} {inner_type}') 142 | else: 143 | cols = ( 144 | self.process(type_api.to_instance(nested_type), **kw) 145 | for nested_type in type_.nested_types 146 | ) 147 | return 'Tuple(%s)' % ', '.join(cols) 148 | 149 | def visit_map(self, type_, **kw): 150 | key_type = type_api.to_instance(type_.key_type) 151 | value_type = type_api.to_instance(type_.value_type) 152 | return 'Map(%s, %s)' % ( 153 | self.process(key_type, **kw), 154 | self.process(value_type, **kw) 155 | ) 156 | 157 | def visit_aggregatefunction(self, type_, **kw): 158 | types = [type_api.to_instance(val) for val in type_.nested_types] 159 | type_strings = [self.process(val, **kw) for val in types] 160 | 161 | if isinstance(type_.agg_func, str): 162 | agg_str = type_.agg_func 163 | else: 164 | agg_str = str(type_.agg_func.compile(dialect=self.dialect)) 165 | 166 | return "AggregateFunction(%s, %s)" % ( 167 | agg_str, ", ".join(type_strings) 168 | ) 169 | 170 | def visit_simpleaggregatefunction(self, type_, **kw): 171 | types = [type_api.to_instance(val) for val in type_.nested_types] 172 | type_strings = [self.process(val, **kw) for val in types] 173 | 174 | if isinstance(type_.agg_func, str): 175 | agg_str = type_.agg_func 176 | else: 177 | agg_str = str(type_.agg_func.compile(dialect=self.dialect)) 178 | 179 | return "SimpleAggregateFunction(%s, %s)" % ( 180 | agg_str, ", ".join(type_strings) 181 | ) 182 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/drivers/http/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/base.py: -------------------------------------------------------------------------------- 1 | import sqlalchemy as sa 2 | from sqlalchemy.util import asbool 3 | 4 | from ..base import ClickHouseDialect, ClickHouseExecutionContextBase 5 | from . import connector 6 | 7 | 8 | # Export connector version 9 | VERSION = (0, 0, 2, None) 10 | 11 | 12 | class ClickHouseExecutionContext(ClickHouseExecutionContextBase): 13 | def pre_exec(self): 14 | pass 15 | 16 | 17 | class ClickHouseDialect_http(ClickHouseDialect): 18 | driver = 'http' 19 | execution_ctx_cls = ClickHouseExecutionContext 20 | 21 | supports_statement_cache = True 22 | 23 | @classmethod 24 | def import_dbapi(cls): 25 | return connector 26 | 27 | def create_connect_args(self, url): 28 | kwargs = {} 29 | protocol = url.query.get('protocol', 'http') 30 | port = url.port or 8123 31 | db_name = url.database or 'default' 32 | endpoint = url.query.get('endpoint', '') 33 | 34 | query = dict(url.query) 35 | self.engine_reflection = asbool( 36 | query.pop('engine_reflection', 'true') 37 | ) 38 | url = url.set(query=query) 39 | 40 | kwargs.update(query) 41 | if kwargs.get('verify') and kwargs['verify'] in ('False', 'false'): 42 | kwargs['verify'] = False 43 | 44 | db_url = '%s://%s:%d/%s' % (protocol, url.host, port, endpoint) 45 | 46 | return (db_url, db_name, url.username, url.password), kwargs 47 | 48 | def _execute(self, connection, sql, scalar=False, **kwargs): 49 | if isinstance(sql, str): 50 | # Makes sure the query will go through the 51 | # `ClickHouseExecutionContext` logic. 52 | sql = sa.sql.elements.TextClause(sql) 53 | f = connection.scalar if scalar else connection.execute 54 | return f(sql, kwargs) 55 | 56 | 57 | dialect = ClickHouseDialect_http 58 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/escaper.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | from decimal import Decimal 3 | import enum 4 | import uuid 5 | 6 | 7 | class Escaper(object): 8 | 9 | number_types = (int, float, ) 10 | 11 | escape_chars = { 12 | "\b": "\\b", 13 | "\f": "\\f", 14 | "\r": "\\r", 15 | "\n": "\\n", 16 | "\t": "\\t", 17 | "\0": "\\0", 18 | "\\": "\\\\", 19 | "'": "\\'" 20 | } 21 | 22 | def escape_string(self, value): 23 | value = ''.join(self.escape_chars.get(c, c) for c in value) 24 | return "'" + value + "'" 25 | 26 | def escape(self, parameters): 27 | if isinstance(parameters, dict): 28 | return {k: self.escape_item(v) for k, v in parameters.items()} 29 | elif isinstance(parameters, (list, tuple)): 30 | return "[" + ",".join( 31 | [str(self.escape_item(x)) for x in parameters]) + "]" 32 | else: 33 | raise Exception("Unsupported param format: {}".format(parameters)) 34 | 35 | def escape_number(self, item): 36 | return item 37 | 38 | def escape_date(self, item): 39 | # XXX: shouldn't this be `toDate(...)`? 40 | return self.escape_string(item.strftime('%Y-%m-%d')) 41 | 42 | def escape_datetime(self, item): 43 | # XXX: shouldn't this be `toDateTime(...)`? 44 | return self.escape_string(item.strftime('%Y-%m-%d %H:%M:%S')) 45 | 46 | def escape_datetime64(self, item): 47 | # XXX: shouldn't this be `toDateTime64(...)`? 48 | return self.escape_string(item.strftime('%Y-%m-%d %H:%M:%S.%f')) 49 | 50 | def escape_decimal(self, item): 51 | return float(item) 52 | 53 | def escape_uuid(self, item): 54 | return str(item) 55 | 56 | def escape_item(self, item): 57 | if item is None: 58 | return 'NULL' 59 | elif isinstance(item, self.number_types): 60 | return self.escape_number(item) 61 | elif isinstance(item, datetime): 62 | return self.escape_datetime(item) 63 | elif isinstance(item, date): 64 | return self.escape_date(item) 65 | elif isinstance(item, Decimal): 66 | return self.escape_decimal(item) 67 | elif isinstance(item, str): 68 | return self.escape_string(item) 69 | elif isinstance(item, (list, tuple)): 70 | return "[" + ", ".join( 71 | [str(self.escape_item(x)) for x in item] 72 | ) + "]" 73 | elif isinstance(item, dict): 74 | return "{" + ", ".join( 75 | ["{}: {}".format( 76 | self.escape_item(k), 77 | self.escape_item(v) 78 | ) for k, v in item.items()] 79 | ) + "}" 80 | elif isinstance(item, enum.Enum): 81 | return self.escape_string(item.name) 82 | elif isinstance(item, uuid.UUID): 83 | return self.escape_uuid(item) 84 | else: 85 | raise Exception("Unsupported object {}".format(item)) 86 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class HTTPException(Exception): 3 | code = None 4 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/transport.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from datetime import datetime 4 | from decimal import Decimal 5 | from functools import partial 6 | 7 | from ipaddress import IPv4Address, IPv6Address 8 | 9 | import requests 10 | 11 | from ...exceptions import DatabaseException 12 | from .exceptions import HTTPException 13 | from .utils import parse_tsv 14 | 15 | 16 | DEFAULT_DDL_TIMEOUT = None 17 | DATE_NULL = '0000-00-00' 18 | DATETIME_NULL = '0000-00-00 00:00:00' 19 | 20 | EXTRACT_SUBTYPE_RE = re.compile(r'^[^\(]+\((.+)\)$') 21 | 22 | 23 | def date_converter(x): 24 | if x != DATE_NULL: 25 | return datetime.strptime(x, '%Y-%m-%d').date() 26 | return None 27 | 28 | 29 | def datetime_converter(x): 30 | if x == DATETIME_NULL: 31 | return None 32 | elif len(x) > 19: 33 | return datetime.strptime(x, '%Y-%m-%d %H:%M:%S.%f') 34 | else: 35 | return datetime.strptime(x, '%Y-%m-%d %H:%M:%S') 36 | 37 | 38 | def nullable_converter(subtype_str, x): 39 | if x is None: 40 | return None 41 | 42 | converter = _get_type(subtype_str) 43 | return converter(x) if converter else x 44 | 45 | 46 | def nothing_converter(x): 47 | return None 48 | 49 | 50 | converters = { 51 | 'Int8': int, 52 | 'UInt8': int, 53 | 'Int16': int, 54 | 'UInt16': int, 55 | 'Int32': int, 56 | 'UInt32': int, 57 | 'Int64': int, 58 | 'UInt64': int, 59 | 'Int128': int, 60 | 'UInt128': int, 61 | 'Int256': int, 62 | 'UInt256': int, 63 | 'Float32': float, 64 | 'Float64': float, 65 | 'Decimal': Decimal, 66 | 'Date': date_converter, 67 | 'DateTime': datetime_converter, 68 | 'DateTime64': datetime_converter, 69 | 'IPv4': IPv4Address, 70 | 'IPv6': IPv6Address, 71 | 'Nullable': nullable_converter, 72 | 'Nothing': nothing_converter, 73 | } 74 | 75 | 76 | def _get_type(type_str): 77 | result = converters.get(type_str) 78 | if result is not None: 79 | return result 80 | # sometimes type_str is DateTime64(x) 81 | if type_str.startswith('DateTime64'): 82 | return converters['DateTime64'] 83 | if type_str.startswith('Decimal'): 84 | return converters['Decimal'] 85 | if type_str.startswith('Nullable('): 86 | subtype_str = EXTRACT_SUBTYPE_RE.match(type_str).group(1) 87 | return partial(converters['Nullable'], subtype_str) 88 | return None 89 | 90 | 91 | class RequestsTransport(object): 92 | 93 | def __init__( 94 | self, 95 | db_url, db_name, username, password, 96 | timeout=None, ch_settings=None, 97 | **kwargs): 98 | 99 | self.db_url = db_url 100 | self.db_name = db_name 101 | self.auth = (username, password) 102 | self.timeout = float(timeout) if timeout is not None else None 103 | self.verify = kwargs.pop('verify', True) 104 | self.cert = kwargs.pop('cert', None) 105 | self.headers = { 106 | key[8:]: value 107 | for key, value in kwargs.items() 108 | if key.startswith('header__') 109 | } 110 | 111 | self.unicode_errors = kwargs.pop('unicode_errors', 'replace') 112 | 113 | ch_settings = dict(ch_settings or {}) 114 | self.ch_settings = ch_settings 115 | 116 | self.ch_settings['default_format'] = 'TabSeparatedWithNamesAndTypes' 117 | 118 | ddl_timeout = kwargs.pop('ddl_timeout', DEFAULT_DDL_TIMEOUT) 119 | if ddl_timeout is not None: 120 | self.ch_settings['distributed_ddl_task_timeout'] = int(ddl_timeout) 121 | 122 | # By default, keep connection open between queries. 123 | http = kwargs.pop('http_session', requests.Session) 124 | self.http = http() if callable(http) else http 125 | 126 | super(RequestsTransport, self).__init__() 127 | 128 | def execute(self, query, params=None): 129 | """ 130 | Query is returning rows and these rows should be parsed or 131 | there is nothing to return. 132 | """ 133 | r = self._send(query, params=params, stream=True) 134 | lines = r.iter_lines() 135 | try: 136 | names = parse_tsv(next(lines), self.unicode_errors) 137 | types = parse_tsv(next(lines), self.unicode_errors) 138 | except StopIteration: 139 | # Empty result; e.g. a DDL request. 140 | return 141 | 142 | convs = [_get_type(type_) for type_ in types] 143 | 144 | yield names 145 | yield types 146 | 147 | for line in lines: 148 | yield [ 149 | (conv(x) if conv else x) 150 | for x, conv in zip(parse_tsv(line, self.unicode_errors), convs) 151 | ] 152 | 153 | def raw(self, query, params=None, stream=False): 154 | """ 155 | Performs raw query to database. Returns its output 156 | :param query: Query to execute 157 | :param params: Additional params should be passed during query. 158 | :param stream: If flag is true, Http response from ClickHouse will be 159 | streamed. 160 | :return: Query execution result 161 | """ 162 | return self._send(query, params=params, stream=stream).text 163 | 164 | def _send(self, data, params=None, stream=False): 165 | data = data.encode('utf-8') 166 | params = params or {} 167 | params['database'] = self.db_name 168 | params.update(self.ch_settings) 169 | 170 | # TODO: retries, prepared requests 171 | r = self.http.post( 172 | self.db_url, auth=self.auth, params=params, data=data, 173 | stream=stream, timeout=self.timeout, headers=self.headers, 174 | verify=self.verify, cert=self.cert 175 | ) 176 | if r.status_code != 200: 177 | orig = HTTPException(r.text) 178 | orig.code = r.status_code 179 | raise DatabaseException(orig) 180 | return r 181 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/http/utils.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | 3 | 4 | def unescape(value, errors=None): 5 | if errors is None: 6 | errors = 'replace' 7 | return codecs.escape_decode(value)[0].decode('utf-8', errors=errors) 8 | 9 | 10 | def parse_tsv(line, errors=None): 11 | return [ 12 | (unescape(x, errors) if x != b'\\N' else None) 13 | for x in line.split(b'\t') 14 | ] 15 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/native/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/drivers/native/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/native/base.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import quote 2 | 3 | from sqlalchemy.sql.elements import TextClause 4 | from sqlalchemy.util import asbool 5 | 6 | from . import connector 7 | from ..base import ( 8 | ClickHouseDialect, ClickHouseExecutionContextBase, ClickHouseSQLCompiler, 9 | ) 10 | from sqlalchemy.engine.interfaces import ExecuteStyle 11 | from sqlalchemy import __version__ as sqlalchemy_version 12 | 13 | # Export connector version 14 | VERSION = (0, 0, 2, None) 15 | 16 | sqlalchemy_version = tuple( 17 | (int(x) if x.isdigit() else x) for x in sqlalchemy_version.split('.') 18 | ) 19 | 20 | 21 | class ClickHouseExecutionContext(ClickHouseExecutionContextBase): 22 | def pre_exec(self): 23 | # Always do executemany on INSERT with VALUES clause. 24 | if (self.isinsert and self.compiled.statement.select is None and 25 | self.parameters != [{}]): 26 | self.execute_style = ExecuteStyle.EXECUTEMANY 27 | 28 | 29 | class ClickHouseNativeSQLCompiler(ClickHouseSQLCompiler): 30 | 31 | def visit_insert(self, insert_stmt, asfrom=False, **kw): 32 | rv = super(ClickHouseNativeSQLCompiler, self).visit_insert( 33 | insert_stmt, asfrom=asfrom, **kw) 34 | 35 | if kw.get('literal_binds') or insert_stmt._values: 36 | return rv 37 | 38 | pos = rv.lower().rfind('values (') 39 | # Remove (%s)-templates from VALUES clause if exists. 40 | # ClickHouse server since version 19.3.3 parse query after VALUES and 41 | # allows inplace parameters. 42 | # Example: INSERT INTO test (x) VALUES (1), (2). 43 | if pos != -1: 44 | rv = rv[:pos + 6] 45 | return rv 46 | 47 | 48 | class ClickHouseDialect_native(ClickHouseDialect): 49 | driver = 'native' 50 | execution_ctx_cls = ClickHouseExecutionContext 51 | statement_compiler = ClickHouseNativeSQLCompiler 52 | 53 | supports_statement_cache = True 54 | 55 | @classmethod 56 | def import_dbapi(cls): 57 | return connector 58 | 59 | def create_connect_args(self, url): 60 | use_quote = sqlalchemy_version < (2, 0, 24) 61 | 62 | url = url.set(drivername='clickhouse') 63 | if url.username: 64 | username = quote(url.username) if use_quote else url.username 65 | url = url.set(username=username) 66 | 67 | if url.password: 68 | password = quote(url.password) if use_quote else url.password 69 | url = url.set(password=password) 70 | 71 | query = dict(url.query) 72 | self.engine_reflection = asbool( 73 | query.pop('engine_reflection', 'true') 74 | ) 75 | url = url.set(query=query) 76 | 77 | return (url.render_as_string(hide_password=False), ), {} 78 | 79 | def _execute(self, connection, sql, scalar=False, **kwargs): 80 | if isinstance(sql, str): 81 | # Makes sure the query will go through the 82 | # `ClickHouseExecutionContext` logic. 83 | sql = TextClause(sql) 84 | f = connection.scalar if scalar else connection.execute 85 | return f(sql, kwargs) 86 | 87 | 88 | dialect = ClickHouseDialect_native 89 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/reflection.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.engine import reflection 2 | 3 | from clickhouse_sqlalchemy import Table, engines 4 | 5 | 6 | class ClickHouseInspector(reflection.Inspector): 7 | def reflect_table(self, table, *args, **kwargs): 8 | # This check is necessary to support direct instantiation of 9 | # `clickhouse_sqlalchemy.Table` and then reflection of it. 10 | if not isinstance(table, Table): 11 | table.metadata.remove(table) 12 | ch_table = Table._make_from_standard( 13 | table, _extend_on=kwargs.get('_extend_on') 14 | ) 15 | else: 16 | ch_table = table 17 | 18 | super(ClickHouseInspector, self).reflect_table( 19 | ch_table, *args, **kwargs 20 | ) 21 | 22 | with self._operation_context() as conn: 23 | schema = conn.schema_for_object(ch_table) 24 | 25 | self._reflect_engine(ch_table.name, schema, ch_table) 26 | 27 | def _reflect_engine(self, table_name, schema, table): 28 | should_reflect = ( 29 | self.dialect.supports_engine_reflection and 30 | self.dialect.engine_reflection 31 | ) 32 | if not should_reflect: 33 | return 34 | 35 | engine_cls_by_name = {e.__name__: e for e in engines.__all__} 36 | 37 | e = self.get_engine(table_name, schema=table.schema) 38 | if not e: 39 | raise ValueError("Cannot find engine for table '%s'" % table_name) 40 | 41 | engine_cls = engine_cls_by_name.get(e['engine']) 42 | if engine_cls is not None: 43 | engine = engine_cls.reflect(table, **e) 44 | engine._set_parent(table) 45 | else: 46 | table.engine = None 47 | 48 | def get_engine(self, table_name, schema=None, **kw): 49 | with self._operation_context() as conn: 50 | return self.dialect.get_engine( 51 | conn, table_name, schema=schema, info_cache=self.info_cache, 52 | **kw 53 | ) 54 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/drivers/util.py: -------------------------------------------------------------------------------- 1 | 2 | def get_inner_spec(spec): 3 | brackets = 0 4 | offset = spec.find('(') 5 | if offset == -1: 6 | return '' 7 | i = offset 8 | for i, ch in enumerate(spec[offset:], offset): 9 | if ch == '(': 10 | brackets += 1 11 | 12 | elif ch == ')': 13 | brackets -= 1 14 | 15 | if brackets == 0: 16 | break 17 | 18 | return spec[offset + 1:i] 19 | 20 | 21 | def parse_arguments(param_string): 22 | """ 23 | Given a string of function arguments, parse them into a tuple. 24 | """ 25 | params = [] 26 | bracket_level = 0 27 | current_param = '' 28 | 29 | for char in param_string: 30 | if char == '(': 31 | bracket_level += 1 32 | elif char == ')': 33 | bracket_level -= 1 34 | elif char == ',' and bracket_level == 0: 35 | params.append(current_param.strip()) 36 | current_param = '' 37 | continue 38 | 39 | current_param += char 40 | 41 | # Append the last parameter 42 | if current_param: 43 | params.append(current_param.strip()) 44 | 45 | return tuple(params) 46 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/engines/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .mergetree import ( 3 | MergeTree, AggregatingMergeTree, GraphiteMergeTree, CollapsingMergeTree, 4 | VersionedCollapsingMergeTree, ReplacingMergeTree, SummingMergeTree 5 | ) 6 | from .misc import ( 7 | Distributed, View, MaterializedView, 8 | Buffer, TinyLog, Log, Memory, Null, File 9 | ) 10 | from .replicated import ( 11 | ReplicatedMergeTree, ReplicatedAggregatingMergeTree, 12 | ReplicatedCollapsingMergeTree, ReplicatedVersionedCollapsingMergeTree, 13 | ReplicatedReplacingMergeTree, ReplicatedSummingMergeTree 14 | ) 15 | 16 | 17 | __all__ = ( 18 | MergeTree, 19 | AggregatingMergeTree, 20 | GraphiteMergeTree, 21 | CollapsingMergeTree, 22 | VersionedCollapsingMergeTree, 23 | SummingMergeTree, 24 | ReplacingMergeTree, 25 | Distributed, 26 | ReplicatedMergeTree, 27 | ReplicatedAggregatingMergeTree, 28 | ReplicatedCollapsingMergeTree, 29 | ReplicatedVersionedCollapsingMergeTree, 30 | ReplicatedReplacingMergeTree, 31 | ReplicatedSummingMergeTree, 32 | View, 33 | MaterializedView, 34 | Buffer, 35 | TinyLog, 36 | Log, 37 | Memory, 38 | Null, 39 | File 40 | ) 41 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/engines/base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql import ClauseElement 2 | from sqlalchemy.sql.schema import ColumnCollectionMixin, SchemaItem, Constraint 3 | 4 | 5 | class Engine(Constraint): 6 | __visit_name__ = 'engine' 7 | 8 | def __init__(self, *args, **kwargs): 9 | pass 10 | 11 | def get_parameters(self): 12 | return [] 13 | 14 | def extend_parameters(self, *params): 15 | rv = [] 16 | for param in params: 17 | if isinstance(param, (tuple, list)): 18 | rv.extend(param) 19 | elif param is not None: 20 | rv.append(param) 21 | return rv 22 | 23 | @property 24 | def name(self): 25 | return self.__class__.__name__ 26 | 27 | def _set_parent(self, parent, **kwargs): 28 | self.parent = parent 29 | parent.engine = self 30 | 31 | @classmethod 32 | def reflect(cls, table, engine_full, **kwargs): 33 | raise NotImplementedError 34 | 35 | 36 | class TableCol(ColumnCollectionMixin, SchemaItem): 37 | def __init__(self, column, **kwargs): 38 | super(TableCol, self).__init__(*[column], **kwargs) 39 | 40 | def get_column(self): 41 | return list(self.columns)[0] 42 | 43 | 44 | class KeysExpressionOrColumn(ColumnCollectionMixin, SchemaItem): 45 | def __init__(self, *expressions, **kwargs): 46 | self.expressions = [] 47 | 48 | super(KeysExpressionOrColumn, self).__init__( 49 | *expressions, _gather_expressions=self.expressions, **kwargs 50 | ) 51 | 52 | def _set_parent(self, table, **kw): 53 | ColumnCollectionMixin._set_parent(self, table) 54 | 55 | self.table = table 56 | 57 | expressions = self.expressions 58 | col_expressions = self._col_expressions(table) 59 | assert len(expressions) == len(col_expressions) 60 | self.expressions = [ 61 | expr if isinstance(expr, ClauseElement) else colexpr 62 | for expr, colexpr in zip(expressions, col_expressions) 63 | ] 64 | 65 | def get_expressions_or_columns(self): 66 | return self.expressions 67 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/engines/misc.py: -------------------------------------------------------------------------------- 1 | 2 | from .base import Engine 3 | from .util import parse_columns 4 | 5 | 6 | class Distributed(Engine): 7 | def __init__(self, logs, default, hits, sharding_key=None): 8 | self.logs = logs 9 | self.default = default 10 | self.hits = hits 11 | self.sharding_key = sharding_key 12 | super(Distributed, self).__init__() 13 | 14 | def get_parameters(self): 15 | params = [ 16 | self.logs, 17 | self.default, 18 | self.hits 19 | ] 20 | 21 | if self.sharding_key is not None: 22 | params.append(self.sharding_key) 23 | 24 | return params 25 | 26 | @classmethod 27 | def reflect(cls, table, engine_full, **kwargs): 28 | engine = parse_columns(engine_full, delimeter=' ')[0] 29 | columns = engine.split('(', 1)[1][:-1] 30 | columns = parse_columns(columns) 31 | 32 | return cls(*columns) 33 | 34 | 35 | class Buffer(Engine): 36 | def __init__(self, database, table, num_layers=16, 37 | min_time=10, max_time=100, min_rows=10000, max_rows=1000000, 38 | min_bytes=10000000, max_bytes=100000000): 39 | self.database = database 40 | self.table_name = table 41 | self.num_layers = num_layers 42 | self.min_time = min_time 43 | self.max_time = max_time 44 | self.min_rows = min_rows 45 | self.max_rows = max_rows 46 | self.min_bytes = min_bytes 47 | self.max_bytes = max_bytes 48 | super(Buffer, self).__init__() 49 | 50 | def get_parameters(self): 51 | return [ 52 | self.database, 53 | self.table_name, 54 | self.num_layers, 55 | self.min_time, 56 | self.max_time, 57 | self.min_rows, 58 | self.max_rows, 59 | self.min_bytes, 60 | self.max_bytes 61 | ] 62 | 63 | @classmethod 64 | def reflect(cls, table, engine_full, **kwargs): 65 | engine = parse_columns(engine_full, delimeter=' ')[0] 66 | params = parse_columns(engine[len(cls.__name__):].strip("()")) 67 | 68 | database = params[0] 69 | table_name = params[1] 70 | num_layers = int(params[2]) 71 | min_time = int(params[3]) 72 | max_time = int(params[4]) 73 | min_rows = int(params[5]) 74 | max_rows = int(params[6]) 75 | min_bytes = int(params[7]) 76 | max_bytes = int(params[8]) 77 | 78 | return cls( 79 | database, table_name, num_layers, min_time, max_time, 80 | min_rows, max_rows, min_bytes, max_bytes 81 | ) 82 | 83 | 84 | class _NoParamsEngine(Engine): 85 | @classmethod 86 | def reflect(cls, table, engine_full, **kwargs): 87 | return cls() 88 | 89 | 90 | class View(_NoParamsEngine): 91 | pass 92 | 93 | 94 | class MaterializedView(_NoParamsEngine): 95 | pass 96 | 97 | 98 | class TinyLog(_NoParamsEngine): 99 | pass 100 | 101 | 102 | class Log(_NoParamsEngine): 103 | pass 104 | 105 | 106 | class Memory(_NoParamsEngine): 107 | pass 108 | 109 | 110 | class Null(_NoParamsEngine): 111 | pass 112 | 113 | 114 | class File(Engine): 115 | supported_data_formats = { 116 | 'tabseparated': 'TabSeparated', 117 | 'tabseparatedwithnames': 'TabSeparatedWithNames', 118 | 'tabseparatedwithnamesandtypes': 'TabSeparatedWithNamesAndTypes', 119 | 'template': 'Template', 120 | 'csv': 'CSV', 121 | 'csvwithnames': 'CSVWithNames', 122 | 'customseparated': 'CustomSeparated', 123 | 'values': 'Values', 124 | 'jsoneachrow': 'JSONEachRow', 125 | 'tskv': 'TSKV', 126 | 'protobuf': 'Protobuf', 127 | 'parquet': 'Parquet', 128 | 'rowbinary': 'RowBinary', 129 | 'rowbinarywithnamesandtypes': 'RowBinaryWithNamesAndTypes', 130 | 'native': 'Native' 131 | } 132 | 133 | def __init__(self, data_format): 134 | fmt = self.supported_data_formats.get(data_format.lower()) 135 | if not fmt: 136 | raise ValueError( 137 | 'Format {0} not supported for File engine'.format( 138 | data_format 139 | ) 140 | ) 141 | self.data_format = fmt 142 | super(File, self).__init__() 143 | 144 | def get_parameters(self): 145 | return (self.data_format, ) 146 | 147 | @classmethod 148 | def reflect(cls, table, engine_full, **kwargs): 149 | fmt = engine_full.split('(', 1)[1][:-1].strip("'") 150 | return cls(fmt) 151 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/engines/util.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def parse_columns(str_columns, delimeter=',', quote_symbol='`', 4 | escape_symbol='\\'): 5 | if not str_columns: 6 | return [] 7 | 8 | in_column = False 9 | quoted = False 10 | prev_symbol = None 11 | brackets_count = 0 12 | 13 | rv = [] 14 | col = '' 15 | for i, x in enumerate(str_columns + delimeter): 16 | if x == delimeter and not quoted and brackets_count == 0: 17 | in_column = False 18 | rv.append(col) 19 | col = '' 20 | 21 | elif x == ' ' and not in_column: 22 | continue 23 | 24 | elif x == '(': 25 | brackets_count += 1 26 | col += x 27 | 28 | elif x == ')': 29 | brackets_count -= 1 30 | col += x 31 | 32 | else: 33 | if x == quote_symbol: 34 | if prev_symbol != escape_symbol: 35 | if not quoted: 36 | quoted = True 37 | in_column = True 38 | else: 39 | quoted = False 40 | in_column = False 41 | else: 42 | col = col[:-1] + x 43 | 44 | else: 45 | if not in_column: 46 | in_column = True 47 | 48 | col += x 49 | 50 | prev_symbol = x 51 | 52 | return rv 53 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class DatabaseException(Exception): 4 | def __init__(self, orig): 5 | self.orig = orig 6 | super(DatabaseException, self).__init__(orig) 7 | 8 | def __str__(self): 9 | return 'Orig exception: {}'.format(self.orig) 10 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/clickhouse_sqlalchemy/ext/__init__.py -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/ext/clauses.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import exc 2 | from sqlalchemy.sql import type_api, roles 3 | from sqlalchemy.sql.elements import ( 4 | BindParameter, 5 | ColumnElement, 6 | ClauseList 7 | ) 8 | from sqlalchemy.sql.util import _offset_or_limit_clause 9 | from sqlalchemy.sql.visitors import Visitable 10 | 11 | 12 | class SampleParam(BindParameter): 13 | pass 14 | 15 | 16 | def sample_clause(element): 17 | """Convert the given value to an "sample" clause. 18 | 19 | This handles incoming element to an expression; if 20 | an expression is already given, it is passed through. 21 | 22 | """ 23 | if element is None: 24 | return None 25 | elif hasattr(element, '__clause_element__'): 26 | return element.__clause_element__() 27 | elif isinstance(element, Visitable): 28 | return element 29 | else: 30 | return SampleParam(None, element, unique=True) 31 | 32 | 33 | class LimitByClause: 34 | 35 | def __init__(self, by_clauses, limit, offset): 36 | self.by_clauses = ClauseList( 37 | *by_clauses, _literal_as_text_role=roles.ByOfRole 38 | ) 39 | self.offset = _offset_or_limit_clause(offset) 40 | self.limit = _offset_or_limit_clause(limit) 41 | 42 | def __bool__(self): 43 | return bool(self.by_clauses.clauses) 44 | 45 | 46 | class Lambda(ColumnElement): 47 | """Represent a lambda function, ``Lambda(lambda x: 2 * x)``.""" 48 | 49 | __visit_name__ = 'lambda' 50 | 51 | def __init__(self, func): 52 | if not callable(func): 53 | raise exc.ArgumentError('func must be callable') 54 | 55 | self.type = type_api.NULLTYPE 56 | self.func = func 57 | 58 | 59 | class ArrayJoin(ClauseList): 60 | __visit_name__ = 'array_join' 61 | 62 | 63 | class LeftArrayJoin(ClauseList): 64 | __visit_name__ = 'left_array_join' 65 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/ext/declarative.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from sqlalchemy import Column 4 | from sqlalchemy.ext.declarative import DeclarativeMeta 5 | from sqlalchemy.orm import declarative_base 6 | 7 | from ..sql.schema import Table 8 | 9 | 10 | class ClickHouseDeclarativeMeta(DeclarativeMeta): 11 | """ 12 | Generates __tablename__ automatically. Taken from flask-sqlalchemy. 13 | Also adds custom __table_cls__. 14 | """ 15 | _camelcase_re = re.compile(r'([A-Z]+)(?=[a-z0-9])') 16 | 17 | def __new__(cls, name, bases, d): 18 | tablename = d.get('__tablename__') 19 | 20 | has_pks = any( 21 | v.primary_key for k, v in d.items() if isinstance(v, Column) 22 | ) 23 | 24 | # generate a table name automatically if it's missing and the 25 | # class dictionary declares a primary key. We cannot always 26 | # attach a primary key to support model inheritance that does 27 | # not use joins. We also don't want a table name if a whole 28 | # table is defined 29 | if not tablename and d.get('__table__') is None and has_pks: 30 | def _join(match): 31 | word = match.group() 32 | if len(word) > 1: 33 | return ('_%s_%s' % (word[:-1], word[-1])).lower() 34 | return '_' + word.lower() 35 | d['__tablename__'] = cls._camelcase_re.sub(_join, name).lstrip('_') 36 | 37 | if '__table_cls__' not in d: 38 | d['__table_cls__'] = Table 39 | 40 | return DeclarativeMeta.__new__(cls, name, bases, d) 41 | 42 | 43 | def get_declarative_base(metadata=None): 44 | return declarative_base( 45 | metadata=metadata, metaclass=ClickHouseDeclarativeMeta 46 | ) 47 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/orm/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .session import make_session 3 | 4 | 5 | __all__ = ('make_session', ) 6 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/orm/query.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | from sqlalchemy import exc 4 | from sqlalchemy.sql.base import _generative 5 | from sqlalchemy.orm.query import Query as BaseQuery 6 | 7 | from ..ext.clauses import ( 8 | ArrayJoin, 9 | LeftArrayJoin, 10 | LimitByClause, 11 | sample_clause, 12 | ) 13 | 14 | 15 | def _compile_state_factory(orig_compile_state_factory, query, statement, 16 | *args, **kwargs): 17 | rv = orig_compile_state_factory(statement, *args, **kwargs) 18 | new_stmt = rv.statement 19 | new_stmt._with_cube = query._with_cube 20 | new_stmt._with_rollup = query._with_rollup 21 | new_stmt._with_totals = query._with_totals 22 | new_stmt._final_clause = query._final 23 | new_stmt._sample_clause = sample_clause(query._sample) 24 | new_stmt._limit_by_clause = query._limit_by 25 | new_stmt._array_join = query._array_join 26 | return rv 27 | 28 | 29 | class Query(BaseQuery): 30 | _with_cube = False 31 | _with_rollup = False 32 | _with_totals = False 33 | _final = None 34 | _sample = None 35 | _limit_by = None 36 | _array_join = None 37 | 38 | def _statement_20(self, *args, **kwargs): 39 | statement = super(Query, self)._statement_20(*args, **kwargs) 40 | statement._compile_state_factory = partial( 41 | _compile_state_factory, statement._compile_state_factory, self 42 | ) 43 | 44 | return statement 45 | 46 | @_generative 47 | def with_cube(self): 48 | if not self._group_by_clauses: 49 | raise exc.InvalidRequestError( 50 | "Query.with_cube() can be used only with specified " 51 | "GROUP BY, call group_by()" 52 | ) 53 | if self._with_rollup: 54 | raise exc.InvalidRequestError( 55 | "Query.with_cube() and Query.with_rollup() are mutually " 56 | "exclusive" 57 | ) 58 | 59 | self._with_cube = True 60 | return self 61 | 62 | @_generative 63 | def with_rollup(self): 64 | if not self._group_by_clauses: 65 | raise exc.InvalidRequestError( 66 | "Query.with_rollup() can be used only with specified " 67 | "GROUP BY, call group_by()" 68 | ) 69 | if self._with_cube: 70 | raise exc.InvalidRequestError( 71 | "Query.with_cube() and Query.with_rollup() are mutually " 72 | "exclusive" 73 | ) 74 | 75 | self._with_rollup = True 76 | return self 77 | 78 | @_generative 79 | def with_totals(self): 80 | if not self._group_by_clauses: 81 | raise exc.InvalidRequestError( 82 | "Query.with_totals() can be used only with specified " 83 | "GROUP BY, call group_by()" 84 | ) 85 | 86 | self._with_totals = True 87 | return self 88 | 89 | def _add_array_join(self, columns, left): 90 | join_type = ArrayJoin if not left else LeftArrayJoin 91 | self._array_join = join_type(*columns) 92 | 93 | @_generative 94 | def array_join(self, *columns, **kwargs): 95 | left = kwargs.get("left", False) 96 | self._add_array_join(columns, left=left) 97 | return self 98 | 99 | @_generative 100 | def left_array_join(self, *columns): 101 | self._add_array_join(columns, left=True) 102 | return self 103 | 104 | @_generative 105 | def final(self): 106 | self._final = True 107 | return self 108 | 109 | @_generative 110 | def sample(self, sample): 111 | self._sample = sample 112 | return self 113 | 114 | @_generative 115 | def limit_by(self, by_clauses, limit, offset=None): 116 | self._limit_by = LimitByClause(by_clauses, limit, offset) 117 | return self 118 | 119 | def join(self, target, onclause=None, **kwargs): 120 | spec = { 121 | 'type': kwargs.pop('type', None), 122 | 'strictness': kwargs.pop('strictness', None), 123 | 'distribution': kwargs.pop('distribution', None) 124 | } 125 | rv = super().join(target, onclause, **kwargs) 126 | x = rv._setup_joins[-1] 127 | x_spec = dict(spec) 128 | # use 'full' key to pass extra flags 129 | x_spec['full'] = x[-1]['full'] 130 | x[-1]['full'] = tuple(x_spec.items()) 131 | 132 | return rv 133 | 134 | def outerjoin(self, *props, **kwargs): 135 | kwargs['type'] = kwargs.get('type') or 'LEFT OUTER' 136 | return self.join(*props, **kwargs) 137 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/orm/session.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import sessionmaker, Session 2 | from sqlalchemy.ext.asyncio import AsyncSession 3 | 4 | from .query import Query 5 | 6 | 7 | def make_session(engine, is_async=False): 8 | session_class = Session 9 | if is_async: 10 | session_class = AsyncSession 11 | 12 | factory = sessionmaker(bind=engine, class_=session_class) 13 | 14 | return factory(query_cls=Query) 15 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/sql/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .schema import Table, MaterializedView 3 | from .selectable import Select, select 4 | 5 | 6 | __all__ = ('Table', 'MaterializedView', 'Select', 'select') 7 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/sql/ddl.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql.ddl import ( 2 | SchemaDropper as SchemaDropperBase, DropTable as DropTableBase, 3 | SchemaGenerator as SchemaGeneratorBase, _CreateDropBase 4 | ) 5 | from sqlalchemy.sql.expression import UnaryExpression 6 | from sqlalchemy.sql.operators import custom_op 7 | 8 | 9 | class DropTable(DropTableBase): 10 | def __init__(self, element, bind=None, if_exists=False): 11 | self.on_cluster = element.dialect_options['clickhouse']['cluster'] 12 | super(DropTable, self).__init__(element, 13 | if_exists=if_exists) 14 | 15 | 16 | class DropView(DropTableBase): 17 | def __init__(self, element, bind=None, if_exists=False): 18 | self.on_cluster = element.cluster 19 | super(DropView, self).__init__(element, if_exists=if_exists) 20 | 21 | 22 | class SchemaDropper(SchemaDropperBase): 23 | def __init__(self, dialect, connection, if_exists=False, **kwargs): 24 | self.if_exists = if_exists 25 | super(SchemaDropper, self).__init__(dialect, connection, **kwargs) 26 | 27 | def visit_table(self, table, **kwargs): 28 | table.dispatch.before_drop( 29 | table, 30 | self.connection, 31 | checkfirst=self.checkfirst, 32 | _ddl_runner=self, 33 | ) 34 | 35 | self.connection.execute(DropTable(table, if_exists=self.if_exists)) 36 | 37 | table.dispatch.after_drop( 38 | table, 39 | self.connection, 40 | checkfirst=self.checkfirst, 41 | _ddl_runner=self, 42 | ) 43 | 44 | def visit_materialized_view(self, table, **kwargs): 45 | self.connection.execute(DropView(table, if_exists=self.if_exists)) 46 | 47 | 48 | class CreateMaterializedView(_CreateDropBase): 49 | """Represent a CREATE MATERIALIZED VIEW statement.""" 50 | 51 | __visit_name__ = "create_materialized_view" 52 | 53 | def __init__(self, element, if_not_exists=False): 54 | self.if_not_exists = if_not_exists 55 | super(CreateMaterializedView, self).__init__(element) 56 | 57 | 58 | class SchemaGenerator(SchemaGeneratorBase): 59 | def __init__(self, dialect, connection, if_not_exists=False, **kwargs): 60 | self.if_not_exists = if_not_exists 61 | super(SchemaGenerator, self).__init__(dialect, connection, **kwargs) 62 | 63 | def visit_materialized_view(self, table, **kwargs): 64 | self.connection.execute( 65 | CreateMaterializedView(table, if_not_exists=self.if_not_exists) 66 | ) 67 | 68 | 69 | def ttl_delete(expr): 70 | return UnaryExpression(expr, modifier=custom_op('DELETE')) 71 | 72 | 73 | def ttl_to_disk(expr, disk): 74 | assert isinstance(disk, str), 'Disk must be str' 75 | return expr.op('TO DISK')(disk) 76 | 77 | 78 | def ttl_to_volume(expr, volume): 79 | assert isinstance(volume, str), 'Volume must be str' 80 | return expr.op('TO VOLUME')(volume) 81 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/sql/functions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any, TypeVar 4 | 5 | from sqlalchemy.ext.compiler import compiles 6 | from sqlalchemy.sql import coercions, roles 7 | from sqlalchemy.sql.elements import ColumnElement 8 | from sqlalchemy.sql.functions import GenericFunction 9 | 10 | from clickhouse_sqlalchemy import types 11 | 12 | if TYPE_CHECKING: 13 | from sqlalchemy.sql._typing import _ColumnExpressionArgument 14 | 15 | _T = TypeVar('_T', bound=Any) 16 | 17 | 18 | class quantile(GenericFunction[_T]): 19 | inherit_cache = True 20 | 21 | def __init__( 22 | self, level: float, expr: _ColumnExpressionArgument[Any], 23 | condition: _ColumnExpressionArgument[Any] = None, **kwargs: Any 24 | ): 25 | arg: ColumnElement[Any] = coercions.expect( 26 | roles.ExpressionElementRole, expr, apply_propagate_attrs=self 27 | ) 28 | 29 | args = [arg] 30 | if condition is not None: 31 | condition = coercions.expect( 32 | roles.ExpressionElementRole, condition, 33 | apply_propagate_attrs=self 34 | ) 35 | args.append(condition) 36 | 37 | self.level = level 38 | 39 | if isinstance(arg.type, (types.Decimal, types.Float, types.Int)): 40 | return_type = types.Float64 41 | elif isinstance(arg.type, types.DateTime): 42 | return_type = types.DateTime 43 | elif isinstance(arg.type, types.Date): 44 | return_type = types.Date 45 | else: 46 | return_type = types.Float64 47 | 48 | kwargs['type_'] = return_type 49 | kwargs['_parsed_args'] = args 50 | super().__init__(arg, **kwargs) 51 | 52 | 53 | class quantileIf(quantile[_T]): 54 | inherit_cache = True 55 | 56 | 57 | @compiles(quantile, 'clickhouse') 58 | @compiles(quantileIf, 'clickhouse') 59 | def compile_quantile(element, compiler, **kwargs): 60 | args_str = compiler.function_argspec(element, **kwargs) 61 | return f'{element.name}({element.level}){args_str}' 62 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/sql/schema.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Table as TableBase 2 | from sqlalchemy.sql.base import ( 3 | DialectKWArgs, Immutable 4 | ) 5 | from sqlalchemy.sql.schema import SchemaItem 6 | from sqlalchemy.sql.selectable import FromClause 7 | from sqlalchemy.sql.selectable import Join 8 | 9 | from clickhouse_sqlalchemy.sql.selectable import Select 10 | 11 | from . import ddl 12 | 13 | 14 | class Table(TableBase): 15 | def drop(self, bind=None, checkfirst=False, if_exists=False): 16 | if bind is None: 17 | bind = self.bind 18 | bind._run_ddl_visitor(ddl.SchemaDropper, self, 19 | checkfirst=checkfirst, if_exists=if_exists) 20 | 21 | def join(self, right, onclause=None, isouter=False, full=False, 22 | type=None, strictness=None, distribution=None): 23 | flags = tuple({ 24 | 'full': full, 25 | 'type': type, 26 | 'strictness': strictness, 27 | 'distribution': distribution 28 | }.items()) 29 | return Join(self, right, onclause=onclause, isouter=isouter, 30 | full=flags) 31 | 32 | def select(self, whereclause=None, **params): 33 | if whereclause: 34 | return Select(self, whereclause, **params) 35 | return Select(self, **params) 36 | 37 | @classmethod 38 | def _make_from_standard(cls, std_table, _extend_on=None): 39 | ch_table = cls(std_table.name, std_table.metadata) 40 | ch_table.schema = std_table.schema 41 | ch_table.fullname = std_table.fullname 42 | ch_table.implicit_returning = std_table.implicit_returning 43 | ch_table.comment = std_table.comment 44 | ch_table.info = std_table.info 45 | ch_table._prefixes = std_table._prefixes 46 | ch_table.dialect_options = std_table.dialect_options 47 | 48 | if _extend_on is None: 49 | ch_table._columns = std_table._columns 50 | ch_table.columns = std_table.columns 51 | ch_table.c = std_table.c 52 | 53 | return ch_table 54 | 55 | 56 | class MaterializedView(DialectKWArgs, SchemaItem, Immutable, FromClause): 57 | __visit_name__ = 'materialized_view' 58 | 59 | def __init__(self, *args, **kwargs): 60 | pass 61 | 62 | @property 63 | def bind(self): 64 | return self.metadata.bind 65 | 66 | @property 67 | def metadata(self): 68 | return self.inner_table.metadata 69 | 70 | def __new__(cls, inner_model, selectable, if_not_exists=False, 71 | cluster=None, populate=False, use_to=None, 72 | mv_suffix='_mv', name=None): 73 | rv = object.__new__(cls) 74 | rv.__init__() 75 | 76 | rv.mv_selectable = selectable 77 | rv.inner_table = inner_model.__table__ 78 | rv.if_not_exists = if_not_exists 79 | rv.cluster = cluster 80 | rv.populate = populate 81 | rv.to = use_to 82 | 83 | table = inner_model.__table__ 84 | metadata = rv.inner_table.metadata 85 | 86 | if use_to: 87 | if name is None: 88 | name = table.name + mv_suffix 89 | else: 90 | name = table.name 91 | 92 | rv.name = name 93 | 94 | metadata.info.setdefault('mat_views', set()).add(name) 95 | if not hasattr(metadata, 'mat_views'): 96 | metadata.mat_views = {} 97 | metadata.mat_views[name] = rv 98 | 99 | table.info['mv_storage'] = True 100 | 101 | return rv 102 | 103 | def __repr__(self): 104 | args = [repr(self.name), repr(self.metadata)] 105 | 106 | if self.to: 107 | args += ['TO ' + repr(self.inner_table.name)] 108 | else: 109 | args += ( 110 | [repr(x) for x in self.inner_table.columns] 111 | + [repr(self.inner_table.engine)] 112 | + ['%s=%s' % (k, repr(getattr(self, k))) for k in ['schema']] 113 | ) 114 | 115 | args += ['AS ' + str(self.mv_selectable)] 116 | 117 | return 'MaterializedView(%s)' % ', '.join(args) 118 | 119 | def create(self, bind=None, checkfirst=False, if_not_exists=False): 120 | if bind is None: 121 | bind = self.bind 122 | bind._run_ddl_visitor(ddl.SchemaGenerator, self, checkfirst=checkfirst, 123 | if_not_exists=if_not_exists) 124 | 125 | def drop(self, bind=None, checkfirst=False, if_exists=False): 126 | if bind is None: 127 | bind = self.bind 128 | bind._run_ddl_visitor(ddl.SchemaDropper, self, checkfirst=checkfirst, 129 | if_exists=if_exists) 130 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/sql/selectable.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql.base import _generative 2 | from sqlalchemy.sql.selectable import ( 3 | Select as StandardSelect, 4 | ) 5 | 6 | from ..ext.clauses import ( 7 | ArrayJoin, 8 | LeftArrayJoin, 9 | LimitByClause, 10 | sample_clause, 11 | ) 12 | 13 | 14 | __all__ = ('Select', 'select') 15 | 16 | 17 | class Select(StandardSelect): 18 | _with_cube = False 19 | _with_rollup = False 20 | _with_totals = False 21 | _final_clause = None 22 | _sample_clause = None 23 | _limit_by_clause = None 24 | _array_join = None 25 | 26 | @_generative 27 | def with_cube(self): 28 | self._with_cube = True 29 | return self 30 | 31 | @_generative 32 | def with_rollup(self): 33 | self._with_rollup = True 34 | return self 35 | 36 | @_generative 37 | def with_totals(self): 38 | self._with_totals = True 39 | return self 40 | 41 | @_generative 42 | def final(self): 43 | self._final_clause = True 44 | return self 45 | 46 | @_generative 47 | def sample(self, sample): 48 | self._sample_clause = sample_clause(sample) 49 | return self 50 | 51 | @_generative 52 | def limit_by(self, by_clauses, limit, offset=None): 53 | self._limit_by_clause = LimitByClause(by_clauses, limit, offset) 54 | return self 55 | 56 | def _add_array_join(self, columns, left): 57 | join_type = ArrayJoin if not left else LeftArrayJoin 58 | self._array_join = join_type(*columns) 59 | 60 | @_generative 61 | def array_join(self, *columns, **kwargs): 62 | left = kwargs.get("left", False) 63 | self._add_array_join(columns, left=left) 64 | return self 65 | 66 | @_generative 67 | def left_array_join(self, *columns): 68 | self._add_array_join(columns, left=True) 69 | return self 70 | 71 | def join(self, right, onclause=None, isouter=False, full=False, type=None, 72 | strictness=None, distribution=None): 73 | flags = tuple({ 74 | 'full': full, 75 | 'type': type, 76 | 'strictness': strictness, 77 | 'distribution': distribution 78 | }.items()) 79 | return super().join(right, onclause=onclause, isouter=isouter, 80 | full=flags) 81 | 82 | 83 | select = Select 84 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/types/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | 'String', 3 | 'Int', 4 | 'Float', 5 | 'Boolean', 6 | 'Array', 7 | 'Nullable', 8 | 'UUID', 9 | 'LowCardinality', 10 | 'Int8', 11 | 'UInt8', 12 | 'Int16', 13 | 'UInt16', 14 | 'Int32', 15 | 'UInt32', 16 | 'Int64', 17 | 'UInt64', 18 | 'Int128', 19 | 'UInt128', 20 | 'Int256', 21 | 'UInt256', 22 | 'Float32', 23 | 'Float64', 24 | 'Date', 25 | 'Date32', 26 | 'DateTime', 27 | 'DateTime64', 28 | 'Enum', 29 | 'Enum8', 30 | 'Enum16', 31 | 'Decimal', 32 | 'IPv4', 33 | 'IPv6', 34 | 'JSON', 35 | 'Nested', 36 | 'Tuple', 37 | 'Map', 38 | 'AggregateFunction', 39 | 'SimpleAggregateFunction', 40 | ] 41 | 42 | from .common import String 43 | from .common import Int 44 | from .common import Float 45 | from .common import Boolean 46 | from .common import Array 47 | from .common import Nullable 48 | from .common import UUID 49 | from .common import LowCardinality 50 | from .common import Int8 51 | from .common import UInt8 52 | from .common import Int16 53 | from .common import UInt16 54 | from .common import Int32 55 | from .common import UInt32 56 | from .common import Int64 57 | from .common import UInt64 58 | from .common import Int128 59 | from .common import UInt128 60 | from .common import Int256 61 | from .common import UInt256 62 | from .common import Float32 63 | from .common import Float64 64 | from .common import Date 65 | from .common import Date32 66 | from .common import DateTime 67 | from .common import DateTime64 68 | from .common import Enum 69 | from .common import Enum8 70 | from .common import Enum16 71 | from .common import Decimal 72 | from .common import JSON 73 | from .common import Tuple 74 | from .common import Map 75 | from .common import AggregateFunction 76 | from .common import SimpleAggregateFunction 77 | from .ip import IPv4 78 | from .ip import IPv6 79 | from .nested import Nested 80 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/types/ip.py: -------------------------------------------------------------------------------- 1 | from ipaddress import IPv4Network, IPv6Network 2 | 3 | from sqlalchemy import or_, and_, types, func 4 | from sqlalchemy.sql.type_api import UserDefinedType 5 | 6 | 7 | class BaseIPComparator(UserDefinedType.Comparator): 8 | network_class = None 9 | 10 | def _wrap_to_ip(self, x): 11 | raise NotImplementedError() 12 | 13 | def _split_other(self, other): 14 | """ 15 | Split values between addresses and networks 16 | This allows to generate complex filters with both addresses 17 | and networks in the same IN 18 | ie in_('10.0.0.0/24', '192.168.0.1') 19 | """ 20 | addresses = [] 21 | networks = [] 22 | for sub in other: 23 | sub = self.network_class(sub) 24 | if sub.prefixlen == sub.max_prefixlen: 25 | # this is an address 26 | addresses.append(sub.network_address) 27 | else: 28 | networks.append(sub) 29 | return addresses, networks 30 | 31 | def in_(self, other): 32 | if isinstance(other, (list, tuple)): 33 | addresses, networks = self._split_other(other) 34 | addresses_clause = super(BaseIPComparator, self).in_( 35 | self._wrap_to_ip(x) for x in addresses 36 | ) if addresses else None 37 | networks_clause = or_(*[ 38 | and_( 39 | self >= self._wrap_to_ip(net[0]), 40 | self <= self._wrap_to_ip(net[-1]) 41 | ) 42 | for net in networks 43 | ]) if networks else None 44 | if addresses_clause is not None and networks_clause is not None: 45 | return or_(addresses_clause, networks_clause) 46 | elif addresses_clause is not None and networks_clause is None: 47 | return addresses_clause 48 | elif networks_clause is not None and addresses_clause is None: 49 | return networks_clause 50 | else: 51 | # other is an empty array 52 | return super(BaseIPComparator, self).in_(other) 53 | 54 | if not isinstance(other, self.network_class): 55 | other = self.network_class(other) 56 | 57 | return and_( 58 | self >= self._wrap_to_ip(other[0]), 59 | self <= self._wrap_to_ip(other[-1]) 60 | ) 61 | 62 | def not_in(self, other): 63 | if isinstance(other, (list, tuple)): 64 | addresses, networks = self._split_other(other) 65 | addresses_clause = super(BaseIPComparator, self).notin_( 66 | self._wrap_to_ip(x) for x in addresses 67 | ) if addresses else None 68 | networks_clause = and_(*[ 69 | or_( 70 | self < self._wrap_to_ip(net[0]), 71 | self > self._wrap_to_ip(net[-1]) 72 | ) 73 | for net in networks 74 | ]) if networks else None 75 | if addresses_clause is not None and networks_clause is not None: 76 | return and_(addresses_clause, networks_clause) 77 | elif addresses_clause is not None and networks_clause is None: 78 | return addresses_clause 79 | elif networks_clause is not None and addresses_clause is None: 80 | return networks_clause 81 | else: 82 | # other is an empty array 83 | return super(BaseIPComparator, self).notin_(other) 84 | 85 | if not isinstance(other, self.network_class): 86 | other = self.network_class(other) 87 | 88 | return or_( 89 | self < self._wrap_to_ip(other[0]), 90 | self > self._wrap_to_ip(other[-1]) 91 | ) 92 | 93 | 94 | class IPv4(types.UserDefinedType): 95 | __visit_name__ = "ipv4" 96 | 97 | cache_ok = True 98 | 99 | def bind_processor(self, dialect): 100 | def process(value): 101 | return str(value) 102 | 103 | return process 104 | 105 | def literal_processor(self, dialect): 106 | bp = self.bind_processor(dialect) 107 | 108 | def process(value): 109 | return "'%s'" % bp(value) 110 | 111 | return process 112 | 113 | def bind_expression(self, bindvalue): 114 | if isinstance(bindvalue.value, (list, tuple)): 115 | bindvalue.value = ([func.toIPv4(x) for x in bindvalue.value]) 116 | return bindvalue 117 | return func.toIPv4(bindvalue) 118 | 119 | class comparator_factory(BaseIPComparator): 120 | network_class = IPv4Network 121 | 122 | def _wrap_to_ip(self, x): 123 | return func.toIPv4(str(x)) 124 | 125 | 126 | class IPv6(types.UserDefinedType): 127 | __visit_name__ = "ipv6" 128 | 129 | cache_ok = True 130 | 131 | def bind_processor(self, dialect): 132 | def process(value): 133 | return str(value) 134 | 135 | return process 136 | 137 | def literal_processor(self, dialect): 138 | bp = self.bind_processor(dialect) 139 | 140 | def process(value): 141 | return "'%s'" % bp(value) 142 | 143 | return process 144 | 145 | def bind_expression(self, bindvalue): 146 | if isinstance(bindvalue.value, (list, tuple)): 147 | bindvalue.value = ([func.toIPv6(x) for x in bindvalue.value]) 148 | return bindvalue 149 | return func.toIPv6(bindvalue) 150 | 151 | class comparator_factory(BaseIPComparator): 152 | network_class = IPv6Network 153 | 154 | def _wrap_to_ip(self, x): 155 | return func.toIPv6(str(x)) 156 | -------------------------------------------------------------------------------- /clickhouse_sqlalchemy/types/nested.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import types 2 | from sqlalchemy.ext.compiler import compiles 3 | from sqlalchemy.sql.elements import Label, ColumnClause 4 | from sqlalchemy.sql.type_api import UserDefinedType 5 | 6 | from .common import Array 7 | 8 | 9 | class Nested(types.TypeEngine): 10 | __visit_name__ = 'nested' 11 | 12 | def __init__(self, *columns): 13 | if not columns: 14 | raise ValueError('columns must be specified for nested type') 15 | self.columns = columns 16 | self._columns_dict = {col.name: col for col in columns} 17 | super(Nested, self).__init__() 18 | 19 | class Comparator(UserDefinedType.Comparator): 20 | def __getattr__(self, key): 21 | str_key = key.rstrip("_") 22 | try: 23 | sub = self.type._columns_dict[str_key] 24 | except KeyError: 25 | raise AttributeError(key) 26 | else: 27 | original_type = sub.type 28 | try: 29 | sub.type = Array(sub.type) 30 | expr = NestedColumn(self.expr, sub) 31 | return expr 32 | finally: 33 | sub.type = original_type 34 | 35 | comparator_factory = Comparator 36 | 37 | 38 | class NestedColumn(ColumnClause): 39 | def __init__(self, parent, sub_column): 40 | self.parent = parent 41 | self.sub_column = sub_column 42 | if isinstance(self.parent, Label): 43 | table = self.parent.element.table 44 | else: 45 | table = self.parent.table 46 | super(NestedColumn, self).__init__( 47 | sub_column.name, 48 | sub_column.type, 49 | _selectable=table 50 | ) 51 | 52 | 53 | @compiles(NestedColumn) 54 | def _comp(element, compiler, **kw): 55 | from_labeled_label = False 56 | if isinstance(element.parent, Label): 57 | from_labeled_label = True 58 | return "%s.%s" % ( 59 | compiler.process(element.parent, 60 | from_labeled_label=from_labeled_label, 61 | within_label_clause=False, 62 | within_columns_clause=True), 63 | compiler.visit_column(element, include_table=False), 64 | ) 65 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = -W 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | 2 | table.table-small-text { 3 | font-size: small; 4 | } 5 | 6 | 7 | table.table-center-header thead tr th { 8 | text-align: center; 9 | } 10 | 11 | 12 | table.table-right-text-align-results tbody tr td { 13 | text-align: right; 14 | } 15 | 16 | table.table-right-text-align-results tbody tr td:first-child { 17 | text-align: inherit; 18 | } 19 | -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block extrahead %} 3 | 4 | {% endblock %} 5 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | Changelog is available in `github repo `_. 5 | -------------------------------------------------------------------------------- /docs/connection.rst: -------------------------------------------------------------------------------- 1 | .. _connection: 2 | 3 | Connection configuration 4 | ======================== 5 | 6 | ClickHouse SQLAlchemy uses the following syntax for the connection string: 7 | 8 | .. code-block:: 9 | 10 | clickhouse<+driver>://:@:/[?key=value..] 11 | 12 | Where: 13 | 14 | - **driver** is driver to use. Possible choices: ``http``, ``native``, ``asynch``. 15 | ``http`` is default. When you omit driver http is used. 16 | - **database** is database connect to. Default is ``default``. 17 | - **user** is database user. Defaults to ``'default'``. 18 | - **password** of the user. Defaults to ``''`` (no password). 19 | - **port** can be customized if ClickHouse server is listening on non-standard 20 | port. 21 | 22 | Additional parameters are passed to driver. 23 | 24 | Common options 25 | -------------- 26 | 27 | - **engine_reflection** controls table engine reflection during table reflection. 28 | Engine reflection can be very slow if you have thousand of tables. You can 29 | disable reflection by setting this parameter to ``false``. Possible choices: 30 | ``true``/``false``. Default is ``true``. 31 | - **server_version** can be used for eliminating initialization 32 | ``select version()`` query. Generally you shouldn't set this parameter and 33 | server version will be detected automatically. 34 | 35 | 36 | Driver options 37 | -------------- 38 | 39 | There are several options can be specified in query string. 40 | 41 | HTTP 42 | ~~~~ 43 | 44 | - **port** is port ClickHouse server is bound to. Default is ``8123``. 45 | - **timeout** in seconds. There is no timeout by default. 46 | - **protocol** to use. Possible choices: ``http``, ``https``. ``http`` is default. 47 | - **verify** controls certificate verification in ``https`` protocol. 48 | Possible choices: ``true``/``false``. Default is ``true``. 49 | 50 | Simple DSN example: 51 | 52 | .. code-block:: RST 53 | 54 | clickhouse+http://host/db 55 | 56 | DSN example for ClickHouse https port: 57 | 58 | .. code-block:: RST 59 | 60 | clickhouse+http://user:password@host:8443/db?protocol=https 61 | 62 | When you are using `nginx` as proxy server for ClickHouse server connection 63 | string might look like: 64 | 65 | .. code-block:: RST 66 | 67 | clickhouse+http://user:password@host:8124/test?protocol=https 68 | 69 | Where ``8124`` is proxy port. 70 | 71 | If you need control over the underlying HTTP connection, pass a `requests.Session 72 | `_ instance 73 | to ``create_engine()``, like so: 74 | 75 | .. code-block:: python 76 | 77 | from sqlalchemy import create_engine 78 | from requests import Session 79 | 80 | engine = create_engine( 81 | 'clickhouse+http://localhost/test', 82 | connect_args={'http_session': Session()} 83 | ) 84 | 85 | 86 | Native 87 | ~~~~~~ 88 | 89 | Please note that native connection **is not encrypted**. All data including 90 | user/password is transferred in plain text. You should use this connection over 91 | SSH or VPN (for example) while communicating over untrusted network. 92 | 93 | Simple DSN example: 94 | 95 | .. code-block:: RST 96 | 97 | clickhouse+native://host/db 98 | 99 | All connection string parameters are proxied to ``clickhouse-driver``. 100 | See it's `parameters `__. 101 | 102 | Example DSN with LZ4 compression secured with Let's Encrypt certificate on server side: 103 | 104 | .. code-block:: python 105 | 106 | import certify 107 | 108 | dsn = ( 109 | 'clickhouse+native://user:pass@host/db?compression=lz4&' 110 | 'secure=True&ca_certs={}'.format(certify.where()) 111 | ) 112 | 113 | Example with multiple hosts 114 | 115 | .. code-block:: RST 116 | 117 | clickhouse+native://wronghost/default?alt_hosts=localhost:9000 118 | 119 | 120 | Asynch 121 | ~~~~~~ 122 | 123 | Same as Native. 124 | 125 | Simple DSN example: 126 | 127 | .. code-block:: RST 128 | 129 | clickhouse+asynch://host/db 130 | 131 | All connection string parameters are proxied to ``asynch``. 132 | See it's `parameters `__. 133 | -------------------------------------------------------------------------------- /docs/contents.rst.inc: -------------------------------------------------------------------------------- 1 | User's Guide 2 | ------------ 3 | 4 | This part of the documentation focuses on step-by-step instructions for development with clickhouse-sqlalchemy. 5 | 6 | It assumes that you have experience with SQLAlchemy. Consider `its `_ docs at the first, if that is not so. 7 | Experience with ClickHouse is also highly recommended. 8 | 9 | ClickHouse server provides a lot of interfaces. This dialect supports: 10 | 11 | * HTTP interface (port 8123 by default); 12 | * Native (TCP) interface (port 9000 by default). 13 | 14 | Each interface has it's own support by corresponding "driver": 15 | 16 | - **http** via ``requests`` 17 | - **native** via ``clickhouse-driver`` or via ``asynch`` for async support 18 | 19 | Native driver is recommended due to rich ``clickhouse-driver`` support. HTTP 20 | driver has poor development support compare to native driver. 21 | However default driver is still ``http``. 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | 26 | installation 27 | quickstart 28 | connection 29 | features 30 | types 31 | migrations 32 | 33 | Additional Notes 34 | ---------------- 35 | 36 | Legal information, changelog and contributing are here for the interested. 37 | 38 | .. toctree:: 39 | :maxdepth: 2 40 | 41 | development 42 | changelog 43 | license 44 | contributing 45 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/development.rst: -------------------------------------------------------------------------------- 1 | .. _development: 2 | 3 | Development 4 | =========== 5 | 6 | Test configuration 7 | ------------------ 8 | 9 | In ``setup.cfg`` you can find ClickHouse server ports, credentials, logging 10 | level and another options than can be tuned during local testing. 11 | 12 | Running tests locally 13 | --------------------- 14 | 15 | Install desired Python version with system package manager/pyenv/another manager. 16 | 17 | Install test requirements and build package: 18 | 19 | .. code-block:: bash 20 | 21 | python testsrequire.py && python setup.py develop 22 | 23 | ClickHouse on host machine 24 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 25 | 26 | Install desired versions of ``clickhouse-server`` and ``clickhouse-client`` on 27 | your machine. 28 | 29 | Run tests: 30 | 31 | .. code-block:: bash 32 | 33 | pytest -v 34 | 35 | ClickHouse in docker 36 | ^^^^^^^^^^^^^^^^^^^^ 37 | 38 | Create container desired version of ``clickhouse-server``: 39 | 40 | .. code-block:: bash 41 | 42 | docker run --rm -p 127.0.0.1:9000:9000 -p 127.0.0.1:8123:8123 --name test-clickhouse-server clickhouse/clickhouse-server:$VERSION 43 | 44 | Create container with the same version of ``clickhouse-client``: 45 | 46 | .. code-block:: bash 47 | 48 | docker run --rm --entrypoint "/bin/sh" --name test-clickhouse-client --link test-clickhouse-server:clickhouse-server clickhouse/clickhouse-client:$VERSION -c 'while :; do sleep 1; done' 49 | 50 | Create ``clickhouse-client`` script on your host machine: 51 | 52 | .. code-block:: bash 53 | 54 | echo -e '#!/bin/bash\n\ndocker exec test-clickhouse-client clickhouse-client "$@"' | sudo tee /usr/local/bin/clickhouse-client > /dev/null 55 | sudo chmod +x /usr/local/bin/clickhouse-client 56 | 57 | After it container ``test-clickhouse-client`` will communicate with 58 | ``test-clickhouse-server`` transparently from host machine. 59 | 60 | Set ``host=clickhouse-server`` in ``setup.cfg``. 61 | 62 | Add entry in hosts file: 63 | 64 | .. code-block:: bash 65 | 66 | echo '127.0.0.1 clickhouse-server' | sudo tee -a /etc/hosts > /dev/null 67 | 68 | And run tests: 69 | 70 | .. code-block:: bash 71 | 72 | pytest -v 73 | 74 | ``pip`` will automatically install all required modules for testing. 75 | 76 | GitHub Actions in forked repository 77 | ----------------------------------- 78 | 79 | Workflows in forked repositories can be used for running tests. 80 | 81 | Workflows don't run in forked repositories by default. 82 | You must enable GitHub Actions in the **Actions** tab of the forked repository. 83 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to clickhouse-sqlalchemy 2 | ================================ 3 | 4 | Release |release|. 5 | 6 | Supported SQLAlchemy: 1.4. 7 | 8 | Welcome to clickhouse-sqlalchemy's documentation. Get started with :ref:`installation` 9 | and then get an overview with the :ref:`quickstart` where common queries are described. 10 | 11 | 12 | .. include:: contents.rst.inc 13 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | Installation 4 | ============ 5 | 6 | Python Version 7 | -------------- 8 | 9 | Clickhouse-sqlalchemy supports Python 2.7 and newer. 10 | 11 | Dependencies 12 | ------------ 13 | 14 | These distributions will be installed automatically when installing 15 | clickhouse-sqlalchemy: 16 | 17 | * `clickhouse-driver`_ ClickHouse Python Driver with native (TCP) interface support. 18 | * `requests`_ a simple and elegant HTTP library. 19 | * `ipaddress`_ backport ipaddress module. 20 | * `asynch`_ An asyncio ClickHouse Python Driver with native (TCP) interface support. 21 | 22 | .. _clickhouse-driver: https://pypi.org/project/clickhouse-driver/ 23 | .. _requests: https://pypi.org/project/requests/ 24 | .. _ipaddress: https://pypi.org/project/ipaddress/ 25 | .. _asynch: https://pypi.org/project/asynch/ 26 | 27 | If you are planning to use ``clickhouse-driver`` with compression you should 28 | also install compression extras as well. See clickhouse-driver `documentation `_. 29 | 30 | .. _installation-pypi: 31 | 32 | Installation from PyPI 33 | ---------------------- 34 | 35 | The package can be installed using ``pip``: 36 | 37 | .. code-block:: bash 38 | 39 | pip install clickhouse-sqlalchemy 40 | 41 | 42 | Installation from github 43 | ------------------------ 44 | 45 | Development version can be installed directly from github: 46 | 47 | .. code-block:: bash 48 | 49 | pip install git+https://github.com/xzkostyan/clickhouse-sqlalchemy@master#egg=clickhouse-sqlalchemy 50 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ======= 3 | 4 | clickhouse-sqlalchemy is distributed under the `MIT license 5 | `_. 6 | 7 | -------------------------------------------------------------------------------- /docs/migrations.rst: -------------------------------------------------------------------------------- 1 | Migrations 2 | ========== 3 | 4 | Since version 0.1.10 clickhouse-sqlalchemy has alembic support. This support 5 | allows autogenerate migrations from source code with some limitations. 6 | This is the main advantage comparing to other migration tools when you need to 7 | write plain migrations by yourself. 8 | 9 | .. note:: 10 | 11 | It is necessary to notice that ClickHouse doesn't have transactions. 12 | Therefore migration will not rolled back after some command with an error. 13 | And schema will remain in partially migrated state. 14 | 15 | Autogenerate **will detect**: 16 | 17 | * Table and materialized view additions, removals. 18 | * Column additions, removals. 19 | * Column and table comment additions, removals. 20 | 21 | Example project with migrations https://github.com/xzkostyan/clickhouse-sqlalchemy-alembic-example. 22 | 23 | Requirements 24 | ------------ 25 | 26 | Minimal versions: 27 | 28 | * ClickHouse server 21.11.11.1 29 | * clickhouse-sqlalchemy 0.1.10 30 | * alembic 1.5.x 31 | 32 | You can always write you migrations with pure alembic's ``op.execute`` if 33 | autogenerate is not possible for your schema objects or your are using 34 | ``clickhouse-sqlalchemy<0.1.10``. 35 | 36 | Limitations 37 | ----------- 38 | 39 | Common limitations: 40 | 41 | * Engines are not added into ``op.create_table``. 42 | * ``Nullable(T)`` columns generation via ``Column(..., nullable=True)`` is not 43 | supported. 44 | 45 | Currently ``ATTACH MATERIALIZED VIEW`` with modified ``SELECT`` statement 46 | doesn't work for ``Atomic`` engine. 47 | 48 | Migration adjusting 49 | ------------------- 50 | 51 | You can and should adjust migrations after autogeneration. 52 | 53 | Following parameters can be specified for: 54 | 55 | * ``op.detach_mat_view``: ``if_exists``, ``on_cluster``, ``permanently``. 56 | * ``op.attach_mat_view``: ``if_not_exists``, ``on_cluster``. 57 | * ``op.create_mat_view``: ``if_not_exists``, ``on_cluster``, ``populate``. 58 | 59 | See ClickHouse's Materialized View documentation. 60 | 61 | For ``op.add_column`` you can add: 62 | 63 | * ``AFTER name_after``: ``op.add_column(..., sa.Column(..., clickhouse_after=sa.text('my_column')))``. 64 | -------------------------------------------------------------------------------- /docs/quickstart.rst: -------------------------------------------------------------------------------- 1 | .. _quickstart: 2 | 3 | Quickstart 4 | ========== 5 | 6 | This page gives a good introduction to clickhouse-sqlalchemy. 7 | It assumes you already have clickhouse-sqlalchemy installed. 8 | If you do not, head over to the :ref:`installation` section. 9 | 10 | It should be pointed that session must be created with 11 | ``clickhouse_sqlalchemy.make_session``. Otherwise ``session.query`` and 12 | ``session.execute`` will not have ClickHouse SQL extensions. The same is 13 | applied to ``Table`` and ``get_declarative_base``. 14 | 15 | 16 | Let's define some table, insert data into it and query inserted data. 17 | 18 | .. code-block:: python 19 | 20 | from sqlalchemy import create_engine, Column, MetaData 21 | 22 | from clickhouse_sqlalchemy import ( 23 | Table, make_session, get_declarative_base, types, engines 24 | ) 25 | 26 | uri = 'clickhouse+native://localhost/default' 27 | 28 | engine = create_engine(uri) 29 | session = make_session(engine) 30 | metadata = MetaData(bind=engine) 31 | 32 | Base = get_declarative_base(metadata=metadata) 33 | 34 | class Rate(Base): 35 | day = Column(types.Date, primary_key=True) 36 | value = Column(types.Int32) 37 | 38 | __table_args__ = ( 39 | engines.Memory(), 40 | ) 41 | 42 | # Emits CREATE TABLE statement 43 | Rate.__table__.create() 44 | 45 | 46 | Now it's time to insert some data 47 | 48 | .. code-block:: python 49 | 50 | from datetime import date, timedelta 51 | 52 | from sqlalchemy import func 53 | 54 | today = date.today() 55 | rates = [ 56 | {'day': today - timedelta(i), 'value': 200 - i} 57 | for i in range(100) 58 | ] 59 | 60 | 61 | Let's query inserted data 62 | 63 | .. code-block:: python 64 | 65 | session.execute(Rate.__table__.insert(), rates) 66 | 67 | session.query(func.count(Rate.day)) \ 68 | .filter(Rate.day > today - timedelta(20)) \ 69 | .scalar() 70 | 71 | Now you are ready to :ref:`configure your connection` and see more 72 | ClickHouse :ref:`features` support. -------------------------------------------------------------------------------- /docs/types.rst: -------------------------------------------------------------------------------- 1 | 2 | Types 3 | ===== 4 | 5 | The following ClickHouse types are supported by clickhouse-sqlalchemy. 6 | 7 | TODO. 8 | 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [db] 2 | host=localhost 3 | port=9000 4 | http_port=8123 5 | database=test 6 | user=default 7 | password= 8 | 9 | [log] 10 | level=ERROR 11 | 12 | [metadata] 13 | license_file = LICENSE 14 | 15 | [tool:pytest] 16 | asyncio_mode=strict 17 | 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | from codecs import open 4 | 5 | from setuptools import setup, find_packages 6 | 7 | here = os.path.abspath(os.path.dirname(__file__)) 8 | 9 | 10 | with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: 11 | long_description = f.read() 12 | 13 | 14 | def read_version(): 15 | regexp = re.compile(r'^VERSION\W*=\W*\(([^\(\)]*)\)') 16 | init_py = os.path.join(here, 'clickhouse_sqlalchemy', '__init__.py') 17 | with open(init_py, encoding='utf-8') as f: 18 | for line in f: 19 | match = regexp.match(line) 20 | if match is not None: 21 | return match.group(1).replace(', ', '.') 22 | else: 23 | raise RuntimeError( 24 | 'Cannot find version in clickhouse_sqlalchemy/__init__.py' 25 | ) 26 | 27 | 28 | dialects = [ 29 | 'clickhouse{}=clickhouse_sqlalchemy.drivers.{}'.format(driver, d_path) 30 | 31 | for driver, d_path in [ 32 | ('', 'http.base:ClickHouseDialect_http'), 33 | ('.http', 'http.base:ClickHouseDialect_http'), 34 | ('.native', 'native.base:ClickHouseDialect_native'), 35 | ('.asynch', 'asynch.base:ClickHouseDialect_asynch'), 36 | ] 37 | ] 38 | 39 | github_url = 'https://github.com/xzkostyan/clickhouse-sqlalchemy' 40 | 41 | setup( 42 | name='clickhouse-sqlalchemy', 43 | version=read_version(), 44 | 45 | description='Simple ClickHouse SQLAlchemy Dialect', 46 | long_description=long_description, 47 | 48 | url=github_url, 49 | 50 | author='Konstantin Lebedev', 51 | author_email='kostyan.lebedev@gmail.com', 52 | 53 | license='MIT', 54 | 55 | classifiers=[ 56 | 'Development Status :: 4 - Beta', 57 | 58 | 59 | 'Environment :: Console', 60 | 61 | 62 | 'Intended Audience :: Developers', 63 | 'Intended Audience :: Information Technology', 64 | 65 | 66 | 'License :: OSI Approved :: MIT License', 67 | 68 | 69 | 'Operating System :: OS Independent', 70 | 71 | 72 | 'Programming Language :: SQL', 73 | 'Programming Language :: Python :: 3', 74 | 'Programming Language :: Python :: 3.6', 75 | 'Programming Language :: Python :: 3.7', 76 | 'Programming Language :: Python :: 3.8', 77 | 'Programming Language :: Python :: 3.9', 78 | 'Programming Language :: Python :: 3.10', 79 | 'Programming Language :: Python :: 3.11', 80 | 'Programming Language :: Python :: 3.12', 81 | 82 | 'Topic :: Database', 83 | 'Topic :: Software Development', 84 | 'Topic :: Software Development :: Libraries', 85 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 86 | 'Topic :: Software Development :: Libraries :: Python Modules', 87 | 'Topic :: Scientific/Engineering :: Information Analysis' 88 | ], 89 | 90 | keywords='ClickHouse db database cloud analytics', 91 | 92 | project_urls={ 93 | 'Documentation': 'https://clickhouse-sqlalchemy.readthedocs.io', 94 | 'Changes': github_url + '/blob/master/CHANGELOG.md' 95 | }, 96 | packages=find_packages('.', exclude=["tests*"]), 97 | python_requires='>=3.7, <4', 98 | install_requires=[ 99 | 'sqlalchemy>=2.0.0,<2.1.0', 100 | 'requests', 101 | 'clickhouse-driver>=0.1.2', 102 | 'asynch>=0.2.2,<=0.2.4', 103 | ], 104 | # Registering `clickhouse` as dialect. 105 | entry_points={ 106 | 'sqlalchemy.dialects': dialects 107 | }, 108 | test_suite='pytest' 109 | ) 110 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/__init__.py -------------------------------------------------------------------------------- /tests/alembic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/alembic/__init__.py -------------------------------------------------------------------------------- /tests/alembic/test_default_schema_comparators.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from clickhouse_sqlalchemy.alembic.comparators import ( 4 | comparators, compare_mat_view 5 | ) 6 | 7 | 8 | class AlembicComparatorsTestCase(TestCase): 9 | def test_default_schema_comparators(self): 10 | default_schema = comparators._registry[('schema', 'default')] 11 | clickhouse_schema = comparators._registry[('schema', 'clickhouse')] 12 | 13 | for comparator in default_schema: 14 | self.assertIn(comparator, clickhouse_schema) 15 | 16 | self.assertNotIn(compare_mat_view, default_schema) 17 | -------------------------------------------------------------------------------- /tests/config.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | 3 | from sqlalchemy.dialects import registry 4 | 5 | from tests import log 6 | 7 | 8 | registry.register( 9 | "clickhouse", "clickhouse_sqlalchemy.drivers.http.base", "dialect" 10 | ) 11 | registry.register( 12 | "clickhouse.native", "clickhouse_sqlalchemy.drivers.native.base", "dialect" 13 | ) 14 | registry.register( 15 | "clickhouse.asynch", "clickhouse_sqlalchemy.drivers.asynch.base", "dialect" 16 | ) 17 | 18 | file_config = configparser.ConfigParser() 19 | file_config.read(['setup.cfg']) 20 | 21 | log.configure(file_config.get('log', 'level')) 22 | 23 | host = file_config.get('db', 'host') 24 | port = file_config.getint('db', 'port') 25 | http_port = file_config.getint('db', 'http_port') 26 | database = file_config.get('db', 'database') 27 | user = file_config.get('db', 'user') 28 | password = file_config.get('db', 'password') 29 | 30 | uri_template = '{schema}://{user}:{password}@{host}:{port}/{database}' 31 | 32 | http_uri = uri_template.format( 33 | schema='clickhouse+http', user=user, password=password, host=host, 34 | port=http_port, database=database) 35 | native_uri = uri_template.format( 36 | schema='clickhouse+native', user=user, password=password, host=host, 37 | port=port, database=database) 38 | asynch_uri = uri_template.format( 39 | schema='clickhouse+asynch', user=user, password=password, host=host, 40 | port=port, database=database) 41 | 42 | system_http_uri = uri_template.format( 43 | schema='clickhouse+http', user=user, password=password, host=host, 44 | port=http_port, database='system') 45 | system_native_uri = uri_template.format( 46 | schema='clickhouse+native', user=user, password=password, host=host, 47 | port=port, database='system') 48 | system_asynch_uri = uri_template.format( 49 | schema='clickhouse+asynch', user=user, password=password, host=host, 50 | port=port, database='system') 51 | -------------------------------------------------------------------------------- /tests/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | clickhouse-server: 5 | image: "$ORG/clickhouse-server:$VERSION" 6 | container_name: test-clickhouse-server 7 | ports: 8 | - "127.0.0.1:9000:9000" 9 | - "127.0.0.1:8123:8123" 10 | -------------------------------------------------------------------------------- /tests/drivers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/drivers/__init__.py -------------------------------------------------------------------------------- /tests/drivers/asynch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/drivers/asynch/__init__.py -------------------------------------------------------------------------------- /tests/drivers/asynch/test_base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.engine.url import URL 2 | 3 | from clickhouse_sqlalchemy.drivers.asynch.base import ClickHouseDialect_asynch 4 | from tests.testcase import BaseTestCase 5 | 6 | 7 | class TestConnectArgs(BaseTestCase): 8 | def setUp(self): 9 | self.dialect = ClickHouseDialect_asynch() 10 | 11 | def test_simple_url(self): 12 | url = URL.create( 13 | drivername='clickhouse+asynch', 14 | host='localhost', 15 | database='default', 16 | ) 17 | connect_args = self.dialect.create_connect_args(url) 18 | self.assertEqual( 19 | str(connect_args[0][0]), 'clickhouse://localhost/default' 20 | ) 21 | 22 | def test_secure_false(self): 23 | url = URL.create( 24 | drivername='clickhouse+asynch', 25 | username='default', 26 | password='default', 27 | host='localhost', 28 | port=9001, 29 | database='default', 30 | query={'secure': 'False'} 31 | ) 32 | connect_args = self.dialect.create_connect_args(url) 33 | self.assertEqual( 34 | str(connect_args[0][0]), 35 | 'clickhouse://default:default@localhost:9001/default?secure=False' 36 | ) 37 | 38 | def test_no_auth(self): 39 | url = URL.create( 40 | drivername='clickhouse+asynch', 41 | host='localhost', 42 | port=9001, 43 | database='default', 44 | ) 45 | connect_args = self.dialect.create_connect_args(url) 46 | self.assertEqual( 47 | str(connect_args[0][0]), 'clickhouse://localhost:9001/default' 48 | ) 49 | -------------------------------------------------------------------------------- /tests/drivers/asynch/test_cursor.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import text 2 | from sqlalchemy.util.concurrency import greenlet_spawn 3 | 4 | from tests.testcase import AsynchSessionTestCase 5 | 6 | 7 | class CursorTestCase(AsynchSessionTestCase): 8 | async def test_execute_without_context(self): 9 | raw = await self.session.bind.raw_connection() 10 | cur = await greenlet_spawn(lambda: raw.cursor()) 11 | 12 | await greenlet_spawn( 13 | lambda: cur.execute('SELECT * FROM system.numbers LIMIT 1') 14 | ) 15 | rv = cur.fetchall() 16 | 17 | self.assertEqual(len(rv), 1) 18 | 19 | raw.close() 20 | 21 | async def test_execute_with_context(self): 22 | rv = await self.session.execute( 23 | text('SELECT * FROM system.numbers LIMIT 1') 24 | ) 25 | 26 | self.assertEqual(len(rv.fetchall()), 1) 27 | 28 | async def test_check_iter_cursor(self): 29 | rv = await self.session.execute( 30 | text('SELECT number FROM system.numbers LIMIT 5') 31 | ) 32 | 33 | self.assertListEqual(list(rv), [(x,) for x in range(5)]) 34 | 35 | async def test_execute_with_stream(self): 36 | async with self.connection.stream( 37 | text("SELECT * FROM system.numbers LIMIT 10"), 38 | execution_options={'max_block_size': 1} 39 | ) as result: 40 | idx = 0 41 | async for r in result: 42 | self.assertEqual(r[0], idx) 43 | idx += 1 44 | 45 | self.assertEqual(idx, 10) 46 | -------------------------------------------------------------------------------- /tests/drivers/asynch/test_insert.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, func, text 2 | 3 | from clickhouse_sqlalchemy import engines, types, Table 4 | from asynch.errors import TypeMismatchError 5 | 6 | from tests.testcase import AsynchSessionTestCase 7 | 8 | 9 | class NativeInsertTestCase(AsynchSessionTestCase): 10 | async def test_rowcount_return1(self): 11 | metadata = self.metadata() 12 | table = Table( 13 | 'test', metadata, 14 | Column('x', types.UInt32, primary_key=True), 15 | engines.Memory() 16 | ) 17 | await self.run_sync(metadata.drop_all) 18 | await self.run_sync(metadata.create_all) 19 | 20 | rv = await self.session.execute( 21 | table.insert(), 22 | [{'x': x} for x in range(5)] 23 | ) 24 | 25 | self.assertEqual(rv.rowcount, 5) 26 | self.assertEqual( 27 | await self.session.run_sync( 28 | lambda sc: sc.query(func.count()).select_from(table).scalar() 29 | ), 30 | 5 31 | ) 32 | 33 | rv = await self.session.execute( 34 | text('INSERT INTO test SELECT * FROM system.numbers LIMIT 5') 35 | ) 36 | self.assertEqual(rv.rowcount, -1) 37 | 38 | async def test_types_check(self): 39 | metadata = self.metadata() 40 | table = Table( 41 | 'test', metadata, 42 | Column('x', types.UInt32, primary_key=True), 43 | engines.Memory() 44 | ) 45 | await self.run_sync(metadata.drop_all) 46 | await self.run_sync(metadata.create_all) 47 | 48 | with self.assertRaises(TypeMismatchError) as ex: 49 | await self.session.execute( 50 | table.insert(), 51 | [{'x': -1}], 52 | execution_options=dict(types_check=True), 53 | ) 54 | self.assertIn('-1 for column "x"', str(ex.exception)) 55 | 56 | with self.assertRaises(TypeMismatchError) as ex: 57 | await self.session.execute(table.insert(), {'x': -1}) 58 | self.assertIn( 59 | 'Repeat query with types_check=True for detailed info', 60 | str(ex.exception) 61 | ) 62 | -------------------------------------------------------------------------------- /tests/drivers/asynch/test_select.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | 3 | from clickhouse_sqlalchemy import engines, types, Table 4 | from tests.session import mocked_engine 5 | from tests.testcase import AsynchSessionTestCase 6 | 7 | 8 | class SanityTestCase(AsynchSessionTestCase): 9 | async def test_sanity(self): 10 | with mocked_engine(self.session) as engine: 11 | metadata = self.metadata() 12 | Table( 13 | 't1', metadata, 14 | Column('x', types.Int32, primary_key=True), 15 | engines.Memory() 16 | ) 17 | 18 | prev_has_table = engine.dialect_cls.has_table 19 | engine.dialect_cls.has_table = lambda *args, **kwargs: True 20 | 21 | await self.run_sync(metadata.drop_all) 22 | 23 | self.assertEqual(engine.history, ['DROP TABLE t1']) 24 | 25 | engine.dialect_cls.has_table = prev_has_table 26 | -------------------------------------------------------------------------------- /tests/drivers/http/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/drivers/http/__init__.py -------------------------------------------------------------------------------- /tests/drivers/http/test_cursor.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import text 2 | 3 | from tests.testcase import HttpSessionTestCase, HttpEngineTestCase 4 | from tests.util import require_server_version 5 | 6 | 7 | class CursorTestCase(HttpSessionTestCase, HttpEngineTestCase): 8 | def test_check_iter_cursor_by_session(self): 9 | rv = self.session.execute( 10 | text('SELECT number FROM system.numbers LIMIT 5') 11 | ) 12 | self.assertListEqual(list(rv), [(x,) for x in range(5)]) 13 | 14 | def test_check_iter_cursor_by_engine(self): 15 | with self.engine.connect() as conn: 16 | rv = conn.execute( 17 | text('SELECT number FROM system.numbers LIMIT 5') 18 | ) 19 | self.assertListEqual(list(rv), [(x,) for x in range(5)]) 20 | 21 | @require_server_version(23, 2, 1) 22 | def test_with_settings_in_execution_options(self): 23 | rv = self.session.execute( 24 | text("SELECT number FROM system.numbers LIMIT 5"), 25 | execution_options={"settings": {"final": 1}} 26 | ) 27 | self.assertEqual( 28 | dict(rv.context.execution_options), {"settings": {"final": 1}} 29 | ) 30 | self.assertListEqual(list(rv), [(x,) for x in range(5)]) 31 | -------------------------------------------------------------------------------- /tests/drivers/http/test_escaping.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | from datetime import date 3 | import uuid 4 | 5 | from sqlalchemy import Column, literal 6 | 7 | from clickhouse_sqlalchemy import types, engines, Table 8 | from clickhouse_sqlalchemy.drivers.http.escaper import Escaper 9 | from tests.testcase import HttpSessionTestCase 10 | 11 | 12 | class EscapingTestCase(HttpSessionTestCase): 13 | def escaped_compile(self, clause, **kwargs): 14 | return str(self._compile(clause, **kwargs)) 15 | 16 | def test_select_escaping(self): 17 | query = self.session.query(literal('\t')) 18 | self.assertEqual( 19 | self.escaped_compile(query, literal_binds=True), 20 | "SELECT '\t' AS anon_1" 21 | ) 22 | 23 | def test_escaper(self): 24 | e = Escaper() 25 | self.assertEqual(e.escape([None]), '[NULL]') 26 | self.assertEqual(e.escape([[None]]), '[[NULL]]') 27 | self.assertEqual(e.escape([[123]]), '[[123]]') 28 | self.assertEqual(e.escape({'x': 'str'}), {'x': "'str'"}) 29 | self.assertEqual(e.escape([Decimal('10')]), '[10.0]') 30 | self.assertEqual(e.escape([10.0]), '[10.0]') 31 | self.assertEqual(e.escape([date(2017, 1, 2)]), "['2017-01-02']") 32 | self.assertEqual(e.escape(dict(x=10, y=20)), {'x': 10, 'y': 20}) 33 | self.assertEqual( 34 | e.escape([uuid.UUID("ef3e3d4b-c782-4993-83fc-894ff0aba8ff")]), 35 | '[ef3e3d4b-c782-4993-83fc-894ff0aba8ff]' 36 | ) 37 | with self.assertRaises(Exception) as ex: 38 | e.escape([object()]) 39 | 40 | self.assertIn('Unsupported object', str(ex.exception)) 41 | 42 | with self.assertRaises(Exception) as ex: 43 | e.escape('str') 44 | 45 | self.assertIn('Unsupported param format', str(ex.exception)) 46 | 47 | def test_escape_binary_mod(self): 48 | query = self.session.query(literal(1) % literal(2)) 49 | self.assertEqual( 50 | self.compile(query, literal_binds=True), 51 | 'SELECT 1 %% 2 AS anon_1' 52 | ) 53 | 54 | table = Table( 55 | 't', self.metadata(), 56 | Column('x', types.Int32, primary_key=True), 57 | engines.Memory() 58 | ) 59 | 60 | query = self.session.query(table.c.x % table.c.x) 61 | self.assertEqual( 62 | self.compile(query, literal_binds=True), 63 | 'SELECT t.x %% t.x AS anon_1 FROM t' 64 | ) 65 | -------------------------------------------------------------------------------- /tests/drivers/http/test_select.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | 3 | from clickhouse_sqlalchemy import types, Table 4 | from tests.testcase import HttpSessionTestCase 5 | 6 | 7 | class FormatSectionTestCase(HttpSessionTestCase): 8 | 9 | @property 10 | def execution_ctx_cls(self): 11 | return self.session.bind.dialect.execution_ctx_cls 12 | 13 | def _compile(self, clause, bind=None, **kwargs): 14 | if bind is None: 15 | bind = self.session.bind 16 | statement = super(FormatSectionTestCase, self)._compile( 17 | clause, bind=bind, **kwargs 18 | ) 19 | 20 | context = self.execution_ctx_cls._init_compiled( 21 | bind.dialect, bind, bind, {}, statement, [], clause, [] 22 | ) 23 | context.pre_exec() 24 | 25 | return context.statement 26 | 27 | def test_select_format_clause(self): 28 | metadata = self.metadata() 29 | 30 | bind = self.session.bind 31 | bind.cursor = lambda: None 32 | 33 | table = Table( 34 | 't1', metadata, 35 | Column('x', types.Int32, primary_key=True) 36 | ) 37 | 38 | statement = self.compile(self.session.query(table.c.x), bind=bind) 39 | self.assertEqual( 40 | statement, 41 | 'SELECT t1.x AS t1_x FROM t1' 42 | ) 43 | 44 | def test_insert_from_select_no_format_clause(self): 45 | metadata = self.metadata() 46 | 47 | bind = self.session.bind 48 | bind.cursor = lambda: None 49 | 50 | t1 = Table( 51 | 't1', metadata, 52 | Column('x', types.Int32, primary_key=True) 53 | ) 54 | 55 | t2 = Table( 56 | 't2', metadata, 57 | Column('x', types.Int32, primary_key=True) 58 | ) 59 | 60 | query = t2.insert().from_select(['x'], self.session.query(t1.c.x)) 61 | statement = self.compile(query, bind=bind) 62 | self.assertEqual( 63 | statement, 'INSERT INTO t2 (x) SELECT t1.x AS t1_x FROM t1' 64 | ) 65 | -------------------------------------------------------------------------------- /tests/drivers/http/test_stream.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import text 2 | 3 | from tests.testcase import HttpSessionTestCase 4 | from tests.session import http_stream_session 5 | 6 | 7 | class StreamingHttpTestCase(HttpSessionTestCase): 8 | 9 | session = http_stream_session 10 | power = 4 11 | 12 | def make_query(self, power=None): 13 | if power is None: 14 | power = self.power 15 | # `10**power` rows. 16 | # Timing order of magnitude: 17 | # est. 1 second on 10**5 lines for TSV, 18 | # 2.5 seconds on uncythonized native protocol. 19 | query = 'select ' + ', '.join( 20 | ("arrayJoin([" 21 | "'{0}', '2', '3', '4', '5', " 22 | "'6', '7', '8', '9', '10']) as a{0}").format(num) 23 | for num in range(power)) 24 | return query 25 | 26 | def test_streaming(self): 27 | power = self.power 28 | query = self.make_query(power=power) 29 | res = self.session.execute(text(query)) 30 | count = sum(1 for _ in res) 31 | self.assertEqual(count, 10 ** power) 32 | 33 | def test_fetchmany(self): 34 | power = self.power - 1 35 | query = self.make_query(power=power) 36 | res = self.session.execute(text(query)) 37 | 38 | count = 0 39 | while True: 40 | block = res.fetchmany(1000) 41 | if not block: 42 | break 43 | count += sum(1 for _ in block) 44 | 45 | self.assertEqual(count, 10 ** power) 46 | 47 | def test_fetchall(self): 48 | power = self.power - 2 49 | if power < 1: 50 | raise Exception( 51 | "Misconfigured test case:" 52 | " `power` should be at least 3.") 53 | query = self.make_query(power=power) 54 | res = self.session.execute(text(query)) 55 | 56 | count = 0 57 | 58 | block = res.fetchmany(7) 59 | count += sum(1 for _ in block) 60 | 61 | block = res.fetchall() 62 | count += sum(1 for _ in block) 63 | 64 | self.assertEqual(count, 10 ** power) 65 | -------------------------------------------------------------------------------- /tests/drivers/http/test_utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from clickhouse_sqlalchemy.drivers.http.utils import unescape, parse_tsv 4 | from tests.testcase import BaseTestCase 5 | 6 | 7 | class HttpUtilsTestCase(BaseTestCase): 8 | def test_unescape(self): 9 | test_values = [b'', b'a', b'\xff'] 10 | actual = [unescape(t) for t in test_values] 11 | self.assertListEqual(actual, ['', 'a', '\ufffd']) 12 | 13 | def test_unescape_surrogates(self): 14 | test_values = [b'', b'a', b'\xc3\xa3\xff\xf6\0\x00\n', b'a\n\\0'] 15 | actual = [unescape(t, errors='surrogateescape') for t in test_values] 16 | expected = ['', 'a', 'ã\udcff\udcf6\x00\x00\n', 'a\n\x00'] 17 | self.assertListEqual(actual, expected) 18 | 19 | def test_reverse_surrogates(self): 20 | # What's stored in the database: 21 | expected = [b'', b'a', b'\xc3\xa3\xff\x55\xf6\0\x00\x45\0\n'] 22 | 23 | # What comes over the wire: 24 | test_values = [b'', b'a', b'\xc3\xa3\xff\x55\xf6\0\x00\x45\\0\n'] 25 | 26 | escaped = [unescape(t, errors='surrogateescape') for t in test_values] 27 | actual = [t.encode('utf-8', errors='surrogateescape') for t in escaped] 28 | self.assertListEqual(actual, expected) 29 | 30 | def test_parse_tsv(self): 31 | test_values = [b'', b'a\tb\tc\n', b'a\tb\t\xff'] 32 | expected = [[''], ['a', 'b', 'c\n'], ['a', 'b', '\ufffd']] 33 | try: 34 | actual = [parse_tsv(value) for value in test_values] 35 | except IndexError: 36 | self.fail('"parse_tsv" raised IndexError exception!') 37 | except TypeError: 38 | self.fail('"parse_tsv" raised TypeError exception!') 39 | 40 | self.assertListEqual(actual, expected) 41 | -------------------------------------------------------------------------------- /tests/drivers/native/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/drivers/native/__init__.py -------------------------------------------------------------------------------- /tests/drivers/native/test_base.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import quote 2 | 3 | from sqlalchemy.engine.url import URL, make_url 4 | 5 | from clickhouse_sqlalchemy.drivers.native.base import ClickHouseDialect_native 6 | from tests.testcase import BaseTestCase 7 | 8 | 9 | class TestConnectArgs(BaseTestCase): 10 | def setUp(self): 11 | self.dialect = ClickHouseDialect_native() 12 | 13 | def test_simple_url(self): 14 | url = URL.create( 15 | drivername='clickhouse+native', 16 | host='localhost', 17 | database='default', 18 | ) 19 | connect_args = self.dialect.create_connect_args(url) 20 | self.assertEqual( 21 | str(connect_args[0][0]), 'clickhouse://localhost/default' 22 | ) 23 | 24 | def test_secure_false(self): 25 | url = URL.create( 26 | drivername='clickhouse+native', 27 | username='default', 28 | password='default', 29 | host='localhost', 30 | port=9001, 31 | database='default', 32 | query={'secure': 'False'} 33 | ) 34 | connect_args = self.dialect.create_connect_args(url) 35 | self.assertEqual( 36 | str(connect_args[0][0]), 37 | 'clickhouse://default:default@localhost:9001/default?secure=False' 38 | ) 39 | 40 | def test_no_auth(self): 41 | url = URL.create( 42 | drivername='clickhouse+native', 43 | host='localhost', 44 | port=9001, 45 | database='default', 46 | ) 47 | connect_args = self.dialect.create_connect_args(url) 48 | self.assertEqual( 49 | str(connect_args[0][0]), 'clickhouse://localhost:9001/default' 50 | ) 51 | 52 | def test_quoting(self): 53 | user = "us#er" 54 | password = 'pass#word' 55 | part = '{}:{}@host/database'.format(user, password) 56 | quote_user = quote(user) 57 | quote_password = quote(password) 58 | quote_part = '{}:{}@host/database'.format(quote_user, quote_password) 59 | 60 | # test with unquote user and password 61 | url = make_url('clickhouse+native://' + part) 62 | connect_args = self.dialect.create_connect_args(url) 63 | self.assertEqual( 64 | str(connect_args[0][0]), 'clickhouse://' + quote_part 65 | ) 66 | 67 | # test with quote user and password 68 | url = make_url('clickhouse+native://' + quote_part) 69 | connect_args = self.dialect.create_connect_args(url) 70 | self.assertEqual( 71 | str(connect_args[0][0]), 'clickhouse://' + quote_part 72 | ) 73 | -------------------------------------------------------------------------------- /tests/drivers/native/test_cursor.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from sqlalchemy import text 4 | 5 | from tests.testcase import NativeSessionTestCase 6 | from tests.util import require_server_version 7 | 8 | 9 | class CursorTestCase(NativeSessionTestCase): 10 | def test_execute_without_context(self): 11 | raw = self.session.bind.raw_connection() 12 | cur = raw.cursor() 13 | 14 | cur.execute("SELECT * FROM system.numbers LIMIT 1") 15 | rv = cur.fetchall() 16 | self.assertEqual(len(rv), 1) 17 | 18 | def test_execute_with_context(self): 19 | rv = self.session.execute(text("SELECT * FROM system.numbers LIMIT 1")) 20 | 21 | self.assertEqual(len(rv.fetchall()), 1) 22 | 23 | def test_check_iter_cursor(self): 24 | rv = self.session.execute( 25 | text('SELECT number FROM system.numbers LIMIT 5') 26 | ) 27 | self.assertListEqual(list(rv), [(x,) for x in range(5)]) 28 | 29 | def test_execute_with_stream(self): 30 | rv = self.session.execute( 31 | text("SELECT * FROM system.numbers LIMIT 1") 32 | ).yield_per(10) 33 | 34 | self.assertEqual(len(rv.fetchall()), 1) 35 | 36 | def test_with_stream_results(self): 37 | rv = self.session.execute(text("SELECT * FROM system.numbers LIMIT 1"), 38 | execution_options={"stream_results": True}) 39 | 40 | self.assertEqual(len(rv.fetchall()), 1) 41 | 42 | @require_server_version(23, 2, 1) 43 | def test_with_settings_in_execution_options(self): 44 | rv = self.session.execute( 45 | text("SELECT * FROM system.numbers LIMIT 1"), 46 | execution_options={"settings": {"final": 1}} 47 | ) 48 | self.assertEqual( 49 | dict(rv.context.execution_options), {"settings": {"final": 1}} 50 | ) 51 | self.assertEqual(len(rv.fetchall()), 1) 52 | 53 | def test_set_query_id(self): 54 | query_id = str(uuid.uuid4()) 55 | rv = self.session.execute( 56 | text( 57 | f"SELECT query_id " 58 | f"FROM system.processes " 59 | f"WHERE query_id = '{query_id}'" 60 | ), execution_options={'query_id': query_id} 61 | ) 62 | self.assertEqual(rv.fetchall()[0][0], query_id) 63 | -------------------------------------------------------------------------------- /tests/drivers/native/test_insert.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, func, text 2 | 3 | from clickhouse_sqlalchemy import engines, types, Table 4 | from clickhouse_sqlalchemy.exceptions import DatabaseException 5 | from tests.testcase import NativeSessionTestCase 6 | 7 | 8 | class NativeInsertTestCase(NativeSessionTestCase): 9 | def test_rowcount_return(self): 10 | table = Table( 11 | 'test', self.metadata(), 12 | Column('x', types.Int32, primary_key=True), 13 | engines.Memory() 14 | ) 15 | table.drop(bind=self.session.bind, if_exists=True) 16 | table.create(bind=self.session.bind) 17 | 18 | rv = self.session.execute(table.insert(), [{'x': x} for x in range(5)]) 19 | self.assertEqual(rv.rowcount, 5) 20 | self.assertEqual( 21 | self.session.query(func.count()).select_from(table).scalar(), 5 22 | ) 23 | 24 | rv = self.session.execute( 25 | text('INSERT INTO test SELECT * FROM system.numbers LIMIT 5') 26 | ) 27 | self.assertEqual(rv.rowcount, -1) 28 | 29 | def test_types_check(self): 30 | table = Table( 31 | 'test', self.metadata(), 32 | Column('x', types.UInt32, primary_key=True), 33 | engines.Memory() 34 | ) 35 | table.drop(bind=self.session.bind, if_exists=True) 36 | table.create(bind=self.session.bind) 37 | 38 | with self.assertRaises(DatabaseException) as ex: 39 | self.session.execute( 40 | table.insert().execution_options(types_check=True), 41 | [{'x': -1}] 42 | ) 43 | self.assertIn('-1 for column "x"', str(ex.exception.orig)) 44 | 45 | with self.assertRaises(DatabaseException) as ex: 46 | self.session.execute(table.insert(), [{'x': -1}]) 47 | self.assertIn( 48 | 'Repeat query with types_check=True for detailed info', 49 | str(ex.exception.orig) 50 | ) 51 | -------------------------------------------------------------------------------- /tests/drivers/native/test_select.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | 3 | from clickhouse_sqlalchemy import engines, types, Table 4 | from tests.session import mocked_engine 5 | from tests.testcase import NativeSessionTestCase 6 | 7 | 8 | class SanityTestCase(NativeSessionTestCase): 9 | def test_sanity(self): 10 | with mocked_engine(self.session) as engine: 11 | table = Table( 12 | 't1', self.metadata(), 13 | Column('x', types.Int32, primary_key=True), 14 | engines.Memory() 15 | ) 16 | table.drop(bind=engine.session.bind, if_exists=True) 17 | self.assertEqual(engine.history, ['DROP TABLE IF EXISTS t1']) 18 | -------------------------------------------------------------------------------- /tests/drivers/test_util.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from clickhouse_sqlalchemy.drivers.util import get_inner_spec, parse_arguments 4 | 5 | 6 | class GetInnerSpecTestCase(TestCase): 7 | def test_get_inner_spec(self): 8 | self.assertEqual( 9 | get_inner_spec("DateTime('Europe/Paris')"), "'Europe/Paris'" 10 | ) 11 | self.assertEqual(get_inner_spec('Decimal(18, 2)'), "18, 2") 12 | self.assertEqual(get_inner_spec('DateTime64(3)'), "3") 13 | 14 | 15 | class ParseArgumentsTestCase(TestCase): 16 | def test_parse_arguments(self): 17 | self.assertEqual( 18 | parse_arguments('uniq, UInt64'), ('uniq', 'UInt64') 19 | ) 20 | self.assertEqual( 21 | parse_arguments('anyIf, String, UInt8'), 22 | ('anyIf', 'String', 'UInt8') 23 | ) 24 | self.assertEqual( 25 | parse_arguments('quantiles(0.5, 0.9), UInt64'), 26 | ('quantiles(0.5, 0.9)', 'UInt64') 27 | ) 28 | self.assertEqual( 29 | parse_arguments('sum, Int64, Int64'), ('sum', 'Int64', 'Int64') 30 | ) 31 | self.assertEqual( 32 | parse_arguments('sum, Nullable(Int64), Int64'), 33 | ('sum', 'Nullable(Int64)', 'Int64') 34 | ) 35 | self.assertEqual( 36 | parse_arguments('Float32, Decimal(18, 2)'), 37 | ('Float32', 'Decimal(18, 2)') 38 | ) 39 | self.assertEqual( 40 | parse_arguments('sum, Float32, Decimal(18, 2)'), 41 | ('sum', 'Float32', 'Decimal(18, 2)') 42 | ) 43 | self.assertEqual( 44 | parse_arguments('quantiles(0.5, 0.9), UInt64'), 45 | ('quantiles(0.5, 0.9)', 'UInt64') 46 | ) 47 | self.assertEqual( 48 | parse_arguments("sumIf(total, status = 'accepted'), Float32"), 49 | ("sumIf(total, status = 'accepted')", "Float32") 50 | ) 51 | -------------------------------------------------------------------------------- /tests/engines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/engines/__init__.py -------------------------------------------------------------------------------- /tests/engines/test_parse_columns.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from clickhouse_sqlalchemy.engines.util import parse_columns 4 | 5 | 6 | class ParseColumnsTestCase(TestCase): 7 | def test_empty(self): 8 | self.assertEqual(parse_columns(''), []) 9 | 10 | def test_simple(self): 11 | self.assertEqual(parse_columns('x, y, z'), ['x', 'y', 'z']) 12 | self.assertEqual(parse_columns('xx, yy, zz'), ['xx', 'yy', 'zz']) 13 | 14 | def test_one(self): 15 | self.assertEqual(parse_columns('x'), ['x']) 16 | 17 | def test_quoted(self): 18 | self.assertEqual(parse_columns('` , `, ` , `'), [' , ', ' , ']) 19 | 20 | def test_escaped(self): 21 | self.assertEqual( 22 | parse_columns('` \\`, `, ` \\`, `'), 23 | [' `, ', ' `, '] 24 | ) 25 | self.assertEqual(parse_columns('` \\`\\` `'), [' `` ']) 26 | 27 | def test_brackets(self): 28 | self.assertEqual( 29 | parse_columns('test(a, b), test(c, d)'), 30 | ['test(a, b)', 'test(c, d)'] 31 | ) 32 | 33 | self.assertEqual( 34 | parse_columns('x, (y, z)'), ['x', '(y, z)'] 35 | ) 36 | -------------------------------------------------------------------------------- /tests/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/ext/__init__.py -------------------------------------------------------------------------------- /tests/ext/test_declative.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | from sqlalchemy.sql.ddl import CreateTable 3 | 4 | from clickhouse_sqlalchemy import types, engines, get_declarative_base 5 | from tests.testcase import BaseTestCase 6 | 7 | 8 | class DeclarativeTestCase(BaseTestCase): 9 | def test_create_table(self): 10 | base = get_declarative_base() 11 | 12 | class TestTable(base): 13 | x = Column(types.Int32, primary_key=True) 14 | y = Column(types.String) 15 | 16 | __table_args__ = ( 17 | engines.Memory(), 18 | ) 19 | 20 | self.assertEqual( 21 | self.compile(CreateTable(TestTable.__table__)), 22 | 'CREATE TABLE test_table (x Int32, y String) ENGINE = Memory' 23 | ) 24 | 25 | def test_create_table_custom_name(self): 26 | base = get_declarative_base() 27 | 28 | class TestTable(base): 29 | __tablename__ = 'testtable' 30 | 31 | x = Column(types.Int32, primary_key=True) 32 | y = Column(types.String) 33 | 34 | __table_args__ = ( 35 | engines.Memory(), 36 | ) 37 | 38 | self.assertEqual( 39 | self.compile(CreateTable(TestTable.__table__)), 40 | 'CREATE TABLE testtable (x Int32, y String) ENGINE = Memory' 41 | ) 42 | -------------------------------------------------------------------------------- /tests/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/functions/__init__.py -------------------------------------------------------------------------------- /tests/functions/test_count.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, func 2 | 3 | from clickhouse_sqlalchemy import types, Table 4 | from tests.testcase import CompilationTestCase 5 | 6 | 7 | class CountTestCaseBase(CompilationTestCase): 8 | table = Table( 9 | 't1', CompilationTestCase.metadata(), 10 | Column('x', types.Int32, primary_key=True) 11 | ) 12 | 13 | def test_count(self): 14 | self.assertEqual( 15 | self.compile(self.session.query(func.count(self.table.c.x))), 16 | 'SELECT count(t1.x) AS count_1 FROM t1' 17 | ) 18 | 19 | def test_count_distinct(self): 20 | query = self.session.query(func.count(func.distinct(self.table.c.x))) 21 | self.assertEqual( 22 | self.compile(query), 23 | 'SELECT count(distinct(t1.x)) AS count_1 FROM t1' 24 | ) 25 | 26 | def test_count_no_column_specified(self): 27 | query = self.session.query(func.count()).select_from(self.table) 28 | self.assertEqual( 29 | self.compile(query), 30 | 'SELECT count(*) AS count_1 FROM t1' 31 | ) 32 | -------------------------------------------------------------------------------- /tests/functions/test_extract.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test EXTRACT 3 | """ 4 | from sqlalchemy import Column, extract 5 | 6 | from clickhouse_sqlalchemy import types 7 | from tests.testcase import BaseTestCase 8 | 9 | 10 | def get_date_column(name): 11 | """ 12 | Return types.Date column 13 | :param name: Column name 14 | :return: sqlalchemy.Column 15 | """ 16 | return Column(name, types.Date) 17 | 18 | 19 | class ExtractTestCase(BaseTestCase): 20 | def test_extract_year(self): 21 | self.assertEqual( 22 | self.compile(extract('year', get_date_column('x'))), 23 | 'toYear(x)' 24 | ) 25 | 26 | def test_extract_month(self): 27 | self.assertEqual( 28 | self.compile(extract('month', get_date_column('x'))), 29 | 'toMonth(x)' 30 | ) 31 | 32 | def test_extract_day(self): 33 | self.assertEqual( 34 | self.compile(extract('day', get_date_column('x'))), 35 | 'toDayOfMonth(x)' 36 | ) 37 | 38 | def test_extract_unknown(self): 39 | self.assertEqual( 40 | self.compile(extract('test', get_date_column('x'))), 41 | 'x' 42 | ) 43 | -------------------------------------------------------------------------------- /tests/functions/test_has.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import func 2 | 3 | from tests.testcase import BaseTestCase 4 | 5 | 6 | class HasTestCase(BaseTestCase): 7 | def test_has_any(self): 8 | self.assertEqual( 9 | self.compile(func.has([1, 2], 1), literal_binds=True), 10 | 'has([1, 2], 1)' 11 | ) 12 | 13 | self.assertEqual( 14 | self.compile(func.hasAny([1], []), literal_binds=True), 15 | 'hasAny([1], [])' 16 | ) 17 | 18 | self.assertEqual( 19 | self.compile(func.hasAll(['a', 'b'], ['a']), literal_binds=True), 20 | "hasAll(['a', 'b'], ['a'])" 21 | ) 22 | -------------------------------------------------------------------------------- /tests/functions/test_if.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import literal, func, text 2 | 3 | from tests.testcase import BaseTestCase 4 | 5 | 6 | class IfTestCase(BaseTestCase): 7 | def test_if(self): 8 | expression = func.if_( 9 | literal(1) > literal(2), 10 | text('a'), 11 | text('b'), 12 | ) 13 | 14 | self.assertEqual( 15 | self.compile(expression, literal_binds=True), 16 | 'if(1 > 2, a, b)' 17 | ) 18 | -------------------------------------------------------------------------------- /tests/log.py: -------------------------------------------------------------------------------- 1 | from logging.config import dictConfig 2 | 3 | 4 | def configure(level): 5 | dictConfig({ 6 | 'version': 1, 7 | 'disable_existing_loggers': False, 8 | 'formatters': { 9 | 'standard': { 10 | 'format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s' 11 | }, 12 | }, 13 | 'handlers': { 14 | 'default': { 15 | 'level': level, 16 | 'formatter': 'standard', 17 | 'class': 'logging.StreamHandler', 18 | }, 19 | }, 20 | 'loggers': { 21 | '': { 22 | 'handlers': ['default'], 23 | 'level': level, 24 | 'propagate': True 25 | }, 26 | } 27 | }) 28 | -------------------------------------------------------------------------------- /tests/orm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/orm/__init__.py -------------------------------------------------------------------------------- /tests/session.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from sqlalchemy import create_engine 4 | from sqlalchemy.ext.asyncio import create_async_engine 5 | 6 | from clickhouse_sqlalchemy import make_session 7 | from tests.config import http_uri, native_uri, system_native_uri, asynch_uri, \ 8 | system_asynch_uri 9 | 10 | http_engine = create_engine(http_uri) 11 | http_session = make_session(http_engine) 12 | http_stream_session = make_session(create_engine(http_uri + '?stream=1')) 13 | native_engine = create_engine(native_uri) 14 | native_session = make_session(native_engine) 15 | asynch_engine = create_async_engine(asynch_uri) 16 | asynch_session = make_session(asynch_engine, is_async=True) 17 | 18 | system_native_session = make_session(create_engine(system_native_uri)) 19 | system_asynch_session = make_session( 20 | create_async_engine(system_asynch_uri), 21 | is_async=True 22 | ) 23 | 24 | 25 | class MockedEngine(object): 26 | 27 | prev_do_execute = None 28 | prev_do_executemany = None 29 | prev_get_server_version_info = None 30 | prev_get_default_schema_name = None 31 | 32 | def __init__(self, session=None): 33 | self._buffer = [] 34 | 35 | if session is None: 36 | session = make_session(create_engine(http_uri)) 37 | 38 | self.session = session 39 | self.dialect_cls = session.bind.dialect.__class__ 40 | 41 | @property 42 | def history(self): 43 | return [re.sub(r'[\n\t]', '', str(ssql)) for ssql in self._buffer] 44 | 45 | def __enter__(self): 46 | self.prev_do_execute = self.dialect_cls.do_execute 47 | self.prev_do_executemany = self.dialect_cls.do_executemany 48 | self.prev_get_server_version_info = \ 49 | self.dialect_cls._get_server_version_info 50 | self.prev_get_default_schema_name = \ 51 | self.dialect_cls._get_default_schema_name 52 | 53 | def do_executemany( 54 | instance, cursor, statement, parameters, context=None): 55 | self._buffer.append(statement) 56 | 57 | def do_execute(instance, cursor, statement, parameters, context=None): 58 | self._buffer.append(statement) 59 | 60 | def get_server_version_info(*args, **kwargs): 61 | return (19, 16, 2, 2) 62 | 63 | def get_default_schema_name(*args, **kwargs): 64 | return 'test' 65 | 66 | self.dialect_cls.do_execute = do_execute 67 | self.dialect_cls.do_executemany = do_executemany 68 | self.dialect_cls._get_server_version_info = get_server_version_info 69 | self.dialect_cls._get_default_schema_name = get_default_schema_name 70 | 71 | return self 72 | 73 | def __exit__(self, *exc_info): 74 | self.dialect_cls.do_execute = self.prev_do_execute 75 | self.dialect_cls.do_executemany = self.prev_do_executemany 76 | self.dialect_cls._get_server_version_info = \ 77 | self.prev_get_server_version_info 78 | self.dialect_cls._get_default_schema_name = \ 79 | self.prev_get_default_schema_name 80 | 81 | 82 | mocked_engine = MockedEngine 83 | -------------------------------------------------------------------------------- /tests/sql/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/sql/__init__.py -------------------------------------------------------------------------------- /tests/sql/test_case.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import literal, case 2 | 3 | from tests.testcase import BaseTestCase 4 | 5 | 6 | class CaseTestCase(BaseTestCase): 7 | def test_else_required(self): 8 | expression = case((literal(1), 0)) 9 | 10 | self.assertEqual( 11 | self.compile(expression, literal_binds=True), 12 | 'CASE WHEN 1 THEN 0 END' 13 | ) 14 | 15 | def test_case(self): 16 | expression = case((literal(1), 0), else_=1) 17 | self.assertEqual( 18 | self.compile(expression, literal_binds=True), 19 | 'CASE WHEN 1 THEN 0 ELSE 1 END' 20 | ) 21 | 22 | expression = case((literal(1), 0), (literal(2), 1), else_=1) 23 | self.assertEqual( 24 | self.compile(expression, literal_binds=True), 25 | 'CASE WHEN 1 THEN 0 WHEN 2 THEN 1 ELSE 1 END' 26 | ) 27 | -------------------------------------------------------------------------------- /tests/sql/test_delete.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, exc, delete 2 | 3 | from clickhouse_sqlalchemy import types, Table, engines 4 | from tests.testcase import NativeSessionTestCase 5 | from tests.util import mock_object_attr 6 | 7 | 8 | class DeleteTestCase(NativeSessionTestCase): 9 | def test_delete(self): 10 | t1 = Table( 11 | 't1', self.metadata(), 12 | Column('x', types.Int32, primary_key=True), 13 | engines.MergeTree('x', order_by=('x', )) 14 | ) 15 | 16 | query = t1.delete().where(t1.c.x == 25) 17 | statement = self.compile(query, literal_binds=True) 18 | self.assertEqual(statement, 'ALTER TABLE t1 DELETE WHERE x = 25') 19 | 20 | query = delete(t1).where(t1.c.x == 25) 21 | statement = self.compile(query, literal_binds=True) 22 | self.assertEqual(statement, 'ALTER TABLE t1 DELETE WHERE x = 25') 23 | 24 | def test_delete_without_where(self): 25 | t1 = Table( 26 | 't1', self.metadata(), 27 | Column('x', types.Int32, primary_key=True), 28 | engines.MergeTree('x', order_by=('x', )) 29 | ) 30 | 31 | query = t1.delete() 32 | with self.assertRaises(exc.CompileError) as ex: 33 | self.compile(query, literal_binds=True) 34 | 35 | self.assertEqual(str(ex.exception), 'WHERE clause is required') 36 | 37 | query = delete(t1) 38 | with self.assertRaises(exc.CompileError) as ex: 39 | self.compile(query, literal_binds=True) 40 | 41 | self.assertEqual(str(ex.exception), 'WHERE clause is required') 42 | 43 | def test_delete_unsupported(self): 44 | t1 = Table( 45 | 't1', self.metadata(), 46 | Column('x', types.Int32, primary_key=True), 47 | engines.MergeTree('x', order_by=('x', )) 48 | ) 49 | t1.drop(bind=self.session.bind, if_exists=True) 50 | t1.create(bind=self.session.bind) 51 | 52 | with self.assertRaises(exc.CompileError) as ex: 53 | dialect = self.session.bind.dialect 54 | with mock_object_attr(dialect, 'supports_delete', False): 55 | self.session.execute(t1.delete().where(t1.c.x == 25)) 56 | 57 | self.assertEqual( 58 | str(ex.exception), 59 | 'ALTER DELETE is not supported by this server version' 60 | ) 61 | 62 | with self.assertRaises(exc.CompileError) as ex: 63 | dialect = self.session.bind.dialect 64 | with mock_object_attr(dialect, 'supports_delete', False): 65 | self.session.execute(delete(t1).where(t1.c.x == 25)) 66 | 67 | self.assertEqual( 68 | str(ex.exception), 69 | 'ALTER DELETE is not supported by this server version' 70 | ) 71 | -------------------------------------------------------------------------------- /tests/sql/test_functions.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, func 2 | 3 | from clickhouse_sqlalchemy import types, Table 4 | 5 | from tests.testcase import CompilationTestCase 6 | 7 | 8 | class FunctionTestCase(CompilationTestCase): 9 | table = Table( 10 | 't1', CompilationTestCase.metadata(), 11 | Column('x', types.Int32, primary_key=True), 12 | Column('time', types.DateTime) 13 | ) 14 | 15 | def test_quantile(self): 16 | func0 = func.quantile(0.5, self.table.c.x) 17 | self.assertIsInstance(func0.type, types.Float64) 18 | func1 = func.quantile(0.5, self.table.c.time) 19 | self.assertIsInstance(func1.type, types.DateTime) 20 | self.assertEqual( 21 | self.compile(self.session.query(func0)), 22 | 'SELECT quantile(0.5)(t1.x) AS quantile_1 FROM t1' 23 | ) 24 | 25 | func2 = func.quantileIf(0.5, self.table.c.x, self.table.c.x > 10) 26 | 27 | self.assertEqual( 28 | self.compile( 29 | self.session.query(func2) 30 | ), 31 | 'SELECT quantileIf(0.5)(t1.x, t1.x > %(x_1)s) AS ' + 32 | '"quantileIf_1" FROM t1' 33 | ) 34 | -------------------------------------------------------------------------------- /tests/sql/test_ilike.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | from clickhouse_sqlalchemy import types, Table 3 | 4 | from tests.testcase import BaseTestCase 5 | 6 | 7 | class ILike(BaseTestCase): 8 | table = Table( 9 | 't1', 10 | BaseTestCase.metadata(), 11 | Column('x', types.Int32, primary_key=True), 12 | Column('y', types.String) 13 | ) 14 | 15 | def test_ilike(self): 16 | query = ( 17 | self.session.query(self.table.c.x) 18 | .where(self.table.c.y.ilike('y')) 19 | ) 20 | 21 | self.assertEqual( 22 | self.compile(query, literal_binds=True), 23 | "SELECT t1.x AS t1_x FROM t1 WHERE t1.y ILIKE 'y'" 24 | ) 25 | 26 | def test_not_ilike(self): 27 | query = ( 28 | self.session.query(self.table.c.x) 29 | .where(self.table.c.y.not_ilike('y')) 30 | ) 31 | 32 | self.assertEqual( 33 | self.compile(query, literal_binds=True), 34 | "SELECT t1.x AS t1_x FROM t1 WHERE t1.y NOT ILIKE 'y'" 35 | ) 36 | -------------------------------------------------------------------------------- /tests/sql/test_insert.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, literal_column, select 2 | 3 | from clickhouse_sqlalchemy import types, Table, engines 4 | from tests.testcase import NativeSessionTestCase 5 | from tests.util import require_server_version 6 | 7 | 8 | class InsertTestCase(NativeSessionTestCase): 9 | @require_server_version(19, 3, 3) 10 | def test_insert(self): 11 | table = Table( 12 | 't', self.metadata(), 13 | Column('x', types.String, primary_key=True), 14 | engines.Log() 15 | ) 16 | 17 | with self.create_table(table): 18 | query = table.insert().values(x=literal_column("'test'")) 19 | self.session.execute(query) 20 | 21 | rv = self.session.execute(select(table.c.x)).scalar() 22 | self.assertEqual(rv, 'test') 23 | 24 | @require_server_version(21, 1, 3) 25 | def test_insert_map(self): 26 | table = Table( 27 | 't', self.metadata(), 28 | Column('x', types.Map(types.String, types.Int32), 29 | primary_key=True), 30 | engines.Memory() 31 | ) 32 | 33 | with self.create_table(table): 34 | dict_map = dict(key1=1, Key2=2) 35 | x = [ 36 | {'x': dict_map} 37 | ] 38 | self.session.execute(table.insert(), x) 39 | 40 | rv = self.session.execute(select(table.c.x)).scalar() 41 | self.assertEqual(rv, dict_map) 42 | -------------------------------------------------------------------------------- /tests/sql/test_is_distinct_from.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from parameterized import parameterized 4 | from sqlalchemy import select, literal 5 | 6 | from tests.testcase import BaseTestCase 7 | 8 | 9 | class IsDistinctFromTestCase(BaseTestCase): 10 | def _select_bool(self, expr): 11 | query = select(expr) 12 | (result,), = self.session.execute(query).fetchall() 13 | return bool(result) 14 | 15 | @parameterized.expand([ 16 | (1, 2), 17 | (1, None), 18 | (None, "NULL"), 19 | (None, u"ᴺᵁᴸᴸ"), 20 | ((1, None), (2, None)), 21 | ((1, (1, None)), (1, (2, None))) 22 | ]) 23 | def test_is_distinct_from(self, a, b): 24 | self.assertTrue(self._select_bool(literal(a).is_distinct_from(b))) 25 | self.assertFalse(self._select_bool(literal(a).isnot_distinct_from(b))) 26 | 27 | @parameterized.expand([ 28 | (None, ), 29 | ((1, None), ), 30 | ((None, None), ), 31 | ((1, (1, None)), ) 32 | ]) 33 | def test_is_self_distinct_from(self, v): 34 | self.assertTrue(self._select_bool(literal(v).isnot_distinct_from(v))) 35 | self.assertFalse(self._select_bool(literal(v).is_distinct_from(v))) 36 | -------------------------------------------------------------------------------- /tests/sql/test_lambda.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import exc 2 | 3 | from clickhouse_sqlalchemy.ext.clauses import Lambda 4 | from tests.testcase import BaseTestCase 5 | 6 | 7 | class LabmdaTestCase(BaseTestCase): 8 | def test_lambda_is_callable(self): 9 | with self.assertRaises(exc.ArgumentError) as ex: 10 | self.compile(Lambda(1)) 11 | 12 | self.assertEqual(str(ex.exception), 'func must be callable') 13 | 14 | def test_lambda_no_arg_kwargs(self): 15 | with self.assertRaises(exc.CompileError) as ex: 16 | self.compile(Lambda(lambda x, *args: x * 2)) 17 | 18 | self.assertEqual( 19 | str(ex.exception), 20 | 'Lambdas with *args are not supported' 21 | ) 22 | 23 | with self.assertRaises(exc.CompileError) as ex: 24 | self.compile(Lambda(lambda x, **kwargs: x * 2)) 25 | 26 | self.assertEqual( 27 | str(ex.exception), 28 | 'Lambdas with **kwargs are not supported' 29 | ) 30 | 31 | def test_lambda_ok(self): 32 | self.assertEqual( 33 | self.compile(Lambda(lambda x: x * 2), literal_binds=True), 34 | 'x -> x * 2' 35 | ) 36 | 37 | def mul(x): 38 | return x * 2 39 | 40 | self.assertEqual( 41 | self.compile(Lambda(mul), literal_binds=True), 42 | 'x -> x * 2' 43 | ) 44 | -------------------------------------------------------------------------------- /tests/sql/test_limit.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, exc 2 | 3 | from clickhouse_sqlalchemy import types, Table 4 | from tests.testcase import BaseTestCase 5 | from tests.util import with_native_and_http_sessions 6 | 7 | 8 | @with_native_and_http_sessions 9 | class LimitTestCase(BaseTestCase): 10 | table = Table( 11 | 't1', BaseTestCase.metadata(), 12 | Column('x', types.Int32, primary_key=True) 13 | ) 14 | 15 | def test_limit(self): 16 | query = self.session.query(self.table.c.x).limit(10) 17 | self.assertEqual( 18 | self.compile(query, literal_binds=True), 19 | 'SELECT t1.x AS t1_x FROM t1 LIMIT 10' 20 | ) 21 | 22 | def test_limit_with_offset(self): 23 | query = self.session.query(self.table.c.x).limit(10).offset(5) 24 | self.assertEqual( 25 | self.compile(query, literal_binds=True), 26 | 'SELECT t1.x AS t1_x FROM t1 LIMIT 5, 10' 27 | ) 28 | 29 | query = self.session.query(self.table.c.x).offset(5).limit(10) 30 | self.assertEqual( 31 | self.compile(query, literal_binds=True), 32 | 'SELECT t1.x AS t1_x FROM t1 LIMIT 5, 10' 33 | ) 34 | 35 | def test_offset_without_limit(self): 36 | with self.assertRaises(exc.CompileError) as ex: 37 | query = self.session.query(self.table.c.x).offset(5) 38 | self.compile(query, literal_binds=True) 39 | 40 | self.assertEqual( 41 | str(ex.exception), 42 | 'OFFSET without LIMIT is not supported' 43 | ) 44 | -------------------------------------------------------------------------------- /tests/sql/test_regexp_match.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, not_ 2 | from clickhouse_sqlalchemy import types, Table 3 | 4 | from tests.testcase import BaseTestCase 5 | 6 | 7 | class RegexpMatch(BaseTestCase): 8 | table = Table( 9 | 't1', 10 | BaseTestCase.metadata(), 11 | Column('x', types.Int32, primary_key=True), 12 | Column('y', types.String) 13 | ) 14 | 15 | def test_regex_match(self): 16 | query = ( 17 | self.session.query(self.table.c.x) 18 | .where(self.table.c.y.regexp_match('^s.*')) 19 | ) 20 | 21 | self.assertEqual( 22 | self.compile(query, literal_binds=True), 23 | "SELECT t1.x AS t1_x FROM t1 WHERE match(t1.y, '^s.*')" 24 | ) 25 | 26 | def test_not_regex_match(self): 27 | query = ( 28 | self.session.query(self.table.c.x) 29 | .where(not_(self.table.c.y.regexp_match('^s.*'))) 30 | ) 31 | 32 | self.assertEqual( 33 | self.compile(query, literal_binds=True), 34 | "SELECT t1.x AS t1_x FROM t1 WHERE NOT match(t1.y, '^s.*')" 35 | ) 36 | -------------------------------------------------------------------------------- /tests/sql/test_schema.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, text, Table, inspect 2 | from sqlalchemy.sql.elements import TextClause 3 | 4 | from clickhouse_sqlalchemy import types, Table as CHTable, engines 5 | from tests.testcase import BaseTestCase 6 | from tests.util import with_native_and_http_sessions 7 | 8 | 9 | @with_native_and_http_sessions 10 | class SchemaTestCase(BaseTestCase): 11 | 12 | def test_reflect(self): 13 | # checking, that after call metadata.reflect() 14 | # we have a clickhouse-specific table, which has overridden join 15 | # methods 16 | 17 | session = self.session 18 | metadata = self.metadata() 19 | 20 | # ### Maybe: ### 21 | # # session = native_session # database=test 22 | # session = self.__class__.session 23 | # same as in `self.metadata()` # database=default 24 | # unbound_metadata = MetaData(bind=session.bind) 25 | 26 | table = CHTable( 27 | 'test_reflect', 28 | metadata, 29 | Column('x', types.Int32), 30 | engines.Log() 31 | ) 32 | 33 | table.drop(session.bind, if_exists=True) 34 | table.create(session.bind) 35 | 36 | # Sub-test: ensure the `metadata.reflect` makes a CHTable 37 | metadata.clear() # reflect from clean state 38 | self.assertFalse(metadata.tables) 39 | metadata.reflect(bind=session.bind, only=[table.name]) 40 | table2 = metadata.tables.get(table.name) 41 | self.assertIsNotNone(table2) 42 | self.assertListEqual([c.name for c in table2.columns], ['x']) 43 | self.assertTrue(isinstance(table2, CHTable)) 44 | 45 | # Sub-test: ensure `CHTable(..., autoload=True)` works too 46 | metadata.clear() 47 | table3 = CHTable('test_reflect', metadata, autoload_with=session.bind) 48 | self.assertListEqual([c.name for c in table3.columns], ['x']) 49 | 50 | # Sub-test: check that they all reflected the same. 51 | for table_x in [table, table2, table3]: 52 | query = table_x.select().select_from(table_x.join( 53 | text('another_table'), 54 | table.c.x == 'xxx', 55 | type='INNER', 56 | strictness='ALL', 57 | distribution='GLOBAL' 58 | )) 59 | self.assertEqual( 60 | self.compile(query), 61 | "SELECT test_reflect.x FROM test_reflect " 62 | "GLOBAL ALL INNER JOIN another_table " 63 | "ON test_reflect.x = %(x_1)s" 64 | ) 65 | 66 | def test_reflect_generic_table(self): 67 | # checking, that generic table columns are reflected properly 68 | 69 | metadata = self.metadata() 70 | 71 | table = Table( 72 | 'test_reflect', 73 | metadata, 74 | Column('x', types.Int32), 75 | engines.Log() 76 | ) 77 | 78 | self.session.execute(text('DROP TABLE IF EXISTS test_reflect')) 79 | table.create(self.session.bind) 80 | 81 | # Sub-test: ensure the `metadata.reflect` makes a CHTable 82 | metadata.clear() # reflect from clean state 83 | self.assertFalse(metadata.tables) 84 | 85 | table = Table( 86 | 'test_reflect', 87 | metadata, 88 | autoload_with=self.session.bind 89 | ) 90 | self.assertListEqual([c.name for c in table.columns], ['x']) 91 | 92 | def test_reflect_subquery(self): 93 | table_node_sql = ( 94 | '(select arrayJoin([1, 2]) as a, arrayJoin([3, 4]) as b)') 95 | table_node = TextClause(table_node_sql) 96 | 97 | # Cannot use `Table` as it only works with a simple string. 98 | columns = inspect(self.session.bind).get_columns(table_node) 99 | self.assertListEqual( 100 | sorted([col['name'] for col in columns]), 101 | ['a', 'b']) 102 | 103 | def test_get_schema_names(self): 104 | insp = inspect(self.session.bind) 105 | schema_names = insp.get_schema_names() 106 | self.assertGreater(len(schema_names), 0) 107 | 108 | def test_get_table_names(self): 109 | table = Table( 110 | 'test_reflect', 111 | self.metadata(), 112 | Column('x', types.Int32), 113 | engines.Log() 114 | ) 115 | 116 | self.session.execute(text('DROP TABLE IF EXISTS test_reflect')) 117 | table.create(self.session.bind) 118 | 119 | insp = inspect(self.session.bind) 120 | self.assertListEqual(insp.get_table_names(), ['test_reflect']) 121 | -------------------------------------------------------------------------------- /tests/sql/test_update.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, exc, update, func 2 | 3 | from clickhouse_sqlalchemy import types, Table, engines 4 | from tests.testcase import NativeSessionTestCase 5 | from tests.util import mock_object_attr 6 | 7 | 8 | class UpdateTestCase(NativeSessionTestCase): 9 | def test_update(self): 10 | t1 = Table( 11 | 't1', self.metadata(), 12 | Column('x', types.Int32, primary_key=True), 13 | engines.MergeTree('x', order_by=('x', )) 14 | ) 15 | 16 | query = t1.update().where(t1.c.x == 25).values(x=5) 17 | statement = self.compile(query, literal_binds=True) 18 | self.assertEqual(statement, 'ALTER TABLE t1 UPDATE x=5 WHERE x = 25') 19 | 20 | query = update(t1).where(t1.c.x == 25).values(x=5) 21 | statement = self.compile(query, literal_binds=True) 22 | self.assertEqual(statement, 'ALTER TABLE t1 UPDATE x=5 WHERE x = 25') 23 | 24 | def test_update_without_where(self): 25 | t1 = Table( 26 | 't1', self.metadata(), 27 | Column('x', types.Int32, primary_key=True), 28 | engines.MergeTree('x', order_by=('x', )) 29 | ) 30 | 31 | query = t1.update().values(x=5) 32 | with self.assertRaises(exc.CompileError) as ex: 33 | self.compile(query, literal_binds=True) 34 | 35 | self.assertEqual(str(ex.exception), 'WHERE clause is required') 36 | 37 | query = update(t1).values(x=5) 38 | with self.assertRaises(exc.CompileError) as ex: 39 | self.compile(query, literal_binds=True) 40 | 41 | self.assertEqual(str(ex.exception), 'WHERE clause is required') 42 | 43 | def test_update_unsupported(self): 44 | t1 = Table( 45 | 't1', self.metadata(), 46 | Column('x', types.Int32, primary_key=True), 47 | engines.MergeTree(order_by=(func.tuple(), )) 48 | ) 49 | t1.drop(bind=self.session.bind, if_exists=True) 50 | t1.create(bind=self.session.bind) 51 | 52 | with self.assertRaises(exc.CompileError) as ex: 53 | dialect = self.session.bind.dialect 54 | with mock_object_attr(dialect, 'supports_update', False): 55 | self.session.execute( 56 | t1.update().where(t1.c.x == 25).values(x=5) 57 | ) 58 | 59 | self.assertEqual( 60 | str(ex.exception), 61 | 'ALTER UPDATE is not supported by this server version' 62 | ) 63 | 64 | with self.assertRaises(exc.CompileError) as ex: 65 | dialect = self.session.bind.dialect 66 | with mock_object_attr(dialect, 'supports_update', False): 67 | self.session.execute( 68 | update(t1).where(t1.c.x == 25).values(x=5) 69 | ) 70 | 71 | self.assertEqual( 72 | str(ex.exception), 73 | 'ALTER UPDATE is not supported by this server version' 74 | ) 75 | -------------------------------------------------------------------------------- /tests/test_compiler.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from sqlalchemy import sql, Column, literal, literal_column 3 | 4 | from clickhouse_sqlalchemy import types, Table, engines 5 | from tests.testcase import CompilationTestCase, NativeSessionTestCase 6 | 7 | 8 | class VisitTestCase(CompilationTestCase): 9 | def test_true_false(self): 10 | self.assertEqual(self.compile(sql.false()), 'false') 11 | self.assertEqual(self.compile(sql.true()), 'true') 12 | 13 | def test_array(self): 14 | self.assertEqual( 15 | self.compile(types.Array(types.Int32())), 16 | 'Array(Int32)' 17 | ) 18 | self.assertEqual( 19 | self.compile(types.Array(types.Array(types.Int32()))), 20 | 'Array(Array(Int32))' 21 | ) 22 | 23 | def test_enum(self): 24 | class MyEnum(enum.Enum): 25 | __order__ = 'foo bar' 26 | foo = 100 27 | bar = 500 28 | 29 | self.assertEqual( 30 | self.compile(types.Enum(MyEnum)), 31 | "Enum('foo' = 100, 'bar' = 500)" 32 | ) 33 | 34 | self.assertEqual( 35 | self.compile(types.Enum16(MyEnum)), 36 | "Enum16('foo' = 100, 'bar' = 500)" 37 | ) 38 | 39 | MyEnum = enum.Enum('MyEnum', [" ' t = ", "test"]) 40 | 41 | self.assertEqual( 42 | self.compile(types.Enum8(MyEnum)), 43 | "Enum8(' \\' t = ' = 1, 'test' = 2)" 44 | ) 45 | 46 | def test_do_not_allow_execution(self): 47 | with self.assertRaises(TypeError): 48 | self.session.execute('SHOW TABLES') 49 | 50 | with self.assertRaises(TypeError): 51 | self.session.query(literal(0)).all() 52 | 53 | 54 | class VisitNativeTestCase(NativeSessionTestCase): 55 | def test_insert_no_templates_after_value(self): 56 | # Optimized non-templating insert test (native protocol only). 57 | table = Table( 58 | 't1', self.metadata(), 59 | Column('x', types.Int32), 60 | engines.Memory() 61 | ) 62 | self.assertEqual( 63 | self.compile(table.insert()), 64 | 'INSERT INTO t1 (x) VALUES' 65 | ) 66 | 67 | def test_insert_inplace_values(self): 68 | table = Table( 69 | 't1', self.metadata(), 70 | Column('x', types.Int32), 71 | engines.Memory() 72 | ) 73 | self.assertEqual( 74 | self.compile( 75 | table.insert().values(x=literal_column(str(42))), 76 | literal_binds=True 77 | ), 'INSERT INTO t1 (x) VALUES (42)' 78 | ) 79 | -------------------------------------------------------------------------------- /tests/testcase.py: -------------------------------------------------------------------------------- 1 | import re 2 | from contextlib import contextmanager 3 | from unittest import TestCase 4 | 5 | from sqlalchemy import MetaData, text 6 | from sqlalchemy.orm import Query 7 | 8 | from tests.config import database, host, port, http_port, user, password 9 | from tests.session import http_session, native_session, \ 10 | system_native_session, http_engine, asynch_session, system_asynch_session 11 | from tests.util import skip_by_server_version, run_async 12 | 13 | 14 | class BaseAbstractTestCase(object): 15 | """ Supporting code for tests """ 16 | required_server_version = None 17 | server_version = None 18 | 19 | host = host 20 | port = None 21 | database = database 22 | user = user 23 | password = password 24 | 25 | strip_spaces = re.compile(r'[\n\t]') 26 | session = native_session 27 | 28 | @classmethod 29 | def metadata(cls): 30 | return MetaData() 31 | 32 | def _compile(self, clause, bind=None, literal_binds=False, 33 | render_postcompile=False): 34 | if bind is None: 35 | bind = self.session.bind 36 | if isinstance(clause, Query): 37 | clause = clause._statement_20() 38 | 39 | kw = {} 40 | compile_kwargs = {} 41 | if literal_binds: 42 | compile_kwargs['literal_binds'] = True 43 | if render_postcompile: 44 | compile_kwargs['render_postcompile'] = True 45 | 46 | if compile_kwargs: 47 | kw['compile_kwargs'] = compile_kwargs 48 | 49 | return clause.compile(dialect=bind.dialect, **kw) 50 | 51 | def compile(self, clause, **kwargs): 52 | return self.strip_spaces.sub( 53 | '', str(self._compile(clause, **kwargs)) 54 | ) 55 | 56 | @contextmanager 57 | def create_table(self, table): 58 | table.drop(bind=self.session.bind, if_exists=True) 59 | table.create(bind=self.session.bind) 60 | try: 61 | yield 62 | finally: 63 | table.drop(bind=self.session.bind) 64 | 65 | 66 | class BaseTestCase(BaseAbstractTestCase, TestCase): 67 | """ Actually tests """ 68 | 69 | @classmethod 70 | def setUpClass(cls): 71 | # System database is always present. 72 | system_native_session.execute( 73 | text('DROP DATABASE IF EXISTS {}'.format(cls.database)) 74 | ) 75 | system_native_session.execute( 76 | text('CREATE DATABASE {}'.format(cls.database)) 77 | ) 78 | 79 | version = system_native_session.execute( 80 | text('SELECT version()') 81 | ).fetchall() 82 | cls.server_version = tuple(int(x) for x in version[0][0].split('.')) 83 | 84 | super().setUpClass() 85 | 86 | def setUp(self): 87 | super(BaseTestCase, self).setUp() 88 | 89 | required = self.required_server_version 90 | 91 | if required and required > self.server_version: 92 | skip_by_server_version(self, self.required_server_version) 93 | 94 | 95 | class BaseAsynchTestCase(BaseTestCase): 96 | session = asynch_session 97 | 98 | @classmethod 99 | def setUpClass(cls): 100 | # System database is always present. 101 | run_async(system_asynch_session.execute)( 102 | text('DROP DATABASE IF EXISTS {}'.format(cls.database)) 103 | ) 104 | run_async(system_asynch_session.execute)( 105 | text('CREATE DATABASE {}'.format(cls.database)) 106 | ) 107 | 108 | version = ( 109 | run_async(system_asynch_session.execute)(text('SELECT version()')) 110 | ).fetchall() 111 | cls.server_version = tuple(int(x) for x in version[0][0].split('.')) 112 | 113 | def setUp(self): 114 | self.connection = run_async(self.session.connection)() 115 | super(BaseAsynchTestCase, self).setUp() 116 | 117 | def _callTestMethod(self, method): 118 | return run_async(method)() 119 | 120 | async def run_sync(self, f): 121 | return await self.connection.run_sync(f) 122 | 123 | 124 | class HttpSessionTestCase(BaseTestCase): 125 | """ Explicitly HTTP-based session Test Case """ 126 | 127 | port = http_port 128 | session = http_session 129 | 130 | 131 | class HttpEngineTestCase(BaseTestCase): 132 | """ Explicitly HTTP-based session Test Case """ 133 | 134 | port = http_port 135 | engine = http_engine 136 | 137 | 138 | class NativeSessionTestCase(BaseTestCase): 139 | """ Explicitly Native-protocol-based session Test Case """ 140 | 141 | port = port 142 | session = native_session 143 | 144 | 145 | class AsynchSessionTestCase(BaseAsynchTestCase): 146 | """ Explicitly Native-protocol-based async session Test Case """ 147 | 148 | port = port 149 | session = asynch_session 150 | 151 | 152 | class CompilationTestCase(BaseTestCase): 153 | """ Test Case that should be used only for SQL generation """ 154 | 155 | session = native_session 156 | 157 | @classmethod 158 | def setUpClass(cls): 159 | super(CompilationTestCase, cls).setUpClass() 160 | 161 | cls._session_connection = cls.session.connection 162 | cls._session_execute = cls.session.execute 163 | 164 | cls.session.execute = None 165 | cls.session.connection = None 166 | 167 | @classmethod 168 | def tearDownClass(cls): 169 | cls.session.execute = cls._session_execute 170 | cls.session.connection = cls._session_connection 171 | 172 | super(CompilationTestCase, cls).tearDownClass() 173 | -------------------------------------------------------------------------------- /tests/types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xzkostyan/clickhouse-sqlalchemy/3dc8df9da598ac51e20e9b7bb110ae97e250fab7/tests/types/__init__.py -------------------------------------------------------------------------------- /tests/types/test_boolean.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column, Boolean 2 | from sqlalchemy.sql.ddl import CreateTable 3 | 4 | from clickhouse_sqlalchemy import engines, Table 5 | from tests.testcase import CompilationTestCase 6 | 7 | 8 | class BooleanCompilationTestCase(CompilationTestCase): 9 | table = Table( 10 | 'test', CompilationTestCase.metadata(), 11 | Column('x', Boolean), 12 | engines.Memory() 13 | ) 14 | 15 | def test_create_table(self): 16 | self.assertEqual( 17 | self.compile(CreateTable(self.table)), 18 | 'CREATE TABLE test (x Bool) ENGINE = Memory' 19 | ) 20 | 21 | def test_literals(self): 22 | query = self.session.query(self.table.c.x).filter(self.table.c.x) 23 | self.assertEqual( 24 | self.compile(query), 25 | 'SELECT test.x AS test_x FROM test WHERE test.x' 26 | ) 27 | -------------------------------------------------------------------------------- /tests/types/test_date32.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from sqlalchemy import Column 4 | from sqlalchemy.sql.ddl import CreateTable 5 | 6 | from clickhouse_sqlalchemy import types, engines, Table 7 | from tests.testcase import BaseTestCase, CompilationTestCase 8 | from tests.util import with_native_and_http_sessions 9 | 10 | 11 | class Date32CompilationTestCase(CompilationTestCase): 12 | required_server_version = (21, 9, 0) 13 | 14 | def test_create_table(self): 15 | table = Table( 16 | 'test', CompilationTestCase.metadata(), 17 | Column('x', types.Date32, primary_key=True), 18 | engines.Memory() 19 | ) 20 | 21 | self.assertEqual( 22 | self.compile(CreateTable(table)), 23 | 'CREATE TABLE test (x Date32) ENGINE = Memory' 24 | ) 25 | 26 | 27 | @with_native_and_http_sessions 28 | class Date32TestCase(BaseTestCase): 29 | required_server_version = (21, 9, 0) 30 | 31 | table = Table( 32 | 'test', BaseTestCase.metadata(), 33 | Column('x', types.Date32, primary_key=True), 34 | engines.Memory() 35 | ) 36 | 37 | def test_select_insert(self): 38 | # Use a date before epoch to validate dates before epoch can be stored. 39 | date = datetime.date(1925, 1, 1) 40 | with self.create_table(self.table): 41 | self.session.execute(self.table.insert(), [{'x': date}]) 42 | result = self.session.execute(self.table.select()).scalar() 43 | if isinstance(result, datetime.date): 44 | self.assertEqual(result, date) 45 | else: 46 | self.assertEqual(result, date.isoformat()) 47 | -------------------------------------------------------------------------------- /tests/types/test_datetime.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from sqlalchemy import Column 4 | from sqlalchemy.sql.ddl import CreateTable 5 | 6 | from clickhouse_sqlalchemy import types, engines, Table 7 | from tests.testcase import BaseTestCase, CompilationTestCase 8 | from tests.util import with_native_and_http_sessions 9 | 10 | 11 | class DateTimeCompilationTestCase(CompilationTestCase): 12 | def test_create_table(self): 13 | table = Table( 14 | 'test', CompilationTestCase.metadata(), 15 | Column('x', types.DateTime, primary_key=True), 16 | engines.Memory() 17 | ) 18 | 19 | self.assertEqual( 20 | self.compile(CreateTable(table)), 21 | 'CREATE TABLE test (x DateTime) ENGINE = Memory' 22 | ) 23 | 24 | def test_create_table_with_timezone(self): 25 | table = Table( 26 | 'test', CompilationTestCase.metadata(), 27 | Column('x', types.DateTime('Europe/Moscow'), primary_key=True), 28 | engines.Memory() 29 | ) 30 | 31 | self.assertEqual( 32 | self.compile(CreateTable(table)), 33 | "CREATE TABLE test (x DateTime('Europe/Moscow')) ENGINE = Memory" 34 | ) 35 | 36 | 37 | @with_native_and_http_sessions 38 | class DateTimeTestCase(BaseTestCase): 39 | table = Table( 40 | 'test', BaseTestCase.metadata(), 41 | Column('x', types.DateTime, primary_key=True), 42 | engines.Memory() 43 | ) 44 | 45 | def test_select_insert(self): 46 | dt = datetime(2018, 1, 1, 15, 20) 47 | 48 | with self.create_table(self.table): 49 | self.session.execute(self.table.insert(), [{'x': dt}]) 50 | self.assertEqual(self.session.query(self.table.c.x).scalar(), dt) 51 | -------------------------------------------------------------------------------- /tests/types/test_datetime64.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | from sqlalchemy.sql.ddl import CreateTable 3 | 4 | from clickhouse_sqlalchemy import types, engines, Table 5 | from tests.testcase import CompilationTestCase 6 | 7 | 8 | class DateTime64CompilationTestCase(CompilationTestCase): 9 | table = Table( 10 | 'test', CompilationTestCase.metadata(), 11 | Column('x', types.DateTime64, primary_key=True), 12 | engines.Memory() 13 | ) 14 | 15 | def test_create_table(self): 16 | self.assertEqual( 17 | self.compile(CreateTable(self.table)), 18 | 'CREATE TABLE test (x DateTime64(3)) ENGINE = Memory' 19 | ) 20 | 21 | 22 | class DateTime64CompilationTestCasePrecision(CompilationTestCase): 23 | table = Table( 24 | 'test', CompilationTestCase.metadata(), 25 | Column('x', types.DateTime64(4), primary_key=True), 26 | engines.Memory() 27 | ) 28 | 29 | def test_create_table_precision(self): 30 | self.assertEqual( 31 | self.compile(CreateTable(self.table)), 32 | 'CREATE TABLE test (x DateTime64(4)) ENGINE = Memory' 33 | ) 34 | 35 | 36 | class DateTime64CompilationTestCaseTimezone(CompilationTestCase): 37 | table = Table( 38 | 'test', CompilationTestCase.metadata(), 39 | Column( 40 | 'x', types.DateTime64(4, 'Pacific/Honolulu'), primary_key=True, 41 | ), 42 | engines.Memory() 43 | ) 44 | 45 | def test_create_table_precision(self): 46 | self.assertEqual( 47 | self.compile(CreateTable(self.table)), 48 | 'CREATE TABLE test (' 49 | 'x DateTime64(4, \'Pacific/Honolulu\')' 50 | ') ENGINE = Memory' 51 | ) 52 | -------------------------------------------------------------------------------- /tests/types/test_enum16.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from sqlalchemy import Column 4 | from sqlalchemy.sql.ddl import CreateTable 5 | 6 | from clickhouse_sqlalchemy import types, engines, Table 7 | from tests.testcase import BaseTestCase, CompilationTestCase 8 | from tests.util import with_native_and_http_sessions 9 | 10 | 11 | class TestEnum(enum.IntEnum): 12 | First = 1 13 | Second = 2 14 | 15 | 16 | class Enum16CompilationTestCase(CompilationTestCase): 17 | def test_create_table(self): 18 | table = Table( 19 | "test", 20 | BaseTestCase.metadata(), 21 | Column("x", types.Enum16(TestEnum)), 22 | engines.Memory(), 23 | ) 24 | 25 | self.assertEqual( 26 | self.compile(CreateTable(table)), 27 | "CREATE TABLE test (x Enum16('First' = 1, 'Second' = 2)) " 28 | "ENGINE = Memory" 29 | ) 30 | 31 | 32 | @with_native_and_http_sessions 33 | class Enum8TestCase(BaseTestCase): 34 | table = Table( 35 | 'test', BaseTestCase.metadata(), 36 | Column('x', types.Enum16(TestEnum)), 37 | engines.Memory() 38 | ) 39 | 40 | def test_select_insert(self): 41 | value = TestEnum.First 42 | with self.create_table(self.table): 43 | self.session.execute(self.table.insert(), [{'x': value}]) 44 | self.assertEqual(self.session.query(self.table.c.x).scalar(), 45 | value) 46 | -------------------------------------------------------------------------------- /tests/types/test_enum8.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | from sqlalchemy import Column 4 | from sqlalchemy.sql.ddl import CreateTable 5 | 6 | from clickhouse_sqlalchemy import types, engines, Table 7 | from tests.testcase import BaseTestCase, CompilationTestCase 8 | from tests.util import with_native_and_http_sessions 9 | 10 | 11 | class TestEnum(enum.IntEnum): 12 | First = 1 13 | Second = 2 14 | 15 | 16 | class Enum8CompilationTestCase(CompilationTestCase): 17 | def test_create_table(self): 18 | table = Table( 19 | "test", 20 | BaseTestCase.metadata(), 21 | Column("x", types.Enum8(TestEnum)), 22 | engines.Memory(), 23 | ) 24 | 25 | self.assertEqual( 26 | self.compile(CreateTable(table)), 27 | "CREATE TABLE test (x Enum8('First' = 1, 'Second' = 2)) " 28 | "ENGINE = Memory" 29 | ) 30 | 31 | 32 | @with_native_and_http_sessions 33 | class Enum8TestCase(BaseTestCase): 34 | table = Table( 35 | 'test', BaseTestCase.metadata(), 36 | Column('x', types.Enum8(TestEnum)), 37 | engines.Memory() 38 | ) 39 | 40 | def test_select_insert(self): 41 | value = TestEnum.First 42 | with self.create_table(self.table): 43 | self.session.execute(self.table.insert(), [{'x': value}]) 44 | self.assertEqual(self.session.query(self.table.c.x).scalar(), 45 | value) 46 | -------------------------------------------------------------------------------- /tests/types/test_int128.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | from sqlalchemy.sql.ddl import CreateTable 3 | 4 | from clickhouse_sqlalchemy import types, engines, Table 5 | from tests.testcase import BaseTestCase, CompilationTestCase 6 | from tests.util import with_native_and_http_sessions 7 | 8 | 9 | class Int128CompilationTestCase(CompilationTestCase): 10 | required_server_version = (21, 6, 0) 11 | 12 | table = Table( 13 | 'test', CompilationTestCase.metadata(), 14 | Column('x', types.Int128), 15 | Column('y', types.UInt128), 16 | engines.Memory() 17 | ) 18 | 19 | def test_create_table(self): 20 | self.assertEqual( 21 | self.compile(CreateTable(self.table)), 22 | 'CREATE TABLE test (x Int128, y UInt128) ENGINE = Memory' 23 | ) 24 | 25 | 26 | @with_native_and_http_sessions 27 | class Int128TestCase(BaseTestCase): 28 | required_server_version = (21, 6, 0) 29 | 30 | table = Table( 31 | 'test', BaseTestCase.metadata(), 32 | Column('x', types.Int128), 33 | Column('y', types.UInt128), 34 | engines.Memory() 35 | ) 36 | 37 | def test_select_insert(self): 38 | x = -2 ** 127 39 | y = 2 ** 128 - 1 40 | 41 | with self.create_table(self.table): 42 | self.session.execute(self.table.insert(), [{'x': x, 'y': y}]) 43 | self.assertEqual(self.session.query(self.table.c.x).scalar(), x) 44 | self.assertEqual(self.session.query(self.table.c.y).scalar(), y) 45 | -------------------------------------------------------------------------------- /tests/types/test_int256.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Column 2 | from sqlalchemy.sql.ddl import CreateTable 3 | 4 | from clickhouse_sqlalchemy import types, engines, Table 5 | from tests.testcase import BaseTestCase, CompilationTestCase 6 | from tests.util import with_native_and_http_sessions 7 | 8 | 9 | class Int256CompilationTestCase(CompilationTestCase): 10 | required_server_version = (21, 6, 0) 11 | 12 | table = Table( 13 | 'test', CompilationTestCase.metadata(), 14 | Column('x', types.Int256), 15 | Column('y', types.UInt256), 16 | engines.Memory() 17 | ) 18 | 19 | def test_create_table(self): 20 | self.assertEqual( 21 | self.compile(CreateTable(self.table)), 22 | 'CREATE TABLE test (x Int256, y UInt256) ENGINE = Memory' 23 | ) 24 | 25 | 26 | @with_native_and_http_sessions 27 | class Int256TestCase(BaseTestCase): 28 | required_server_version = (21, 6, 0) 29 | 30 | table = Table( 31 | 'test', BaseTestCase.metadata(), 32 | Column('x', types.Int256), 33 | Column('y', types.UInt256), 34 | engines.Memory() 35 | ) 36 | 37 | def test_select_insert(self): 38 | x = -2 ** 255 39 | y = 2 ** 256 - 1 40 | 41 | with self.create_table(self.table): 42 | self.session.execute(self.table.insert(), [{'x': x, 'y': y}]) 43 | self.assertEqual(self.session.query(self.table.c.x).scalar(), x) 44 | self.assertEqual(self.session.query(self.table.c.y).scalar(), y) 45 | -------------------------------------------------------------------------------- /tests/types/test_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | from sqlalchemy import Column, text, inspect, func 3 | from sqlalchemy.sql.ddl import CreateTable 4 | 5 | from clickhouse_sqlalchemy import types, engines, Table 6 | from tests.testcase import BaseTestCase, CompilationTestCase 7 | from tests.util import class_name_func 8 | from parameterized import parameterized_class 9 | from tests.session import native_session 10 | 11 | 12 | class JSONCompilationTestCase(CompilationTestCase): 13 | def test_create_table(self): 14 | table = Table( 15 | 'test', CompilationTestCase.metadata(), 16 | Column('x', types.JSON), 17 | engines.Memory() 18 | ) 19 | 20 | self.assertEqual( 21 | self.compile(CreateTable(table)), 22 | 'CREATE TABLE test (x JSON) ENGINE = Memory' 23 | ) 24 | 25 | 26 | @parameterized_class( 27 | [{'session': native_session}], 28 | class_name_func=class_name_func 29 | ) 30 | class JSONTestCase(BaseTestCase): 31 | required_server_version = (22, 6, 1) 32 | 33 | table = Table( 34 | 'test', BaseTestCase.metadata(), 35 | Column('x', types.JSON), 36 | engines.Memory() 37 | ) 38 | 39 | def test_select_insert(self): 40 | data = {'k1': 1, 'k2': '2', 'k3': True} 41 | 42 | self.table.drop(bind=self.session.bind, if_exists=True) 43 | try: 44 | # http session is unsupport 45 | self.session.execute( 46 | text('SET allow_experimental_object_type = 1;') 47 | ) 48 | self.session.execute(text(self.compile(CreateTable(self.table)))) 49 | self.session.execute(self.table.insert(), [{'x': data}]) 50 | coltype = inspect(self.session.bind).get_columns('test')[0]['type'] 51 | self.assertIsInstance(coltype, types.JSON) 52 | # https://clickhouse.com/docs/en/sql-reference/functions/json-functions#tojsonstring 53 | # The json type returns a tuple of values by default, 54 | # which needs to be converted to json using the 55 | # toJSONString function. 56 | res = self.session.query( 57 | func.toJSONString(self.table.c.x) 58 | ).scalar() 59 | self.assertEqual(json.loads(res), data) 60 | finally: 61 | self.table.drop(bind=self.session.bind, if_exists=True) 62 | -------------------------------------------------------------------------------- /tests/types/test_numeric.py: -------------------------------------------------------------------------------- 1 | from decimal import Decimal 2 | 3 | from sqlalchemy import Column, Numeric 4 | from sqlalchemy.sql.ddl import CreateTable 5 | 6 | from clickhouse_sqlalchemy import types, engines, Table 7 | from clickhouse_sqlalchemy.exceptions import DatabaseException 8 | from tests.testcase import ( 9 | BaseTestCase, CompilationTestCase, 10 | HttpSessionTestCase, NativeSessionTestCase, 11 | ) 12 | 13 | 14 | class NumericCompilationTestCase(CompilationTestCase): 15 | table = Table( 16 | 'test', CompilationTestCase.metadata(), 17 | Column('x', Numeric(10, 2)), 18 | engines.Memory() 19 | ) 20 | 21 | def test_create_table(self): 22 | self.assertEqual( 23 | self.compile(CreateTable(self.table)), 24 | 'CREATE TABLE test (x Decimal(10, 2)) ENGINE = Memory' 25 | ) 26 | 27 | def test_create_table_decimal_symlink(self): 28 | table = Table( 29 | 'test', CompilationTestCase.metadata(), 30 | Column('x', types.Decimal(10, 2)), 31 | engines.Memory() 32 | ) 33 | 34 | self.assertEqual( 35 | self.compile(CreateTable(table)), 36 | 'CREATE TABLE test (x Decimal(10, 2)) ENGINE = Memory' 37 | ) 38 | 39 | 40 | class NumericTestCase(BaseTestCase): 41 | table = Table( 42 | 'test', BaseTestCase.metadata(), 43 | Column('x', Numeric(10, 2)), 44 | engines.Memory() 45 | ) 46 | 47 | def test_select_insert(self): 48 | x = Decimal('12345678.12') 49 | 50 | with self.create_table(self.table): 51 | self.session.execute(self.table.insert(), [{'x': x}]) 52 | self.assertEqual(self.session.query(self.table.c.x).scalar(), x) 53 | 54 | def test_insert_overflow(self): 55 | value = Decimal('12345678901234567890.1234567890') 56 | 57 | with self.create_table(self.table): 58 | with self.assertRaises(DatabaseException) as ex: 59 | self.session.execute(self.table.insert(), [{'x': value}]) 60 | 61 | # 'Too many digits' is written in the CH response; 62 | # 'out of range' is raised from `struct` within 63 | # `clickhouse_driver`, 64 | # before the query is sent to CH. 65 | ex_text = str(ex.exception) 66 | self.assertTrue( 67 | 'out of range' in ex_text or 'format requires' in ex_text or 68 | 'Too many digits' in ex_text 69 | ) 70 | 71 | 72 | class NumericNativeTestCase(NativeSessionTestCase): 73 | table = Table( 74 | 'test', NativeSessionTestCase.metadata(), 75 | Column('x', Numeric(10, 2)), 76 | engines.Memory() 77 | ) 78 | 79 | def test_insert_truncate(self): 80 | value = Decimal('123.129999') 81 | expected = Decimal('123.12') 82 | 83 | with self.create_table(self.table): 84 | self.session.execute(self.table.insert(), [{'x': value}]) 85 | self.assertEqual( 86 | self.session.query(self.table.c.x).scalar(), 87 | expected 88 | ) 89 | 90 | 91 | class NumericHttpTestCase(HttpSessionTestCase): 92 | table = Table( 93 | 'test', HttpSessionTestCase.metadata(), 94 | Column('x', Numeric(10, 2)), 95 | engines.Memory() 96 | ) 97 | 98 | def test_insert_truncate(self): 99 | value = Decimal('123.129999') 100 | expected = Decimal('123.12') 101 | 102 | with self.create_table(self.table): 103 | if self.server_version >= (20, 5, 2): 104 | self.session.execute(self.table.insert(), [{'x': value}]) 105 | self.assertEqual( 106 | self.session.query(self.table.c.x).scalar(), 107 | expected 108 | ) 109 | else: 110 | with self.assertRaises(DatabaseException) as ex: 111 | self.session.execute(self.table.insert(), [{'x': value}]) 112 | self.assertIn('value is too small', str(ex.exception)) 113 | -------------------------------------------------------------------------------- /tests/util.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from contextlib import contextmanager 3 | from functools import wraps 4 | 5 | from parameterized import parameterized_class 6 | from sqlalchemy.util.concurrency import greenlet_spawn 7 | 8 | from tests.session import http_session, native_session 9 | 10 | 11 | def skip_by_server_version(testcase, version_required): 12 | testcase.skipTest( 13 | 'Mininum revision required: {}'.format( 14 | '.'.join(str(x) for x in version_required) 15 | ) 16 | ) 17 | 18 | 19 | async def _get_version(conn): 20 | cursor = await greenlet_spawn(lambda: conn.cursor()) 21 | await greenlet_spawn(lambda: cursor.execute('SELECT version()')) 22 | version = cursor.fetchall()[0][0].split('.') 23 | return tuple(int(x) for x in version[:3]) 24 | 25 | 26 | def require_server_version(*version_required, is_async=False): 27 | def check(f): 28 | @wraps(f) 29 | def wrapper(*args, **kwargs): 30 | self = args[0] 31 | if not is_async: 32 | conn = self.session.bind.raw_connection() 33 | else: 34 | async def _get_conn(session): 35 | return await session.bind.raw_connection() 36 | 37 | conn = run_async(lambda: _get_conn(self.session))() 38 | 39 | dialect = self.session.bind.dialect.name 40 | if dialect in ['clickhouse+native', 'clickhouse+asynch']: 41 | i = conn.transport.connection.server_info 42 | current = (i.version_major, i.version_minor, i.version_patch) 43 | 44 | else: 45 | if not is_async: 46 | cursor = conn.cursor() 47 | cursor.execute('SELECT version()') 48 | version = cursor.fetchall()[0][0].split('.') 49 | current = tuple(int(x) for x in version[:3]) 50 | 51 | else: 52 | current = run_async(_get_version)(conn) 53 | 54 | conn.close() 55 | if version_required <= current: 56 | return f(*args, **kwargs) 57 | else: 58 | self.skipTest( 59 | 'Mininum revision required: {}'.format( 60 | '.'.join(str(x) for x in version_required) 61 | ) 62 | ) 63 | 64 | return wrapper 65 | 66 | return check 67 | 68 | 69 | @contextmanager 70 | def mock_object_attr(dialect, attr, new_value): 71 | old_value = getattr(dialect, attr) 72 | setattr(dialect, attr, new_value) 73 | 74 | try: 75 | yield 76 | finally: 77 | setattr(dialect, attr, old_value) 78 | 79 | 80 | def class_name_func(cls, num, params_dict): 81 | suffix = 'HTTP' if params_dict['session'] is http_session else 'Native' 82 | return cls.__name__ + suffix 83 | 84 | 85 | def run_async(f): 86 | """ 87 | Decorator to create asyncio context for asyncio methods or functions. 88 | """ 89 | @wraps(f) 90 | def g(*args, **kwargs): 91 | coro = f(*args, **kwargs) 92 | try: 93 | loop = asyncio.get_running_loop() 94 | except RuntimeError: 95 | try: 96 | loop = asyncio.get_event_loop_policy().get_event_loop() 97 | except DeprecationWarning: 98 | loop = asyncio.new_event_loop() 99 | asyncio.set_event_loop(loop) 100 | 101 | return loop.run_until_complete(coro) 102 | return g 103 | 104 | 105 | with_native_and_http_sessions = parameterized_class([ 106 | {'session': http_session}, 107 | {'session': native_session} 108 | ], class_name_func=class_name_func) 109 | -------------------------------------------------------------------------------- /testsrequire.py: -------------------------------------------------------------------------------- 1 | 2 | tests_require = [ 3 | 'pytest', 4 | 'pytest-asyncio', 5 | 'sqlalchemy>=2.0.0,<2.1.0', 6 | 'greenlet>=2.0.1', 7 | 'alembic', 8 | 'requests', 9 | 'responses', 10 | 'parameterized' 11 | ] 12 | 13 | try: 14 | from pip import main as pipmain 15 | except ImportError: 16 | from pip._internal import main as pipmain 17 | 18 | pipmain(['install'] + tests_require) 19 | --------------------------------------------------------------------------------