├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── aioch ├── __init__.py ├── client.py ├── result.py └── utils.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── log.py ├── test_client.py └── testcase.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - VERSION=19.15.3.6 3 | - VERSION=1.1.54276 4 | 5 | language: python 6 | python: 7 | - "3.6" 8 | - "3.7" 9 | - "3.8" 10 | cache: pip 11 | services: 12 | - docker 13 | before_install: 14 | - docker run -d -p 127.0.0.1:9000:9000 --name test-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:$VERSION 15 | - docker run -d --entrypoint "/bin/sh" --name test-clickhouse-client --link test-clickhouse-server:clickhouse-server yandex/clickhouse-client:$VERSION -c 'while :; do sleep 1; done' 16 | - docker ps -a 17 | # Faking clickhouse-client real comminitation with container via docker exec. 18 | - echo -e '#!/bin/bash\n\ndocker exec test-clickhouse-client clickhouse-client "$@"' | sudo tee /usr/local/bin/clickhouse-client > /dev/null 19 | - sudo chmod +x /usr/local/bin/clickhouse-client 20 | # Overriding setup.cfg. Set host=clickhouse-server 21 | - echo -e '[db]\nhost=clickhouse-server\nport=9000\ndatabase=test\nuser=default\npassword=\ncompression=lz4,lz4hc,zstd' > setup.cfg 22 | # Make host think that clickhouse-server is localhost 23 | - echo '127.0.0.1 clickhouse-server' | sudo tee /etc/hosts > /dev/null 24 | install: 25 | pip install flake8 flake8-print coveralls 26 | before_script: 27 | flake8 . 28 | script: 29 | - coverage run --source=aioch setup.py test 30 | after_success: 31 | - coveralls 32 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [Unreleased] 4 | 5 | ## [0.0.2] - 2019-11-22 6 | ### Added 7 | - Python 3.7, 3.8 in Travis CI build matrix. 8 | 9 | ### Fixed 10 | - Interfaces update to clickhouse-driver 0.1.2. 11 | 12 | ### Removed 13 | - Python 3.5 support. 14 | 15 | ## 0.0.1 - 2017-09-27 16 | ### Added 17 | - `execute` / `execute_with_progress`wrappers. 18 | - `loop`, `executor` client params. 19 | 20 | [Unreleased]: https://github.com/mymarilyn/aioch/compare/0.0.2...HEAD 21 | [0.0.2]: https://github.com/mymarilyn/aioch/compare/0.0.1...0.0.2 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is the MIT license: http://www.opensource.org/licenses/mit-license.php 2 | 3 | Copyright (c) 2017 by Konstantin Lebedev. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aioch 2 | **aioch** is a library for accessing a ClickHouse database over native interface from the asyncio. 3 | It wraps features of [clickhouse-driver](https://github.com/mymarilyn/clickhouse-driver) for asynchronous usage. 4 | 5 | [![Coverage Status](https://coveralls.io/repos/github/mymarilyn/aioch/badge.svg?branch=master)](https://coveralls.io/github/mymarilyn/aioch?branch=master) 6 | [![Build Status](https://travis-ci.org/mymarilyn/aioch.svg?branch=master)](https://travis-ci.org/mymarilyn/aioch) 7 | 8 | 9 | ## Installation 10 | 11 | The package can be installed using `pip`: 12 | 13 | ```bash 14 | pip install aioch 15 | ``` 16 | 17 | To install from source: 18 | 19 | ```bash 20 | git clone https://github.com/mymarilyn/aioch 21 | cd aioch 22 | python setup.py install 23 | ``` 24 | 25 | ## Usage 26 | ```python 27 | from datetime import datetime 28 | 29 | import asyncio 30 | from aioch import Client 31 | 32 | 33 | async def exec_progress(): 34 | client = Client('localhost') 35 | 36 | progress = await client.execute_with_progress('LONG AND COMPLICATED QUERY') 37 | timeout = 20 38 | started_at = datetime.now() 39 | 40 | async for num_rows, total_rows in progress: 41 | done = num_rows / total_rows if total_rows else total_rows 42 | now = datetime.now() 43 | # Cancel query if it takes more than 20 seconds to process 50% of rows. 44 | if (now - started_at).total_seconds() > timeout and done < 0.5: 45 | await client.cancel() 46 | break 47 | else: 48 | rv = await progress.get_result() 49 | print(rv) 50 | 51 | 52 | async def exec_no_progress(): 53 | client = Client('localhost') 54 | rv = await client.execute('LONG AND COMPLICATED QUERY') 55 | print(rv) 56 | 57 | 58 | loop = asyncio.get_event_loop() 59 | loop.run_until_complete(asyncio.wait([exec_progress(), exec_no_progress()])) 60 | ``` 61 | 62 | For more information see **clickhouse-driver** usage examples. 63 | 64 | ## Parameters 65 | 66 | * `executor` - instance of custom Executor, if not supplied default executor will be used 67 | * `loop` - asyncio compatible event loop 68 | 69 | Other parameters are passing to wrapped clickhouse-driver's Client. 70 | 71 | ## License 72 | 73 | aioch is distributed under the [MIT license](http://www.opensource.org/licenses/mit-license.php). 74 | -------------------------------------------------------------------------------- /aioch/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from aioch.client import Client 3 | 4 | 5 | VERSION = (0, 0, 2) 6 | __version__ = '.'.join(str(x) for x in VERSION) 7 | 8 | __all__ = ['Client'] 9 | -------------------------------------------------------------------------------- /aioch/client.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from clickhouse_driver import Client as BlockingClient 4 | from clickhouse_driver.result import QueryInfo 5 | from .result import ProgressQueryResult, QueryResult, IterQueryResult 6 | from .utils import run_in_executor 7 | 8 | 9 | class Client(object): 10 | def __init__(self, *args, **kwargs): 11 | self._loop = kwargs.pop('loop', None) or asyncio.get_event_loop() 12 | self._executor = kwargs.pop('executor', None) 13 | 14 | if '_client' not in kwargs: 15 | self._client = BlockingClient(*args, **kwargs) 16 | else: 17 | self._client = kwargs.pop('_client') 18 | 19 | super(Client, self).__init__() 20 | 21 | @classmethod 22 | def from_url(cls, url, loop=None, executor=None): 23 | """ 24 | *New in version 0.0.2.* 25 | """ 26 | 27 | _client = BlockingClient.from_url(url) 28 | return cls(_client=_client, loop=loop, executor=executor) 29 | 30 | def run_in_executor(self, *args, **kwargs): 31 | return run_in_executor(self._executor, self._loop, *args, **kwargs) 32 | 33 | async def disconnect(self): 34 | return await self.run_in_executor(self._client.disconnect) 35 | 36 | async def execute(self, *args, **kwargs): 37 | return await self.run_in_executor(self._client.execute, *args, 38 | **kwargs) 39 | 40 | async def execute_with_progress( 41 | self, query, params=None, with_column_types=False, 42 | external_tables=None, query_id=None, settings=None, 43 | types_check=False, columnar=False): 44 | self._client.make_query_settings(settings) 45 | 46 | await self.run_in_executor(self._client.connection.force_connect) 47 | 48 | self._client.last_query = QueryInfo() 49 | 50 | return await self.process_ordinary_query_with_progress( 51 | query, params=params, with_column_types=with_column_types, 52 | external_tables=external_tables, 53 | query_id=query_id, types_check=types_check, columnar=columnar 54 | ) 55 | 56 | async def execute_iter( 57 | self, query, params=None, with_column_types=False, 58 | external_tables=None, query_id=None, settings=None, 59 | types_check=False): 60 | """ 61 | *New in version 0.0.2.* 62 | """ 63 | 64 | self._client.make_query_settings(settings) 65 | 66 | await self.run_in_executor(self._client.connection.force_connect) 67 | 68 | self._client.last_query = QueryInfo() 69 | 70 | return await self.iter_process_ordinary_query( 71 | query, params=params, with_column_types=with_column_types, 72 | external_tables=external_tables, 73 | query_id=query_id, types_check=types_check 74 | ) 75 | 76 | async def process_ordinary_query_with_progress( 77 | self, query, params=None, with_column_types=False, 78 | external_tables=None, query_id=None, 79 | types_check=False, columnar=False): 80 | 81 | if params is not None: 82 | query = self._client.substitute_params(query, params) 83 | 84 | await self.run_in_executor( 85 | self._client.connection.send_query, query, query_id=query_id 86 | ) 87 | await self.run_in_executor( 88 | self._client.connection.send_external_tables, external_tables, 89 | types_check=types_check 90 | ) 91 | 92 | return await self.receive_result( 93 | with_column_types=with_column_types, progress=True, 94 | columnar=columnar 95 | ) 96 | 97 | async def iter_process_ordinary_query( 98 | self, query, params=None, with_column_types=False, 99 | external_tables=None, query_id=None, 100 | types_check=False): 101 | 102 | if params is not None: 103 | query = self._client.substitute_params(query, params) 104 | 105 | await self.run_in_executor( 106 | self._client.connection.send_query, query, query_id=query_id 107 | ) 108 | await self.run_in_executor( 109 | self._client.connection.send_external_tables, external_tables, 110 | types_check=types_check 111 | ) 112 | return self.iter_receive_result(with_column_types=with_column_types) 113 | 114 | async def cancel(self, with_column_types=False): 115 | # TODO: Add warning if already cancelled. 116 | await self.run_in_executor(self._client.connection.send_cancel) 117 | # Client must still read until END_OF_STREAM packet. 118 | return await self.receive_result(with_column_types=with_column_types) 119 | 120 | async def iter_receive_result(self, with_column_types=False): 121 | gen = self.packet_generator() 122 | rows_gen = IterQueryResult(gen, with_column_types=with_column_types) 123 | 124 | async for rows in rows_gen: 125 | for row in rows: 126 | yield row 127 | 128 | async def packet_generator(self): 129 | receive_packet = self._client.receive_packet 130 | while True: 131 | try: 132 | packet = await self.run_in_executor(receive_packet) 133 | if not packet: 134 | break 135 | 136 | if packet is True: 137 | continue 138 | 139 | yield packet 140 | 141 | except (Exception, KeyboardInterrupt): 142 | await self.disconnect() 143 | raise 144 | 145 | async def receive_result( 146 | self, with_column_types=False, progress=False, columnar=False): 147 | 148 | gen = self.packet_generator() 149 | 150 | if progress: 151 | return ProgressQueryResult( 152 | gen, with_column_types=with_column_types, columnar=columnar 153 | ) 154 | 155 | else: 156 | result = QueryResult( 157 | gen, with_column_types=with_column_types, columnar=columnar 158 | ) 159 | return await result.get_result() 160 | -------------------------------------------------------------------------------- /aioch/result.py: -------------------------------------------------------------------------------- 1 | from clickhouse_driver.progress import Progress 2 | 3 | 4 | class QueryResult(object): 5 | """ 6 | Stores query result from multiple blocks. 7 | """ 8 | 9 | def __init__( 10 | self, packet_generator, 11 | with_column_types=False, columnar=False): 12 | self.packet_generator = packet_generator 13 | self.with_column_types = with_column_types 14 | 15 | self.data = [] 16 | self.columns_with_types = [] 17 | self.columnar = columnar 18 | 19 | super(QueryResult, self).__init__() 20 | 21 | def store(self, packet): 22 | block = getattr(packet, 'block', None) 23 | if block is None: 24 | return 25 | 26 | # Header block contains no rows. Pick columns from it. 27 | if block.get_rows(): 28 | if self.columnar: 29 | columns = block.get_columns() 30 | if self.data: 31 | # Extend corresponding column. 32 | for i, column in enumerate(columns): 33 | self.data[i] += column 34 | else: 35 | self.data.extend(columns) 36 | else: 37 | self.data.extend(block.get_rows()) 38 | 39 | elif not self.columns_with_types: 40 | self.columns_with_types = block.columns_with_types 41 | 42 | async def get_result(self): 43 | """ 44 | :return: Stored query result. 45 | """ 46 | 47 | async for packet in self.packet_generator: 48 | self.store(packet) 49 | 50 | if self.with_column_types: 51 | return self.data, self.columns_with_types 52 | else: 53 | return self.data 54 | 55 | 56 | class ProgressQueryResult(QueryResult): 57 | """ 58 | Stores query result and progress information from multiple blocks. 59 | Provides iteration over query progress. 60 | """ 61 | 62 | def __init__( 63 | self, packet_generator, 64 | with_column_types=False, columnar=False): 65 | self.progress_totals = Progress() 66 | 67 | super(ProgressQueryResult, self).__init__( 68 | packet_generator, with_column_types, columnar 69 | ) 70 | 71 | def __aiter__(self): 72 | return self 73 | 74 | async def __anext__(self): 75 | while True: 76 | packet = await self.packet_generator.__anext__() 77 | progress_packet = getattr(packet, 'progress', None) 78 | if progress_packet: 79 | self.progress_totals.increment(progress_packet) 80 | return ( 81 | self.progress_totals.rows, self.progress_totals.total_rows 82 | ) 83 | else: 84 | self.store(packet) 85 | 86 | async def get_result(self): 87 | # Read all progress packets. 88 | async for _ in self: 89 | pass 90 | 91 | return await super(ProgressQueryResult, self).get_result() 92 | 93 | 94 | class IterQueryResult(object): 95 | """ 96 | Provides iteration over returned data by chunks (streaming by chunks). 97 | """ 98 | 99 | def __init__( 100 | self, packet_generator, 101 | with_column_types=False): 102 | self.packet_generator = packet_generator 103 | self.with_column_types = with_column_types 104 | 105 | self.first_block = True 106 | super(IterQueryResult, self).__init__() 107 | 108 | def __aiter__(self): 109 | return self 110 | 111 | async def __anext__(self): 112 | packet = await self.packet_generator.__anext__() 113 | block = getattr(packet, 'block', None) 114 | if block is None: 115 | return [] 116 | 117 | if self.first_block and self.with_column_types: 118 | self.first_block = False 119 | rv = [block.columns_with_types] 120 | rv.extend(block.get_rows()) 121 | return rv 122 | else: 123 | return block.get_rows() 124 | -------------------------------------------------------------------------------- /aioch/utils.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | 4 | def run_in_executor(executor, loop, func, *args, **kwargs): 5 | if kwargs: 6 | func = partial(func, **kwargs) 7 | 8 | return loop.run_in_executor(executor, func, *args) 9 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [db] 2 | host=localhost 3 | port=9000 4 | database=test 5 | user=default 6 | password= 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from codecs import open 2 | import os 3 | import re 4 | import sys 5 | from setuptools import setup 6 | 7 | 8 | PY_VER = sys.version_info 9 | 10 | 11 | if PY_VER < (3, 6): 12 | raise RuntimeError("aioch doesn't suppport Python earlier than 3.6") 13 | 14 | 15 | here = os.path.abspath(os.path.dirname(__file__)) 16 | 17 | 18 | def read_version(): 19 | regexp = re.compile(r'^VERSION\W*=\W*\(([^\(\)]*)\)') 20 | init_py = os.path.join(here, 'aioch', '__init__.py') 21 | with open(init_py) as f: 22 | for line in f: 23 | match = regexp.match(line) 24 | if match is not None: 25 | return match.group(1).replace(', ', '.') 26 | else: 27 | raise RuntimeError('Cannot find version in aioch/__init__.py') 28 | 29 | 30 | with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 31 | long_description = f.read() 32 | 33 | 34 | setup( 35 | name='aioch', 36 | version=read_version(), 37 | 38 | description=( 39 | 'Library for accessing a ClickHouse database over native interface ' 40 | 'from the asyncio' 41 | ), 42 | long_description=long_description, 43 | long_description_content_type='text/markdown', 44 | 45 | url='https://github.com/mymarilyn/aioch', 46 | 47 | author='Konstantin Lebedev', 48 | author_email='kostyan.lebedev@gmail.com', 49 | 50 | license='MIT', 51 | 52 | classifiers=[ 53 | 'Development Status :: 4 - Beta', 54 | 55 | 56 | 'Environment :: Console', 57 | 58 | 'Intended Audience :: Developers', 59 | 'Intended Audience :: Information Technology', 60 | 61 | 62 | 'License :: OSI Approved :: MIT License', 63 | 64 | 65 | 'Operating System :: OS Independent', 66 | 67 | 68 | 'Programming Language :: SQL', 69 | 'Programming Language :: Python :: 3', 70 | 'Programming Language :: Python :: 3.6', 71 | 'Programming Language :: Python :: 3.7', 72 | 'Programming Language :: Python :: 3.8', 73 | 74 | 'Topic :: Database', 75 | 'Topic :: Software Development', 76 | 'Topic :: Software Development :: Libraries', 77 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 78 | 'Topic :: Software Development :: Libraries :: Python Modules', 79 | 'Topic :: Scientific/Engineering :: Information Analysis' 80 | ], 81 | 82 | keywords='ClickHouse db database cloud analytics asyncio', 83 | 84 | packages=['aioch'], 85 | install_requires=[ 86 | 'clickhouse-driver>=0.1.3' 87 | ], 88 | test_suite='nose.collector', 89 | tests_require=[ 90 | 'nose' 91 | ], 92 | ) 93 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mymarilyn/aioch/5eee23002578ce65c103f77da098b31046d98ba3/tests/__init__.py -------------------------------------------------------------------------------- /tests/log.py: -------------------------------------------------------------------------------- 1 | from logging.config import dictConfig 2 | 3 | 4 | def configure(): 5 | dictConfig({ 6 | 'version': 1, 7 | 'disable_existing_loggers': False, 8 | 'formatters': { 9 | 'standard': { 10 | 'format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s' 11 | }, 12 | }, 13 | 'handlers': { 14 | 'default': { 15 | 'level': 'ERROR', 16 | 'formatter': 'standard', 17 | 'class': 'logging.StreamHandler', 18 | }, 19 | }, 20 | 'loggers': { 21 | '': { 22 | 'handlers': ['default'], 23 | 'level': 'ERROR', 24 | 'propagate': True 25 | }, 26 | } 27 | }) 28 | -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- 1 | import types 2 | 3 | from clickhouse_driver import errors 4 | 5 | from aioch import Client 6 | from tests.testcase import BaseTestCase 7 | 8 | 9 | class PacketsTestCase(BaseTestCase): 10 | def create_client(self): 11 | return Client( 12 | self.host, self.port, self.database, 'wrong_user', loop=self.loop 13 | ) 14 | 15 | def test_exception_on_hello_packet(self): 16 | async def run(): 17 | with self.assertRaises(errors.ServerException) as e: 18 | await self.client.execute('SHOW TABLES') 19 | 20 | # Simple exception formatting checks 21 | exc = e.exception 22 | self.assertIn('Code:', str(exc)) 23 | self.assertIn('Stack trace:', str(exc)) 24 | 25 | self.loop.run_until_complete(run()) 26 | 27 | 28 | class SelectTestCase(BaseTestCase): 29 | def test_simple_select(self): 30 | async def run(): 31 | rv = await self.client.execute('SELECT 2') 32 | self.assertEqual(rv, [(2,)]) 33 | 34 | self.loop.run_until_complete(run()) 35 | 36 | def test_from_url(self): 37 | client = Client.from_url(f'clickhouse://{self.host}', loop=self.loop) 38 | 39 | async def run(): 40 | rv = await client.execute('SELECT 2') 41 | self.assertEqual(rv, [(2,)]) 42 | 43 | self.loop.run_until_complete(run()) 44 | self.loop.run_until_complete(client.disconnect()) 45 | 46 | 47 | class ProgressTestCase(BaseTestCase): 48 | def test_select_with_progress(self): 49 | async def run(): 50 | progress = await self.client.execute_with_progress('SELECT 2') 51 | 52 | progress_rv = [] 53 | async for x in progress: 54 | progress_rv.append(x) 55 | 56 | self.assertEqual(progress_rv, [(1, 0)]) 57 | rv = await progress.get_result() 58 | self.assertEqual(rv, [(2,)]) 59 | 60 | self.loop.run_until_complete(run()) 61 | 62 | def test_select_with_progress_error(self): 63 | async def run(): 64 | with self.assertRaises(errors.ServerException): 65 | progress = await self.client.execute_with_progress( 66 | 'SELECT error' 67 | ) 68 | await progress.get_result() 69 | 70 | self.loop.run_until_complete(run()) 71 | 72 | def test_select_with_progress_no_progress_unwind(self): 73 | async def run(): 74 | progress = await self.client.execute_with_progress('SELECT 2') 75 | self.assertEqual(await progress.get_result(), [(2,)]) 76 | 77 | self.loop.run_until_complete(run()) 78 | 79 | def test_select_with_progress_cancel(self): 80 | async def run(): 81 | await self.client.execute_with_progress('SELECT 2') 82 | rv = await self.client.cancel() 83 | self.assertEqual(rv, [(2,)]) 84 | 85 | self.loop.run_until_complete(run()) 86 | 87 | 88 | class IterTestCase(BaseTestCase): 89 | def test_simple(self): 90 | async def run(): 91 | result = await self.client.execute_iter( 92 | 'SELECT number FROM system.numbers LIMIT 10' 93 | ) 94 | 95 | self.assertIsInstance(result, types.AsyncGeneratorType) 96 | 97 | self.assertEqual([i async for i in result], list(zip(range(10)))) 98 | self.assertEqual([i async for i in result], []) 99 | 100 | self.loop.run_until_complete(run()) 101 | 102 | def test_next(self): 103 | async def run(): 104 | result = await self.client.execute_iter( 105 | 'SELECT number FROM system.numbers LIMIT 3' 106 | ) 107 | 108 | self.assertIsInstance(result, types.AsyncGeneratorType) 109 | 110 | self.assertEqual(await result.__anext__(), (0, )) 111 | self.assertEqual(await result.__anext__(), (1, )) 112 | self.assertEqual(await result.__anext__(), (2, )) 113 | 114 | with self.assertRaises(StopAsyncIteration): 115 | await result.__anext__() 116 | 117 | self.loop.run_until_complete(run()) 118 | -------------------------------------------------------------------------------- /tests/testcase.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import configparser 3 | from contextlib import contextmanager 4 | import subprocess 5 | from unittest import TestCase 6 | 7 | from aioch.client import Client 8 | from tests import log 9 | 10 | 11 | log.configure() 12 | 13 | file_config = configparser.ConfigParser() 14 | file_config.read(['setup.cfg']) 15 | 16 | 17 | class BaseTestCase(TestCase): 18 | host = file_config.get('db', 'host') 19 | port = int(file_config.get('db', 'port')) 20 | database = file_config.get('db', 'database') 21 | user = file_config.get('db', 'user') 22 | password = file_config.get('db', 'password') 23 | 24 | client = None 25 | loop = None 26 | 27 | @classmethod 28 | def emit_cli(cls, statement, database=None): 29 | if database is None: 30 | database = cls.database 31 | 32 | args = [ 33 | 'clickhouse-client', 34 | '--database', database, 35 | '--host', cls.host, 36 | '--port', str(cls.port), 37 | '--query', str(statement) 38 | ] 39 | 40 | process = subprocess.Popen( 41 | args, stdout=subprocess.PIPE, stderr=subprocess.PIPE 42 | ) 43 | output = process.communicate() 44 | out, err = output 45 | 46 | if err: 47 | raise RuntimeError( 48 | 'Error during communication. {}'.format(err) 49 | ) 50 | 51 | return out.decode('utf-8') 52 | 53 | def create_client(self, **kwargs): 54 | kwargs.setdefault('loop', self.loop) 55 | 56 | return Client( 57 | self.host, self.port, self.database, self.user, self.password, 58 | **kwargs 59 | ) 60 | 61 | @classmethod 62 | def setUpClass(cls): 63 | cls.emit_cli( 64 | 'DROP DATABASE IF EXISTS {}'.format(cls.database), 'default' 65 | ) 66 | cls.emit_cli('CREATE DATABASE {}'.format(cls.database), 'default') 67 | super(BaseTestCase, cls).setUpClass() 68 | 69 | @classmethod 70 | def tearDownClass(cls): 71 | cls.emit_cli('DROP DATABASE {}'.format(cls.database)) 72 | super(BaseTestCase, cls).tearDownClass() 73 | 74 | def setUp(self): 75 | self.loop = asyncio.new_event_loop() 76 | self.client = self.create_client() 77 | super(BaseTestCase, self).setUp() 78 | 79 | def tearDown(self): 80 | self.loop.run_until_complete(self.client.disconnect()) 81 | self.loop.stop() 82 | super(BaseTestCase, self).setUp() 83 | 84 | @contextmanager 85 | def create_table(self, columns): 86 | self.emit_cli( 87 | 'CREATE TABLE test ({}) ''ENGINE = Memory'.format(columns) 88 | ) 89 | try: 90 | yield 91 | except Exception: 92 | raise 93 | finally: 94 | self.emit_cli('DROP TABLE test') 95 | --------------------------------------------------------------------------------