├── .gitignore ├── README.rst ├── pykylin ├── __init__.py ├── connection.py ├── cursor.py ├── dialect.py ├── encoding.py ├── errors.py ├── log.py ├── proxy.py └── types.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # remove python bytecode files 2 | *.pyc 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Kylin DBAPI Driver and Sqlalchemy Dialect 2 | ============================== 3 | 4 | Installation 5 | ------------ 6 | 7 | To install this driver run:: 8 | 9 | $ pip install pykylin 10 | 11 | or from source:: 12 | 13 | $ pip install -r ./requirements.txt 14 | $ python setup.py install 15 | 16 | 17 | More info 18 | --------- 19 | 20 | * http://kylin.incubator.apache.org/ 21 | * http://www.sqlalchemy.org/ 22 | 23 | 24 | Authors 25 | ------- 26 | 27 | * Wu Xiang -------------------------------------------------------------------------------- /pykylin/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .connection import connect 4 | from .errors import * 5 | 6 | paramstyle = 'pyformat' 7 | threadsafety = 2 8 | 9 | __all__ = [ 10 | 'connect', 'apilevel', 'threadsafety', 'paramstyle', 11 | 'Warning', 'Error', 'InterfaceError', 'DatabaseError', 'DataError', 'OperationalError', 'IntegrityError', 12 | 'InternalError', 'ProgrammingError', 'NotSupportedError' 13 | ] 14 | -------------------------------------------------------------------------------- /pykylin/connection.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .cursor import Cursor 4 | from .proxy import Proxy 5 | from .log import logger 6 | 7 | class Connection(object): 8 | 9 | def __init__(self, username, password, endpoint, project, **kwargs): 10 | self.endpoint = endpoint 11 | self.username = username 12 | self.password = password 13 | self.project = project 14 | self.proxy = Proxy(self.endpoint) 15 | self.limit = kwargs['limit'] if 'limit' in kwargs else 50000 16 | 17 | self.proxy.login(self.username, self.password) 18 | 19 | def close(self): 20 | logger.debug('Connection close called') 21 | 22 | def commit(self): 23 | logger.warn('Transactional commit is not supported') 24 | 25 | def rollback(self): 26 | logger.warn('Transactional rollback is not supported') 27 | 28 | def list_tables(self): 29 | route = 'tables_and_columns' 30 | params = {'project': self.project} 31 | tables = self.proxy.get(route, params=params) 32 | return [t['table_NAME'] for t in tables] 33 | 34 | def list_columns(self, table_name): 35 | table_NAME = str(table_name).upper() 36 | route = 'tables_and_columns' 37 | params = {'project': self.project} 38 | tables = self.proxy.get(route, params=params) 39 | table = [t for t in tables if t['table_NAME'] == table_NAME][0] 40 | return table['columns'] 41 | 42 | def cursor(self): 43 | return Cursor(self) 44 | 45 | 46 | def connect(username='', password='', endpoint='', project='', **kwargs): 47 | return Connection(username, password, endpoint, project, **kwargs) 48 | -------------------------------------------------------------------------------- /pykylin/cursor.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from dateutil import parser 4 | 5 | from .errors import Error 6 | from .log import logger 7 | 8 | class Cursor(object): 9 | 10 | def __init__(self, connection): 11 | self.connection = connection 12 | self._arraysize = 1 13 | 14 | self.description = None 15 | self.rowcount = -1 16 | self.results = None 17 | self.fetched_rows = 0 18 | 19 | def callproc(self): 20 | raise('Stored procedures not supported in Kylin') 21 | 22 | def close(self): 23 | logger.debug('Cursor close called') 24 | 25 | def execute(self, operation, parameters={}, acceptPartial=True, limit=None, offset=0): 26 | sql = operation % parameters 27 | data = { 28 | 'sql': sql, 29 | 'offset': offset, 30 | 'limit': limit or self.connection.limit, 31 | 'acceptPartial': acceptPartial, 32 | 'project': self.connection.project 33 | } 34 | logger.debug("QUERY KYLIN: %s" % sql) 35 | resp = self.connection.proxy.post('query', json=data) 36 | 37 | column_metas = resp['columnMetas'] 38 | self.description = [ 39 | [c['label'], c['columnTypeName'], 40 | c['displaySize'], 0, 41 | c['precision'], c['scale'], c['isNullable']] 42 | for c in column_metas 43 | ] 44 | 45 | self.results = [self._type_mapped(r) for r in resp['results']] 46 | self.rowcount = len(self.results) 47 | self.fetched_rows = 0 48 | return self.rowcount 49 | 50 | def _type_mapped(self, result): 51 | meta = self.description 52 | size = len(meta) 53 | for i in range(0, size): 54 | column = meta[i] 55 | tpe = column[1] 56 | val = result[i] 57 | if tpe == 'DATE': 58 | val = parser.parse(val) 59 | elif tpe == 'BIGINT' or tpe == 'INT' or tpe == 'TINYINT': 60 | val = int(val) 61 | elif tpe == 'DOUBLE' or tpe == 'FLOAT': 62 | val = float(val) 63 | elif tpe == 'BOOLEAN': 64 | val = (val == 'true') 65 | result[i] = val 66 | return result 67 | 68 | def executemany(self, operation, seq_params=[]): 69 | results = [] 70 | for param in seq_params: 71 | self.execute(operation, param) 72 | results.extend(self.results) 73 | self.results = results 74 | self.rowcount = len(self.results) 75 | self.fetched_rows = 0 76 | return self.rowcount 77 | 78 | def fetchone(self): 79 | if self.fetched_rows < self.rowcount: 80 | row = self.results[self.fetched_rows] 81 | self.fetched_rows += 1 82 | return row 83 | else: 84 | return None 85 | 86 | def fetchmany(self, size=None): 87 | fetched_rows = self.fetched_rows 88 | size = size or self.arraysize 89 | self.fetched_rows = fetched_rows + size 90 | return self.results[fetched_rows:self.fetched_rows] 91 | 92 | def fetchall(self): 93 | fetched_rows = self.fetched_rows 94 | self.fetched_rows = self.rowcount 95 | return self.results[fetched_rows:] 96 | 97 | def nextset(self): 98 | raise Error('Nextset operation not supported in Kylin') 99 | 100 | @property 101 | def arraysize(self): 102 | return self._arraysize 103 | 104 | @arraysize.setter 105 | def arraysize(self, array_size): 106 | self._arraysize = array_size 107 | 108 | def setinputsizes(self): 109 | logger.warn('setinputsize not supported in Kylin') 110 | 111 | def setoutputsize(self): 112 | logger.warn('setoutputsize not supported in Kylin') 113 | -------------------------------------------------------------------------------- /pykylin/dialect.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from sqlalchemy import pool 4 | from sqlalchemy.sql.compiler import * 5 | from sqlalchemy.engine import default 6 | from .types import KYLIN_TYPE_MAP 7 | 8 | class KylinCompiler(SQLCompiler): 9 | 10 | def visit_label(self, label, 11 | add_to_result_map=None, 12 | within_label_clause=False, 13 | within_columns_clause=False, 14 | render_label_as_label=None, 15 | **kw): 16 | # only render labels within the columns clause 17 | # or ORDER BY clause of a select. dialect-specific compilers 18 | # can modify this behavior. 19 | render_label_with_as = (within_columns_clause and not 20 | within_label_clause) 21 | render_label_only = render_label_as_label is label 22 | 23 | if render_label_only or render_label_with_as: 24 | if isinstance(label.name, elements._truncated_label): 25 | labelname = self._truncated_identifier("colident", label.name) 26 | else: 27 | labelname = label.name 28 | 29 | # Hierarchical label is not supported in Kylin 30 | # quote label as a temp hack 31 | if labelname.find('.') >= 0: 32 | labelname = '"%s"' % labelname 33 | 34 | if render_label_with_as: 35 | if add_to_result_map is not None: 36 | add_to_result_map( 37 | labelname, 38 | label.name, 39 | (label, labelname, ) + label._alt_names, 40 | label.type 41 | ) 42 | 43 | return label.element._compiler_dispatch( 44 | self, within_columns_clause=True, 45 | within_label_clause=True, **kw) + \ 46 | OPERATORS[operators.as_] + \ 47 | self.preparer.format_label(label, labelname) 48 | elif render_label_only: 49 | return self.preparer.format_label(label, labelname) 50 | else: 51 | return label.element._compiler_dispatch( 52 | self, within_columns_clause=False, **kw) 53 | 54 | def visit_column(self, column, add_to_result_map=None, 55 | include_table=True, **kwargs): 56 | name = orig_name = column.name 57 | if name is None: 58 | raise exc.CompileError("Cannot compile Column object until " 59 | "its 'name' is assigned.") 60 | 61 | is_literal = column.is_literal 62 | if not is_literal and isinstance(name, elements._truncated_label): 63 | name = self._truncated_identifier("colident", name) 64 | 65 | if add_to_result_map is not None: 66 | add_to_result_map( 67 | name, 68 | orig_name, 69 | (column, name, column.key), 70 | column.type 71 | ) 72 | 73 | if is_literal: 74 | name = self.escape_literal_column(name) 75 | elif name.find('.') >= 0: 76 | # Hierarchical label is not supported in Kylin 77 | # quote label as a temp hack 78 | name = '"%s"' % name 79 | name = self.preparer.quote(name) 80 | else: 81 | name = self.preparer.quote(name) 82 | 83 | table = column.table 84 | if table is None or not include_table or not table.named_with_column: 85 | return name 86 | else: 87 | if table.schema: 88 | schema_prefix = self.preparer.quote_schema(table.schema) + '.' 89 | else: 90 | schema_prefix = '' 91 | tablename = table.name 92 | if isinstance(tablename, elements._truncated_label): 93 | tablename = self._truncated_identifier("alias", tablename) 94 | 95 | return schema_prefix + \ 96 | self.preparer.quote(tablename) + \ 97 | "." + name 98 | 99 | 100 | class KylinIdentifierPreparer(IdentifierPreparer): 101 | # Kylin is case sensitive, temp hack to turn off name quoting 102 | def __init__(self, dialect, initial_quote='', 103 | final_quote=None, escape_quote='', omit_schema=False): 104 | super(KylinIdentifierPreparer, self).__init__(dialect, initial_quote, final_quote, escape_quote, omit_schema) 105 | 106 | 107 | class KylinDialect(default.DefaultDialect): 108 | name = 'kylin' 109 | driver = 'pykylin' 110 | 111 | preparer = KylinIdentifierPreparer 112 | preexecute_pk_sequences = True 113 | supports_pk_autoincrement = True 114 | supports_sequences = True 115 | sequences_optional = True 116 | supports_native_decimal = True 117 | supports_default_values = True 118 | supports_native_boolean = True 119 | poolclass = pool.SingletonThreadPool 120 | supports_unicode_statements = True 121 | 122 | default_paramstyle = 'pyformat' 123 | 124 | def __init__(self, **kwargs): 125 | super(KylinDialect, self).__init__(self, **kwargs) 126 | 127 | @classmethod 128 | def dbapi(cls): 129 | return __import__('pykylin') 130 | 131 | def initialize(self, connection): 132 | self.server_version_info = None 133 | self.default_schema_name = None 134 | self.default_isolation_level = None 135 | self.returns_unicode_strings = True 136 | 137 | def create_connect_args(self, url): 138 | opts = url.translate_connect_args() 139 | args = { 140 | 'username': opts['username'], 141 | 'password': opts['password'], 142 | 'endpoint': 'http://%s:%s/%s' % (opts['host'], opts['port'], opts['database']) 143 | } 144 | args.update(url.query) 145 | return [], args 146 | 147 | def get_table_names(self, connection, schema=None, **kw): 148 | return connection.connection.list_tables() 149 | 150 | def has_table(self, connection, table_name, schema=None): 151 | return table_name in self.get_table_names(connection, table_name, schema) 152 | 153 | def has_sequence(self, connection, sequence_name, schema=None): 154 | return False 155 | 156 | def get_columns(self, connection, table_name, schema=None, **kw): 157 | cols = connection.connection.list_columns(table_name) 158 | return [self._map_column_type(c) for c in cols] 159 | 160 | def _map_column_type(self, column): 161 | tpe_NAME = column['type_NAME'] 162 | if tpe_NAME.startswith('VARCHAR'): 163 | tpe_size = column['column_SIZE'] 164 | args = (tpe_size,) 165 | tpe = KYLIN_TYPE_MAP['VARCHAR'] 166 | elif tpe_NAME == 'DECIMAL': 167 | digit_size = column['decimal_DIGITS'] 168 | args = (digit_size,) 169 | tpe = KYLIN_TYPE_MAP['DECIMAL'] 170 | else: 171 | args = () 172 | tpe = KYLIN_TYPE_MAP[tpe_NAME] 173 | column_tpe = tpe(*args) 174 | return { 175 | 'name': column['column_NAME'].lower(), 176 | 'type': column_tpe 177 | } 178 | 179 | def get_foreign_keys(self, connection, table_name, schema=None, **kw): 180 | return [] 181 | 182 | def get_indexes(self, connection, table_name, schema=None, **kw): 183 | return [] 184 | 185 | def get_view_names(self, connection, schema=None, **kw): 186 | return [] 187 | 188 | def get_pk_constraint(self, conn, table_name, schema=None, **kw): 189 | return {} 190 | 191 | def get_unique_constraints( 192 | self, connection, table_name, schema=None, **kw): 193 | return [] 194 | 195 | -------------------------------------------------------------------------------- /pykylin/encoding.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from json import JSONEncoder, JSONDecoder, loads 4 | from json.decoder import WHITESPACE 5 | 6 | class KylinJSONEncoder(JSONEncoder): 7 | 8 | def default(self, o): 9 | return super(KylinJSONEncoder, self).default(self, o) 10 | 11 | class KylinJSONDecoder(JSONDecoder): 12 | 13 | def decode(self, s, _w=WHITESPACE.match): 14 | return super(KylinJSONDecoder, self).decode(s) 15 | 16 | def decode(s): 17 | return loads(s, cls=KylinJSONDecoder) 18 | -------------------------------------------------------------------------------- /pykylin/errors.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | class Error(Exception): 4 | 5 | def __init__(self, msg): 6 | super(Error, self).__init__(msg) 7 | self.msg = msg 8 | -------------------------------------------------------------------------------- /pykylin/log.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | import sys 5 | 6 | logger = logging.getLogger('Kylin') 7 | logger.setLevel(logging.DEBUG) 8 | ch = logging.StreamHandler(sys.stdout) 9 | ch.setLevel(logging.DEBUG) 10 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 11 | ch.setFormatter(formatter) 12 | logger.addHandler(ch) 13 | -------------------------------------------------------------------------------- /pykylin/proxy.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import requests 4 | from requests.auth import HTTPBasicAuth 5 | 6 | from .encoding import decode 7 | from .errors import Error 8 | from .log import logger 9 | 10 | class Proxy(object): 11 | 12 | def __init__(self, base_url): 13 | self.base_url = base_url 14 | self.cookies = {} 15 | self.user = None 16 | self.password = None 17 | self.auth = None 18 | self.headers = {'Content-Type': 'application/json;charset=UTF-8'} 19 | 20 | def login(self, user, password): 21 | route = 'user/authentication' 22 | url = '%s/%s' % (self.base_url, route) 23 | self.user = user 24 | self.password = user 25 | self.auth = HTTPBasicAuth(user, password) 26 | resp = requests.post(url, auth=self.auth, headers=self.headers) 27 | 28 | if resp.status_code != 200: 29 | raise Error('Login failed with username: "%s"' % self.user) 30 | else: 31 | logger.info('Login success with username: "%s"' % self.user) 32 | logger.debug('Authority details: %s', resp.text) 33 | 34 | jsesion_guid = resp.cookies['JSESSIONID'] 35 | self.set_cookie('JSESSIONID', jsesion_guid) 36 | 37 | def request(self, method, route, **kwargs): 38 | url = '%s/%s' % (self.base_url, route) 39 | resp = requests.request(method, url, headers=self.headers, cookies=self.cookies, auth=self.auth, **kwargs) 40 | 41 | if resp.status_code != 200: 42 | exception = 'Unknown' 43 | try: 44 | err = decode(resp.text) 45 | exception = err['exception'] 46 | except ValueError: 47 | pass 48 | finally: 49 | raise Error('Error when requesting: "%s", exception: "%s"' % (route, exception)) 50 | 51 | return decode(resp.text) 52 | 53 | def post(self, route, **kwargs): 54 | return self.request('post', route, **kwargs) 55 | 56 | def get(self, route, **kwargs): 57 | return self.request('get', route, **kwargs) 58 | 59 | def set_cookie(self, cookie_key, cookie_value): 60 | self.cookies[cookie_key] = cookie_value 61 | 62 | def clear_cookie(self): 63 | self.cookies.clear() 64 | -------------------------------------------------------------------------------- /pykylin/types.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from sqlalchemy import types as sqltypes 4 | from sqlalchemy.types import INTEGER, BIGINT, SMALLINT, VARCHAR, CHAR, \ 5 | FLOAT, DATE, BOOLEAN 6 | 7 | class DOUBLE(sqltypes.Float): 8 | __visit_name__ = 'DOUBLE' 9 | 10 | 11 | class TINYINT(sqltypes.Integer): 12 | __visit_name__ = "TINYINT" 13 | 14 | 15 | KYLIN_TYPE_MAP = { 16 | 'TINYINT': TINYINT, 17 | 'BIGINT': BIGINT, 18 | 'BOOLEAN': BOOLEAN, 19 | 'CHAR': CHAR, 20 | 'DATE': DATE, 21 | 'DOUBLE': DOUBLE, 22 | 'INT': INTEGER, 23 | 'INTEGER': INTEGER, 24 | 'FLOAT': FLOAT, 25 | 'SMALLINT': SMALLINT, 26 | 'VARCHAR': VARCHAR, 27 | } 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.8.1 2 | python-dateutil==2.4.2 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | import os 3 | 4 | version = "0.0.1" 5 | 6 | readme = open(os.path.join(os.path.dirname(__file__), 'README.rst')).read() 7 | 8 | req_file = os.path.join(os.path.dirname(__file__), 'requirements.txt') 9 | requirements = [i.strip() for i in open(req_file).readlines()] 10 | 11 | setup_params = dict( 12 | name="pykylin", 13 | version=version, 14 | description="Kylin DBAPI Driver", 15 | author="Wu Xiang", 16 | author_email="w.xiang7@gmail.com", 17 | long_description=readme, 18 | classifiers=[ 19 | "Development Status :: 3 - Alpha", 20 | 'Environment :: Console', 21 | 'Intended Audience :: Developers', 22 | 'Programming Language :: Python', 23 | 'Programming Language :: Python :: 3', 24 | 'Programming Language :: Python :: Implementation :: CPython', 25 | 'Programming Language :: Python :: Implementation :: PyPy', 26 | 'Topic :: Database :: Front-Ends', 27 | ], 28 | keywords='Kylin SQLAlchemy', 29 | packages=find_packages(), 30 | include_package_data=True, 31 | zip_safe=False, 32 | entry_points={ 33 | "sqlalchemy.dialects": 34 | ["kylin = pykylin.dialect:KylinDialect"] 35 | }, 36 | install_requires=requirements 37 | ) 38 | 39 | if __name__ == '__main__': 40 | setup(**setup_params) 41 | --------------------------------------------------------------------------------