├── .gitignore ├── MANIFEST.in ├── README.rst ├── README.txt ├── aiotarantool.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── benchmark.py ├── benchmark_hot.py ├── test_aiotarantool.py └── tnt_graphite.lua /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.pyc 3 | .idea 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.txt 2 | include README.rst 3 | include setup.py 4 | include aiotarantool.py 5 | recursive-include tests/ *.py 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Tarantool connection driver for work with asyncio 2 | ---------------------------------------------------------- 3 | Connector required tarantool version 1.6: 4 | 5 | $ pip install aiotarantool 6 | 7 | Try it example: 8 | 9 | .. code:: python 10 | 11 | import asyncio 12 | import aiotarantool 13 | 14 | cnt = 0 15 | 16 | async def insert_job(tnt): 17 | global cnt 18 | 19 | for it in range(2500): 20 | cnt += 1 21 | r = await tnt.insert("tester", (cnt, cnt)) 22 | 23 | loop = asyncio.get_event_loop() 24 | 25 | tnt = aiotarantool.connect("127.0.0.1", 3301) 26 | tasks = [loop.create_task(insert_job(tnt)) 27 | for _ in range(40)] 28 | 29 | loop.run_until_complete(asyncio.wait(tasks)) 30 | loop.run_until_complete(tnt.close()) 31 | loop.close() 32 | 33 | Under this scheme the aiotarantool driver makes a smaller number of read/write tarantool socket. 34 | 35 | See benchmark results time for insert/select/delete 100K tuples on 1.5KBytes: 36 | 37 | ========= ========= ========== 38 | call tarantool aiotarantool 39 | ========= ========= ========== 40 | insert 35.938047 12.701088 41 | select 24.389748 12.746204 42 | delete 35.224515 13.905095 43 | ========= ========= ========== 44 | 45 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | Tarantool connection driver for work with asyncio 2 | ---------------------------------------------------------- 3 | Connector required tarantool version 1.6: 4 | 5 | $ pip install aiotarantool 6 | 7 | Try it example: 8 | 9 | .. code:: python 10 | 11 | import asyncio 12 | import aiotarantool 13 | 14 | cnt = 0 15 | 16 | async def insert_job(tnt): 17 | global cnt 18 | 19 | for it in range(2500): 20 | cnt += 1 21 | r = await tnt.insert("tester", (cnt, cnt)) 22 | 23 | loop = asyncio.get_event_loop() 24 | 25 | tnt = aiotarantool.connect("127.0.0.1", 3301) 26 | tasks = [loop.create_task(insert_job(tnt)) 27 | for _ in range(40)] 28 | 29 | loop.run_until_complete(asyncio.wait(tasks)) 30 | loop.run_until_complete(tnt.close()) 31 | loop.close() 32 | 33 | Under this scheme the aiotarantool driver makes a smaller number of read/write tarantool socket. 34 | 35 | See benchmark results time for insert/select/delete 100K tuples on 1.5KBytes: 36 | 37 | ========= ========= ========== 38 | call tarantool aiotarantool 39 | ========= ========= ========== 40 | insert 35.938047 12.701088 41 | select 24.389748 12.746204 42 | delete 35.224515 13.905095 43 | ========= ========= ========== 44 | 45 | -------------------------------------------------------------------------------- /aiotarantool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __version__ = "1.1.5" 4 | 5 | import asyncio 6 | import socket 7 | import errno 8 | import msgpack 9 | import base64 10 | 11 | import tarantool 12 | from tarantool.response import Response 13 | from tarantool.request import ( 14 | Request, 15 | RequestCall, 16 | RequestDelete, 17 | RequestEval, 18 | RequestInsert, 19 | RequestJoin, 20 | RequestReplace, 21 | RequestPing, 22 | RequestSelect, 23 | RequestSubscribe, 24 | RequestUpdate, 25 | RequestUpsert, 26 | RequestAuthenticate) 27 | 28 | from tarantool.schema import SchemaIndex, SchemaSpace 29 | from tarantool.error import SchemaError 30 | import tarantool.const 31 | from tarantool.utils import check_key 32 | 33 | from tarantool.error import ( 34 | NetworkError, 35 | DatabaseError, 36 | SchemaReloadException) 37 | 38 | from tarantool.const import ( 39 | REQUEST_TYPE_OK, 40 | REQUEST_TYPE_ERROR, 41 | IPROTO_GREETING_SIZE, 42 | IPROTO_SYNC) 43 | 44 | try: 45 | from tarantool.const import ENCODING_DEFAULT 46 | except ImportError: 47 | from tarantool.utils import ENCODING_DEFAULT 48 | 49 | 50 | import logging 51 | 52 | logger = logging.getLogger(__package__) 53 | 54 | 55 | def connect(host, port, user=None, password=None, loop=None, encoding=ENCODING_DEFAULT, **kwargs): 56 | conn = Connection(host, port, user=user, password=password, loop=loop, encoding=encoding, **kwargs) 57 | 58 | return conn 59 | 60 | 61 | class Schema(object): 62 | def __init__(self, con): 63 | self.schema = {} 64 | self.con = con 65 | 66 | async def get_space(self, space): 67 | try: 68 | return self.schema[space] 69 | except KeyError: 70 | pass 71 | 72 | if not self.con.connected: 73 | await self.con.connect() 74 | 75 | async with self.con.lock: 76 | if space in self.schema: 77 | return self.schema[space] 78 | 79 | _index = (tarantool.const.INDEX_SPACE_NAME 80 | if isinstance(space, str) 81 | else tarantool.const.INDEX_SPACE_PRIMARY) 82 | 83 | array = await self.con.select(tarantool.const.SPACE_SPACE, space, index=_index) 84 | if len(array) > 1: 85 | raise SchemaError('Some strange output from server: \n' + array) 86 | 87 | if len(array) == 0 or not len(array[0]): 88 | temp_name = ('name' if isinstance(space, str) else 'id') 89 | raise SchemaError( 90 | "There's no space with {1} '{0}'".format(space, temp_name)) 91 | 92 | array = array[0] 93 | return SchemaSpace(array, self.schema) 94 | 95 | async def get_index(self, space, index): 96 | _space = await self.get_space(space) 97 | try: 98 | return _space.indexes[index] 99 | except KeyError: 100 | pass 101 | 102 | if not self.con.connected: 103 | await self.con.connect() 104 | 105 | async with self.con.lock: 106 | if index in _space.indexes: 107 | return _space.indexes[index] 108 | 109 | _index = (tarantool.const.INDEX_INDEX_NAME 110 | if isinstance(index, str) 111 | else tarantool.const.INDEX_INDEX_PRIMARY) 112 | 113 | array = await self.con.select(tarantool.const.SPACE_INDEX, [_space.sid, index], index=_index) 114 | 115 | if len(array) > 1: 116 | raise SchemaError('Some strange output from server: \n' + array) 117 | 118 | if len(array) == 0 or not len(array[0]): 119 | temp_name = ('name' if isinstance(index, str) else 'id') 120 | raise SchemaError( 121 | "There's no index with {2} '{0}' in space '{1}'".format( 122 | index, _space.name, temp_name)) 123 | 124 | array = array[0] 125 | return SchemaIndex(array, _space) 126 | 127 | def flush(self): 128 | self.schema.clear() 129 | 130 | 131 | class Connection(tarantool.Connection): 132 | DatabaseError = DatabaseError 133 | 134 | def __init__(self, host, port, user=None, password=None, connect_now=False, loop=None, 135 | encoding=ENCODING_DEFAULT, aiobuffer_size=16384, **kwargs): 136 | """just create instance, do not really connect by default""" 137 | 138 | super().__init__(host, port, 139 | user=user, 140 | password=password, 141 | connect_now=connect_now, 142 | encoding=encoding, **kwargs) 143 | 144 | self.aiobuffer_size = aiobuffer_size 145 | assert isinstance(self.aiobuffer_size, int) 146 | 147 | self.loop = loop or asyncio.get_event_loop() 148 | self.lock = asyncio.Lock(loop=self.loop) 149 | self._reader = None 150 | self._writer = None 151 | 152 | self.connect_now = connect_now 153 | self.connected = False 154 | self.req_num = 0 155 | 156 | self._waiters = dict() 157 | self._reader_task = None 158 | self._writer_task = None 159 | self._write_event = None 160 | self._write_buf = None 161 | self._greeting_event = None 162 | self._salt = None 163 | 164 | self.error = False # important not raise exception in response reader 165 | self.schema = Schema(self) # need schema with lock 166 | self.schema_version = 1 167 | 168 | async def connect(self): 169 | if self.connected: 170 | return 171 | 172 | async with self.lock: 173 | if self.connected: 174 | return 175 | 176 | await self._do_connect() 177 | 178 | self.connected = True 179 | 180 | async def _do_connect(self): 181 | logger.log(logging.DEBUG, "connecting to %r" % self) 182 | self._reader, self._writer = await asyncio.open_connection(self.host, self.port, loop=self.loop) 183 | 184 | self._reader_task = self.loop.create_task(self._response_reader()) 185 | self._writer_task = self.loop.create_task(self._response_writer()) 186 | self._write_event = asyncio.Event(loop=self.loop) 187 | self._write_buf = b"" 188 | 189 | self._greeting_event = asyncio.Event(loop=self.loop) 190 | 191 | if self.user and self.password: 192 | await self._greeting_event.wait() 193 | await self._authenticate(self.user, self.password) 194 | 195 | async def _response_writer(self): 196 | while True: 197 | await self._write_event.wait() 198 | 199 | if self._write_buf: 200 | to_write = self._write_buf 201 | self._write_buf = b"" 202 | self._writer.write(to_write) 203 | 204 | self._write_event.clear() 205 | 206 | async def _response_reader(self): 207 | # handshake 208 | greeting = await self._reader.read(IPROTO_GREETING_SIZE) 209 | self._salt = base64.decodestring(greeting[64:])[:20] 210 | self._greeting_event.set() 211 | 212 | buf = b"" 213 | while not self._reader.at_eof(): 214 | tmp_buf = await self._reader.read(self.aiobuffer_size) 215 | if not tmp_buf: 216 | await self._do_close( 217 | NetworkError(socket.error(errno.ECONNRESET, "Lost connection to server during query"))) 218 | 219 | buf += tmp_buf 220 | len_buf = len(buf) 221 | curr = 0 222 | 223 | while len_buf - curr >= 5: 224 | length_pack = buf[curr:curr + 5] 225 | length = msgpack.unpackb(length_pack) 226 | 227 | if len_buf - curr < 5 + length: 228 | break 229 | 230 | body = buf[curr + 5:curr + 5 + length] 231 | curr += 5 + length 232 | try: 233 | response = Response(self, body) # unpack response 234 | except SchemaReloadException as exp: 235 | if self.encoding is not None: 236 | unpacker = msgpack.Unpacker(use_list=True, encoding=self.encoding) 237 | else: 238 | unpacker = msgpack.Unpacker(use_list=True) 239 | 240 | unpacker.feed(body) 241 | header = unpacker.unpack() 242 | sync = header.get(IPROTO_SYNC, 0) 243 | 244 | waiter = self._waiters[sync] 245 | if not waiter.cancelled(): 246 | waiter.set_exception(exp) 247 | 248 | del self._waiters[sync] 249 | 250 | self.schema.flush() 251 | self.schema_version = exp.schema_version 252 | continue 253 | 254 | sync = response.sync 255 | if sync not in self._waiters: 256 | logger.error("aio git happens: {r}", response) 257 | continue 258 | 259 | waiter = self._waiters[sync] 260 | if not waiter.cancelled(): 261 | if response.return_code != 0: 262 | waiter.set_exception(DatabaseError(response.return_code, response.return_message)) 263 | else: 264 | waiter.set_result(response) 265 | 266 | del self._waiters[sync] 267 | 268 | # one cut for buffer 269 | if curr: 270 | buf = buf[curr:] 271 | 272 | await self._do_close(None) 273 | 274 | async def _wait_response(self, sync): 275 | resp = await self._waiters[sync] 276 | # renew request waiter 277 | self._waiters[sync] = asyncio.Future(loop=self.loop) 278 | return resp 279 | 280 | async def _send_request(self, request): 281 | assert isinstance(request, Request) 282 | 283 | if not self.connected: 284 | await self.connect() 285 | return (await self._send_request_no_check_connected(request)) 286 | 287 | async def _send_request_no_check_connected(self, request): 288 | while True: 289 | request_bytes = bytes(request) 290 | sync = request.sync 291 | waiter = self._waiters[sync] 292 | 293 | self._write_buf += request_bytes 294 | self._write_event.set() 295 | 296 | # read response 297 | try: 298 | response = await waiter 299 | except SchemaReloadException: 300 | continue 301 | 302 | return response 303 | 304 | def generate_sync(self): 305 | self.req_num += 1 306 | if self.req_num > 10000000: 307 | self.req_num = 0 308 | 309 | self._waiters[self.req_num] = asyncio.Future(loop=self.loop) 310 | return self.req_num 311 | 312 | async def close(self): 313 | await self._do_close(None) 314 | 315 | async def _do_close(self, exc): 316 | if not self.connected: 317 | return 318 | 319 | async with self.lock: 320 | self.connected = False 321 | self._writer.transport.close() 322 | self._reader_task.cancel() 323 | self._reader_task = None 324 | 325 | self._writer_task.cancel() 326 | self._writer_task = None 327 | self._write_event = None 328 | self._write_buf = None 329 | 330 | self._writer = None 331 | self._reader = None 332 | 333 | self._greeting_event = None 334 | 335 | for waiter in self._waiters.values(): 336 | if exc is None: 337 | waiter.cancel() 338 | else: 339 | waiter.set_exception(exc) 340 | 341 | self._waiters = dict() 342 | 343 | def __repr__(self): 344 | return "aiotarantool.Connection(host=%r, port=%r)" % (self.host, self.port) 345 | 346 | async def call(self, func_name, *args): 347 | assert isinstance(func_name, str) 348 | 349 | if len(args) == 1 and isinstance(args[0], (list, tuple)): 350 | args = args[0] 351 | 352 | co_varnames = RequestCall.__init__.__code__.co_varnames 353 | params = { 354 | "conn": self, 355 | "name": func_name, 356 | "args": args, 357 | } 358 | 359 | if "call_16" in co_varnames: 360 | # tarantool-python >= 0.6.1 361 | # 362 | params["call_16"] = None 363 | 364 | resp = await self._send_request(RequestCall(**params)) 365 | return resp 366 | 367 | async def eval(self, expr, *args): 368 | assert isinstance(expr, str) 369 | 370 | if len(args) == 1 and isinstance(args[0], (list, tuple)): 371 | args = args[0] 372 | 373 | resp = await self._send_request(RequestEval(self, expr, args)) 374 | return resp 375 | 376 | async def replace(self, space_name, values): 377 | if isinstance(space_name, str): 378 | sp = await self.schema.get_space(space_name) 379 | space_name = sp.sid 380 | 381 | resp = await self._send_request(RequestReplace(self, space_name, values)) 382 | return resp 383 | 384 | async def authenticate(self, user, password): 385 | self.user = user 386 | self.password = password 387 | 388 | if not self.connected: 389 | await self.connect() # connects and authorizes 390 | return 391 | else: # need only to authenticate 392 | await self._authenticate(user, password) 393 | 394 | async def _authenticate(self, user, password): 395 | assert self._salt, 'Server salt hasn\'t been received.' 396 | resp = await self._send_request_no_check_connected(RequestAuthenticate(self, self._salt, user, password)) 397 | return resp 398 | 399 | async def join(self, server_uuid): 400 | request = RequestJoin(self, server_uuid) 401 | resp = await self._send_request(request) 402 | while True: 403 | await resp 404 | if resp.code == REQUEST_TYPE_OK or resp.code >= REQUEST_TYPE_ERROR: 405 | return 406 | 407 | resp = await self._wait_response(request.sync) 408 | 409 | # close connection after JOIN 410 | await self.close() 411 | 412 | async def subscribe(self, cluster_uuid, server_uuid, vclock=None): 413 | vclock = vclock or dict() 414 | request = RequestSubscribe(self, cluster_uuid, server_uuid, vclock) 415 | resp = await self._send_request(request) 416 | while True: 417 | await resp 418 | if resp.code >= REQUEST_TYPE_ERROR: 419 | return 420 | 421 | resp = await self._wait_response(request.sync) 422 | 423 | # close connection after SUBSCRIBE 424 | await self.close() 425 | 426 | async def insert(self, space_name, values): 427 | if isinstance(space_name, str): 428 | sp = await self.schema.get_space(space_name) 429 | space_name = sp.sid 430 | 431 | res = await self._send_request(RequestInsert(self, space_name, values)) 432 | return res 433 | 434 | async def select(self, space_name, key=None, **kwargs): 435 | offset = kwargs.get("offset", 0) 436 | limit = kwargs.get("limit", 0xffffffff) 437 | index_name = kwargs.get("index", 0) 438 | iterator_type = kwargs.get("iterator", 0) 439 | 440 | key = check_key(key, select=True) 441 | 442 | if isinstance(space_name, str): 443 | sp = await self.schema.get_space(space_name) 444 | space_name = sp.sid 445 | 446 | if isinstance(index_name, str): 447 | idx = await self.schema.get_index(space_name, index_name) 448 | index_name = idx.iid 449 | 450 | res = await self._send_request( 451 | RequestSelect(self, space_name, index_name, key, offset, limit, iterator_type)) 452 | 453 | return res 454 | 455 | async def delete(self, space_name, key, **kwargs): 456 | index_name = kwargs.get("index", 0) 457 | 458 | key = check_key(key) 459 | if isinstance(space_name, str): 460 | sp = await self.schema.get_space(space_name) 461 | space_name = sp.sid 462 | 463 | if isinstance(index_name, str): 464 | idx = await self.schema.get_index(space_name, index_name) 465 | index_name = idx.iid 466 | 467 | res = await self._send_request( 468 | RequestDelete(self, space_name, index_name, key)) 469 | 470 | return res 471 | 472 | async def update(self, space_name, key, op_list, **kwargs): 473 | index_name = kwargs.get("index", 0) 474 | 475 | key = check_key(key) 476 | if isinstance(space_name, str): 477 | sp = await self.schema.get_space(space_name) 478 | space_name = sp.sid 479 | 480 | if isinstance(index_name, str): 481 | idx = await self.schema.get_index(space_name, index_name) 482 | index_name = idx.iid 483 | 484 | res = await self._send_request( 485 | RequestUpdate(self, space_name, index_name, key, op_list)) 486 | 487 | return res 488 | 489 | async def upsert(self, space_name, tuple_value, op_list, **kwargs): 490 | index_name = kwargs.get("index", 0) 491 | 492 | if isinstance(space_name, str): 493 | sp = await self.schema.get_space(space_name) 494 | space_name = sp.sid 495 | 496 | if isinstance(index_name, str): 497 | idx = await self.schema.get_index(space_name, index_name) 498 | index_name = idx.iid 499 | 500 | res = await self._send_request( 501 | RequestUpsert(self, space_name, index_name, tuple_value, op_list)) 502 | 503 | return res 504 | 505 | async def ping(self, notime=False): 506 | request = RequestPing(self) 507 | t0 = self.loop.time() 508 | await self._send_request(request) 509 | t1 = self.loop.time() 510 | 511 | if notime: 512 | return "Success" 513 | 514 | return t1 - t0 515 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tarantool>=0.5.4 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import re 4 | 5 | try: 6 | from setuptools import setup 7 | except ImportError: 8 | from distutils.core import setup 9 | 10 | 11 | def find_version(): 12 | for line in open("aiotarantool.py"): 13 | if line.startswith("__version__"): 14 | return re.match(r"""__version__\s*=\s*(['"])([^'"]+)\1""", line).group(2) 15 | 16 | setup( 17 | name="aiotarantool", 18 | py_modules=["aiotarantool"], 19 | version=find_version(), 20 | author="Dmitry Shveenkov", 21 | author_email="shveenkov@mail.ru", 22 | url="https://github.com/shveenkov/aiotarantool", 23 | classifiers=[ 24 | "Programming Language :: Python :: 3.4", 25 | "Intended Audience :: Developers", 26 | "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)", 27 | "Operating System :: OS Independent", 28 | "Topic :: Software Development :: Libraries :: Python Modules", 29 | "Topic :: Database :: Front-Ends" 30 | ], 31 | install_requires=[ 32 | "tarantool>=0.5.1", 33 | ], 34 | description="Tarantool connection driver for work with asyncio", 35 | long_description=open("README.rst").read() 36 | ) 37 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/benchmark.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import tarantool 5 | 6 | benchmark = { 7 | "tarantool": {}, 8 | "aiotarantool": {}, 9 | } 10 | 11 | cnt = 0 12 | 13 | tnt = tarantool.connect("127.0.0.1", 3301) 14 | 15 | import string 16 | mod_len = len(string.printable) 17 | data = [string.printable[it] * 1536 for it in range(mod_len)] 18 | 19 | # sync benchmark 20 | # insert test 21 | print("tarantool insert test") 22 | t1 = time.time() 23 | for it in range(100000): 24 | r = tnt.insert("tester", (it, data[it % mod_len])) 25 | 26 | t2 = time.time() 27 | benchmark["tarantool"]["insert"] = t2 - t1 28 | 29 | # select test 30 | print("tarantool select test") 31 | t1 = time.time() 32 | for it in range(100000): 33 | r = tnt.select("tester", it) 34 | 35 | t2 = time.time() 36 | benchmark["tarantool"]["select"] = t2 - t1 37 | 38 | # update test 39 | print("tarantool update test") 40 | t1 = time.time() 41 | for it in range(100000): 42 | r = tnt.update("tester", it, [("=", 2, it)]) 43 | 44 | 45 | t2 = time.time() 46 | benchmark["tarantool"]["update"] = t2 - t1 47 | 48 | # delete test 49 | print("tarantool delete test") 50 | t1 = time.time() 51 | for it in range(100000): 52 | r = tnt.delete("tester", it) 53 | 54 | t2 = time.time() 55 | benchmark["tarantool"]["delete"] = t2 - t1 56 | 57 | # gevent benchmark 58 | import asyncio 59 | import aiotarantool 60 | 61 | 62 | async def insert_job(tnt): 63 | global cnt 64 | 65 | for i in range(2500): 66 | cnt += 1 67 | await tnt.insert("tester", (cnt, data[cnt % mod_len])) 68 | 69 | 70 | async def select_job(tnt): 71 | global cnt 72 | 73 | for i in range(2500): 74 | cnt += 1 75 | await tnt.select("tester", cnt) 76 | 77 | 78 | async def update_job(tnt): 79 | global cnt 80 | 81 | for i in range(2500): 82 | cnt += 1 83 | await tnt.update("tester", cnt, [("=", 2, cnt)]) 84 | 85 | 86 | async def delete_job(tnt): 87 | global cnt 88 | 89 | for i in range(2500): 90 | cnt += 1 91 | await tnt.delete("tester", cnt) 92 | 93 | 94 | loop = asyncio.get_event_loop() 95 | 96 | tnt = aiotarantool.connect("127.0.0.1", 3301) 97 | 98 | # insert test 99 | print("aiotarantool insert test") 100 | t1 = loop.time() 101 | cnt = 0 102 | tasks = [loop.create_task(insert_job(tnt)) 103 | for _ in range(40)] 104 | 105 | loop.run_until_complete(asyncio.wait(tasks)) 106 | t2 = loop.time() 107 | benchmark["aiotarantool"]["insert"] = t2 - t1 108 | 109 | # select test 110 | print("aiotarantool select test") 111 | t1 = loop.time() 112 | cnt = 0 113 | tasks = [loop.create_task(select_job(tnt)) 114 | for _ in range(40)] 115 | 116 | loop.run_until_complete(asyncio.wait(tasks)) 117 | t2 = loop.time() 118 | benchmark["aiotarantool"]["select"] = t2 - t1 119 | 120 | # update test 121 | print("aiotarantool update test") 122 | t1 = loop.time() 123 | cnt = 0 124 | tasks = [loop.create_task(update_job(tnt)) 125 | for _ in range(40)] 126 | 127 | loop.run_until_complete(asyncio.wait(tasks)) 128 | t2 = loop.time() 129 | benchmark["aiotarantool"]["update"] = t2 - t1 130 | 131 | # delete test 132 | print("aiotarantool delete test") 133 | t1 = loop.time() 134 | cnt = 0 135 | tasks = [loop.create_task(delete_job(tnt)) 136 | for _ in range(40)] 137 | 138 | loop.run_until_complete(asyncio.wait(tasks)) 139 | t2 = loop.time() 140 | benchmark["aiotarantool"]["delete"] = t2 - t1 141 | 142 | loop.run_until_complete(tnt.close()) 143 | loop.close() 144 | 145 | print("\nbenchmark results:") 146 | print("call tarantool aiotarantool") 147 | for k in ("insert", "select", "update", "delete"): 148 | print("{2:6}: {0:0.6f} {1:0.6f}".format(benchmark["tarantool"][k], benchmark["aiotarantool"][k], k)) 149 | -------------------------------------------------------------------------------- /tests/benchmark_hot.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import asyncio 5 | import aiotarantool 6 | import string 7 | import multiprocessing 8 | 9 | 10 | class Bench(object): 11 | def __init__(self, aiotnt): 12 | self.tnt = aiotnt 13 | self.mod_len = len(string.printable) 14 | self.data = [string.printable[it] * 1536 for it in range(self.mod_len)] 15 | 16 | self.cnt_i = 0 17 | self.cnt_s = 0 18 | self.cnt_u = 0 19 | self.cnt_d = 0 20 | 21 | self.iter_max = 10000 22 | 23 | async def insert_job(self): 24 | for it in range(self.iter_max): 25 | try: 26 | await self.tnt.insert("tester", (it, self.data[it % self.mod_len])) 27 | self.cnt_i += 1 28 | except self.tnt.DatabaseError: 29 | pass 30 | 31 | async def select_job(self): 32 | for it in range(self.iter_max): 33 | rs = await self.tnt.select("tester", it) 34 | if len(rs): 35 | self.cnt_s += 1 36 | 37 | async def update_job(self): 38 | for it in range(self.iter_max): 39 | try: 40 | await self.tnt.update("tester", it, [("=", 2, it)]) 41 | self.cnt_u += 1 42 | except self.tnt.DatabaseError: 43 | pass 44 | 45 | async def delete_job(self): 46 | for it in range(0, self.iter_max, 2): 47 | rs = await self.tnt.delete("tester", it) 48 | if len(rs): 49 | self.cnt_d += 1 50 | 51 | 52 | def target_bench(loop): 53 | print("run process:", os.getpid()) 54 | tnt = aiotarantool.connect("127.0.0.1", 3301) 55 | bench = Bench(tnt) 56 | 57 | tasks = [] 58 | tasks += [loop.create_task(bench.insert_job()) 59 | for _ in range(20)] 60 | 61 | tasks += [loop.create_task(bench.select_job()) 62 | for _ in range(20)] 63 | 64 | tasks += [loop.create_task(bench.update_job()) 65 | for _ in range(20)] 66 | 67 | tasks += [loop.create_task(bench.delete_job()) 68 | for _ in range(20)] 69 | 70 | t1 = loop.time() 71 | loop.run_until_complete(asyncio.wait(tasks)) 72 | t2 = loop.time() 73 | 74 | loop.run_until_complete(tnt.close()) 75 | 76 | print("select=%d; insert=%d; update=%d; delete=%d; total=%d" % ( 77 | bench.cnt_s, bench.cnt_i, bench.cnt_u, bench.cnt_d, t2 - t1)) 78 | 79 | loop.close() 80 | 81 | 82 | loop = asyncio.get_event_loop() 83 | 84 | workers = [multiprocessing.Process(target=target_bench, args=(loop,)) 85 | for _ in range(22)] 86 | 87 | for worker in workers: 88 | worker.start() 89 | 90 | for worker in workers: 91 | worker.join() 92 | -------------------------------------------------------------------------------- /tests/test_aiotarantool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import asyncio 4 | import aiotarantool 5 | 6 | import logging 7 | logging.basicConfig(level=logging.DEBUG) 8 | 9 | import string 10 | 11 | mod_len = len(string.printable) 12 | data = [string.printable[it] * 1536 for it in range(mod_len)] 13 | 14 | cnt = 0 15 | 16 | 17 | async def insert_job(tnt): 18 | global cnt 19 | 20 | for it in range(2500): 21 | cnt += 1 22 | r = await tnt.insert("tester", (cnt, data[it % mod_len])) 23 | 24 | async def delete_job(tnt): 25 | global cnt 26 | 27 | for it in range(2500): 28 | cnt += 1 29 | r = await tnt.delete("tester", cnt) 30 | 31 | loop = asyncio.get_event_loop() 32 | 33 | # create tnt instance, not real connect 34 | tnt = aiotarantool.connect("127.0.0.1", 3301) 35 | 36 | tasks = [loop.create_task(delete_job(tnt)) 37 | for _ in range(40)] 38 | 39 | t1 = loop.time() 40 | loop.run_until_complete(asyncio.wait(tasks)) 41 | 42 | # close tnt connection 43 | loop.run_until_complete(tnt.close()) 44 | t2 = loop.time() 45 | 46 | print("total:", t2 - t1) 47 | 48 | loop.close() 49 | 50 | -------------------------------------------------------------------------------- /tests/tnt_graphite.lua: -------------------------------------------------------------------------------- 1 | 2 | fiber = require('fiber') 3 | socket = require('socket') 4 | log = require('log') 5 | 6 | local host = '127.0.0.1' 7 | local port = 2003 8 | 9 | fstat = function() 10 | local sock = socket('AF_INET', 'SOCK_DGRAM', 'udp') 11 | while true do 12 | local ts = tostring(math.floor(fiber.time())) 13 | info = { 14 | insert = box.stat.INSERT.rps, 15 | select = box.stat.SELECT.rps, 16 | update = box.stat.UPDATE.rps, 17 | delete = box.stat.DELETE.rps 18 | } 19 | 20 | for k, v in pairs(info) do 21 | sock:sendto(host, port, 'tnt.' .. k .. ' ' .. tostring(v) .. ' ' .. ts) 22 | end 23 | 24 | fiber.sleep(1) 25 | log.info('send stat ' .. ts) 26 | end 27 | end 28 | 29 | fiber.create(fstat) 30 | --------------------------------------------------------------------------------