├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pymysqlpool ├── __init__.py └── pool.py ├── requirements.txt ├── setup.py └── test ├── __init__.py └── test_pool.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | build/* 3 | dist/* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "3.4" 5 | - "3.5" 6 | - "3.5-dev" 7 | - "3.6" 8 | - "3.6-dev" 9 | - "3.7-dev" 10 | 11 | env: 12 | - CODECOV_TOKEN="e57b350a-75fb-40fd-964a-9c0e85dfa0ff" 13 | 14 | before_install: 15 | - "pip install -U pip" 16 | - "export PYTHONPATH=$PYTHONPATH:$(pwd)" 17 | 18 | install: 19 | - pip install -r requirements.txt 20 | - pip install codecov 21 | 22 | script: 23 | - coverage run test/test_pool.py 24 | 25 | after_success: 26 | - codecov 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 LICENSE 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyMySQLPool 2 | 3 | [![build status](https://travis-ci.org/prprprus/PyMySQLPool.svg?branch=master)](https://travis-ci.org/zongzhenh/PyMySQLPool.svg?branch=master) [![codecov](https://codecov.io/gh/zongzhenh/PyMySQLPool/branch/master/graph/badge.svg)](https://codecov.io/gh/zongzhenh/PyMySQLPool) [![pip version](https://img.shields.io/badge/pip-v18.1-blue.svg)](https://img.shields.io/badge/pip-v18.1-blue.svg) [![license](https://img.shields.io/dub/l/vibe-d.svg)](./LICENSE) 4 | 5 | PyMySQLPool is a pymysql-based database connection pool, simple and lightweight. 6 | 7 | Table of content 8 | 9 | - [Features](https://github.com/zongzhenh/PyMySQLPool#features) 10 | - [Requirements](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#requirements) 11 | - [Installation](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#installation) 12 | - [Example](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#example) 13 | - [Roadmap](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#roadmap) 14 | - [Resources](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#resources) 15 | - [License](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#license) 16 | - [Contributing](https://github.com/zongzhenh/PyMySQLPool/blob/master/README.md#contributing) 17 | 18 | ## Features 19 | 20 | - Maintain a minimum number of connection pools by default. 21 | - If a number of unuse connections less than zero, dynamically add connections to pool until the current number of inuse connections equal maximum of the pool. 22 | - Release the idle connections in regular until a number of unuse connections equal minimum of the pool. 23 | - Support auto-commit mode. 24 | - Support for ping check to get healthy connections. 25 | 26 | ## Requirements 27 | 28 | - Python 29 | - CPython : >= 3.4 30 | - MySQL Server -- one of the following: 31 | - MySQL >= 5.5 32 | - MariaDB >= 5.5 33 | - PyMySQL: >= 0.9.2 34 | 35 | ## Installation 36 | 37 | Package is uploaded on [PyPI](https://pypi.org/project/pymysql-pooling/) 38 | 39 | You can install with pip 40 | 41 | ``` 42 | $ pip install pymysql-pooling 43 | ``` 44 | 45 | ## Example 46 | 47 | Make use of a simple table (Example in [MySQL doc](https://dev.mysql.com/doc/refman/8.0/en/creating-tables.html)) 48 | 49 | ```mysql 50 | mysql> CREATE TABLE pet (name VARCHAR(20), owner VARCHAR(20), 51 | -> species VARCHAR(20), sex CHAR(1), birth DATE, death DATE); 52 | 53 | mysql> INSERT INTO pet 54 | -> VALUES ("Puffball", "Diane", "hamster", "f", "1999-03-30", NULL); 55 | ``` 56 | 57 | ```python 58 | from pymysqlpool.pool import Pool 59 | 60 | 61 | # Note: you can also add any parameters relates to `pymysql.connections.Connection` object 62 | pool = Pool(host=HOST, port=PORT, user=USER, password=PASSWORD, db=DB) 63 | pool.init() 64 | 65 | connection = pool.get_conn() 66 | cur = connection.cursor() 67 | cur.execute('SELECT * FROM `pet` WHERE `name`=%s', args=("Puffball", )) 68 | print(cur.fetchone()) 69 | 70 | pool.release(connection) 71 | ``` 72 | 73 | This example will print: 74 | 75 | ``` 76 | ('Puffball', 'Diane', 'hamster', 'f', datetime.date(1999, 3, 30), None) 77 | ``` 78 | 79 | Support autocommit mode, as following: 80 | 81 | ```python 82 | pool = Pool(host=HOST, port=PORT, user=USER, password=PASSWORD, db=DB, autocommit=True) 83 | ``` 84 | 85 | That's all. 86 | 87 | ## Roadmap 88 | 89 | - [x] Connection Pool 90 | - [x] Dynamically Create 91 | - [x] Dynamically Release 92 | - [ ] Monitor Web Interface 93 | 94 | ## Resources 95 | 96 | - [PyMySQL Documenation](https://pymysql.readthedocs.io/en/latest/index.html) 97 | - [MySQL Reference Manuals](https://dev.mysql.com/doc/refman/8.0/en/) 98 | 99 | ## License 100 | 101 | PyMySQLPool is released under the MIT License. See LICENSE for more information. 102 | 103 | ## Contributing 104 | 105 | Thank you for your interest in the contribution of PyMySQLPool, your help and contribution is very valuable. 106 | 107 | You can submit an issue and pull requests, please submit an issue before submitting pull requests. 108 | -------------------------------------------------------------------------------- /pymysqlpool/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Tiger. All Rights Reserved. 2 | # Licensed to MIT under a Contributor Agreement. 3 | 4 | __author__ = 'Tiger huangzongzhuan@gmail.com' 5 | -------------------------------------------------------------------------------- /pymysqlpool/pool.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | from time import time 3 | from threading import Thread, Lock, Condition, Timer 4 | 5 | 6 | class Error(Exception): 7 | """Base class for all pool-related exceptions""" 8 | pass 9 | 10 | 11 | class TimeoutError(Error): 12 | """The operation exceeded the given deadline""" 13 | pass 14 | 15 | 16 | def resize_pool(interval=600.0, stati_num=3, multiple=4, counter=0, 17 | accumulation=0, pool=None): 18 | """Resize connection pool in cycle""" 19 | assert (pool is not None) 20 | with pool.lock: 21 | if counter >= stati_num: 22 | avg = accumulation / stati_num 23 | if avg >= multiple: 24 | num = len(pool.unuse_list) - pool.min_size 25 | for _ in range(num): 26 | c = pool.unuse_list.pop() 27 | c.close() 28 | counter = 0 29 | accumulation = 0 30 | else: 31 | accumulation += (len(pool.unuse_list) + 1) / (len(pool.inuse_list) + 1) 32 | counter += 1 33 | t = Timer(interval, resize_pool, args=(interval, stati_num, multiple, 34 | counter, accumulation, pool)) 35 | t.start() 36 | 37 | 38 | class Pool(object): 39 | """ 40 | Connection pool for pymysql. 41 | 42 | The initialization parameters are as follows: 43 | :param host: Host of MySQL server 44 | :param port: Port of MySQL server 45 | :param user: User of MySQL server 46 | :param password: Password of MySQL server 47 | :param unix_socket: Optionally, you can use a unix socket rather than TCP/IP. 48 | :param db: Database of MySQL server 49 | :param charset: Charset of MySQL server 50 | :param cursorclass: Class of MySQL Cursor 51 | :param autocommit: auto commit mode 52 | :param min_size: Minimum size of connection pool 53 | :param max_size: Maximum size of connection pool 54 | :param timeout: Watting time in the multi-thread environment 55 | :param interval: Statistical cycle time 56 | :param stati_mun: Statistical frequency 57 | :param multiple: Regulation standard 58 | :param counter: Counter 59 | :param accumulation: Statiscal result 60 | :param ping_check: Verify if the conn is healthy after some amount of seconds (or always, or never). 61 | """ 62 | 63 | def __init__(self, 64 | host="localhost", 65 | port=3306, 66 | user=None, 67 | password=None, 68 | unix_socket=None, 69 | db=None, 70 | charset="utf8", 71 | cursorclass=pymysql.cursors.DictCursor, 72 | autocommit=False, 73 | min_size=1, 74 | max_size=3, 75 | timeout=10.0, 76 | interval=600.0, 77 | stati_num=3, 78 | multiple=4, 79 | counter=0, 80 | accumulation=0, 81 | ping_check: (int, bool) = False, 82 | **configs): 83 | self.host = host 84 | self.port = port 85 | self.user = user 86 | self.password = password 87 | self.db = db 88 | self.charset = charset 89 | self.cursorclass = cursorclass 90 | self.autocommit = autocommit 91 | 92 | self.min_size = min_size 93 | self.max_size = max_size 94 | self.current_size = 0 95 | self.timeout = timeout 96 | 97 | self.unuse_list = set() 98 | self.inuse_list = set() 99 | self.lock = Lock() 100 | self.cond = Condition(self.lock) 101 | 102 | self.interval = interval 103 | self.stati_num = stati_num 104 | self.multiple = multiple 105 | self.counter = 0 106 | self.accumulation = 0 107 | self.ping_check = ping_check 108 | 109 | self.unix_socket=unix_socket 110 | self.configs=configs 111 | 112 | def create_conn(self): 113 | """Create mysql connection by pymysql and to add unuse_list""" 114 | c = pymysql.connect( 115 | host=self.host, 116 | port=self.port, 117 | user=self.user, 118 | password=self.password, 119 | db=self.db, 120 | charset=self.charset, 121 | cursorclass=self.cursorclass, 122 | autocommit=self.autocommit, 123 | unix_socket=self.unix_socket, 124 | **self.configs 125 | ) 126 | self.unuse_list.add(c) 127 | 128 | def _start(self): 129 | """Start thread for resize pool""" 130 | t = Thread(target=resize_pool, 131 | args=(self.interval, self.stati_num, 132 | self.multiple, self.counter, 133 | self.accumulation, self), 134 | daemon=True) 135 | t.start() 136 | 137 | def _init_pool(self): 138 | """Initial minimum size of pool""" 139 | assert (self.min_size <= self.max_size) 140 | for _ in range(self.min_size): 141 | self.create_conn() 142 | 143 | def init(self): 144 | self._init_pool() 145 | self._start() 146 | 147 | def _wait(self): 148 | """Waiting condition""" 149 | return len(self.unuse_list) > 0 150 | 151 | def get_conn(self): 152 | with self.cond: 153 | return self.__get_conn() 154 | 155 | def __get_conn(self, retry_count=0): 156 | # Lack of resources and wait 157 | if len(self.unuse_list) <= 0 and \ 158 | self.current_size >= self.max_size: 159 | # note: TimeoutError mean release operation exception 160 | # or max_size much less than concurrence 161 | self.cond.wait_for(self._wait, self.timeout) 162 | if len(self.unuse_list) <= 0: 163 | raise TimeoutError 164 | # Lack of resources but can created 165 | if len(self.unuse_list) <= 0 and \ 166 | self.current_size < self.max_size: 167 | self.create_conn() 168 | 169 | return self.__get_safe_conn(retry_count) 170 | 171 | def __get_safe_conn(self, retry_count): 172 | self.current_size += 1 173 | c = self.unuse_list.pop() 174 | if self.ping_check: 175 | now = int(time()) 176 | timeout = now 177 | if isinstance(self.ping_check, int): 178 | timeout = timeout - self.ping_check 179 | if not hasattr(c, '__ping_check_timestamp'): 180 | c.__ping_check_timestamp = now 181 | try: 182 | if c.__ping_check_timestamp < timeout: 183 | c.__ping_check_timestamp = now 184 | c.ping() 185 | except: 186 | self.current_size -= 1 187 | if retry_count < 10: c = self.__get_conn(retry_count+1) 188 | if c: self.inuse_list.add(c) 189 | return c 190 | 191 | def get_pool_size(self): 192 | """Get current pool size""" 193 | return self.current_size 194 | 195 | def release(self, c): 196 | """Release connection from inuse_list to unuse_list""" 197 | with self.cond: 198 | self.current_size -= 1 199 | self.inuse_list.remove(c) 200 | self.unuse_list.add(c) 201 | self.cond.notify_all() 202 | 203 | def destroy(self): 204 | """Destroy pool""" 205 | for _ in range(len(self.unuse_list)): 206 | c = self.unuse_list.pop() 207 | c.close() 208 | for _ in range(len(self.inuse_list)): 209 | c = self.inuse_list.pop() 210 | c.close() 211 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyMySQL==0.9.2 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="pymysql-pooling", 8 | version="1.0.6", 9 | description="pymysql-based database connection pool", 10 | long_description=long_description, 11 | long_description_content_type="text/markdown", 12 | url="https://github.com/prprprus/PyMySQLPool", 13 | packages=setuptools.find_packages(), 14 | install_requires=["PyMySQL"], 15 | classifiers=[ 16 | "Programming Language :: Python :: 3", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent", 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/prprprus/PyMySQLPool/212110afaaa01f1d0d3faa8fe099ad19f48a8fbe/test/__init__.py -------------------------------------------------------------------------------- /test/test_pool.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest.mock import patch 3 | from pymysqlpool import pool 4 | from threading import Thread 5 | 6 | 7 | class MockMySQLConnection(object): 8 | 9 | def __init__(self, host='localhost', port=3306, user='root', 10 | password='None', db='None', charset='utf8'): 11 | self.host = host 12 | self.port = port 13 | self.user = user 14 | self.password = password 15 | self.db = db 16 | self.charset = charset 17 | 18 | def close(self): 19 | return 'Mock pymysql close method.' 20 | 21 | 22 | # mock method of pool 23 | def mock_create_conn(self): 24 | c = MockMySQLConnection() 25 | self.unuse_list.add(c) 26 | 27 | 28 | def mock__init_pool(self): 29 | assert (self.min_size <= self.max_size) 30 | for _ in range(self.min_size): 31 | mock_create_conn(self) 32 | 33 | 34 | def mock__start(self): 35 | pass 36 | 37 | 38 | # test case 39 | class TestPool(unittest.TestCase): 40 | 41 | def setUp(self): 42 | self.concurrence = 1000 43 | # unequal pre 44 | self.p = pool.Pool(min_size=10, max_size=90, timeout=10.0) 45 | 46 | def tearDown(self): 47 | pass 48 | 49 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 50 | def test_create_conn(self): 51 | assert (mock_create_conn == pool.Pool.create_conn) 52 | self.p.create_conn() 53 | assert (len(self.p.unuse_list) == 1) 54 | assert (filter(lambda x: isinstance(x, MockMySQLConnection), self.p.unuse_list)) 55 | 56 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 57 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 58 | @patch.object(pool.Pool, '_start', new=mock__start) 59 | def test_init(self): 60 | self.p.init() 61 | assert (len(self.p.unuse_list) == self.p.min_size) 62 | 63 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 64 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 65 | @patch.object(pool.Pool, '_start', new=mock__start) 66 | def test_get_conn(self): 67 | self.p.init() 68 | c = self.p.get_conn() 69 | assert (isinstance(c, MockMySQLConnection)) 70 | assert (len(self.p.unuse_list) == self.p.min_size - 1) 71 | assert (len(self.p.inuse_list) == 1) 72 | assert (filter(lambda x: isinstance(x, MockMySQLConnection), self.p.unuse_list)) 73 | assert (filter(lambda x: isinstance(x, MockMySQLConnection), self.p.inuse_list)) 74 | 75 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 76 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 77 | @patch.object(pool.Pool, '_start', new=mock__start) 78 | def test_release(self): 79 | self.p.init() 80 | c = self.p.get_conn() 81 | self.p.release(c) 82 | assert (len(self.p.unuse_list) == self.p.min_size) 83 | assert (len(self.p.inuse_list) == 0) 84 | 85 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 86 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 87 | @patch.object(pool.Pool, '_start', new=mock__start) 88 | def test_destroy(self): 89 | self.p.init() 90 | self.p.inuse_list.add(MockMySQLConnection()) 91 | self.p.destroy() 92 | assert (len(self.p.unuse_list) == 0) 93 | assert (len(self.p.inuse_list) == 0) 94 | 95 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 96 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 97 | @patch.object(pool.Pool, '_start', new=mock__start) 98 | def pure_get_and_release(self): 99 | c = self.p.get_conn() 100 | self.p.release(c) 101 | 102 | @patch.object(pool.Pool, 'create_conn', new=mock_create_conn) 103 | @patch.object(pool.Pool, '_init_pool', new=mock__init_pool) 104 | @patch.object(pool.Pool, '_start', new=mock__start) 105 | def test_get_conn_by_concurrence(self): 106 | self.p.init() 107 | for i in range(self.concurrence): 108 | t = Thread(target=self.pure_get_and_release) 109 | t.start() 110 | assert (len(self.p.unuse_list) == self.p.min_size) 111 | assert (len(self.p.inuse_list) == 0) 112 | assert (filter(lambda x: isinstance(x, MockMySQLConnection), self.p.unuse_list)) 113 | 114 | 115 | if __name__ == '__main__': 116 | unittest.main(verbosity=2) 117 | --------------------------------------------------------------------------------