├── db_pool ├── __init__.py └── db │ ├── __init__.py │ ├── backends │ ├── __init__.py │ └── mysql │ │ ├── __init__.py │ │ └── base.py │ └── pool.py ├── demo ├── example │ ├── __init__.py │ ├── app │ │ ├── __init__.py │ │ ├── migrations │ │ │ ├── __init__.py │ │ │ └── 0001_initial.py │ │ ├── urls.py │ │ ├── models.py │ │ └── views.py │ ├── db_pool │ │ ├── __init__.py │ │ ├── db │ │ │ ├── __init__.py │ │ │ ├── backends │ │ │ │ ├── __init__.py │ │ │ │ └── mysql │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── base.py │ │ │ └── pool.py │ │ ├── README.md │ │ └── OTHER.md │ ├── tests │ │ ├── __init__.py │ │ ├── test.py │ │ └── tool.py │ ├── example │ │ ├── __init__.py │ │ ├── wsgi.py │ │ ├── urls.py │ │ └── settings.py │ ├── requirements.txt │ └── manage.py ├── docker-compose.yml └── Dockerfile ├── .gitignore └── README.md /db_pool/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_pool/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_pool/db/backends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/db_pool/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_pool/db/backends/mysql/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/db_pool/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/app/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/db_pool/db/backends/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/db_pool/db/backends/mysql/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demo/example/example/__init__.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | 3 | pymysql.install_as_MySQLdb() 4 | -------------------------------------------------------------------------------- /demo/example/requirements.txt: -------------------------------------------------------------------------------- 1 | ipython==7.4.0 2 | Django==2.0.5 3 | PyMySQL==0.9.3 4 | DBUtils==1.3 5 | 6 | requests==2.25.0 7 | -------------------------------------------------------------------------------- /demo/example/app/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from .views import TestView, TestTransactionView 4 | 5 | urlpatterns = [ 6 | path('view', TestView.as_view(), name='test'), 7 | path('transaction_view', TestTransactionView.as_view(), name='transaction'), 8 | path('threading_view', TestTransactionView.as_view(), name='threading'), 9 | 10 | ] 11 | -------------------------------------------------------------------------------- /demo/example/app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class User(models.Model): 5 | name = models.CharField(unique=True, max_length=50) 6 | age = models.IntegerField() 7 | 8 | 9 | class Category(models.Model): 10 | index = models.IntegerField(verbose_name='序号', default=0) 11 | name = models.CharField(verbose_name='分类', unique=True, max_length=50) 12 | -------------------------------------------------------------------------------- /demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | 3 | services: 4 | mysql: 5 | image: mysql:5.7 6 | restart: always 7 | environment: 8 | MYSQL_DATABASE: example 9 | MYSQL_ROOT_PASSWORD: devpwd123 10 | 11 | ports: 12 | - "3306:3306" 13 | 14 | django_db_pool: 15 | build: . 16 | 17 | volumes: 18 | - ./example:/opt/src 19 | 20 | ports: 21 | - "0.0.0.0:8888:8888" 22 | 23 | command: sleep 10000000000 24 | -------------------------------------------------------------------------------- /demo/example/example/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for example project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /demo/example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "example.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError as exc: 10 | raise ImportError( 11 | "Couldn't import Django. Are you sure it's installed and " 12 | "available on your PYTHONPATH environment variable? Did you " 13 | "forget to activate a virtual environment?") from exc 14 | execute_from_command_line(sys.argv) 15 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | # django APP 2 | # do not operate database in APP's docker 3 | # for there would be several apps, sharing one database 4 | # 5 | FROM python:3.6 6 | 7 | RUN curl -s http://ip-api.com | grep "China" > /dev/null && \ 8 | curl -s http://mirrors.163.com/.help/sources.list.jessie > /etc/apt/sources.list || true 9 | 10 | RUN apt-get update;\ 11 | apt-get install -y vim gettext;\ 12 | true 13 | 14 | COPY example /opt/src 15 | WORKDIR /opt/src 16 | 17 | RUN curl -s http://ip-api.com | grep "China" > /dev/null && \ 18 | pip install -r requirements.txt -i https://pypi.doubanio.com/simple --trusted-host pypi.doubanio.com || \ 19 | pip install -r requirements.txt 20 | -------------------------------------------------------------------------------- /demo/example/tests/test.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | import threading 4 | 5 | 6 | def test_view(): 7 | for i in range(1000): 8 | requests.get('http://localhost:8000/view') 9 | 10 | 11 | def test_transaction_view(): 12 | for i in range(1000): 13 | requests.get('http://localhost:8000/transaction_view') 14 | 15 | 16 | def test_threading_view(): 17 | for i in range(1000): 18 | requests.get('http://localhost:8000/threading_view') 19 | 20 | 21 | if __name__ == '__main__': 22 | threads = [] 23 | for target in [test_view, test_transaction_view, test_threading_view]: 24 | threads.append(threading.Thread(target=target)) 25 | 26 | for t in threads: 27 | t.start() 28 | 29 | for t in threads: 30 | t.join() 31 | -------------------------------------------------------------------------------- /demo/example/example/urls.py: -------------------------------------------------------------------------------- 1 | """example URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/2.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.urls import include, path 17 | 18 | urlpatterns = [ 19 | path('', include('app.urls')), 20 | ] 21 | -------------------------------------------------------------------------------- /demo/example/app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.5 on 2020-12-12 13:08 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Category', 16 | fields=[ 17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('index', models.IntegerField(default=0, verbose_name='序号')), 19 | ('name', models.CharField(max_length=50, unique=True, verbose_name='分类')), 20 | ], 21 | ), 22 | migrations.CreateModel( 23 | name='User', 24 | fields=[ 25 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 26 | ('name', models.CharField(max_length=50, unique=True)), 27 | ('age', models.IntegerField()), 28 | ], 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /demo/example/db_pool/README.md: -------------------------------------------------------------------------------- 1 | # django使用DBUtils实现连接池 2 | 3 | ## DBUtils 4 | DBUtils 是一套用于管理数据库连接池的Python包，为高频度高并发的数据库访问提供更好的性能，可以自动管理连接对象的创建和释放。并允许对非线程安全的数据库接口进行线程安全包装。 5 | 6 | DBUtils提供两种外部接口： 7 | 8 | * PersistentDB ：提供线程专用的数据库连接，并自动管理连接。 9 | * PooledDB ：提供线程间可共享的数据库连接，并自动管理连接。 10 | 11 | 12 | 实测证明 PersistentDB 的速度是最高的，但是在某些特殊情况下，数据库的连接过程可能异常缓慢，而此时的PooledDB则可以提供相对来说平均连接时间比较短的管理方式。 13 | 14 | ## django settings 15 | ``` 16 | DATABASES = { 17 | "default": { 18 | "ENGINE": "db_pool.db.backends.mysql", 19 | "NAME": "xxx", 20 | "USER": "xxx", 21 | "PASSWORD": "xxx", 22 | "HOST": "mysql", 23 | "PORT": "3306", 24 | "ATOMIC_REQUESTS": True, 25 | "CHARSET": "utf8", 26 | "COLLATION": "utf8_bin", 27 | "POOL": { 28 | "mincached": 5, 29 | "maxcached ": 500, 30 | } 31 | } 32 | } 33 | ``` 34 | 35 | ## Usages: 36 | sql: 37 | ``` 38 | from django.db import connection 39 | 40 | # 执行sql 41 | with connection.cursor() as cursor: 42 | cursor.execute("SELECT * from test_user",) 43 | row = cursor.fetchone() 44 | 45 | ``` 46 | -------------------------------------------------------------------------------- /db_pool/db/pool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from DBUtils.PooledDB import PooledDB 3 | from DBUtils.SteadyDB import SteadyDBConnection 4 | 5 | 6 | class DBPoolWrapper(object): 7 | def __init__(self, db_module): 8 | self._connection = None 9 | self._db_module = db_module 10 | self._pool = {} 11 | 12 | def __getattr__(self, item): 13 | return getattr(self._db_module, item) 14 | 15 | def connect(self, **kwargs): 16 | db = kwargs.get("db", "") 17 | db_pool = kwargs.pop('db_pool', {}) 18 | if db not in self._pool: 19 | self._pool[db] = PooledDB(creator=self._db_module, **db_pool, **kwargs) 20 | self._connection = self._pool[db].connection() 21 | return self._connection 22 | 23 | 24 | def autocommit(self, *args, **kwargs): 25 | self._con.autocommit(*args, **kwargs) 26 | 27 | 28 | def get_server_info(self): 29 | return self._con.get_server_info() 30 | 31 | 32 | @property 33 | def encoders(self): 34 | return self._con.encoders 35 | 36 | 37 | setattr(SteadyDBConnection, "autocommit", autocommit) 38 | setattr(SteadyDBConnection, "get_server_info", get_server_info) 39 | setattr(SteadyDBConnection, "encoders", encoders) 40 | -------------------------------------------------------------------------------- /demo/example/db_pool/db/pool.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from DBUtils.PooledDB import PooledDB 3 | from DBUtils.SteadyDB import SteadyDBConnection 4 | 5 | 6 | class DBPoolWrapper(object): 7 | def __init__(self, db_module): 8 | self._connection = None 9 | self._db_module = db_module 10 | self._pool = {} 11 | 12 | def __getattr__(self, item): 13 | return getattr(self._db_module, item) 14 | 15 | def connect(self, **kwargs): 16 | db = kwargs.get("db", "") 17 | db_pool = kwargs.pop('db_pool', {}) 18 | if db not in self._pool: 19 | self._pool[db] = PooledDB(creator=self._db_module, **db_pool, **kwargs) 20 | self._connection = self._pool[db].connection() 21 | return self._connection 22 | 23 | 24 | def autocommit(self, *args, **kwargs): 25 | self._con.autocommit(*args, **kwargs) 26 | 27 | 28 | def get_server_info(self): 29 | return self._con.get_server_info() 30 | 31 | 32 | @property 33 | def encoders(self): 34 | return self._con.encoders 35 | 36 | 37 | setattr(SteadyDBConnection, "autocommit", autocommit) 38 | setattr(SteadyDBConnection, "get_server_info", get_server_info) 39 | setattr(SteadyDBConnection, "encoders", encoders) 40 | -------------------------------------------------------------------------------- /db_pool/db/backends/mysql/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.utils import six 3 | from django.core.exceptions import ImproperlyConfigured 4 | from django.db.backends.mysql.base import DatabaseWrapper as _DatabaseWrapper 5 | from django.utils.safestring import SafeBytes, SafeText 6 | 7 | try: 8 | import MySQLdb as Database 9 | except ImportError as err: 10 | try: 11 | import pymysql as Database 12 | except ImportError: 13 | raise ImproperlyConfigured("Error loading MySQLdb or pymsql module.\n Did you install mysqlclient or pymysql") 14 | 15 | from db_pool.db.pool import DBPoolWrapper 16 | 17 | 18 | Database = DBPoolWrapper(Database) 19 | 20 | 21 | class DatabaseWrapper(_DatabaseWrapper): 22 | 23 | Database = Database 24 | 25 | def get_connection_params(self): 26 | params = super(DatabaseWrapper, self).get_connection_params() 27 | params['db_pool'] = self.settings_dict.get('POOL', {}) 28 | 29 | return params 30 | 31 | def get_new_connection(self, conn_params): 32 | conn = Database.connect(**conn_params) 33 | conn.encoders[SafeText] = conn.encoders[six.text_type] 34 | conn.encoders[SafeBytes] = conn.encoders[bytes] 35 | return conn 36 | -------------------------------------------------------------------------------- /demo/example/db_pool/db/backends/mysql/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.utils import six 3 | from django.core.exceptions import ImproperlyConfigured 4 | from django.db.backends.mysql.base import DatabaseWrapper as _DatabaseWrapper 5 | from django.utils.safestring import SafeBytes, SafeText 6 | 7 | try: 8 | import MySQLdb as Database 9 | except ImportError as err: 10 | try: 11 | import pymysql as Database 12 | except ImportError: 13 | raise ImproperlyConfigured("Error loading MySQLdb or pymsql module.\n Did you install mysqlclient or pymysql") 14 | 15 | from db_pool.db.pool import DBPoolWrapper 16 | 17 | 18 | Database = DBPoolWrapper(Database) 19 | 20 | 21 | class DatabaseWrapper(_DatabaseWrapper): 22 | 23 | Database = Database 24 | 25 | def get_connection_params(self): 26 | params = super(DatabaseWrapper, self).get_connection_params() 27 | params['db_pool'] = self.settings_dict.get('POOL', {}) 28 | 29 | return params 30 | 31 | def get_new_connection(self, conn_params): 32 | conn = Database.connect(**conn_params) 33 | conn.encoders[SafeText] = conn.encoders[six.text_type] 34 | conn.encoders[SafeBytes] = conn.encoders[bytes] 35 | return conn 36 | -------------------------------------------------------------------------------- /demo/example/tests/tool.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | 4 | 5 | """ 6 | show processlist; 7 | 8 | 9 | select concat('KILL ',id,';') from information_schema.processlist where user='root'; 10 | 11 | show status like ‘%变量名称%’； 12 | 13 | Aborted_clients 由于客户没有正确关闭连接已经死掉，已经放弃的连接数量。 14 | Aborted_connects 尝试已经失败的MySQL服务器的连接的次数。 15 | Connections 试图连接MySQL服务器的次数。 16 | Created_tmp_tables 当执行语句时，已经被创造了的隐含临时表的数量。 17 | Delayed_insert_threads 正在使用的延迟插入处理器线程的数量。 18 | Delayed_writes 用INSERT DELAYED写入的行数。 19 | Delayed_errors 用INSERT DELAYED写入的发生某些错误(可能重复键值)的行数。 20 | Flush_commands 执行FLUSH命令的次数。 21 | Handler_delete 请求从一张表中删除行的次数。 22 | Handler_read_first 请求读入表中第一行的次数。 23 | Handler_read_key 请求数字基于键读行。 24 | Handler_read_next 请求读入基于一个键的一行的次数。 25 | Handler_read_rnd 请求读入基于一个固定位置的一行的次数。 26 | Handler_update 请求更新表中一行的次数。 27 | Handler_write 请求向表中插入一行的次数。 28 | Key_blocks_used 用于关键字缓存的块的数量。 29 | Key_read_requests 请求从缓存读入一个键值的次数。 30 | Key_reads 从磁盘物理读入一个键值的次数。 31 | Key_write_requests 请求将一个关键字块写入缓存次数。 32 | Key_writes 将一个键值块物理写入磁盘的次数。 33 | Max_used_connections 同时使用的连接的最大数目。 34 | Not_flushed_key_blocks 在键缓存中已经改变但是还没被清空到磁盘上的键块。 35 | Not_flushed_delayed_rows 在INSERT DELAY队列中等待写入的行的数量。 36 | Open_tables 打开表的数量。 37 | Open_files 打开文件的数量。 38 | Open_streams 打开流的数量(主要用于日志记载） 39 | Opened_tables 已经打开的表的数量。 40 | Questions 发往服务器的查询的数量。 41 | Slow_queries 要花超过long_query_time时间的查询数量。 42 | Threads_connected 当前打开的连接的数量。 43 | Threads_running 不在睡眠的线程数量。 44 | Uptime 服务器工作了多少秒。 45 | 46 | """ 47 | 48 | 49 | def request_api(): 50 | for i in range(1000): 51 | requests.get('http://localhost:8000/view') 52 | 53 | 54 | if __name__ == '__main__': 55 | request_api() 56 | -------------------------------------------------------------------------------- /demo/example/app/views.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import uuid 3 | import random 4 | import time 5 | import threading 6 | from django.db import connection, transaction 7 | from django.shortcuts import HttpResponse 8 | from django.views import View 9 | 10 | from app.models import User, Category 11 | 12 | 13 | class TestView(View): 14 | def get(self, request, *args, **kwargs): 15 | """ORM和原生sql 16 | """ 17 | User.objects.create(name=str(uuid.uuid4()), age=22) 18 | with connection.cursor() as cursor: 19 | cursor.execute("SELECT * from app_user",) 20 | row = cursor.fetchone() 21 | 22 | return HttpResponse('good') 23 | 24 | 25 | class TestTransactionView(View): 26 | def get(self, request, *args, **kwargs): 27 | """数据库事务 28 | """ 29 | with transaction.atomic(): 30 | User.objects.create(name=str(uuid.uuid4()), age=18) 31 | Category.objects.create(index=int(random.random() * 1000), name=str(uuid.uuid4())) 32 | 33 | return HttpResponse('transaction good') 34 | 35 | 36 | class TestThreadingView(View): 37 | def get(self, request, *args, **kwargs): 38 | """多线程操作数据库 39 | """ 40 | def operator_thread(): 41 | User.objects.create(name=str(uuid.uuid4()), age=50) 42 | Category.objects.create(index=int(random.random() * 1000), name=str(uuid.uuid4())) 43 | time.sleep(5) 44 | 45 | threads = [] 46 | for x in range(10): 47 | threads.append(threading.Thread(target=operator_thread)) 48 | 49 | for t in threads: 50 | t.start() 51 | 52 | for t in threads: 53 | t.join() 54 | 55 | return HttpResponse('transaction good') 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # others 104 | .idea/ 105 | .DS_Store 106 | .pytest_cache/ 107 | 108 | *,cover 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django使用DBUtils实现连接池 2 | 3 | **Blog** 4 | - [Django数据库连接池（mysql）](https://pushiqiang.blog.csdn.net/article/details/51152755) 5 | 6 | - [关于在python项目中使用连接池的问题](https://pushiqiang.blog.csdn.net/article/details/125768647) 7 | 8 | 9 | 10 | #### 由于django没有连接池的功能，当并发量大时可能造成数据库连接超限。 11 | 12 | #### DBUtils 是一套用于管理数据库连接池的Python包，为高频度高并发的数据库访问提供更好的性能，可以自动管理连接对象的创建和释放。并允许对非线程安全的数据库接口进行线程安全包装。 13 | 14 | #### django结合DBUtils使用，数据库连接管理完全交给DBUtils处理，平滑切换db backends。 15 | 16 | ### django使用连接的过程： 17 | * 请求开始： 新建连接（从DBUtils管理的连接池中获取连接） 18 | * 请求结束： 关闭连接（实际由DBUtils将连接返还连接池） 19 | 20 | ## DBUtils 21 | DBUtils提供两种外部接口： 22 | 23 | * PersistentDB ：提供线程专用的数据库连接，并自动管理连接。 24 | * PooledDB ：提供线程间可共享的数据库连接，并自动管理连接。 25 | 26 | 实测证明 PersistentDB 的速度是最高的，但是在某些特殊情况下，数据库的连接过程可能异常缓慢，而此时的PooledDB则可以提供相对来说平均连接时间比较短的管理方式。 27 | 28 | ## django settings 29 | ``` 30 | DATABASES = { 31 | "default": { 32 | "ENGINE": "db_pool.db.backends.mysql", 33 | "NAME": "xxx", 34 | "USER": "xxx", 35 | "PASSWORD": "xxx", 36 | "HOST": "mysql", 37 | "PORT": "3306", 38 | "ATOMIC_REQUESTS": True, 39 | "CHARSET": "utf8", 40 | "COLLATION": "utf8_bin", 41 | "POOL": { 42 | "mincached": 5, 43 | "maxcached ": 500, 44 | } 45 | } 46 | } 47 | ``` 48 | ##### 其中连接池（POOL）配置参见 DBUtils的 PooledDB参数： 49 | ``` 50 | mincached: initial number of idle connections in the pool 51 | (0 means no connections are made at startup) 52 | maxcached: maximum number of idle connections in the pool 53 | (0 or None means unlimited pool size) 54 | maxshared: maximum number of shared connections 55 | (0 or None means all connections are dedicated) 56 | When this maximum number is reached, connections are 57 | shared if they have been requested as shareable. 58 | maxconnections: maximum number of connections generally allowed 59 | (0 or None means an arbitrary number of connections) 60 | blocking: determines behavior when exceeding the maximum 61 | (if this is set to true, block and wait until the number of 62 | connections decreases, otherwise an error will be reported) 63 | maxusage: maximum number of reuses of a single connection 64 | (0 or None means unlimited reuse) 65 | When this maximum usage number of the connection is reached, 66 | the connection is automatically reset (closed and reopened). 67 | setsession: optional list of SQL commands that may serve to prepare 68 | the session, e.g. ["set datestyle to ...", "set time zone ..."] 69 | reset: how connections should be reset when returned to the pool 70 | (False or None to rollback transcations started with begin(), 71 | True to always issue a rollback for safety's sake) 72 | failures: an optional exception class or a tuple of exception classes 73 | for which the connection failover mechanism shall be applied, 74 | if the default (OperationalError, InternalError) is not adequate 75 | ping: determines when the connection should be checked with ping() 76 | (0 = None = never, 1 = default = whenever fetched from the pool, 77 | 2 = when a cursor is created, 4 = when a query is executed, 78 | 7 = always, and all other bit combinations of these values) 79 | ``` 80 | -------------------------------------------------------------------------------- /demo/example/example/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for example project. 3 | 4 | Generated by 'django-admin startproject' using Django 2.0.5. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/2.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/2.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | # Quick-start development settings - unsuitable for production 19 | # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ 20 | 21 | # SECURITY WARNING: keep the secret key used in production secret! 22 | SECRET_KEY = '0)0tln&63-49=c!+(gp4j%%n^()27y3gyy!+-h=+fxtx^(i(tw' 23 | 24 | # SECURITY WARNING: don't run with debug turned on in production! 25 | DEBUG = True 26 | 27 | ALLOWED_HOSTS = ['*'] 28 | 29 | # Application definition 30 | 31 | INSTALLED_APPS = [ 32 | 'django.contrib.admin', 33 | 'django.contrib.auth', 34 | 'django.contrib.contenttypes', 35 | 'django.contrib.sessions', 36 | 'django.contrib.messages', 37 | 'app', 38 | ] 39 | 40 | MIDDLEWARE = [ 41 | 'django.middleware.security.SecurityMiddleware', 42 | 'django.contrib.sessions.middleware.SessionMiddleware', 43 | 'django.middleware.common.CommonMiddleware', 44 | 'django.middleware.csrf.CsrfViewMiddleware', 45 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 46 | 'django.contrib.messages.middleware.MessageMiddleware', 47 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 48 | ] 49 | 50 | ROOT_URLCONF = 'example.urls' 51 | 52 | 53 | WSGI_APPLICATION = 'example.wsgi.application' 54 | 55 | DATABASES = { 56 | "default": { 57 | "ENGINE": "db_pool.db.backends.mysql", 58 | "NAME": "example", 59 | "USER": "root", 60 | "PASSWORD": "devpwd123", 61 | "HOST": "172.19.0.2", 62 | "PORT": "3306", 63 | "ATOMIC_REQUESTS": True, 64 | "CHARSET": "utf8", 65 | "COLLATION": "utf8_bin", 66 | "POOL": { 67 | "mincached": 5, 68 | "maxcached": 500, 69 | } 70 | } 71 | } 72 | 73 | # Password validation 74 | # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators 75 | 76 | AUTH_PASSWORD_VALIDATORS = [ 77 | { 78 | 'NAME': 79 | 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 80 | }, 81 | { 82 | 'NAME': 83 | 'django.contrib.auth.password_validation.MinimumLengthValidator', 84 | }, 85 | { 86 | 'NAME': 87 | 'django.contrib.auth.password_validation.CommonPasswordValidator', 88 | }, 89 | { 90 | 'NAME': 91 | 'django.contrib.auth.password_validation.NumericPasswordValidator', 92 | }, 93 | ] 94 | 95 | # Internationalization 96 | # https://docs.djangoproject.com/en/2.0/topics/i18n/ 97 | 98 | LANGUAGE_CODE = 'en-us' 99 | 100 | TIME_ZONE = 'UTC' 101 | 102 | USE_I18N = True 103 | 104 | USE_L10N = False 105 | 106 | USE_TZ = True 107 | 108 | # Static files (CSS, JavaScript, Images) 109 | # https://docs.djangoproject.com/en/2.0/howto/static-files/ 110 | -------------------------------------------------------------------------------- /demo/example/db_pool/OTHER.md: -------------------------------------------------------------------------------- 1 | 都知道django每次请求都会连接数据库和释放数据库连接。Django为每个请求使用新的数据库连接。一开始这个方法行得通。然而随着服务器上的负载的增加,创建/销毁连接数据库开始花大量的时间。要避免这个，你可以使用数据库连接池，这里使用SQLAlchemy的连接池。使Django持久化数据库连接。 2 | 3 | ###但这种方法会改变django的代码。对框架有侵入 4 | 5 | 6 | #### 方法 1 7 | 实现方法如下： 8 | 　　把django/db/backends/mysql文件夹全部拷贝出来，放在项目的一个libs/mysql下面，然后修改base.py文件。 9 | 　　或者把django/db/backends/mysql文件夹在django/db/backends/下面复制为mysql_pool文件夹，将base.py中所以import中的mysql替换为mysql_pool，这样可以直接在settings.py中设置'ENGINE':'django.db.backends.mysql_pool' 10 | 找到 11 | ``` 12 | try: 13 | import MySQLdb as Database 14 | except ImportError as e: 15 | from django.core.exceptions import ImproperlyConfigured 16 | raise ImproperlyConfigured("Error loading MySQLdb module: %s" % e) 17 | ``` 18 | 这段代码，在下面添加： 19 | ``` 20 | from sqlalchemy import pool 21 | Database = pool.manage(Database[,recycle=DATABASE_WAIT_TIMEOUT-1]) 22 | #其中DATABASE_WAIT_TIMEOUT为你定义的连接超时时间，必须小于等于mysql里面的wait_timeout（） 23 | ``` 24 | 结果如下 25 | ``` 26 | try: 27 | import MySQLdb as Database 28 | except ImportError as e: 29 | from django.core.exceptions import ImproperlyConfigured 30 | raise ImproperlyConfigured("Error loading MySQLdb module: %s" % e) 31 | from sqlalchemy import pool 32 | Database = pool.manage(Database) 33 | ``` 34 | 然后找到get_connection_params(self)函数代码： 35 | ``` 36 | def get_connection_params(self): 37 | kwargs = { 38 | 'conv':django_conversions, 39 | 'charset':utf8 40 | } 41 | ... 42 | ``` 43 | 修改为： 44 | ``` 45 | def get_connection_params(self): 46 | kwargs = { 47 | 'charset':utf8 48 | } 49 | ... 50 | ``` 51 | 注意：如果不改变此处的kwargs，将会出现：TypeError:unhashable type:'dict' 的错误。 52 | 原样用kwargs传的话，sqlalchemy的pool会报unhashable错误，那是因为kwargs中有个key（conv）对应的value（django_conversions）是个字典，在pool中会把（key,value）组成元组作为新的key保存在pool中，但是因为value（django_conversions）是dict，不允许作为key的 53 | 54 | 在mysql里使用 show status 或 show processlist查看连接情况 55 | 56 | #### 方法 2 57 | 直接在settings.py同级目录下的__init__.py文件中添加如下代码 58 | ``` 59 | from django.conf import settings 60 | from django.db.utils import load_backend 61 | import sqlalchemy.pool as pool 62 | import logging 63 | pool_initialized=False 64 | 65 | def init_pool(): 66 | if not globals().get('pool_initialized', False): 67 | global pool_initialized 68 | pool_initialized = True 69 | try: 70 | backendname = settings.DATABASES['default']['ENGINE'] 71 | backend = load_backend(backendname) 72 | 73 | #replace the database object with a proxy. 74 | backend.Database = pool.manage(backend.Database) 75 | 76 | backend.DatabaseError = backend.Database.DatabaseError 77 | backend.IntegrityError = backend.Database.IntegrityError 78 | logging.info("Connection Pool initialized") 79 | except: 80 | logging.exception("Connection Pool initialization error") 81 | 82 | init_pool() 83 | ``` 84 | 然后修改django/db/backends/mysql/base.py文件 85 | 找到get_connection_params(self)函数代码： 86 | 修改为： 87 | ``` 88 | def get_connection_params(self): 89 | kwargs = { 90 | 'charset':utf8 91 | } 92 | ... 93 | ``` 94 | 同理，不修改kwargs将会出现：TypeError:unhashable type:'dict' 的错误。 95 | 96 | 以上两种方法都要改变django的代码，有一定入侵性，第二种方法改变要小一点 97 | 98 | 99 | django 1.7 100 | python 2.7 101 | sqlalchemy 1.0 102 | 103 | 104 | --------------------------------------------------------------------------------